From dac61d7c1e4b90ec3030ab1827be0f9ee5987ee7 Mon Sep 17 00:00:00 2001 From: frank chen Date: Fri, 3 Dec 2021 10:50:35 +0800 Subject: [PATCH 001/403] Support hex() on UUID Signed-off-by: frank chen --- src/Functions/FunctionsBinaryRepr.cpp | 70 +++++++++++++++++-- .../queries/0_stateless/00396_uuid.reference | 1 + tests/queries/0_stateless/00396_uuid.sql | 5 ++ 3 files changed, 69 insertions(+), 7 deletions(-) diff --git a/src/Functions/FunctionsBinaryRepr.cpp b/src/Functions/FunctionsBinaryRepr.cpp index 20b2acac88a..4f5a215a251 100644 --- a/src/Functions/FunctionsBinaryRepr.cpp +++ b/src/Functions/FunctionsBinaryRepr.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -42,7 +43,7 @@ struct HexImpl static constexpr size_t word_size = 2; template - static void executeOneUInt(T x, char *& out) + static void executeOneUInt(T x, char *& out, bool auto_close = true) { bool was_nonzero = false; for (int offset = (sizeof(T) - 1) * 8; offset >= 0; offset -= 8) @@ -57,8 +58,11 @@ struct HexImpl writeHexByteUppercase(byte, out); out += word_size; } - *out = '\0'; - ++out; + if (auto_close) + { + *out = '\0'; + ++out; + } } static void executeOneString(const UInt8 * pos, const UInt8 * end, char *& out) @@ -130,7 +134,7 @@ struct BinImpl static constexpr size_t word_size = 8; template - static void executeOneUInt(T x, char *& out) + static void executeOneUInt(T x, char *& out, bool auto_close = true) { bool was_nonzero = false; for (int offset = (sizeof(T) - 1) * 8; offset >= 0; offset -= 8) @@ -145,8 +149,11 @@ struct BinImpl writeBinByte(byte, out); out += word_size; } - *out = '\0'; - ++out; + if (auto_close) + { + *out = '\0'; + ++out; + } } template @@ -275,6 +282,7 @@ public: !which.isUInt() && !which.isFloat() && !which.isDecimal() && + !which.isUUID() && !which.isAggregateFunction()) throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); @@ -306,7 +314,8 @@ public: tryExecuteFloat(column, res_column) || tryExecuteDecimal(column, res_column) || tryExecuteDecimal(column, res_column) || - tryExecuteDecimal(column, res_column)) + tryExecuteDecimal(column, res_column) || + tryExecuteUUID(column, res_column)) return res_column; throw Exception("Illegal column " + arguments[0].column->getName() @@ -480,6 +489,53 @@ public: return false; } } + + bool tryExecuteUUID(const IColumn * col, ColumnPtr & col_res) const + { + const ColumnUUID * col_vec = checkAndGetColumn(col); + + static constexpr size_t MAX_LENGTH = sizeof(UUID) * word_size + 1; /// Including trailing zero byte. + + if (col_vec) + { + auto col_str = ColumnString::create(); + ColumnString::Chars & out_vec = col_str->getChars(); + ColumnString::Offsets & out_offsets = col_str->getOffsets(); + + const typename ColumnUUID::Container & in_vec = col_vec->getData(); + const UUID* uuid = in_vec.data(); + + size_t size = in_vec.size(); + out_offsets.resize(size); + out_vec.resize(size * (word_size+1) + MAX_LENGTH); /// word_size+1 is length of one byte in hex/bin plus zero byte. + + size_t pos = 0; + for (size_t i = 0; i < size; ++i) + { + /// Manual exponential growth, so as not to rely on the linear amortized work time of `resize` (no one guarantees it). + if (pos + MAX_LENGTH > out_vec.size()) + out_vec.resize(out_vec.size() * word_size + MAX_LENGTH); + + char * begin = reinterpret_cast(&out_vec[pos]); + char * end = begin; + + // index 1 stores the higher 64 bit + Impl::executeOneUInt(uuid[i].toUnderType().items[1], end, false); + Impl::executeOneUInt(uuid[i].toUnderType().items[0], end, true); + + pos += end - begin; + out_offsets[i] = pos; + } + out_vec.resize(pos); + + col_res = std::move(col_str); + return true; + } + else + { + return false; + } + } }; /// Decode number or string from string with binary or hexadecimal representation diff --git a/tests/queries/0_stateless/00396_uuid.reference b/tests/queries/0_stateless/00396_uuid.reference index d70322ec4c1..5ea857b8c5c 100644 --- a/tests/queries/0_stateless/00396_uuid.reference +++ b/tests/queries/0_stateless/00396_uuid.reference @@ -6,3 +6,4 @@ 01234567-89ab-cdef-0123-456789abcdef 01234567-89ab-cdef-0123-456789abcdef 01234567-89ab-cdef-0123-456789abcdef 3f1ed72e-f7fe-4459-9cbe-95fe9298f845 1 +1 diff --git a/tests/queries/0_stateless/00396_uuid.sql b/tests/queries/0_stateless/00396_uuid.sql index 9d8b48bddb0..2941bd70877 100644 --- a/tests/queries/0_stateless/00396_uuid.sql +++ b/tests/queries/0_stateless/00396_uuid.sql @@ -11,3 +11,8 @@ with generateUUIDv4() as uuid, identity(lower(hex(reverse(reinterpretAsString(uuid))))) as str, reinterpretAsUUID(reverse(unhex(str))) as uuid2 select uuid = uuid2; + +with generateUUIDv4() as uuid, + hex(reverse(reinterpretAsString(uuid))) as str1, + hex(uuid) as str2 +select str1 = str2 From d4fd3dd2e0f73ff30cdcd7d7f35a1d2ef4aa62bb Mon Sep 17 00:00:00 2001 From: frank chen Date: Fri, 3 Dec 2021 11:27:23 +0800 Subject: [PATCH 002/403] Add test for bin() Signed-off-by: frank chen --- tests/queries/0_stateless/00396_uuid.reference | 1 + tests/queries/0_stateless/00396_uuid.sql | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00396_uuid.reference b/tests/queries/0_stateless/00396_uuid.reference index 5ea857b8c5c..912f7a4544e 100644 --- a/tests/queries/0_stateless/00396_uuid.reference +++ b/tests/queries/0_stateless/00396_uuid.reference @@ -7,3 +7,4 @@ 3f1ed72e-f7fe-4459-9cbe-95fe9298f845 1 1 +1 diff --git a/tests/queries/0_stateless/00396_uuid.sql b/tests/queries/0_stateless/00396_uuid.sql index 2941bd70877..f92bc72a768 100644 --- a/tests/queries/0_stateless/00396_uuid.sql +++ b/tests/queries/0_stateless/00396_uuid.sql @@ -15,4 +15,9 @@ select uuid = uuid2; with generateUUIDv4() as uuid, hex(reverse(reinterpretAsString(uuid))) as str1, hex(uuid) as str2 -select str1 = str2 +select str1 = str2; + +with generateUUIDv4() as uuid, + bin(reverse(reinterpretAsString(uuid))) as bin_str1, + bin(uuid) as bin_str2 +select bin_str1 = bin_str2; \ No newline at end of file From c6f1d7d2921b929f3908be53e19489f065db4e20 Mon Sep 17 00:00:00 2001 From: frank chen Date: Fri, 3 Dec 2021 11:27:41 +0800 Subject: [PATCH 003/403] Add examples in doc Signed-off-by: frank chen --- .../functions/encoding-functions.md | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/en/sql-reference/functions/encoding-functions.md b/docs/en/sql-reference/functions/encoding-functions.md index 69dd14da1bf..f3beddac1bd 100644 --- a/docs/en/sql-reference/functions/encoding-functions.md +++ b/docs/en/sql-reference/functions/encoding-functions.md @@ -93,6 +93,8 @@ For [String](../../sql-reference/data-types/string.md) and [FixedString](../../s Values of [Float](../../sql-reference/data-types/float.md) and [Decimal](../../sql-reference/data-types/decimal.md) types are encoded as their representation in memory. As we support little-endian architecture, they are encoded in little-endian. Zero leading/trailing bytes are not omitted. +Values of [UUID](../data-types/uuid.md) type are encoded as big-endian order string. + **Arguments** - `arg` — A value to convert to hexadecimal. Types: [String](../../sql-reference/data-types/string.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md), [Decimal](../../sql-reference/data-types/decimal.md), [Date](../../sql-reference/data-types/date.md) or [DateTime](../../sql-reference/data-types/datetime.md). @@ -147,6 +149,21 @@ Result: └──────────────────┘ ``` +Query: + +``` sql +SELECT lower(hex(toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0'))) as big_endian_hex +``` + +Result: + +``` text +┌─big_endian_hex───────────────────┐ +│ 907ba6006ad3dba061f0c4045cb311e7 │ +└──────────────────────────────────┘ +``` + + ## unhex {#unhexstr} Performs the opposite operation of [hex](#hex). It interprets each pair of hexadecimal digits (in the argument) as a number and converts it to the byte represented by the number. The return value is a binary string (BLOB). @@ -224,6 +241,8 @@ For [String](../../sql-reference/data-types/string.md) and [FixedString](../../s Values of [Float](../../sql-reference/data-types/float.md) and [Decimal](../../sql-reference/data-types/decimal.md) types are encoded as their representation in memory. As we support little-endian architecture, they are encoded in little-endian. Zero leading/trailing bytes are not omitted. +Values of [UUID](../data-types/uuid.md) type are encoded as big-endian order string. + **Arguments** - `arg` — A value to convert to binary. [String](../../sql-reference/data-types/string.md), [FixedString](../../sql-reference/data-types/fixedstring.md), [UInt](../../sql-reference/data-types/int-uint.md), [Float](../../sql-reference/data-types/float.md), [Decimal](../../sql-reference/data-types/decimal.md), [Date](../../sql-reference/data-types/date.md), or [DateTime](../../sql-reference/data-types/datetime.md). @@ -280,6 +299,21 @@ Result: └──────────────────────────────────────────────────────────────────┘ ``` +Query: + +``` sql +SELECT lower(bin(toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0'))) as big_endian_bin +``` + +Result: + +``` text +┌─big_endian_bin───────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ 10010000011110111010011000000000011010101101001111011011101000000110000111110000110001000000010001011100101100110001000111100111 │ +└──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + + ## unbin {#unbinstr} Interprets each pair of binary digits (in the argument) as a number and converts it to the byte represented by the number. The functions performs the opposite operation to [bin](#bin). From b122f3095d7c2b44328a6aeb6d7e6229bb1627f3 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Fri, 3 Dec 2021 12:46:25 +0300 Subject: [PATCH 004/403] Update 00396_uuid.sql --- tests/queries/0_stateless/00396_uuid.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00396_uuid.sql b/tests/queries/0_stateless/00396_uuid.sql index f92bc72a768..15242f4c0b4 100644 --- a/tests/queries/0_stateless/00396_uuid.sql +++ b/tests/queries/0_stateless/00396_uuid.sql @@ -20,4 +20,4 @@ select str1 = str2; with generateUUIDv4() as uuid, bin(reverse(reinterpretAsString(uuid))) as bin_str1, bin(uuid) as bin_str2 -select bin_str1 = bin_str2; \ No newline at end of file +select bin_str1 = bin_str2; From 0b6949172c2bf6b2814c19e614eb665f0e55bf2c Mon Sep 17 00:00:00 2001 From: frank chen Date: Mon, 6 Dec 2021 17:12:34 +0800 Subject: [PATCH 005/403] keep leading zeros for UUID Signed-off-by: frank chen --- src/Functions/FunctionsBinaryRepr.cpp | 12 ++++++------ tests/queries/0_stateless/00396_uuid.reference | 2 ++ tests/queries/0_stateless/00396_uuid.sql | 6 ++++++ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/Functions/FunctionsBinaryRepr.cpp b/src/Functions/FunctionsBinaryRepr.cpp index 4f5a215a251..59ba722f3c0 100644 --- a/src/Functions/FunctionsBinaryRepr.cpp +++ b/src/Functions/FunctionsBinaryRepr.cpp @@ -43,7 +43,7 @@ struct HexImpl static constexpr size_t word_size = 2; template - static void executeOneUInt(T x, char *& out, bool auto_close = true) + static void executeOneUInt(T x, char *& out, bool skip_leading_zero = true, bool auto_close = true) { bool was_nonzero = false; for (int offset = (sizeof(T) - 1) * 8; offset >= 0; offset -= 8) @@ -51,7 +51,7 @@ struct HexImpl UInt8 byte = x >> offset; /// Skip leading zeros - if (byte == 0 && !was_nonzero && offset) //-V560 + if (byte == 0 && !was_nonzero && offset && skip_leading_zero) //-V560 continue; was_nonzero = true; @@ -134,7 +134,7 @@ struct BinImpl static constexpr size_t word_size = 8; template - static void executeOneUInt(T x, char *& out, bool auto_close = true) + static void executeOneUInt(T x, char *& out, bool skip_leading_zero = true, bool auto_close = true) { bool was_nonzero = false; for (int offset = (sizeof(T) - 1) * 8; offset >= 0; offset -= 8) @@ -142,7 +142,7 @@ struct BinImpl UInt8 byte = x >> offset; /// Skip leading zeros - if (byte == 0 && !was_nonzero && offset) //-V560 + if (byte == 0 && !was_nonzero && offset && skip_leading_zero) //-V560 continue; was_nonzero = true; @@ -520,8 +520,8 @@ public: char * end = begin; // index 1 stores the higher 64 bit - Impl::executeOneUInt(uuid[i].toUnderType().items[1], end, false); - Impl::executeOneUInt(uuid[i].toUnderType().items[0], end, true); + Impl::executeOneUInt(uuid[i].toUnderType().items[1], end, false, false); + Impl::executeOneUInt(uuid[i].toUnderType().items[0], end, false, true); pos += end - begin; out_offsets[i] = pos; diff --git a/tests/queries/0_stateless/00396_uuid.reference b/tests/queries/0_stateless/00396_uuid.reference index 912f7a4544e..dea85d84298 100644 --- a/tests/queries/0_stateless/00396_uuid.reference +++ b/tests/queries/0_stateless/00396_uuid.reference @@ -7,4 +7,6 @@ 3f1ed72e-f7fe-4459-9cbe-95fe9298f845 1 1 +00009d773a2fd3190000000080e746f8 1 +128 diff --git a/tests/queries/0_stateless/00396_uuid.sql b/tests/queries/0_stateless/00396_uuid.sql index 15242f4c0b4..1a14fe41864 100644 --- a/tests/queries/0_stateless/00396_uuid.sql +++ b/tests/queries/0_stateless/00396_uuid.sql @@ -17,7 +17,13 @@ with generateUUIDv4() as uuid, hex(uuid) as str2 select str1 = str2; +-- hex on UUID always generate 32 characters even there're leading zeros +select lower(hex(toUUID('00000000-80e7-46f8-0000-9d773a2fd319'))); + with generateUUIDv4() as uuid, bin(reverse(reinterpretAsString(uuid))) as bin_str1, bin(uuid) as bin_str2 select bin_str1 = bin_str2; + +-- bin on UUID always generate 128 characters even there're leading zeros +select length(bin(toUUID('00000000-80e7-46f8-0000-9d773a2fd319'))); \ No newline at end of file From e4604c10d99935d49e3dde6b853f4c61446f1ac0 Mon Sep 17 00:00:00 2001 From: frank chen Date: Thu, 9 Dec 2021 18:42:54 +0800 Subject: [PATCH 006/403] Move tests into new files Signed-off-by: frank chen --- tests/queries/0_stateless/00396_uuid.reference | 4 ---- tests/queries/0_stateless/00396_uuid.sql | 16 ---------------- .../02128_hex_bin_on_uuid.reference | 5 +++++ .../0_stateless/02128_hex_bin_on_uuid.sql | 18 ++++++++++++++++++ 4 files changed, 23 insertions(+), 20 deletions(-) create mode 100644 tests/queries/0_stateless/02128_hex_bin_on_uuid.reference create mode 100644 tests/queries/0_stateless/02128_hex_bin_on_uuid.sql diff --git a/tests/queries/0_stateless/00396_uuid.reference b/tests/queries/0_stateless/00396_uuid.reference index dea85d84298..d70322ec4c1 100644 --- a/tests/queries/0_stateless/00396_uuid.reference +++ b/tests/queries/0_stateless/00396_uuid.reference @@ -6,7 +6,3 @@ 01234567-89ab-cdef-0123-456789abcdef 01234567-89ab-cdef-0123-456789abcdef 01234567-89ab-cdef-0123-456789abcdef 3f1ed72e-f7fe-4459-9cbe-95fe9298f845 1 -1 -00009d773a2fd3190000000080e746f8 -1 -128 diff --git a/tests/queries/0_stateless/00396_uuid.sql b/tests/queries/0_stateless/00396_uuid.sql index 1a14fe41864..9d8b48bddb0 100644 --- a/tests/queries/0_stateless/00396_uuid.sql +++ b/tests/queries/0_stateless/00396_uuid.sql @@ -11,19 +11,3 @@ with generateUUIDv4() as uuid, identity(lower(hex(reverse(reinterpretAsString(uuid))))) as str, reinterpretAsUUID(reverse(unhex(str))) as uuid2 select uuid = uuid2; - -with generateUUIDv4() as uuid, - hex(reverse(reinterpretAsString(uuid))) as str1, - hex(uuid) as str2 -select str1 = str2; - --- hex on UUID always generate 32 characters even there're leading zeros -select lower(hex(toUUID('00000000-80e7-46f8-0000-9d773a2fd319'))); - -with generateUUIDv4() as uuid, - bin(reverse(reinterpretAsString(uuid))) as bin_str1, - bin(uuid) as bin_str2 -select bin_str1 = bin_str2; - --- bin on UUID always generate 128 characters even there're leading zeros -select length(bin(toUUID('00000000-80e7-46f8-0000-9d773a2fd319'))); \ No newline at end of file diff --git a/tests/queries/0_stateless/02128_hex_bin_on_uuid.reference b/tests/queries/0_stateless/02128_hex_bin_on_uuid.reference new file mode 100644 index 00000000000..673bca894f5 --- /dev/null +++ b/tests/queries/0_stateless/02128_hex_bin_on_uuid.reference @@ -0,0 +1,5 @@ +32 +1 +00009d773a2fd3190000000080e746f8 +1 +128 diff --git a/tests/queries/0_stateless/02128_hex_bin_on_uuid.sql b/tests/queries/0_stateless/02128_hex_bin_on_uuid.sql new file mode 100644 index 00000000000..6f6bc64542a --- /dev/null +++ b/tests/queries/0_stateless/02128_hex_bin_on_uuid.sql @@ -0,0 +1,18 @@ +-- length should be 32 +select length(hex(generateUUIDv4())); + +with generateUUIDv4() as uuid, + hex(reverse(reinterpretAsString(uuid))) as str1, + hex(uuid) as str2 +select str1 = str2; + +-- hex on UUID always generate 32 characters even there're leading zeros +select lower(hex(toUUID('00000000-80e7-46f8-0000-9d773a2fd319'))); + +with generateUUIDv4() as uuid, + bin(reverse(reinterpretAsString(uuid))) as bin_str1, + bin(uuid) as bin_str2 +select bin_str1 = bin_str2; + +-- bin on UUID always generate 128 characters even there're leading zeros +select length(bin(toUUID('00000000-80e7-46f8-0000-9d773a2fd319'))); \ No newline at end of file From a83832c03e64b4abdc72fab07e322c7ba87b6015 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Mon, 27 Dec 2021 18:54:28 +0300 Subject: [PATCH 007/403] fix flushing of in-memory parts --- src/Storages/StorageMergeTree.cpp | 4 ++++ src/Storages/StorageMergeTree.h | 1 + src/Storages/StorageReplicatedMergeTree.cpp | 13 +++++++++++++ src/Storages/StorageReplicatedMergeTree.h | 4 ++++ 4 files changed, 22 insertions(+) diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 03ac27d0e46..02ea036c8ea 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -141,6 +141,10 @@ void StorageMergeTree::startup() void StorageMergeTree::flush() { + if (flush_called) + return; + + flush_called = true; flushAllInMemoryPartsIfNeeded(); } diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h index ee99b412f59..4f600404c72 100644 --- a/src/Storages/StorageMergeTree.h +++ b/src/Storages/StorageMergeTree.h @@ -140,6 +140,7 @@ private: std::map, UInt64> updated_version_by_block_range; std::atomic shutdown_called {false}; + std::atomic flush_called {false}; private: void loadMutations(); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index a1f82e14868..a46a7c2f66f 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4120,9 +4120,22 @@ void StorageReplicatedMergeTree::startup() } } +void StorageReplicatedMergeTree::flush() +{ + if (flush_called) + return; + + flush_called = true; + flushAllInMemoryPartsIfNeeded(); +} void StorageReplicatedMergeTree::shutdown() { + if (shutdown_called) + return; + + shutdown_called = true; + /// Cancel fetches, merges and mutations to force the queue_task to finish ASAP. fetcher.blocker.cancelForever(); merger_mutator.merges_blocker.cancelForever(); diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 6861d89f070..cdee22899c4 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -86,6 +86,7 @@ class StorageReplicatedMergeTree final : public shared_ptr_helper shutdown_called {false}; + std::atomic flush_called {false}; + int metadata_version = 0; /// Threads. From 4ebf61b809707d776a85d9b7e6206769d5deb7f8 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 29 Dec 2021 01:03:55 +0300 Subject: [PATCH 008/403] proper checking of atomic flags --- src/Storages/StorageMergeTree.cpp | 6 ++---- src/Storages/StorageReplicatedMergeTree.cpp | 7 ++----- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 02ea036c8ea..d14ee672727 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -141,18 +141,16 @@ void StorageMergeTree::startup() void StorageMergeTree::flush() { - if (flush_called) + if (flush_called.exchange(true)) return; - flush_called = true; flushAllInMemoryPartsIfNeeded(); } void StorageMergeTree::shutdown() { - if (shutdown_called) + if (shutdown_called.exchange(true)) return; - shutdown_called = true; /// Unlock all waiting mutations { diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index a46a7c2f66f..5f829db72f8 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4122,20 +4122,17 @@ void StorageReplicatedMergeTree::startup() void StorageReplicatedMergeTree::flush() { - if (flush_called) + if (flush_called.exchange(true)) return; - flush_called = true; flushAllInMemoryPartsIfNeeded(); } void StorageReplicatedMergeTree::shutdown() { - if (shutdown_called) + if (shutdown_called.exchange(true)) return; - shutdown_called = true; - /// Cancel fetches, merges and mutations to force the queue_task to finish ASAP. fetcher.blocker.cancelForever(); merger_mutator.merges_blocker.cancelForever(); From 97788b9c21308fe912a356497be35ad8b439233a Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 29 Dec 2021 21:03:15 +0300 Subject: [PATCH 009/403] Allow to create new files on insert for File/S3/HDFS engines --- src/Common/ErrorCodes.cpp | 1 + src/Core/Settings.h | 5 + src/Formats/FormatFactory.cpp | 20 +++ src/Formats/FormatFactory.h | 13 ++ .../Formats/Impl/ArrowBlockOutputFormat.cpp | 2 + .../Impl/CustomSeparatedRowOutputFormat.cpp | 5 + .../Formats/Impl/JSONRowOutputFormat.cpp | 2 + .../Formats/Impl/ORCBlockOutputFormat.cpp | 1 + .../Formats/Impl/ParquetBlockOutputFormat.cpp | 1 + .../Impl/TemplateBlockOutputFormat.cpp | 14 ++ .../Formats/Impl/XMLRowOutputFormat.cpp | 1 + src/Storages/HDFS/StorageHDFS.cpp | 127 ++++++++++++--- src/Storages/HDFS/StorageHDFS.h | 16 +- src/Storages/HDFS/WriteBufferFromHDFS.cpp | 13 +- src/Storages/StorageFile.cpp | 49 +++++- src/Storages/StorageFile.h | 2 + src/Storages/StorageS3.cpp | 150 +++++++++++++++--- src/Storages/StorageS3.h | 17 +- tests/integration/test_storage_hdfs/test.py | 37 +++++ tests/integration/test_storage_s3/test.py | 42 +++++ ..._inserts_for_formats_with_suffix.reference | 100 ++++++++++++ ...ltiple_inserts_for_formats_with_suffix.sql | 39 +++++ 22 files changed, 592 insertions(+), 65 deletions(-) create mode 100644 tests/queries/0_stateless/02155_multiple_inserts_for_formats_with_suffix.reference create mode 100644 tests/queries/0_stateless/02155_multiple_inserts_for_formats_with_suffix.sql diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 16f85fcae61..f568955e531 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -605,6 +605,7 @@ M(634, MONGODB_ERROR) \ M(635, CANNOT_POLL) \ M(636, CANNOT_EXTRACT_TABLE_STRUCTURE) \ + M(637, CANNOT_APPEND_TO_FILE) \ \ M(999, KEEPER_EXCEPTION) \ M(1000, POCO_EXCEPTION) \ diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 6e53fa4342c..ad43c2a04d7 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -75,7 +75,11 @@ class IColumn; M(UInt64, s3_max_single_read_retries, 4, "The maximum number of retries during single S3 read.", 0) \ M(UInt64, s3_max_redirects, 10, "Max number of S3 redirects hops allowed.", 0) \ M(UInt64, s3_max_connections, 1024, "The maximum number of connections per server.", 0) \ + M(Bool, s3_truncate_on_insert, false, "", 0) \ + M(Bool, s3_create_new_file_on_insert, false, "", 0) \ M(UInt64, hdfs_replication, 0, "The actual number of replications can be specified when the hdfs file is created.", 0) \ + M(Bool, hdfs_truncate_on_insert, false, "", 0) \ + M(Bool, hdfs_create_new_file_on_insert, false, "", 0) \ M(UInt64, hsts_max_age, 0, "Expired time for hsts. 0 means disable HSTS.", 0) \ M(Bool, extremes, false, "Calculate minimums and maximums of the result columns. They can be output in JSON-formats.", IMPORTANT) \ M(Bool, use_uncompressed_cache, false, "Whether to use the cache of uncompressed blocks.", 0) \ @@ -490,6 +494,7 @@ class IColumn; \ M(Bool, engine_file_empty_if_not_exists, false, "Allows to select data from a file engine table without file", 0) \ M(Bool, engine_file_truncate_on_insert, false, "Enables or disables truncate before insert in file engine tables", 0) \ + M(Bool, engine_file_allow_create_multiple_files, false, ".", 0) \ M(Bool, allow_experimental_database_replicated, false, "Allow to create databases with Replicated engine", 0) \ M(UInt64, database_replicated_initial_query_timeout_sec, 300, "How long initial DDL query should wait for Replicated database to precess previous DDL queue entries", 0) \ M(UInt64, max_distributed_depth, 5, "Maximum distributed query depth", 0) \ diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index a0a5550627d..baf825c2bbc 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -383,6 +383,26 @@ void FormatFactory::registerNonTrivialPrefixAndSuffixChecker(const String & name target = std::move(non_trivial_prefix_and_suffix_checker); } +void FormatFactory::registerSuffixChecker(const String & name, SuffixChecker suffix_checker) +{ + auto & target = dict[name].suffix_checker; + if (target) + throw Exception("FormatFactory: Suffix checker " + name + " is already registered", ErrorCodes::LOGICAL_ERROR); + target = std::move(suffix_checker); +} + +void FormatFactory::markFormatWithSuffix(const String & name) +{ + registerSuffixChecker(name, [](const FormatSettings &){ return true; }); +} + +bool FormatFactory::checkIfFormatHasSuffix(const String & name, ContextPtr context, const std::optional & format_settings_) +{ + auto format_settings = format_settings_ ? *format_settings_ : getFormatSettings(context); + auto & suffix_checker = dict[name].suffix_checker; + return suffix_checker && suffix_checker(format_settings); +} + void FormatFactory::registerOutputFormat(const String & name, OutputCreator output_creator) { auto & target = dict[name].output_creator; diff --git a/src/Formats/FormatFactory.h b/src/Formats/FormatFactory.h index a62b32da0cc..860b1e19ae6 100644 --- a/src/Formats/FormatFactory.h +++ b/src/Formats/FormatFactory.h @@ -92,6 +92,10 @@ private: /// The checker should return true if parallel parsing should be disabled. using NonTrivialPrefixAndSuffixChecker = std::function; + /// Some formats can have suffix after data depending on settings. + /// The checker should return true if format will write some suffix after data. + using SuffixChecker = std::function; + using SchemaReaderCreator = std::function; using ExternalSchemaReaderCreator = std::function; @@ -105,6 +109,7 @@ private: bool supports_parallel_formatting{false}; bool is_column_oriented{false}; NonTrivialPrefixAndSuffixChecker non_trivial_prefix_and_suffix_checker; + SuffixChecker suffix_checker; }; using FormatsDictionary = std::unordered_map; @@ -165,6 +170,14 @@ public: void registerNonTrivialPrefixAndSuffixChecker(const String & name, NonTrivialPrefixAndSuffixChecker non_trivial_prefix_and_suffix_checker); + void registerSuffixChecker(const String & name, SuffixChecker suffix_checker); + + /// If format always contains suffix, you an use this method instead of + /// registerSuffixChecker with suffix_checker that always returns true. + void markFormatWithSuffix(const String & name); + + bool checkIfFormatHasSuffix(const String & name, ContextPtr context, const std::optional & format_settings_ = std::nullopt); + /// Register format by its name. void registerInputFormat(const String & name, InputCreator input_creator); void registerOutputFormat(const String & name, OutputCreator output_creator); diff --git a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp index 692f17f843a..b60d38f317c 100644 --- a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp @@ -93,6 +93,7 @@ void registerOutputFormatArrow(FormatFactory & factory) { return std::make_shared(buf, sample, false, format_settings); }); + factory.markFormatWithSuffix("Arrow"); factory.registerOutputFormat( "ArrowStream", @@ -103,6 +104,7 @@ void registerOutputFormatArrow(FormatFactory & factory) { return std::make_shared(buf, sample, true, format_settings); }); + factory.markFormatWithSuffix("ArrowStream"); } } diff --git a/src/Processors/Formats/Impl/CustomSeparatedRowOutputFormat.cpp b/src/Processors/Formats/Impl/CustomSeparatedRowOutputFormat.cpp index 21cb549d4cb..3fb7c0a36a3 100644 --- a/src/Processors/Formats/Impl/CustomSeparatedRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/CustomSeparatedRowOutputFormat.cpp @@ -91,6 +91,11 @@ void registerOutputFormatCustomSeparated(FormatFactory & factory) }); factory.markOutputFormatSupportsParallelFormatting(format_name); + + factory.registerSuffixChecker(format_name, [](const FormatSettings & settings) + { + return !settings.custom.result_after_delimiter.empty(); + }); }; registerWithNamesAndTypes("CustomSeparated", register_func); diff --git a/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp index 8e2b2617c4c..577efdb1a21 100644 --- a/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp @@ -284,6 +284,7 @@ void registerOutputFormatJSON(FormatFactory & factory) }); factory.markOutputFormatSupportsParallelFormatting("JSON"); + factory.markFormatWithSuffix("JSON"); factory.registerOutputFormat("JSONStrings", []( WriteBuffer & buf, @@ -295,6 +296,7 @@ void registerOutputFormatJSON(FormatFactory & factory) }); factory.markOutputFormatSupportsParallelFormatting("JSONStrings"); + factory.markFormatWithSuffix("JSONStrings"); } } diff --git a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp index 651b9545c81..33d4b3e568d 100644 --- a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp @@ -526,6 +526,7 @@ void registerOutputFormatORC(FormatFactory & factory) { return std::make_shared(buf, sample, format_settings); }); + factory.markFormatWithSuffix("ORC"); } } diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index a10858ee668..96c3a80b732 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -85,6 +85,7 @@ void registerOutputFormatParquet(FormatFactory & factory) { return std::make_shared(buf, sample, format_settings); }); + factory.markFormatWithSuffix("Parquet"); } } diff --git a/src/Processors/Formats/Impl/TemplateBlockOutputFormat.cpp b/src/Processors/Formats/Impl/TemplateBlockOutputFormat.cpp index d981b92e1dd..eded88298e8 100644 --- a/src/Processors/Formats/Impl/TemplateBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/TemplateBlockOutputFormat.cpp @@ -235,5 +235,19 @@ void registerOutputFormatTemplate(FormatFactory & factory) return std::make_shared(sample, buf, settings, resultset_format, row_format, settings.template_settings.row_between_delimiter); }); + + factory.registerSuffixChecker("Template", [](const FormatSettings & settings) + { + if (settings.template_settings.resultset_format.empty()) + return false; + auto resultset_format = ParsedTemplateFormatString( + FormatSchemaInfo(settings.template_settings.resultset_format, "Template", false, + settings.schema.is_server, settings.schema.format_schema_path), + [&](const String & partName) + { + return static_cast(TemplateBlockOutputFormat::stringToResultsetPart(partName)); + }); + return !resultset_format.delimiters.empty() && !resultset_format.delimiters.back().empty(); + }); } } diff --git a/src/Processors/Formats/Impl/XMLRowOutputFormat.cpp b/src/Processors/Formats/Impl/XMLRowOutputFormat.cpp index d96981fc091..56bbbc0673c 100644 --- a/src/Processors/Formats/Impl/XMLRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/XMLRowOutputFormat.cpp @@ -256,6 +256,7 @@ void registerOutputFormatXML(FormatFactory & factory) }); factory.markOutputFormatSupportsParallelFormatting("XML"); + factory.markFormatWithSuffix("XML"); } } diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index f22f6f66ced..36287b92855 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -14,9 +14,8 @@ #include #include -#include +#include -#include #include #include #include @@ -28,7 +27,6 @@ #include #include - #include #include #include @@ -52,6 +50,7 @@ namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ACCESS_DENIED; + extern const int DATABASE_ACCESS_DENIED; extern const int CANNOT_EXTRACT_TABLE_STRUCTURE; } namespace @@ -130,20 +129,23 @@ StorageHDFS::StorageHDFS( ASTPtr partition_by_) : IStorage(table_id_) , WithContext(context_) - , uri(uri_) + , uris({uri_}) , format_name(format_name_) , compression_method(compression_method_) , distributed_processing(distributed_processing_) , partition_by(partition_by_) { - context_->getRemoteHostFilter().checkURL(Poco::URI(uri)); - checkHDFSURL(uri); + context_->getRemoteHostFilter().checkURL(Poco::URI(uri_)); + checkHDFSURL(uri_); + + String path = uri_.substr(uri_.find('/', uri_.find("//") + 2)); + is_path_with_globs = path.find_first_of("*?{") != std::string::npos; StorageInMemoryMetadata storage_metadata; if (columns_.empty()) { - auto columns = getTableStructureFromData(format_name, uri, compression_method, context_); + auto columns = getTableStructureFromData(format_name, uri_, compression_method, context_); storage_metadata.setColumns(columns); } else @@ -208,6 +210,25 @@ private: Strings::iterator uris_iter; }; +class HDFSSource::URISIterator::Impl +{ +public: + Impl(const std::vector & uris_) : uris(uris_), index(0) + { + } + + String next() + { + if (index == uris.size()) + return ""; + return uris[index++]; + } + +private: + const std::vector & uris; + size_t index; +}; + Block HDFSSource::getHeader(const StorageMetadataPtr & metadata_snapshot, bool need_path_column, bool need_file_column) { auto header = metadata_snapshot->getSampleBlock(); @@ -241,6 +262,15 @@ String HDFSSource::DisclosedGlobIterator::next() return pimpl->next(); } +HDFSSource::URISIterator::URISIterator(const std::vector & uris_) + : pimpl(std::make_shared(uris_)) +{ +} + +String HDFSSource::URISIterator::next() +{ + return pimpl->next(); +} HDFSSource::HDFSSource( StorageHDFSPtr storage_, @@ -275,9 +305,8 @@ bool HDFSSource::initialize() current_path = (*file_iterator)(); if (current_path.empty()) return false; - const size_t begin_of_path = current_path.find('/', current_path.find("//") + 2); - const String path_from_uri = current_path.substr(begin_of_path); - const String uri_without_path = current_path.substr(0, begin_of_path); + + const auto [path_from_uri, uri_without_path] = getPathFromUriAndUriWithoutPath(current_path); auto compression = chooseCompressionMethod(path_from_uri, storage->compression_method); read_buf = wrapReadBufferWithCompressionMethod(std::make_unique(uri_without_path, path_from_uri, getContext()->getGlobalContext()->getConfigRef()), compression); @@ -460,15 +489,23 @@ Pipe StorageHDFS::read( return callback(); }); } - else + else if (is_path_with_globs) { /// Iterate through disclosed globs and make a source for each file - auto glob_iterator = std::make_shared(context_, uri); + auto glob_iterator = std::make_shared(context_, uris[0]); iterator_wrapper = std::make_shared([glob_iterator]() { return glob_iterator->next(); }); } + else + { + auto uris_iterator = std::make_shared(uris); + iterator_wrapper = std::make_shared([uris_iterator]() + { + return uris_iterator->next(); + }); + } Pipes pipes; auto this_ptr = std::static_pointer_cast(shared_from_this()); @@ -496,9 +533,44 @@ Pipe StorageHDFS::read( return Pipe::unitePipes(std::move(pipes)); } -SinkToStoragePtr StorageHDFS::write(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr /*context*/) +SinkToStoragePtr StorageHDFS::write(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr context_) { - bool has_wildcards = uri.find(PartitionedSink::PARTITION_ID_WILDCARD) != String::npos; + if (is_path_with_globs) + throw Exception(ErrorCodes::DATABASE_ACCESS_DENIED, "URI '{}' contains globs, so the table is in readonly mode", uris.back()); + + String current_uri = uris.back(); + + const auto [path_from_uri, uri_without_path] = getPathFromUriAndUriWithoutPath(current_uri); + + HDFSBuilderWrapper builder = createHDFSBuilder(uri_without_path + "/", context_->getGlobalContext()->getConfigRef()); + HDFSFSPtr fs = createHDFSFS(builder.get()); + + bool truncate_on_insert = context_->getSettingsRef().hdfs_truncate_on_insert; + if (!truncate_on_insert && !hdfsExists(fs.get(), path_from_uri.c_str())) + { + if (context_->getSettingsRef().hdfs_create_new_file_on_insert) + { + auto pos = uris[0].find_first_of('.', uris[0].find_last_of('/')); + size_t index = uris.size(); + String new_uri; + do + { + new_uri = uris[0].substr(0, pos) + "." + std::to_string(index) + (pos == std::string::npos ? "" : uris[0].substr(pos)); + ++index; + } + while (!hdfsExists(fs.get(), new_uri.c_str())); + uris.push_back(new_uri); + current_uri = new_uri; + } + else + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "File with path {} already exists. If you want to overwrite it, enable setting hdfs_truncate_on_insert, " + "if you want to create new file on each insert, enable setting hdfs_create_new_file_on_insert", + path_from_uri); + } + + bool has_wildcards = current_uri.find(PartitionedSink::PARTITION_ID_WILDCARD) != String::npos; const auto * insert_query = dynamic_cast(query.get()); auto partition_by_ast = insert_query ? (insert_query->partition_by ? insert_query->partition_by : partition_by) : nullptr; bool is_partitioned_implementation = partition_by_ast && has_wildcards; @@ -507,34 +579,37 @@ SinkToStoragePtr StorageHDFS::write(const ASTPtr & query, const StorageMetadataP { return std::make_shared( partition_by_ast, - uri, + current_uri, format_name, metadata_snapshot->getSampleBlock(), - getContext(), - chooseCompressionMethod(uri, compression_method)); + context_, + chooseCompressionMethod(current_uri, compression_method)); } else { - return std::make_shared(uri, + return std::make_shared(current_uri, format_name, metadata_snapshot->getSampleBlock(), - getContext(), - chooseCompressionMethod(uri, compression_method)); + context_, + chooseCompressionMethod(current_uri, compression_method)); } } void StorageHDFS::truncate(const ASTPtr & /* query */, const StorageMetadataPtr &, ContextPtr local_context, TableExclusiveLockHolder &) { - const size_t begin_of_path = uri.find('/', uri.find("//") + 2); - const String path = uri.substr(begin_of_path); - const String url = uri.substr(0, begin_of_path); + const size_t begin_of_path = uris[0].find('/', uris[0].find("//") + 2); + const String url = uris[0].substr(0, begin_of_path); HDFSBuilderWrapper builder = createHDFSBuilder(url + "/", local_context->getGlobalContext()->getConfigRef()); HDFSFSPtr fs = createHDFSFS(builder.get()); - int ret = hdfsDelete(fs.get(), path.data(), 0); - if (ret) - throw Exception(ErrorCodes::ACCESS_DENIED, "Unable to truncate hdfs table: {}", std::string(hdfsGetLastError())); + for (const auto & uri : uris) + { + const String path = uri.substr(begin_of_path); + int ret = hdfsDelete(fs.get(), path.data(), 0); + if (ret) + throw Exception(ErrorCodes::ACCESS_DENIED, "Unable to truncate hdfs table: {}", std::string(hdfsGetLastError())); + } } diff --git a/src/Storages/HDFS/StorageHDFS.h b/src/Storages/HDFS/StorageHDFS.h index 9e845d8fd74..53be640248c 100644 --- a/src/Storages/HDFS/StorageHDFS.h +++ b/src/Storages/HDFS/StorageHDFS.h @@ -31,7 +31,7 @@ public: size_t max_block_size, unsigned num_streams) override; - SinkToStoragePtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, ContextPtr /*context*/) override; + SinkToStoragePtr write(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr context) override; void truncate( const ASTPtr & query, @@ -70,11 +70,12 @@ protected: ASTPtr partition_by = nullptr); private: - const String uri; + std::vector uris; String format_name; String compression_method; const bool distributed_processing; ASTPtr partition_by; + bool is_path_with_globs; Poco::Logger * log = &Poco::Logger::get("StorageHDFS"); }; @@ -95,6 +96,17 @@ public: std::shared_ptr pimpl; }; + class URISIterator + { + public: + URISIterator(const std::vector & uris_); + String next(); + private: + class Impl; + /// shared_ptr to have copy constructor + std::shared_ptr pimpl; + }; + using IteratorWrapper = std::function; using StorageHDFSPtr = std::shared_ptr; diff --git a/src/Storages/HDFS/WriteBufferFromHDFS.cpp b/src/Storages/HDFS/WriteBufferFromHDFS.cpp index 9f5e3c1f7d2..a5050953b7c 100644 --- a/src/Storages/HDFS/WriteBufferFromHDFS.cpp +++ b/src/Storages/HDFS/WriteBufferFromHDFS.cpp @@ -38,11 +38,14 @@ struct WriteBufferFromHDFS::WriteBufferFromHDFSImpl const size_t begin_of_path = hdfs_uri.find('/', hdfs_uri.find("//") + 2); const String path = hdfs_uri.substr(begin_of_path); - if (path.find_first_of("*?{") != std::string::npos) - throw Exception(ErrorCodes::CANNOT_OPEN_FILE, "URI '{}' contains globs, so the table is in readonly mode", hdfs_uri); - - if (!hdfsExists(fs.get(), path.c_str())) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "File {} already exists", path); +// if (path.find_first_of("*?{") != std::string::npos) +// throw Exception(ErrorCodes::CANNOT_OPEN_FILE, "URI '{}' contains globs, so the table is in readonly mode", hdfs_uri); +// +// if (!hdfsExists(fs.get(), path.c_str()) && !truncate_) +// throw Exception( +// ErrorCodes::BAD_ARGUMENTS, +// "File {} already exists. If you want to overwrite it, enable setting hdfs_truncate_on_insert", +// path); fout = hdfsOpenFile(fs.get(), path.c_str(), flags, 0, replication_, 0); /// O_WRONLY meaning create or overwrite i.e., implies O_TRUNCAT here diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index a479f982c70..586b8d97875 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -65,6 +65,7 @@ namespace ErrorCodes extern const int INCOMPATIBLE_COLUMNS; extern const int CANNOT_STAT; extern const int LOGICAL_ERROR; + extern const int CANNOT_APPEND_TO_FILE; extern const int CANNOT_EXTRACT_TABLE_STRUCTURE; } @@ -285,6 +286,7 @@ StorageFile::StorageFile(const std::string & table_path_, const std::string & us { is_db_table = false; paths = getPathsList(table_path_, user_files_path, args.getContext(), total_bytes_to_read); + is_path_with_globs = paths.size() > 1; path_for_partitioned_write = table_path_; setStorageMetadata(args); } @@ -666,10 +668,9 @@ public: } else { - if (paths.size() != 1) - throw Exception("Table '" + table_name_for_log + "' is in readonly mode because of globs in filepath", ErrorCodes::DATABASE_ACCESS_DENIED); + assert(!paths.empty()); flags |= O_WRONLY | O_APPEND | O_CREAT; - naked_buffer = std::make_unique(paths[0], DBMS_DEFAULT_BUFFER_SIZE, flags); + naked_buffer = std::make_unique(paths.back(), DBMS_DEFAULT_BUFFER_SIZE, flags); } /// In case of formats with prefixes if file is not empty we have already written prefix. @@ -827,6 +828,35 @@ SinkToStoragePtr StorageFile::write( { path = paths[0]; fs::create_directories(fs::path(path).parent_path()); + + if (is_path_with_globs) + throw Exception("Table '" + getStorageID().getNameForLogs() + "' is in readonly mode because of globs in filepath", ErrorCodes::DATABASE_ACCESS_DENIED); + + if (!context->getSettingsRef().engine_file_truncate_on_insert && !is_path_with_globs + && FormatFactory::instance().checkIfFormatHasSuffix(format_name, context, format_settings) && fs::exists(paths.back()) + && fs::file_size(paths.back()) != 0) + { + if (context->getSettingsRef().engine_file_allow_create_multiple_files) + { + auto pos = paths[0].find_first_of('.', paths[0].find_last_of('/')); + size_t index = paths.size(); + String new_path; + do + { + new_path = paths[0].substr(0, pos) + "." + std::to_string(index) + (pos == std::string::npos ? "" : paths[0].substr(pos)); + ++index; + } + while (fs::exists(new_path)); + paths.push_back(new_path); + } + else + throw Exception( + ErrorCodes::CANNOT_APPEND_TO_FILE, + "Cannot append data in format {} to file, because this format contains suffix and " + "data can be written to a file only once. You can allow to create a new file " + "on each insert by enabling setting engine_file_allow_create_multiple_files", + format_name); + } } return std::make_shared( @@ -882,7 +912,7 @@ void StorageFile::truncate( ContextPtr /* context */, TableExclusiveLockHolder &) { - if (paths.size() != 1) + if (is_path_with_globs) throw Exception("Can't truncate table '" + getStorageID().getNameForLogs() + "' in readonly mode", ErrorCodes::DATABASE_ACCESS_DENIED); if (use_table_fd) @@ -892,11 +922,14 @@ void StorageFile::truncate( } else { - if (!fs::exists(paths[0])) - return; + for (const auto & path : paths) + { + if (!fs::exists(path)) + continue; - if (0 != ::truncate(paths[0].c_str(), 0)) - throwFromErrnoWithPath("Cannot truncate file " + paths[0], paths[0], ErrorCodes::CANNOT_TRUNCATE_FILE); + if (0 != ::truncate(path.c_str(), 0)) + throwFromErrnoWithPath("Cannot truncate file " + path, path, ErrorCodes::CANNOT_TRUNCATE_FILE); + } } } diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index 6b015976589..8564d93ccc2 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -120,6 +120,8 @@ private: size_t total_bytes_to_read = 0; String path_for_partitioned_write; + + bool is_path_with_globs = false; }; } diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 3d988472b54..ea563937ca6 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -68,7 +68,7 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int S3_ERROR; extern const int UNEXPECTED_EXPRESSION; - extern const int CANNOT_OPEN_FILE; + extern const int DATABASE_ACCESS_DENIED; extern const int CANNOT_EXTRACT_TABLE_STRUCTURE; } @@ -176,6 +176,33 @@ String StorageS3Source::DisclosedGlobIterator::next() return pimpl->next(); } +class StorageS3Source::KeysIterator::Impl +{ +public: + explicit Impl(const std::vector & keys_) : keys(keys_), index(0) + { + } + + String next() + { + if (index == keys.size()) + return ""; + return keys[index++]; + } + +private: + const std::vector & keys; + size_t index; +}; + +StorageS3Source::KeysIterator::KeysIterator(const std::vector & keys_) : pimpl(std::make_shared(keys_)) +{ +} + +String StorageS3Source::KeysIterator::next() +{ + return pimpl->next(); +} Block StorageS3Source::getHeader(Block sample_block, bool with_path_column, bool with_file_column) { @@ -296,6 +323,39 @@ Chunk StorageS3Source::generate() return generate(); } +static bool checkIfObjectExists(const std::shared_ptr & client, const String & bucket, const String & key) +{ + bool is_finished = false; + Aws::S3::Model::ListObjectsV2Request request; + Aws::S3::Model::ListObjectsV2Outcome outcome; + + request.SetBucket(bucket); + request.SetPrefix(key); + while (!is_finished) + { + outcome = client->ListObjectsV2(request); + if (!outcome.IsSuccess()) + throw Exception( + ErrorCodes::S3_ERROR, + "Could not list objects in bucket {} with key {}, S3 exception: {}, message: {}", + quoteString(bucket), + quoteString(key), + backQuote(outcome.GetError().GetExceptionName()), + quoteString(outcome.GetError().GetMessage())); + + const auto & result_batch = outcome.GetResult().GetContents(); + for (const auto & obj : result_batch) + { + if (obj.GetKey() == key) + return true; + } + + request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken()); + is_finished = !outcome.GetResult().GetIsTruncated(); + } + + return false; +} class StorageS3Sink : public SinkToStorage { @@ -315,9 +375,6 @@ public: , sample_block(sample_block_) , format_settings(format_settings_) { - if (key.find_first_of("*?{") != std::string::npos) - throw Exception(ErrorCodes::CANNOT_OPEN_FILE, "S3 key '{}' contains globs, so the table is in readonly mode", key); - write_buf = wrapWriteBufferWithCompressionMethod( std::make_unique(client, bucket, key, min_upload_part_size, max_single_part_upload_size), compression_method, 3); writer = FormatFactory::instance().getOutputFormatParallelIfPossible(format, *write_buf, sample_block, context, {}, format_settings); @@ -419,7 +476,6 @@ private: std::optional format_settings; ExpressionActionsPtr partition_by_expr; - String partition_by_column_name; static void validateBucket(const String & str) { @@ -468,6 +524,7 @@ StorageS3::StorageS3( ASTPtr partition_by_) : IStorage(table_id_) , client_auth{uri_, access_key_id_, secret_access_key_, max_connections_, {}, {}} /// Client and settings will be updated later + , keys({uri_.key}) , format_name(format_name_) , max_single_read_retries(max_single_read_retries_) , min_upload_part_size(min_upload_part_size_) @@ -477,6 +534,7 @@ StorageS3::StorageS3( , distributed_processing(distributed_processing_) , format_settings(format_settings_) , partition_by(partition_by_) + , is_key_with_globs(uri_.key.find_first_of("*?{") != std::string::npos) { context_->getGlobalContext()->getRemoteHostFilter().checkURL(uri_.uri); StorageInMemoryMetadata storage_metadata; @@ -484,7 +542,7 @@ StorageS3::StorageS3( updateClientAndAuthSettings(context_, client_auth); if (columns_.empty()) { - auto columns = getTableStructureFromDataImpl(format_name, client_auth, max_single_read_retries_, compression_method, distributed_processing_, format_settings, context_); + auto columns = getTableStructureFromDataImpl(format_name, client_auth, max_single_read_retries_, compression_method, distributed_processing_, is_key_with_globs, format_settings, context_); storage_metadata.setColumns(columns); } else @@ -495,7 +553,7 @@ StorageS3::StorageS3( setInMemoryMetadata(storage_metadata); } -std::shared_ptr StorageS3::createFileIterator(const ClientAuthentication & client_auth, bool distributed_processing, ContextPtr local_context) +std::shared_ptr StorageS3::createFileIterator(const ClientAuthentication & client_auth, const std::vector & keys, bool is_key_with_globs, bool distributed_processing, ContextPtr local_context) { std::shared_ptr iterator_wrapper{nullptr}; if (distributed_processing) @@ -505,13 +563,23 @@ std::shared_ptr StorageS3::createFileIterator( return callback(); }); } - - /// Iterate through disclosed globs and make a source for each file - auto glob_iterator = std::make_shared(*client_auth.client, client_auth.uri); - return std::make_shared([glob_iterator]() + else if (is_key_with_globs) { - return glob_iterator->next(); - }); + /// Iterate through disclosed globs and make a source for each file + auto glob_iterator = std::make_shared(*client_auth.client, client_auth.uri); + iterator_wrapper = std::make_shared([glob_iterator]() + { + return glob_iterator->next(); + }); + } + else + { + auto keys_iterator = std::make_shared(keys); + iterator_wrapper = std::make_shared([keys_iterator]() + { + return keys_iterator->next(); + }); + } } Pipe StorageS3::read( @@ -536,7 +604,7 @@ Pipe StorageS3::read( need_file_column = true; } - std::shared_ptr iterator_wrapper = createFileIterator(client_auth, distributed_processing, local_context); + std::shared_ptr iterator_wrapper = createFileIterator(client_auth, keys, is_key_with_globs, distributed_processing, local_context); for (size_t i = 0; i < num_streams; ++i) { @@ -566,9 +634,38 @@ SinkToStoragePtr StorageS3::write(const ASTPtr & query, const StorageMetadataPtr { updateClientAndAuthSettings(local_context, client_auth); + if (is_key_with_globs) + throw Exception(ErrorCodes::DATABASE_ACCESS_DENIED, "S3 key '{}' contains globs, so the table is in readonly mode", client_auth.uri.key); + + bool truncate_in_insert = local_context->getSettingsRef().s3_truncate_on_insert; + + if (!truncate_in_insert && checkIfObjectExists(client_auth.client, client_auth.uri.bucket, keys.back())) + { + if (local_context->getSettingsRef().s3_create_new_file_on_insert) + { + size_t index = keys.size(); + auto pos = keys[0].find_first_of('.'); + String new_key; + do + { + new_key = keys[0].substr(0, pos) + "." + std::to_string(index) + (pos == std::string::npos ? "" : keys[0].substr(pos)); + ++index; + } + while (checkIfObjectExists(client_auth.client, client_auth.uri.bucket, new_key)); + keys.push_back(new_key); + } + else + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Object in bucket {} with key {} already exists. If you want to overwrite it, enable setting s3_truncate_on_insert, if you " + "want to create a new file on each insert, enable setting s3_create_new_file_on_insert", + client_auth.uri.bucket, + keys.back()); + } + auto sample_block = metadata_snapshot->getSampleBlock(); - auto chosen_compression_method = chooseCompressionMethod(client_auth.uri.key, compression_method); - bool has_wildcards = client_auth.uri.bucket.find(PARTITION_ID_WILDCARD) != String::npos || client_auth.uri.key.find(PARTITION_ID_WILDCARD) != String::npos; + auto chosen_compression_method = chooseCompressionMethod(keys.back(), compression_method); + bool has_wildcards = client_auth.uri.bucket.find(PARTITION_ID_WILDCARD) != String::npos || keys.back().find(PARTITION_ID_WILDCARD) != String::npos; auto insert_query = std::dynamic_pointer_cast(query); auto partition_by_ast = insert_query ? (insert_query->partition_by ? insert_query->partition_by : partition_by) : nullptr; @@ -585,7 +682,7 @@ SinkToStoragePtr StorageS3::write(const ASTPtr & query, const StorageMetadataPtr chosen_compression_method, client_auth.client, client_auth.uri.bucket, - client_auth.uri.key, + keys.back(), min_upload_part_size, max_single_part_upload_size); } @@ -599,7 +696,7 @@ SinkToStoragePtr StorageS3::write(const ASTPtr & query, const StorageMetadataPtr chosen_compression_method, client_auth.client, client_auth.uri.bucket, - client_auth.uri.key, + keys.back(), min_upload_part_size, max_single_part_upload_size); } @@ -610,11 +707,17 @@ void StorageS3::truncate(const ASTPtr & /* query */, const StorageMetadataPtr &, { updateClientAndAuthSettings(local_context, client_auth); - Aws::S3::Model::ObjectIdentifier obj; - obj.SetKey(client_auth.uri.key); + if (is_key_with_globs) + throw Exception(ErrorCodes::DATABASE_ACCESS_DENIED, "S3 key '{}' contains globs, so the table is in readonly mode", client_auth.uri.key); Aws::S3::Model::Delete delkeys; - delkeys.AddObjects(std::move(obj)); + + for (const auto & key : keys) + { + Aws::S3::Model::ObjectIdentifier obj; + obj.SetKey(key); + delkeys.AddObjects(std::move(obj)); + } Aws::S3::Model::DeleteObjectsRequest request; request.SetBucket(client_auth.uri.bucket); @@ -731,7 +834,7 @@ ColumnsDescription StorageS3::getTableStructureFromData( { ClientAuthentication client_auth{uri, access_key_id, secret_access_key, max_connections, {}, {}}; updateClientAndAuthSettings(ctx, client_auth); - return getTableStructureFromDataImpl(format, client_auth, max_single_read_retries, compression_method, distributed_processing, format_settings, ctx); + return getTableStructureFromDataImpl(format, client_auth, max_single_read_retries, compression_method, distributed_processing, uri.key.find_first_of("*?{") != std::string::npos, format_settings, ctx); } ColumnsDescription StorageS3::getTableStructureFromDataImpl( @@ -740,12 +843,13 @@ ColumnsDescription StorageS3::getTableStructureFromDataImpl( UInt64 max_single_read_retries, const String & compression_method, bool distributed_processing, + bool is_key_with_globs, const std::optional & format_settings, ContextPtr ctx) { auto read_buffer_creator = [&]() { - auto file_iterator = createFileIterator(client_auth, distributed_processing, ctx); + auto file_iterator = createFileIterator(client_auth, {client_auth.uri.key}, is_key_with_globs, distributed_processing, ctx); String current_key = (*file_iterator)(); if (current_key.empty()) throw Exception( diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index 0690040915d..1b766676427 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -44,6 +44,18 @@ public: std::shared_ptr pimpl; }; + class KeysIterator + { + public: + explicit KeysIterator(const std::vector & keys_); + String next(); + + private: + class Impl; + /// shared_ptr to have copy constructor + std::shared_ptr pimpl; + }; + using IteratorWrapper = std::function; static Block getHeader(Block sample_block, bool with_path_column, bool with_file_column); @@ -174,6 +186,7 @@ private: }; ClientAuthentication client_auth; + std::vector keys; String format_name; UInt64 max_single_read_retries; @@ -184,10 +197,11 @@ private: const bool distributed_processing; std::optional format_settings; ASTPtr partition_by; + bool is_key_with_globs = false; static void updateClientAndAuthSettings(ContextPtr, ClientAuthentication &); - static std::shared_ptr createFileIterator(const ClientAuthentication & client_auth, bool distributed_processing, ContextPtr local_context); + static std::shared_ptr createFileIterator(const ClientAuthentication & client_auth, const std::vector & keys, bool is_key_with_globs, bool distributed_processing, ContextPtr local_context); static ColumnsDescription getTableStructureFromDataImpl( const String & format, @@ -195,6 +209,7 @@ private: UInt64 max_single_read_retries, const String & compression_method, bool distributed_processing, + bool is_key_with_globs, const std::optional & format_settings, ContextPtr ctx); }; diff --git a/tests/integration/test_storage_hdfs/test.py b/tests/integration/test_storage_hdfs/test.py index f317fb5429a..d0305d6c932 100644 --- a/tests/integration/test_storage_hdfs/test.py +++ b/tests/integration/test_storage_hdfs/test.py @@ -362,6 +362,43 @@ def test_hdfsCluster(started_cluster): fs.delete(dir, recursive=True) +def test_overwrite(started_cluster): + hdfs_api = started_cluster.hdfs_api + + table_function = f"hdfs('hdfs://hdfs1:9000/data', 'Parquet', 'a Int32, b String')" + node1.query(f"create table test as {table_function}") + node1.query(f"insert into test select number, randomString(100) from numbers(5)") + node1.query_and_get_error(f"insert into test select number, randomString(100) FROM numbers(10)") + node1.query(f"insert into test select number, randomString(100) from numbers(10) settings hdfs_truncate_on_insert=1") + + result = node1.query(f"select count() from test") + assert(int(result) == 10) + + +def test_multiple_inserts(started_cluster): + hdfs_api = started_cluster.hdfs_api + + table_function = f"hdfs('hdfs://hdfs1:9000/data_multiple_inserts', 'Parquet', 'a Int32, b String')" + node1.query(f"create table test_multiple_inserts as {table_function}") + node1.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(10)") + node1.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(20) settings hdfs_create_new_file_on_insert=1") + node1.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(30) settings hdfs_create_new_file_on_insert=1") + + result = node1.query(f"select count() from test_multiple_inserts") + assert(int(result) == 60) + + result = node1.query(f"drop table test_multiple_inserts") + + table_function = f"hdfs('hdfs://hdfs1:9000/data_multiple_inserts.gz', 'Parquet', 'a Int32, b String')" + node1.query(f"create table test_multiple_inserts as {table_function}") + node1.query(f"insert into test_multiple_inserts select number, randomString(100) FROM numbers(10)") + node1.query(f"insert into test_multiple_inserts select number, randomString(100) FROM numbers(20) settings hdfs_create_new_file_on_insert=1") + node1.query(f"insert into test_multiple_inserts select number, randomString(100) FROM numbers(30) settings hdfs_create_new_file_on_insert=1") + + result = node1.query(f"select count() from test_multiple_inserts") + assert(int(result) == 60) + + if __name__ == '__main__': cluster.start() input("Cluster created, press any key to destroy...") diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 885a37f875c..50a9bd3b49d 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -915,3 +915,45 @@ def test_empty_file(started_cluster): result = instance.query(f"SELECT count() FROM {table_function}") assert(int(result) == 0) + +def test_overwrite(started_cluster): + bucket = started_cluster.minio_bucket + instance = started_cluster.instances["dummy"] + + table_function = f"s3(s3_parquet, structure='a Int32, b String', format='Parquet')" + instance.query(f"create table test_overwrite as {table_function}") + instance.query(f"truncate table test_overwrite") + instance.query(f"insert into test_overwrite select number, randomString(100) from numbers(50) settings s3_truncate_on_insert=1") + instance.query_and_get_error(f"insert into test_overwrite select number, randomString(100) from numbers(100)") + instance.query(f"insert into test_overwrite select number, randomString(100) from numbers(200) settings s3_truncate_on_insert=1") + + result = instance.query(f"select count() from test_overwrite") + assert(int(result) == 200) + + +def test_create_new_files_on_insert(started_cluster): + bucket = started_cluster.minio_bucket + instance = started_cluster.instances["dummy"] + + table_function = f"s3(s3_parquet, structure='a Int32, b String', format='Parquet')" + instance.query(f"create table test_multiple_inserts as {table_function}") + instance.query(f"truncate table test_multiple_inserts") + instance.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(10)") + instance.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(20) settings s3_create_new_file_on_insert=1") + instance.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(30) settings s3_create_new_file_on_insert=1") + + result = instance.query(f"select count() from test_multiple_inserts") + assert(int(result) == 60) + + instance.query(f"drop table test_multiple_inserts") + + table_function = f"s3(s3_parquet_gz, structure='a Int32, b String', format='Parquet')" + instance.query(f"create table test_multiple_inserts as {table_function}") + instance.query(f"truncate table test_multiple_inserts") + instance.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(10)") + instance.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(20) settings s3_create_new_file_on_insert=1") + instance.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(30) settings s3_create_new_file_on_insert=1") + + result = instance.query(f"select count() from test_multiple_inserts") + assert(int(result) == 60) + diff --git a/tests/queries/0_stateless/02155_multiple_inserts_for_formats_with_suffix.reference b/tests/queries/0_stateless/02155_multiple_inserts_for_formats_with_suffix.reference new file mode 100644 index 00000000000..beeb89a5947 --- /dev/null +++ b/tests/queries/0_stateless/02155_multiple_inserts_for_formats_with_suffix.reference @@ -0,0 +1,100 @@ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 diff --git a/tests/queries/0_stateless/02155_multiple_inserts_for_formats_with_suffix.sql b/tests/queries/0_stateless/02155_multiple_inserts_for_formats_with_suffix.sql new file mode 100644 index 00000000000..7947536bc79 --- /dev/null +++ b/tests/queries/0_stateless/02155_multiple_inserts_for_formats_with_suffix.sql @@ -0,0 +1,39 @@ +-- Tags: no-fasttest, no-parallel + +drop table if exists test; +create table test (number UInt64) engine=File('Parquet'); +insert into test select * from numbers(10); +insert into test select * from numbers(10, 10); -- { serverError CANNOT_APPEND_TO_FILE } +insert into test select * from numbers(10, 10) settings engine_file_allow_create_multiple_files=1; +select * from test order by number; +truncate table test; +drop table test; + +create table test (number UInt64) engine=File('Parquet', 'test_02155/test1/data.Parquet'); +insert into test select * from numbers(10) settings engine_file_truncate_on_insert=1; +insert into test select * from numbers(10, 10); -- { serverError CANNOT_APPEND_TO_FILE } +insert into test select * from numbers(10, 10) settings engine_file_allow_create_multiple_files=1; +select * from test order by number; +drop table test; + + +insert into table function file(concat(currentDatabase(), '/test2/data.Parquet'), 'Parquet', 'number UInt64') select * from numbers(10) settings engine_file_truncate_on_insert=1; +insert into table function file(concat(currentDatabase(), '/test2/data.Parquet'), 'Parquet', 'number UInt64') select * from numbers(10, 10); -- { serverError CANNOT_APPEND_TO_FILE } +insert into table function file(concat(currentDatabase(), '/test2/data.Parquet'), 'Parquet', 'number UInt64') select * from numbers(10, 10) settings engine_file_allow_create_multiple_files=1; +select * from file(concat(currentDatabase(), '/test2/data.Parquet'), 'Parquet', 'number UInt64'); +select * from file(concat(currentDatabase(), '/test2/data.1.Parquet'), 'Parquet', 'number UInt64'); + +create table test (number UInt64) engine=File('Parquet', 'test_02155/test3/data.Parquet.gz'); +insert into test select * from numbers(10) settings engine_file_truncate_on_insert=1; +; +insert into test select * from numbers(10, 10); -- { serverError CANNOT_APPEND_TO_FILE } +insert into test select * from numbers(10, 10) settings engine_file_allow_create_multiple_files=1; +select * from test order by number; +drop table test; + +insert into table function file(concat(currentDatabase(), '/test4/data.Parquet.gz'), 'Parquet', 'number UInt64') select * from numbers(10) settings engine_file_truncate_on_insert=1; +insert into table function file(concat(currentDatabase(), '/test4/data.Parquet.gz'), 'Parquet', 'number UInt64') select * from numbers(10, 10); -- { serverError CANNOT_APPEND_TO_FILE } +insert into table function file(concat(currentDatabase(), '/test4/data.Parquet.gz'), 'Parquet', 'number UInt64') select * from numbers(10, 10) settings engine_file_allow_create_multiple_files=1; +select * from file(concat(currentDatabase(), '/test4/data.Parquet.gz'), 'Parquet', 'number UInt64'); +select * from file(concat(currentDatabase(), '/test4/data.1.Parquet.gz'), 'Parquet', 'number UInt64'); + From da7209df294679e21c2c38215dc7057233555c48 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 29 Dec 2021 21:25:06 +0300 Subject: [PATCH 010/403] Add description to settings --- src/Core/Settings.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index ad43c2a04d7..18746638621 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -75,11 +75,11 @@ class IColumn; M(UInt64, s3_max_single_read_retries, 4, "The maximum number of retries during single S3 read.", 0) \ M(UInt64, s3_max_redirects, 10, "Max number of S3 redirects hops allowed.", 0) \ M(UInt64, s3_max_connections, 1024, "The maximum number of connections per server.", 0) \ - M(Bool, s3_truncate_on_insert, false, "", 0) \ - M(Bool, s3_create_new_file_on_insert, false, "", 0) \ + M(Bool, s3_truncate_on_insert, false, "Enables or disables truncate before insert in s3 engine tables.", 0) \ + M(Bool, s3_create_new_file_on_insert, false, "Enables or disables creating a new file on each insert in s3 engine tables", 0) \ M(UInt64, hdfs_replication, 0, "The actual number of replications can be specified when the hdfs file is created.", 0) \ - M(Bool, hdfs_truncate_on_insert, false, "", 0) \ - M(Bool, hdfs_create_new_file_on_insert, false, "", 0) \ + M(Bool, hdfs_truncate_on_insert, false, "Enables or disables truncate before insert in s3 engine tables", 0) \ + M(Bool, hdfs_create_new_file_on_insert, false, "Enables or disables creating a new file on each insert in hdfs engine tables", 0) \ M(UInt64, hsts_max_age, 0, "Expired time for hsts. 0 means disable HSTS.", 0) \ M(Bool, extremes, false, "Calculate minimums and maximums of the result columns. They can be output in JSON-formats.", IMPORTANT) \ M(Bool, use_uncompressed_cache, false, "Whether to use the cache of uncompressed blocks.", 0) \ @@ -494,7 +494,7 @@ class IColumn; \ M(Bool, engine_file_empty_if_not_exists, false, "Allows to select data from a file engine table without file", 0) \ M(Bool, engine_file_truncate_on_insert, false, "Enables or disables truncate before insert in file engine tables", 0) \ - M(Bool, engine_file_allow_create_multiple_files, false, ".", 0) \ + M(Bool, engine_file_allow_create_multiple_files, false, "Enables or disables creating a new file on each insert in file engine tables if format has suffix.", 0) \ M(Bool, allow_experimental_database_replicated, false, "Allow to create databases with Replicated engine", 0) \ M(UInt64, database_replicated_initial_query_timeout_sec, 300, "How long initial DDL query should wait for Replicated database to precess previous DDL queue entries", 0) \ M(UInt64, max_distributed_depth, 5, "Maximum distributed query depth", 0) \ From 131d49e6be334050745020675f57ec74798937ca Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 29 Dec 2021 21:26:23 +0300 Subject: [PATCH 011/403] Remove comments --- src/Storages/HDFS/WriteBufferFromHDFS.cpp | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/Storages/HDFS/WriteBufferFromHDFS.cpp b/src/Storages/HDFS/WriteBufferFromHDFS.cpp index a5050953b7c..2950046b633 100644 --- a/src/Storages/HDFS/WriteBufferFromHDFS.cpp +++ b/src/Storages/HDFS/WriteBufferFromHDFS.cpp @@ -38,15 +38,6 @@ struct WriteBufferFromHDFS::WriteBufferFromHDFSImpl const size_t begin_of_path = hdfs_uri.find('/', hdfs_uri.find("//") + 2); const String path = hdfs_uri.substr(begin_of_path); -// if (path.find_first_of("*?{") != std::string::npos) -// throw Exception(ErrorCodes::CANNOT_OPEN_FILE, "URI '{}' contains globs, so the table is in readonly mode", hdfs_uri); -// -// if (!hdfsExists(fs.get(), path.c_str()) && !truncate_) -// throw Exception( -// ErrorCodes::BAD_ARGUMENTS, -// "File {} already exists. If you want to overwrite it, enable setting hdfs_truncate_on_insert", -// path); - fout = hdfsOpenFile(fs.get(), path.c_str(), flags, 0, replication_, 0); /// O_WRONLY meaning create or overwrite i.e., implies O_TRUNCAT here if (fout == nullptr) From f8114126ed44443709e9c7e270884e5fc762ac47 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 30 Dec 2021 19:21:49 +0300 Subject: [PATCH 012/403] Use HashMap and arena with free lists for keeper --- src/Coordination/KeeperSnapshotManager.cpp | 17 +---- src/Coordination/KeeperStateMachine.cpp | 2 +- src/Coordination/KeeperStorage.cpp | 20 +---- src/Coordination/SnapshotableHashTable.h | 88 +++++++++++++++------- src/Coordination/ZooKeeperDataReader.cpp | 21 ++---- src/Coordination/pathUtils.cpp | 38 ++++++++++ src/Coordination/pathUtils.h | 13 ++++ src/Server/KeeperTCPHandler.cpp | 2 +- utils/keeper-bench/Generator.cpp | 51 +++++++++++++ 9 files changed, 177 insertions(+), 75 deletions(-) create mode 100644 src/Coordination/pathUtils.cpp create mode 100644 src/Coordination/pathUtils.h diff --git a/src/Coordination/KeeperSnapshotManager.cpp b/src/Coordination/KeeperSnapshotManager.cpp index 518d569ca67..fc90ce860d0 100644 --- a/src/Coordination/KeeperSnapshotManager.cpp +++ b/src/Coordination/KeeperSnapshotManager.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -41,20 +42,6 @@ namespace return base; } - std::string getBaseName(const String & path) - { - size_t basename_start = path.rfind('/'); - return std::string{&path[basename_start + 1], path.length() - basename_start - 1}; - } - - String parentPath(const String & path) - { - auto rslash_pos = path.rfind('/'); - if (rslash_pos > 0) - return path.substr(0, rslash_pos); - return "/"; - } - void writeNode(const KeeperStorage::Node & node, SnapshotVersion version, WriteBuffer & out) { writeBinary(node.data, out); @@ -292,7 +279,7 @@ void KeeperStorageSnapshot::deserialize(SnapshotDeserializationResult & deserial if (itr.key != "/") { auto parent_path = parentPath(itr.key); - storage.container.updateValue(parent_path, [&path = itr.key] (KeeperStorage::Node & value) { value.children.insert(getBaseName(path)); }); + storage.container.updateValue(parent_path, [path = itr.key] (KeeperStorage::Node & value) { value.children.insert(getBaseName(path)); }); } } diff --git a/src/Coordination/KeeperStateMachine.cpp b/src/Coordination/KeeperStateMachine.cpp index 1ac1a584451..98d522eaa9b 100644 --- a/src/Coordination/KeeperStateMachine.cpp +++ b/src/Coordination/KeeperStateMachine.cpp @@ -155,7 +155,7 @@ bool KeeperStateMachine::apply_snapshot(nuraft::snapshot & s) { /// deserialize and apply snapshot to storage std::lock_guard lock(storage_and_responses_lock); - auto snapshot_deserialization_result = snapshot_manager.deserializeSnapshotFromBuffer(latest_snapshot_buf); + auto snapshot_deserialization_result = snapshot_manager.deserializeSnapshotFromBuffer(latest_snapshot_ptr); storage = std::move(snapshot_deserialization_result.storage); latest_snapshot_meta = snapshot_deserialization_result.snapshot_meta; cluster_config = snapshot_deserialization_result.cluster_config; diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index a64a7d425f6..17f01c3344f 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -11,6 +11,7 @@ #include #include #include +#include namespace DB { @@ -21,20 +22,6 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -static String parentPath(const String & path) -{ - auto rslash_pos = path.rfind('/'); - if (rslash_pos > 0) - return path.substr(0, rslash_pos); - return "/"; -} - -static std::string getBaseName(const String & path) -{ - size_t basename_start = path.rfind('/'); - return std::string{&path[basename_start + 1], path.length() - basename_start - 1}; -} - static String base64Encode(const String & decoded) { std::ostringstream ostr; // STYLE_CHECK_ALLOW_STD_STRING_STREAM @@ -1095,8 +1082,9 @@ KeeperStorage::ResponsesForSessions KeeperStorage::processRequest(const Coordina { --parent.stat.numChildren; ++parent.stat.cversion; - parent.children.erase(getBaseName(ephemeral_path)); - parent.size_bytes -= getBaseName(ephemeral_path).size(); + auto base_name = getBaseName(ephemeral_path); + parent.children.erase(base_name); + parent.size_bytes -= base_name.size(); }); auto responses = processWatchesImpl(ephemeral_path, watches, list_watches, Coordination::Event::DELETED); diff --git a/src/Coordination/SnapshotableHashTable.h b/src/Coordination/SnapshotableHashTable.h index 7704825f830..f400659bb0d 100644 --- a/src/Coordination/SnapshotableHashTable.h +++ b/src/Coordination/SnapshotableHashTable.h @@ -1,18 +1,23 @@ #pragma once #include +#include +#include #include #include #include +#include namespace DB { + template struct ListNode { - std::string key; + StringRef key; V value; bool active_in_map; + bool free_key{false}; }; @@ -23,11 +28,13 @@ private: using ListElem = ListNode; using List = std::list; - using IndexMap = std::unordered_map; + using Mapped = typename List::iterator; + using IndexMap = HashMap; List list; IndexMap map; bool snapshot_mode{false}; + ArenaWithFreeLists arena; uint64_t approximate_data_size{0}; @@ -105,6 +112,17 @@ private: } } + StringRef copyStringInArena(const std::string & value_to_copy) + { + size_t value_to_copy_size = value_to_copy.size(); + char * place_for_key = arena.alloc(value_to_copy_size); + memcpy(reinterpret_cast(place_for_key), reinterpret_cast(value_to_copy.data()), value_to_copy_size); + StringRef updated_value{place_for_key, value_to_copy_size}; + + return updated_value; + } + + public: using iterator = typename List::iterator; @@ -115,12 +133,18 @@ public: bool insert(const std::string & key, const V & value) { - auto it = map.find(key); - if (it == map.end()) + size_t hash_value = map.hash(key); + auto it = map.find(key, hash_value); + + if (!it) { - ListElem elem{key, value, true}; + ListElem elem{copyStringInArena(key), value, true}; auto itr = list.insert(list.end(), elem); - map.emplace(itr->key, itr); + bool inserted; + map.emplace(itr->key, it, inserted, hash_value); + assert(inserted); + + it->getMapped() = itr; updateDataSize(INSERT, key.size(), value.sizeInBytes(), 0); return true; } @@ -128,28 +152,30 @@ public: return false; } - void insertOrReplace(const std::string & key, const V & value) { - auto it = map.find(key); - uint64_t old_value_size = it == map.end() ? 0 : it->second->value.sizeInBytes(); + size_t hash_value = map.hash(key); + auto it = map.find(key, hash_value); + uint64_t old_value_size = it == map.end() ? 0 : it->getMapped()->value.sizeInBytes(); if (it == map.end()) { - ListElem elem{key, value, true}; + ListElem elem{copyStringInArena(key), value, true}; auto itr = list.insert(list.end(), elem); - map.emplace(itr->key, itr); + bool inserted; + map.emplace(itr->key, it, inserted, hash_value); + assert(inserted); + it->getMapped() = itr; } else { - auto list_itr = it->second; + auto list_itr = it->getMapped(); if (snapshot_mode) { - ListElem elem{key, value, true}; + ListElem elem{list_itr->key, value, true}; list_itr->active_in_map = false; auto new_list_itr = list.insert(list.end(), elem); - map.erase(it); - map.emplace(new_list_itr->key, new_list_itr); + it->getMapped() = new_list_itr; } else { @@ -165,17 +191,19 @@ public: if (it == map.end()) return false; - auto list_itr = it->second; + auto list_itr = it->getMapped(); uint64_t old_data_size = list_itr->value.sizeInBytes(); if (snapshot_mode) { list_itr->active_in_map = false; - map.erase(it); + list_itr->free_key = true; + map.erase(it->getKey()); } else { - map.erase(it); + map.erase(it->getKey()); list.erase(list_itr); + arena.free(const_cast(list_itr->key.data), list_itr->key.size); } updateDataSize(ERASE, key.size(), 0, old_data_size); @@ -189,10 +217,11 @@ public: const_iterator updateValue(const std::string & key, ValueUpdater updater) { - auto it = map.find(key); + size_t hash_value = map.hash(key); + auto it = map.find(key, hash_value); assert(it != map.end()); - auto list_itr = it->second; + auto list_itr = it->getMapped(); uint64_t old_value_size = list_itr->value.sizeInBytes(); const_iterator ret; @@ -201,10 +230,10 @@ public: { auto elem_copy = *(list_itr); list_itr->active_in_map = false; - map.erase(it); + updater(elem_copy.value); auto itr = list.insert(list.end(), elem_copy); - map.emplace(itr->key, itr); + it->getMapped() = itr; ret = itr; } else @@ -212,6 +241,7 @@ public: updater(list_itr->value); ret = list_itr; } + updateDataSize(UPDATE_VALUE, key.size(), ret->value.sizeInBytes(), old_value_size); return ret; } @@ -220,15 +250,16 @@ public: { auto map_it = map.find(key); if (map_it != map.end()) - return map_it->second; + return map_it->getMapped(); return list.end(); } + const V & getValue(const std::string & key) const { auto it = map.find(key); - assert(it != map.end()); - return it->second->value; + assert(it); + return it->getMapped()->value; } void clearOutdatedNodes() @@ -239,11 +270,16 @@ public: { if (!itr->active_in_map) { - updateDataSize(CLEAR_OUTDATED_NODES, itr->key.size(), itr->value.sizeInBytes(), 0); + updateDataSize(CLEAR_OUTDATED_NODES, itr->key.size, itr->value.sizeInBytes(), 0); + if (itr->free_key) + arena.free(const_cast(itr->key.data), itr->key.size); itr = list.erase(itr); } else + { + assert(!itr->free_key); itr++; + } } } diff --git a/src/Coordination/ZooKeeperDataReader.cpp b/src/Coordination/ZooKeeperDataReader.cpp index 4f71274291b..3e1a3198b7b 100644 --- a/src/Coordination/ZooKeeperDataReader.cpp +++ b/src/Coordination/ZooKeeperDataReader.cpp @@ -1,10 +1,13 @@ #include + #include #include +#include + #include #include #include -#include +#include namespace DB @@ -16,20 +19,6 @@ namespace ErrorCodes extern const int CORRUPTED_DATA; } -static String parentPath(const String & path) -{ - auto rslash_pos = path.rfind('/'); - if (rslash_pos > 0) - return path.substr(0, rslash_pos); - return "/"; -} - -static std::string getBaseName(const String & path) -{ - size_t basename_start = path.rfind('/'); - return std::string{&path[basename_start + 1], path.length() - basename_start - 1}; -} - int64_t getZxidFromName(const std::string & filename) { std::filesystem::path path(filename); @@ -148,7 +137,7 @@ int64_t deserializeStorageData(KeeperStorage & storage, ReadBuffer & in, Poco::L if (itr.key != "/") { auto parent_path = parentPath(itr.key); - storage.container.updateValue(parent_path, [&path = itr.key] (KeeperStorage::Node & value) { value.children.insert(getBaseName(path)); value.stat.numChildren++; }); + storage.container.updateValue(parent_path, [path = itr.key] (KeeperStorage::Node & value) { value.children.insert(getBaseName(path)); value.stat.numChildren++; }); } } diff --git a/src/Coordination/pathUtils.cpp b/src/Coordination/pathUtils.cpp new file mode 100644 index 00000000000..dbef6d6d2af --- /dev/null +++ b/src/Coordination/pathUtils.cpp @@ -0,0 +1,38 @@ +#include +#include + +namespace DB +{ + +static size_t findLastSlash(StringRef path) +{ + if (path.size == 0) + return std::string::npos; + + for (size_t i = path.size - 1; i > 0; --i) + { + if (path.data[i] == '/') + return i; + } + + if (path.data[0] == '/') + return 0; + + return std::string::npos; +} + +std::string parentPath(StringRef path) +{ + auto rslash_pos = findLastSlash(path); + if (rslash_pos > 0) + return std::string{path.data, rslash_pos}; + return "/"; +} + +std::string getBaseName(StringRef path) +{ + size_t basename_start = findLastSlash(path); + return std::string{path.data + basename_start + 1, path.size - basename_start - 1}; +} + +} diff --git a/src/Coordination/pathUtils.h b/src/Coordination/pathUtils.h new file mode 100644 index 00000000000..c20a786a15c --- /dev/null +++ b/src/Coordination/pathUtils.h @@ -0,0 +1,13 @@ +#pragma once + +#include +#include + +namespace DB +{ + +std::string parentPath(StringRef path); + +std::string getBaseName(StringRef path); + +} diff --git a/src/Server/KeeperTCPHandler.cpp b/src/Server/KeeperTCPHandler.cpp index 0c5d7d93689..02f8d7cc81a 100644 --- a/src/Server/KeeperTCPHandler.cpp +++ b/src/Server/KeeperTCPHandler.cpp @@ -276,7 +276,7 @@ Poco::Timespan KeeperTCPHandler::receiveHandshake(int32_t handshake_length) void KeeperTCPHandler::runImpl() { - setThreadName("TstKprHandler"); + setThreadName("KeeperHandler"); ThreadStatus thread_status; auto global_receive_timeout = global_context->getSettingsRef().receive_timeout; auto global_send_timeout = global_context->getSettingsRef().send_timeout; diff --git a/utils/keeper-bench/Generator.cpp b/utils/keeper-bench/Generator.cpp index 852de07f2e1..0b35144b0ed 100644 --- a/utils/keeper-bench/Generator.cpp +++ b/utils/keeper-bench/Generator.cpp @@ -48,8 +48,59 @@ std::string generateRandomData(size_t size) return generateRandomString(size); } +void removeRecursive(Coordination::ZooKeeper & zookeeper, const std::string & path) +{ + namespace fs = std::filesystem; + + auto promise = std::make_shared>(); + auto future = promise->get_future(); + + Strings children; + auto list_callback = [promise, &children] (const ListResponse & response) + { + children = response.names; + + promise->set_value(); + }; + zookeeper.list(path, list_callback, nullptr); + future.get(); + + while (!children.empty()) + { + Coordination::Requests ops; + for (size_t i = 0; i < MULTI_BATCH_SIZE && !children.empty(); ++i) + { + removeRecursive(zookeeper, fs::path(path) / children.back()); + ops.emplace_back(makeRemoveRequest(fs::path(path) / children.back(), -1)); + children.pop_back(); + } + auto multi_promise = std::make_shared>(); + auto multi_future = multi_promise->get_future(); + + auto multi_callback = [multi_promise] (const MultiResponse &) + { + multi_promise->set_value(); + }; + zookeeper.multi(ops, multi_callback); + multi_future.get(); + } + auto remove_promise = std::make_shared>(); + auto remove_future = remove_promise->get_future(); + + auto remove_callback = [remove_promise] (const RemoveResponse &) + { + remove_promise->set_value(); + }; + + zookeeper.remove(path, -1, remove_callback); + remove_future.get(); +} + + void CreateRequestGenerator::startup(Coordination::ZooKeeper & zookeeper) { + removeRecursive(zookeeper, path_prefix); + auto promise = std::make_shared>(); auto future = promise->get_future(); auto create_callback = [promise] (const CreateResponse & response) From f2e5399c61153dd152acba3363c99d494fc0cbc8 Mon Sep 17 00:00:00 2001 From: RogerYK Date: Sun, 2 Jan 2022 18:40:34 +0800 Subject: [PATCH 013/403] Add function bitSlice for String and FixedString --- src/Functions/bitSlice.cpp | 480 ++++++++++++++++++ src/Functions/registerFunctionsArithmetic.cpp | 2 + .../02154_bit_slice_for_fixedstring.reference | 163 ++++++ .../02154_bit_slice_for_fixedstring.sql | 111 ++++ .../02154_bit_slice_for_string.reference | 163 ++++++ .../02154_bit_slice_for_string.sql | 111 ++++ 6 files changed, 1030 insertions(+) create mode 100644 src/Functions/bitSlice.cpp create mode 100644 tests/queries/0_stateless/02154_bit_slice_for_fixedstring.reference create mode 100644 tests/queries/0_stateless/02154_bit_slice_for_fixedstring.sql create mode 100644 tests/queries/0_stateless/02154_bit_slice_for_string.reference create mode 100644 tests/queries/0_stateless/02154_bit_slice_for_string.sql diff --git a/src/Functions/bitSlice.cpp b/src/Functions/bitSlice.cpp new file mode 100644 index 00000000000..ce6f0187a67 --- /dev/null +++ b/src/Functions/bitSlice.cpp @@ -0,0 +1,480 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +using namespace GatherUtils; + +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ZERO_ARRAY_OR_TUPLE_INDEX; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + +class FunctionBitSlice : public IFunction +{ + const UInt8 word_size = 8; + +public: + static constexpr auto name = "bitSlice"; + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + String getName() const override { return name; } + + bool isVariadic() const override { return true; } + size_t getNumberOfArguments() const override { return 0; } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + const size_t number_of_arguments = arguments.size(); + + if (number_of_arguments < 2 || number_of_arguments > 3) + throw Exception( + "Number of arguments for function " + getName() + " doesn't match: passed " + toString(number_of_arguments) + + ", should be 2 or 3", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + if (!isString(arguments[0]) && !isStringOrFixedString(arguments[0])) + throw Exception( + "Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (arguments[0]->onlyNull()) + return arguments[0]; + + if (!isNativeNumber(arguments[1])) + throw Exception( + "Illegal type " + arguments[1]->getName() + " of second argument of function " + getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + if (number_of_arguments == 3 && !isNativeNumber(arguments[2])) + throw Exception( + "Illegal type " + arguments[2]->getName() + " of second argument of function " + getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + size_t number_of_arguments = arguments.size(); + + ColumnPtr column_string = arguments[0].column; + ColumnPtr column_start = arguments[1].column; + ColumnPtr column_length; + + if (number_of_arguments == 3) + column_length = arguments[2].column; + + const ColumnConst * column_start_const = checkAndGetColumn(column_start.get()); + const ColumnConst * column_length_const = nullptr; + + if (number_of_arguments == 3) + column_length_const = checkAndGetColumn(column_length.get()); + + Int64 start_value = 0; + Int64 length_value = 0; + + if (column_start_const) + start_value = column_start_const->getInt(0); + if (column_length_const) + length_value = column_length_const->getInt(0); + + + if (const ColumnString * col = checkAndGetColumn(column_string.get())) + return executeForSource( + column_start, + column_length, + column_start_const, + column_length_const, + start_value, + length_value, + StringSource(*col), + input_rows_count); + else if (const ColumnFixedString * col_fixed = checkAndGetColumn(column_string.get())) + return executeForSource( + column_start, + column_length, + column_start_const, + column_length_const, + start_value, + length_value, + FixedStringSource(*col_fixed), + input_rows_count); + else if (const ColumnConst * col_const = checkAndGetColumnConst(column_string.get())) + return executeForSource( + column_start, + column_length, + column_start_const, + column_length_const, + start_value, + length_value, + ConstSource(*col_const), + input_rows_count); + else if (const ColumnConst * col_const_fixed = checkAndGetColumnConst(column_string.get())) + return executeForSource( + column_start, + column_length, + column_start_const, + column_length_const, + start_value, + length_value, + ConstSource(*col_const_fixed), + input_rows_count); + else + throw Exception( + "Illegal column " + arguments[0].column->getName() + " of first argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + } + + template + ColumnPtr executeForSource( + const ColumnPtr & column_start, + const ColumnPtr & column_length, + const ColumnConst * column_start_const, + const ColumnConst * column_length_const, + Int64 start_value, + Int64 length_value, + Source && source, + size_t input_rows_count) const + { + auto col_res = ColumnString::create(); + + if (!column_length) + { + if (column_start_const) + { + if (start_value > 0) + bitSliceFromLeftConstantOffsetUnbounded( + source, StringSink(*col_res, input_rows_count), static_cast(start_value - 1)); + else if (start_value < 0) + bitSliceFromRightConstantOffsetUnbounded( + source, StringSink(*col_res, input_rows_count), -static_cast(start_value)); + else + throw Exception("Indices in strings are 1-based", ErrorCodes::ZERO_ARRAY_OR_TUPLE_INDEX); + } + else + bitSliceDynamicOffsetUnbounded(source, StringSink(*col_res, input_rows_count), *column_start); + } + else + { + if (column_start_const && column_length_const) + { + if (start_value > 0) + bitSliceFromLeftConstantOffsetBounded( + source, StringSink(*col_res, input_rows_count), static_cast(start_value - 1), length_value); + else if (start_value < 0) + bitSliceFromRightConstantOffsetBounded( + source, StringSink(*col_res, input_rows_count), -static_cast(start_value), length_value); + else + throw Exception("Indices in strings are 1-based", ErrorCodes::ZERO_ARRAY_OR_TUPLE_INDEX); + } + else + bitSliceDynamicOffsetBounded(source, StringSink(*col_res, input_rows_count), *column_start, *column_length); + } + + return col_res; + } + + void writeSliceWithLeftShift(const StringSource::Slice & slice, StringSink & sink, size_t shift_bit, size_t abandon_last_bit = 0) const + { + if (!shift_bit && !abandon_last_bit) + { + writeSlice(slice, sink); + return; + } + size_t size = slice.size; + if (!size) + return; + bool abandon_last_byte = abandon_last_bit + shift_bit >= word_size; + if (abandon_last_byte) + size--; + sink.elements.resize(sink.current_offset + size); + UInt8 * out = &sink.elements[sink.current_offset]; + const UInt8 * input = slice.data; + + for (size_t i = 0; i < size - 1; i++) + { + out[i] = (input[i] << shift_bit) | (input[i + 1] >> (word_size - shift_bit)); + } + if (abandon_last_byte) + { + out[size - 1] = (input[size - 1] << shift_bit) | (input[size] >> (word_size - shift_bit)); + out[size - 1] = out[size - 1] & (0xFF << (abandon_last_bit + shift_bit - word_size)); + } + else + { + out[size - 1] = (input[size - 1] << shift_bit) & (0xFF << (abandon_last_bit + shift_bit)); + } + + + sink.current_offset += size; + } + + + template + void bitSliceFromLeftConstantOffsetUnbounded(Source && src, StringSink && sink, size_t offset) const + { + size_t offset_byte = offset / word_size; + size_t offset_bit = offset % word_size; + while (!src.isEnd()) + { + auto sl = src.getSliceFromLeft(offset_byte); + if (sl.size) + writeSliceWithLeftShift(sl, sink, offset_bit); + + sink.next(); + src.next(); + } + } + + template + void bitSliceFromRightConstantOffsetUnbounded(Source && src, StringSink && sink, size_t offset) const + { + size_t offset_byte = offset / word_size; + size_t offset_bit = (word_size - (offset % word_size)) % word_size; + if (offset_bit) + offset_byte++; + while (!src.isEnd()) + { + auto slice = src.getSliceFromRight(offset_byte); + size_t size = src.getElementSize(); + bool left_truncate = offset_byte > size; + size_t shift_bit = left_truncate ? 0 : offset_bit; + if (slice.size) + writeSliceWithLeftShift(slice, sink, shift_bit); + + sink.next(); + src.next(); + } + } + + template + void bitSliceDynamicOffsetUnbounded(Source && src, StringSink && sink, const IColumn & offset_column) const + { + const bool is_null = offset_column.onlyNull(); + const auto * nullable = typeid_cast(&offset_column); + const ColumnUInt8::Container * null_map = nullable ? &nullable->getNullMapData() : nullptr; + const IColumn * nested_column = nullable ? &nullable->getNestedColumn() : &offset_column; + + while (!src.isEnd()) + { + auto row_num = src.rowNum(); + bool has_offset = !is_null && (null_map && (*null_map)[row_num]); + Int64 start = has_offset ? nested_column->getInt(row_num) : 1; + if (start != 0) + { + typename std::decay_t::Slice slice; + size_t shift_bit; + + if (start > 0) + { + UInt64 offset = start - 1; + size_t offset_byte = offset / word_size; + size_t offset_bit = offset % word_size; + shift_bit = offset_bit; + slice = src.getSliceFromLeft(offset_byte); + } + else + { + UInt64 offset = -static_cast(start); + size_t offset_byte = offset / word_size; + size_t offset_bit = (word_size - (offset % word_size)) % word_size; + if (offset_bit) + offset_byte++; + size_t size = src.getElementSize(); + bool left_truncate = offset_byte > size; + shift_bit = left_truncate ? 0 : offset_bit; + slice = src.getSliceFromRight(offset_byte); + } + if (slice.size) + writeSliceWithLeftShift(slice, sink, shift_bit); + } + + sink.next(); + src.next(); + } + } + + template + void bitSliceFromLeftConstantOffsetBounded(Source && src, StringSink && sink, size_t offset, ssize_t length) const + { + size_t offset_byte = offset / word_size; + size_t offset_bit = offset % word_size; + size_t shift_bit = offset_bit; + size_t length_byte = 0; + size_t over_bit = 0; + if (length > 0) + { + length_byte = (length + offset_bit) / word_size; + over_bit = (length + offset_bit) % word_size; + if (over_bit && (length_byte || over_bit > offset_bit)) + length_byte++; + } + + while (!src.isEnd()) + { + ssize_t remain_byte = src.getElementSize() - offset_byte; + if (length < 0) + { + length_byte = std::max(remain_byte + (length / word_size), static_cast(0)); + over_bit = word_size + (length % word_size); + if (length_byte == 1 && over_bit <= offset_bit) { + length_byte = 0; + } + } + bool right_truncate = static_cast(length_byte) > remain_byte; + size_t abandon_last_bit = (over_bit && !right_truncate) ? word_size - over_bit : 0; + auto slice = src.getSliceFromLeft(offset_byte, length_byte); + if (slice.size) + writeSliceWithLeftShift(slice, sink, shift_bit, abandon_last_bit); + + sink.next(); + src.next(); + } + } + + + template + void bitSliceFromRightConstantOffsetBounded(Source && src, StringSink && sink, size_t offset, ssize_t length) const + { + size_t offset_byte = offset / word_size; + size_t offset_bit = (word_size - (offset % word_size)) % word_size; + if (offset_bit) + offset_byte++; + size_t length_byte = 0; + size_t over_bit = 0; + if (length > 0) + { + length_byte = (length + offset_bit) / word_size; + over_bit = (length + offset_bit) % word_size; + if (over_bit && (length_byte || over_bit > offset_bit)) + length_byte++; + } + + while (!src.isEnd()) + { + size_t size = src.getElementSize(); + if (length < 0) + { + length_byte = std::max(static_cast(offset_byte) + (length / word_size), static_cast(0)); + over_bit = word_size + (length % word_size); + if (length_byte == 1 && over_bit <= offset_bit) { + length_byte = 0; + } + } + bool left_truncate = offset_byte > size; + bool right_truncate = length_byte > offset_byte; + size_t shift_bit = left_truncate ? 0 : offset_bit; + size_t abandon_last_bit = (over_bit && !right_truncate) ? word_size - over_bit : 0; + auto slice = src.getSliceFromRight(offset_byte, length_byte); + if (slice.size) + writeSliceWithLeftShift(slice, sink, shift_bit, abandon_last_bit); + + sink.next(); + src.next(); + } + } + + template + void bitSliceDynamicOffsetBounded(Source && src, StringSink && sink, const IColumn & offset_column, const IColumn & length_column) const + { + const bool is_offset_null = offset_column.onlyNull(); + const auto * offset_nullable = typeid_cast(&offset_column); + const ColumnUInt8::Container * offset_null_map = offset_nullable ? &offset_nullable->getNullMapData() : nullptr; + const IColumn * offset_nested_column = offset_nullable ? &offset_nullable->getNestedColumn() : &offset_column; + + const bool is_length_null = length_column.onlyNull(); + const auto * length_nullable = typeid_cast(&length_column); + const ColumnUInt8::Container * length_null_map = length_nullable ? &length_nullable->getNullMapData() : nullptr; + const IColumn * length_nested_column = length_nullable ? &length_nullable->getNestedColumn() : &length_column; + + + while (!src.isEnd()) + { + size_t row_num = src.rowNum(); + bool has_offset = !is_offset_null && !(offset_null_map && (*offset_null_map)[row_num]); + bool has_length = !is_length_null && !(length_null_map && (*length_null_map)[row_num]); + Int64 start = has_offset ? offset_nested_column->getInt(row_num) : 1; + Int64 length = has_length ? length_nested_column->getInt(row_num) : static_cast(src.getElementSize()); + + if (start && length) + { + bool left_offset = start > 0; + size_t offset = left_offset ? start - 1 : -start; + size_t offset_byte; + size_t offset_bit; + size_t shift_bit; + size_t length_byte; + size_t over_bit; + size_t size = src.getElementSize(); + + if (left_offset) + { + offset_byte = offset / word_size; + offset_bit = offset % word_size; + shift_bit = offset_bit; + } + else + { + offset_byte = offset / word_size; + offset_bit = (word_size - (offset % word_size)) % word_size; + if (offset_bit) + offset_byte++; + bool left_truncate = offset_byte > size; + shift_bit = left_truncate ? 0 : offset_bit; + } + + ssize_t remain_byte = left_offset ? size - offset_byte : offset_byte; + if (length > 0) + { + length_byte = (length + offset_bit) / word_size; + over_bit = (length + offset_bit) % word_size; + if (over_bit && (length_byte || (over_bit > offset_bit))) + length_byte++; + } + else + { + length_byte = std::max(remain_byte + (static_cast(length) / word_size), static_cast(0)); + over_bit = word_size + (length % word_size); + if (length_byte == 1 && over_bit <= offset_bit) { + length_byte = 0; + } + } + + bool right_truncate = static_cast(length_byte) > remain_byte; + size_t abandon_last_bit = (over_bit && !right_truncate) ? word_size - over_bit : 0; + auto slice = left_offset ? src.getSliceFromLeft(offset_byte, length_byte) : src.getSliceFromRight(offset_byte, length_byte); + if (slice.size) + writeSliceWithLeftShift(slice, sink, shift_bit, abandon_last_bit); + } + + + sink.next(); + src.next(); + } + } + +}; + + +void registerFunctionBitSlice(FunctionFactory & factory) +{ + factory.registerFunction(); +} + + +} diff --git a/src/Functions/registerFunctionsArithmetic.cpp b/src/Functions/registerFunctionsArithmetic.cpp index 857f370fde3..d3d82ca0dd8 100644 --- a/src/Functions/registerFunctionsArithmetic.cpp +++ b/src/Functions/registerFunctionsArithmetic.cpp @@ -20,6 +20,7 @@ void registerFunctionBitXor(FunctionFactory & factory); void registerFunctionBitNot(FunctionFactory & factory); void registerFunctionBitShiftLeft(FunctionFactory & factory); void registerFunctionBitShiftRight(FunctionFactory & factory); +void registerFunctionBitSlice(FunctionFactory & factory); void registerFunctionBitRotateLeft(FunctionFactory & factory); void registerFunctionBitRotateRight(FunctionFactory & factory); void registerFunctionBitCount(FunctionFactory & factory); @@ -64,6 +65,7 @@ void registerFunctionsArithmetic(FunctionFactory & factory) registerFunctionBitRotateLeft(factory); registerFunctionBitRotateRight(factory); registerFunctionBitCount(factory); + registerFunctionBitSlice(factory); registerFunctionLeast(factory); registerFunctionGreatest(factory); registerFunctionBitTest(factory); diff --git a/tests/queries/0_stateless/02154_bit_slice_for_fixedstring.reference b/tests/queries/0_stateless/02154_bit_slice_for_fixedstring.reference new file mode 100644 index 00000000000..d42a3c76ac7 --- /dev/null +++ b/tests/queries/0_stateless/02154_bit_slice_for_fixedstring.reference @@ -0,0 +1,163 @@ +Const Offset +1 01001000011001010110110001101100011011110000000000000000000000000000000000000000 01001000011001010110110001101100011011110000000000000000000000000000000000000000 +2 1001000011001010110110001101100011011110000000000000000000000000000000000000000 10010000110010101101100011011000110111100000000000000000000000000000000000000000 +3 001000011001010110110001101100011011110000000000000000000000000000000000000000 00100001100101011011000110110001101111000000000000000000000000000000000000000000 +4 01000011001010110110001101100011011110000000000000000000000000000000000000000 01000011001010110110001101100011011110000000000000000000000000000000000000000000 +5 1000011001010110110001101100011011110000000000000000000000000000000000000000 10000110010101101100011011000110111100000000000000000000000000000000000000000000 +6 000011001010110110001101100011011110000000000000000000000000000000000000000 00001100101011011000110110001101111000000000000000000000000000000000000000000000 +7 00011001010110110001101100011011110000000000000000000000000000000000000000 00011001010110110001101100011011110000000000000000000000000000000000000000000000 +8 0011001010110110001101100011011110000000000000000000000000000000000000000 00110010101101100011011000110111100000000000000000000000000000000000000000000000 +9 011001010110110001101100011011110000000000000000000000000000000000000000 011001010110110001101100011011110000000000000000000000000000000000000000 +10 11001010110110001101100011011110000000000000000000000000000000000000000 110010101101100011011000110111100000000000000000000000000000000000000000 +11 1001010110110001101100011011110000000000000000000000000000000000000000 100101011011000110110001101111000000000000000000000000000000000000000000 +12 001010110110001101100011011110000000000000000000000000000000000000000 001010110110001101100011011110000000000000000000000000000000000000000000 +13 01010110110001101100011011110000000000000000000000000000000000000000 010101101100011011000110111100000000000000000000000000000000000000000000 +14 1010110110001101100011011110000000000000000000000000000000000000000 101011011000110110001101111000000000000000000000000000000000000000000000 +15 010110110001101100011011110000000000000000000000000000000000000000 010110110001101100011011110000000000000000000000000000000000000000000000 +16 10110110001101100011011110000000000000000000000000000000000000000 101101100011011000110111100000000000000000000000000000000000000000000000 +-1 0 00000000 +-2 00 00000000 +-3 000 00000000 +-4 0000 00000000 +-5 00000 00000000 +-6 000000 00000000 +-7 0000000 00000000 +-8 00000000 00000000 +-9 000000000 0000000000000000 +-10 0000000000 0000000000000000 +-11 00000000000 0000000000000000 +-12 000000000000 0000000000000000 +-13 0000000000000 0000000000000000 +-14 00000000000000 0000000000000000 +-15 000000000000000 0000000000000000 +-16 0000000000000000 0000000000000000 +Const Offset, Const Length +1 1 0 00000000 +2 2 10 10000000 +3 3 001 00100000 +4 4 0100 01000000 +5 5 10000 10000000 +6 6 000011 00001100 +7 7 0001100 00011000 +8 8 00110010 00110010 +9 9 011001010 0110010100000000 +10 10 1100101011 1100101011000000 +11 11 10010101101 1001010110100000 +12 12 001010110110 0010101101100000 +13 13 0101011011000 0101011011000000 +14 14 10101101100011 1010110110001100 +15 15 010110110001101 0101101100011010 +16 16 1011011000110110 1011011000110110 +1 -1 0100100001100101011011000110110001101111000000000000000000000000000000000000000 01001000011001010110110001101100011011110000000000000000000000000000000000000000 +2 -2 10010000110010101101100011011000110111100000000000000000000000000000000000000 10010000110010101101100011011000110111100000000000000000000000000000000000000000 +3 -3 001000011001010110110001101100011011110000000000000000000000000000000000000 00100001100101011011000110110001101111000000000000000000000000000000000000000000 +4 -4 0100001100101011011000110110001101111000000000000000000000000000000000000 01000011001010110110001101100011011110000000000000000000000000000000000000000000 +5 -5 10000110010101101100011011000110111100000000000000000000000000000000000 100001100101011011000110110001101111000000000000000000000000000000000000 +6 -6 000011001010110110001101100011011110000000000000000000000000000000000 000011001010110110001101100011011110000000000000000000000000000000000000 +7 -7 0001100101011011000110110001101111000000000000000000000000000000000 000110010101101100011011000110111100000000000000000000000000000000000000 +8 -8 00110010101101100011011000110111100000000000000000000000000000000 001100101011011000110110001101111000000000000000000000000000000000000000 +9 -9 011001010110110001101100011011110000000000000000000000000000000 0110010101101100011011000110111100000000000000000000000000000000 +10 -10 1100101011011000110110001101111000000000000000000000000000000 1100101011011000110110001101111000000000000000000000000000000000 +11 -11 10010101101100011011000110111100000000000000000000000000000 1001010110110001101100011011110000000000000000000000000000000000 +12 -12 001010110110001101100011011110000000000000000000000000000 0010101101100011011000110111100000000000000000000000000000000000 +13 -13 0101011011000110110001101111000000000000000000000000000 01010110110001101100011011110000000000000000000000000000 +14 -14 10101101100011011000110111100000000000000000000000000 10101101100011011000110111100000000000000000000000000000 +15 -15 010110110001101100011011110000000000000000000000000 01011011000110110001101111000000000000000000000000000000 +16 -16 1011011000110110001101111000000000000000000000000 10110110001101100011011110000000000000000000000000000000 +-1 1 0 00000000 +-2 2 00 00000000 +-3 3 000 00000000 +-4 4 0000 00000000 +-5 5 00000 00000000 +-6 6 000000 00000000 +-7 7 0000000 00000000 +-8 8 00000000 00000000 +-9 9 000000000 0000000000000000 +-10 10 0000000000 0000000000000000 +-11 11 00000000000 0000000000000000 +-12 12 000000000000 0000000000000000 +-13 13 0000000000000 0000000000000000 +-14 14 00000000000000 0000000000000000 +-15 15 000000000000000 0000000000000000 +-16 16 0000000000000000 0000000000000000 +-1 -16 +-2 -15 +-3 -14 +-4 -13 +-5 -12 +-6 -11 +-7 -10 +-8 -9 +-9 -8 0 00000000 +-10 -7 000 00000000 +-11 -6 00000 00000000 +-12 -5 0000000 00000000 +-13 -4 000000000 0000000000000000 +-14 -3 00000000000 0000000000000000 +-15 -2 0000000000000 0000000000000000 +-16 -1 000000000000000 0000000000000000 +Dynamic Offset, Dynamic Length +0 0 +1 1 0 00000000 +2 2 10 10000000 +3 3 001 00100000 +4 4 0100 01000000 +5 5 10000 10000000 +6 6 000011 00001100 +7 7 0001100 00011000 +8 8 00110010 00110010 +9 9 011001010 0110010100000000 +10 10 1100101011 1100101011000000 +11 11 10010101101 1001010110100000 +12 12 001010110110 0010101101100000 +13 13 0101011011000 0101011011000000 +14 14 10101101100011 1010110110001100 +15 15 010110110001101 0101101100011010 +0 0 +1 -1 0100100001100101011011000110110001101111000000000000000000000000000000000000000 01001000011001010110110001101100011011110000000000000000000000000000000000000000 +2 -2 10010000110010101101100011011000110111100000000000000000000000000000000000000 10010000110010101101100011011000110111100000000000000000000000000000000000000000 +3 -3 001000011001010110110001101100011011110000000000000000000000000000000000000 00100001100101011011000110110001101111000000000000000000000000000000000000000000 +4 -4 0100001100101011011000110110001101111000000000000000000000000000000000000 01000011001010110110001101100011011110000000000000000000000000000000000000000000 +5 -5 10000110010101101100011011000110111100000000000000000000000000000000000 100001100101011011000110110001101111000000000000000000000000000000000000 +6 -6 000011001010110110001101100011011110000000000000000000000000000000000 000011001010110110001101100011011110000000000000000000000000000000000000 +7 -7 0001100101011011000110110001101111000000000000000000000000000000000 000110010101101100011011000110111100000000000000000000000000000000000000 +8 -8 00110010101101100011011000110111100000000000000000000000000000000 001100101011011000110110001101111000000000000000000000000000000000000000 +9 -9 011001010110110001101100011011110000000000000000000000000000000 0110010101101100011011000110111100000000000000000000000000000000 +10 -10 1100101011011000110110001101111000000000000000000000000000000 1100101011011000110110001101111000000000000000000000000000000000 +11 -11 10010101101100011011000110111100000000000000000000000000000 1001010110110001101100011011110000000000000000000000000000000000 +12 -12 001010110110001101100011011110000000000000000000000000000 0010101101100011011000110111100000000000000000000000000000000000 +13 -13 0101011011000110110001101111000000000000000000000000000 01010110110001101100011011110000000000000000000000000000 +14 -14 10101101100011011000110111100000000000000000000000000 10101101100011011000110111100000000000000000000000000000 +15 -15 010110110001101100011011110000000000000000000000000 01011011000110110001101111000000000000000000000000000000 +0 -16 +-1 -15 +-2 -14 +-3 -13 +-4 -12 +-5 -11 +-6 -10 +-7 -9 +-8 -8 +-9 -7 00 00000000 +-10 -6 0000 00000000 +-11 -5 000000 00000000 +-12 -4 00000000 00000000 +-13 -3 0000000000 0000000000000000 +-14 -2 000000000000 0000000000000000 +-15 -1 00000000000000 0000000000000000 +0 0 +-1 1 0 00000000 +-2 2 00 00000000 +-3 3 000 00000000 +-4 4 0000 00000000 +-5 5 00000 00000000 +-6 6 000000 00000000 +-7 7 0000000 00000000 +-8 8 00000000 00000000 +-9 9 000000000 0000000000000000 +-10 10 0000000000 0000000000000000 +-11 11 00000000000 0000000000000000 +-12 12 000000000000 0000000000000000 +-13 13 0000000000000 0000000000000000 +-14 14 00000000000000 0000000000000000 +-15 15 000000000000000 0000000000000000 diff --git a/tests/queries/0_stateless/02154_bit_slice_for_fixedstring.sql b/tests/queries/0_stateless/02154_bit_slice_for_fixedstring.sql new file mode 100644 index 00000000000..2d49a1ff1d6 --- /dev/null +++ b/tests/queries/0_stateless/02154_bit_slice_for_fixedstring.sql @@ -0,0 +1,111 @@ +SELECT 'Const Offset'; +select 1, subString(bin(toFixedString('Hello', 10)), 1), bin(bitSlice(toFixedString('Hello', 10), 1)); +select 2, subString(bin(toFixedString('Hello', 10)), 2), bin(bitSlice(toFixedString('Hello', 10), 2)); +select 3, subString(bin(toFixedString('Hello', 10)), 3), bin(bitSlice(toFixedString('Hello', 10), 3)); +select 4, subString(bin(toFixedString('Hello', 10)), 4), bin(bitSlice(toFixedString('Hello', 10), 4)); +select 5, subString(bin(toFixedString('Hello', 10)), 5), bin(bitSlice(toFixedString('Hello', 10), 5)); +select 6, subString(bin(toFixedString('Hello', 10)), 6), bin(bitSlice(toFixedString('Hello', 10), 6)); +select 7, subString(bin(toFixedString('Hello', 10)), 7), bin(bitSlice(toFixedString('Hello', 10), 7)); +select 8, subString(bin(toFixedString('Hello', 10)), 8), bin(bitSlice(toFixedString('Hello', 10), 8)); +select 9, subString(bin(toFixedString('Hello', 10)), 9), bin(bitSlice(toFixedString('Hello', 10), 9)); +select 10, subString(bin(toFixedString('Hello', 10)), 10), bin(bitSlice(toFixedString('Hello', 10), 10)); +select 11, subString(bin(toFixedString('Hello', 10)), 11), bin(bitSlice(toFixedString('Hello', 10), 11)); +select 12, subString(bin(toFixedString('Hello', 10)), 12), bin(bitSlice(toFixedString('Hello', 10), 12)); +select 13, subString(bin(toFixedString('Hello', 10)), 13), bin(bitSlice(toFixedString('Hello', 10), 13)); +select 14, subString(bin(toFixedString('Hello', 10)), 14), bin(bitSlice(toFixedString('Hello', 10), 14)); +select 15, subString(bin(toFixedString('Hello', 10)), 15), bin(bitSlice(toFixedString('Hello', 10), 15)); +select 16, subString(bin(toFixedString('Hello', 10)), 16), bin(bitSlice(toFixedString('Hello', 10), 16)); + +select -1, subString(bin(toFixedString('Hello', 10)), -1), bin(bitSlice(toFixedString('Hello', 10), -1)); +select -2, subString(bin(toFixedString('Hello', 10)), -2), bin(bitSlice(toFixedString('Hello', 10), -2)); +select -3, subString(bin(toFixedString('Hello', 10)), -3), bin(bitSlice(toFixedString('Hello', 10), -3)); +select -4, subString(bin(toFixedString('Hello', 10)), -4), bin(bitSlice(toFixedString('Hello', 10), -4)); +select -5, subString(bin(toFixedString('Hello', 10)), -5), bin(bitSlice(toFixedString('Hello', 10), -5)); +select -6, subString(bin(toFixedString('Hello', 10)), -6), bin(bitSlice(toFixedString('Hello', 10), -6)); +select -7, subString(bin(toFixedString('Hello', 10)), -7), bin(bitSlice(toFixedString('Hello', 10), -7)); +select -8, subString(bin(toFixedString('Hello', 10)), -8), bin(bitSlice(toFixedString('Hello', 10), -8)); +select -9, subString(bin(toFixedString('Hello', 10)), -9), bin(bitSlice(toFixedString('Hello', 10), -9)); +select -10, subString(bin(toFixedString('Hello', 10)), -10), bin(bitSlice(toFixedString('Hello', 10), -10)); +select -11, subString(bin(toFixedString('Hello', 10)), -11), bin(bitSlice(toFixedString('Hello', 10), -11)); +select -12, subString(bin(toFixedString('Hello', 10)), -12), bin(bitSlice(toFixedString('Hello', 10), -12)); +select -13, subString(bin(toFixedString('Hello', 10)), -13), bin(bitSlice(toFixedString('Hello', 10), -13)); +select -14, subString(bin(toFixedString('Hello', 10)), -14), bin(bitSlice(toFixedString('Hello', 10), -14)); +select -15, subString(bin(toFixedString('Hello', 10)), -15), bin(bitSlice(toFixedString('Hello', 10), -15)); +select -16, subString(bin(toFixedString('Hello', 10)), -16), bin(bitSlice(toFixedString('Hello', 10), -16)); + + +SELECT 'Const Offset, Const Length'; +select 1, 1, subString(bin(toFixedString('Hello', 10)), 1, 1), bin(bitSlice(toFixedString('Hello', 10), 1, 1)); +select 2, 2, subString(bin(toFixedString('Hello', 10)), 2, 2), bin(bitSlice(toFixedString('Hello', 10), 2, 2)); +select 3, 3, subString(bin(toFixedString('Hello', 10)), 3, 3), bin(bitSlice(toFixedString('Hello', 10), 3, 3)); +select 4, 4, subString(bin(toFixedString('Hello', 10)), 4, 4), bin(bitSlice(toFixedString('Hello', 10), 4, 4)); +select 5, 5, subString(bin(toFixedString('Hello', 10)), 5, 5), bin(bitSlice(toFixedString('Hello', 10), 5, 5)); +select 6, 6, subString(bin(toFixedString('Hello', 10)), 6, 6), bin(bitSlice(toFixedString('Hello', 10), 6, 6)); +select 7, 7, subString(bin(toFixedString('Hello', 10)), 7, 7), bin(bitSlice(toFixedString('Hello', 10), 7, 7)); +select 8, 8, subString(bin(toFixedString('Hello', 10)), 8, 8), bin(bitSlice(toFixedString('Hello', 10), 8, 8)); +select 9, 9, subString(bin(toFixedString('Hello', 10)), 9, 9), bin(bitSlice(toFixedString('Hello', 10), 9, 9)); +select 10, 10, subString(bin(toFixedString('Hello', 10)), 10, 10), bin(bitSlice(toFixedString('Hello', 10), 10, 10)); +select 11, 11, subString(bin(toFixedString('Hello', 10)), 11, 11), bin(bitSlice(toFixedString('Hello', 10), 11, 11)); +select 12, 12, subString(bin(toFixedString('Hello', 10)), 12, 12), bin(bitSlice(toFixedString('Hello', 10), 12, 12)); +select 13, 13, subString(bin(toFixedString('Hello', 10)), 13, 13), bin(bitSlice(toFixedString('Hello', 10), 13, 13)); +select 14, 14, subString(bin(toFixedString('Hello', 10)), 14, 14), bin(bitSlice(toFixedString('Hello', 10), 14, 14)); +select 15, 15, subString(bin(toFixedString('Hello', 10)), 15, 15), bin(bitSlice(toFixedString('Hello', 10), 15, 15)); +select 16, 16, subString(bin(toFixedString('Hello', 10)), 16, 16), bin(bitSlice(toFixedString('Hello', 10), 16, 16)); + +select 1, -1, subString(bin(toFixedString('Hello', 10)), 1, -1), bin(bitSlice(toFixedString('Hello', 10), 1, -1)); +select 2, -2, subString(bin(toFixedString('Hello', 10)), 2, -2), bin(bitSlice(toFixedString('Hello', 10), 2, -2)); +select 3, -3, subString(bin(toFixedString('Hello', 10)), 3, -3), bin(bitSlice(toFixedString('Hello', 10), 3, -3)); +select 4, -4, subString(bin(toFixedString('Hello', 10)), 4, -4), bin(bitSlice(toFixedString('Hello', 10), 4, -4)); +select 5, -5, subString(bin(toFixedString('Hello', 10)), 5, -5), bin(bitSlice(toFixedString('Hello', 10), 5, -5)); +select 6, -6, subString(bin(toFixedString('Hello', 10)), 6, -6), bin(bitSlice(toFixedString('Hello', 10), 6, -6)); +select 7, -7, subString(bin(toFixedString('Hello', 10)), 7, -7), bin(bitSlice(toFixedString('Hello', 10), 7, -7)); +select 8, -8, subString(bin(toFixedString('Hello', 10)), 8, -8), bin(bitSlice(toFixedString('Hello', 10), 8, -8)); +select 9, -9, subString(bin(toFixedString('Hello', 10)), 9, -9), bin(bitSlice(toFixedString('Hello', 10), 9, -9)); +select 10, -10, subString(bin(toFixedString('Hello', 10)), 10, -10), bin(bitSlice(toFixedString('Hello', 10), 10, -10)); +select 11, -11, subString(bin(toFixedString('Hello', 10)), 11, -11), bin(bitSlice(toFixedString('Hello', 10), 11, -11)); +select 12, -12, subString(bin(toFixedString('Hello', 10)), 12, -12), bin(bitSlice(toFixedString('Hello', 10), 12, -12)); +select 13, -13, subString(bin(toFixedString('Hello', 10)), 13, -13), bin(bitSlice(toFixedString('Hello', 10), 13, -13)); +select 14, -14, subString(bin(toFixedString('Hello', 10)), 14, -14), bin(bitSlice(toFixedString('Hello', 10), 14, -14)); +select 15, -15, subString(bin(toFixedString('Hello', 10)), 15, -15), bin(bitSlice(toFixedString('Hello', 10), 15, -15)); +select 16, -16, subString(bin(toFixedString('Hello', 10)), 16, -16), bin(bitSlice(toFixedString('Hello', 10), 16, -16)); + +select -1, 1, subString(bin(toFixedString('Hello', 10)), -1, 1), bin(bitSlice(toFixedString('Hello', 10), -1, 1)); +select -2, 2, subString(bin(toFixedString('Hello', 10)), -2, 2), bin(bitSlice(toFixedString('Hello', 10), -2, 2)); +select -3, 3, subString(bin(toFixedString('Hello', 10)), -3, 3), bin(bitSlice(toFixedString('Hello', 10), -3, 3)); +select -4, 4, subString(bin(toFixedString('Hello', 10)), -4, 4), bin(bitSlice(toFixedString('Hello', 10), -4, 4)); +select -5, 5, subString(bin(toFixedString('Hello', 10)), -5, 5), bin(bitSlice(toFixedString('Hello', 10), -5, 5)); +select -6, 6, subString(bin(toFixedString('Hello', 10)), -6, 6), bin(bitSlice(toFixedString('Hello', 10), -6, 6)); +select -7, 7, subString(bin(toFixedString('Hello', 10)), -7, 7), bin(bitSlice(toFixedString('Hello', 10), -7, 7)); +select -8, 8, subString(bin(toFixedString('Hello', 10)), -8, 8), bin(bitSlice(toFixedString('Hello', 10), -8, 8)); +select -9, 9, subString(bin(toFixedString('Hello', 10)), -9, 9), bin(bitSlice(toFixedString('Hello', 10), -9, 9)); +select -10, 10, subString(bin(toFixedString('Hello', 10)), -10, 10), bin(bitSlice(toFixedString('Hello', 10), -10, 10)); +select -11, 11, subString(bin(toFixedString('Hello', 10)), -11, 11), bin(bitSlice(toFixedString('Hello', 10), -11, 11)); +select -12, 12, subString(bin(toFixedString('Hello', 10)), -12, 12), bin(bitSlice(toFixedString('Hello', 10), -12, 12)); +select -13, 13, subString(bin(toFixedString('Hello', 10)), -13, 13), bin(bitSlice(toFixedString('Hello', 10), -13, 13)); +select -14, 14, subString(bin(toFixedString('Hello', 10)), -14, 14), bin(bitSlice(toFixedString('Hello', 10), -14, 14)); +select -15, 15, subString(bin(toFixedString('Hello', 10)), -15, 15), bin(bitSlice(toFixedString('Hello', 10), -15, 15)); +select -16, 16, subString(bin(toFixedString('Hello', 10)), -16, 16), bin(bitSlice(toFixedString('Hello', 10), -16, 16)); + +select -1, -16, subString(bin(toFixedString('Hello', 10)), -1, -16), bin(bitSlice(toFixedString('Hello', 10), -1, -16)); +select -2, -15, subString(bin(toFixedString('Hello', 10)), -2, -15), bin(bitSlice(toFixedString('Hello', 10), -2, -15)); +select -3, -14, subString(bin(toFixedString('Hello', 10)), -3, -14), bin(bitSlice(toFixedString('Hello', 10), -3, -14)); +select -4, -13, subString(bin(toFixedString('Hello', 10)), -4, -13), bin(bitSlice(toFixedString('Hello', 10), -4, -13)); +select -5, -12, subString(bin(toFixedString('Hello', 10)), -5, -12), bin(bitSlice(toFixedString('Hello', 10), -5, -12)); +select -6, -11, subString(bin(toFixedString('Hello', 10)), -6, -11), bin(bitSlice(toFixedString('Hello', 10), -6, -11)); +select -7, -10, subString(bin(toFixedString('Hello', 10)), -7, -10), bin(bitSlice(toFixedString('Hello', 10), -7, -10)); +select -8, -9, subString(bin(toFixedString('Hello', 10)), -8, -9), bin(bitSlice(toFixedString('Hello', 10), -8, -9)); +select -9, -8, subString(bin(toFixedString('Hello', 10)), -9, -8), bin(bitSlice(toFixedString('Hello', 10), -9, -8)); +select -10, -7, subString(bin(toFixedString('Hello', 10)), -10, -7), bin(bitSlice(toFixedString('Hello', 10), -10, -7)); +select -11, -6, subString(bin(toFixedString('Hello', 10)), -11, -6), bin(bitSlice(toFixedString('Hello', 10), -11, -6)); +select -12, -5, subString(bin(toFixedString('Hello', 10)), -12, -5), bin(bitSlice(toFixedString('Hello', 10), -12, -5)); +select -13, -4, subString(bin(toFixedString('Hello', 10)), -13, -4), bin(bitSlice(toFixedString('Hello', 10), -13, -4)); +select -14, -3, subString(bin(toFixedString('Hello', 10)), -14, -3), bin(bitSlice(toFixedString('Hello', 10), -14, -3)); +select -15, -2, subString(bin(toFixedString('Hello', 10)), -15, -2), bin(bitSlice(toFixedString('Hello', 10), -15, -2)); +select -16, -1, subString(bin(toFixedString('Hello', 10)), -16, -1), bin(bitSlice(toFixedString('Hello', 10), -16, -1)); + +select 'Dynamic Offset, Dynamic Length'; + +select number, number, subString(bin(toFixedString('Hello', 10)), number, number), bin(bitSlice(toFixedString('Hello', 10), number, number)) from numbers(16); +select number, -number, subString(bin(toFixedString('Hello', 10)), number, -number), bin(bitSlice(toFixedString('Hello', 10), number, -number)) from numbers(16); +select -number, -16+number, subString(bin(toFixedString('Hello', 10)), -number, -16+number), bin(bitSlice(toFixedString('Hello', 10), -number, -16+number)) from numbers(16); +select -number, number, subString(bin(toFixedString('Hello', 10)), -number, number), bin(bitSlice(toFixedString('Hello', 10), -number, number)) from numbers(16); diff --git a/tests/queries/0_stateless/02154_bit_slice_for_string.reference b/tests/queries/0_stateless/02154_bit_slice_for_string.reference new file mode 100644 index 00000000000..272f415a021 --- /dev/null +++ b/tests/queries/0_stateless/02154_bit_slice_for_string.reference @@ -0,0 +1,163 @@ +Const Offset +1 0100100001100101011011000110110001101111 0100100001100101011011000110110001101111 +2 100100001100101011011000110110001101111 1001000011001010110110001101100011011110 +3 00100001100101011011000110110001101111 0010000110010101101100011011000110111100 +4 0100001100101011011000110110001101111 0100001100101011011000110110001101111000 +5 100001100101011011000110110001101111 1000011001010110110001101100011011110000 +6 00001100101011011000110110001101111 0000110010101101100011011000110111100000 +7 0001100101011011000110110001101111 0001100101011011000110110001101111000000 +8 001100101011011000110110001101111 0011001010110110001101100011011110000000 +9 01100101011011000110110001101111 01100101011011000110110001101111 +10 1100101011011000110110001101111 11001010110110001101100011011110 +11 100101011011000110110001101111 10010101101100011011000110111100 +12 00101011011000110110001101111 00101011011000110110001101111000 +13 0101011011000110110001101111 01010110110001101100011011110000 +14 101011011000110110001101111 10101101100011011000110111100000 +15 01011011000110110001101111 01011011000110110001101111000000 +16 1011011000110110001101111 10110110001101100011011110000000 +-1 1 10000000 +-2 11 11000000 +-3 111 11100000 +-4 1111 11110000 +-5 01111 01111000 +-6 101111 10111100 +-7 1101111 11011110 +-8 01101111 01101111 +-9 001101111 0011011110000000 +-10 0001101111 0001101111000000 +-11 10001101111 1000110111100000 +-12 110001101111 1100011011110000 +-13 0110001101111 0110001101111000 +-14 10110001101111 1011000110111100 +-15 110110001101111 1101100011011110 +-16 0110110001101111 0110110001101111 +Const Offset, Const Length +1 1 0 00000000 +2 2 10 10000000 +3 3 001 00100000 +4 4 0100 01000000 +5 5 10000 10000000 +6 6 000011 00001100 +7 7 0001100 00011000 +8 8 00110010 00110010 +9 9 011001010 0110010100000000 +10 10 1100101011 1100101011000000 +11 11 10010101101 1001010110100000 +12 12 001010110110 0010101101100000 +13 13 0101011011000 0101011011000000 +14 14 10101101100011 1010110110001100 +15 15 010110110001101 0101101100011010 +16 16 1011011000110110 1011011000110110 +1 -1 010010000110010101101100011011000110111 0100100001100101011011000110110001101110 +2 -2 1001000011001010110110001101100011011 1001000011001010110110001101100011011000 +3 -3 00100001100101011011000110110001101 0010000110010101101100011011000110100000 +4 -4 010000110010101101100011011000110 0100001100101011011000110110001100000000 +5 -5 1000011001010110110001101100011 10000110010101101100011011000110 +6 -6 00001100101011011000110110001 00001100101011011000110110001000 +7 -7 000110010101101100011011000 00011001010110110001101100000000 +8 -8 0011001010110110001101100 00110010101101100011011000000000 +9 -9 01100101011011000110110 011001010110110001101100 +10 -10 110010101101100011011 110010101101100011011000 +11 -11 1001010110110001101 100101011011000110100000 +12 -12 00101011011000110 001010110110001100000000 +13 -13 010101101100011 0101011011000110 +14 -14 1010110110001 1010110110001000 +15 -15 01011011000 0101101100000000 +16 -16 101101100 1011011000000000 +-1 1 1 10000000 +-2 2 11 11000000 +-3 3 111 11100000 +-4 4 1111 11110000 +-5 5 01111 01111000 +-6 6 101111 10111100 +-7 7 1101111 11011110 +-8 8 01101111 01101111 +-9 9 001101111 0011011110000000 +-10 10 0001101111 0001101111000000 +-11 11 10001101111 1000110111100000 +-12 12 110001101111 1100011011110000 +-13 13 0110001101111 0110001101111000 +-14 14 10110001101111 1011000110111100 +-15 15 110110001101111 1101100011011110 +-16 16 0110110001101111 0110110001101111 +-1 -16 +-2 -15 +-3 -14 +-4 -13 +-5 -12 +-6 -11 +-7 -10 +-8 -9 +-9 -8 0 00000000 +-10 -7 000 00000000 +-11 -6 10001 10001000 +-12 -5 1100011 11000110 +-13 -4 011000110 0110001100000000 +-14 -3 10110001101 1011000110100000 +-15 -2 1101100011011 1101100011011000 +-16 -1 011011000110111 0110110001101110 +Dynamic Offset, Dynamic Length +0 0 +1 1 0 00000000 +2 2 10 10000000 +3 3 001 00100000 +4 4 0100 01000000 +5 5 10000 10000000 +6 6 000011 00001100 +7 7 0001100 00011000 +8 8 00110010 00110010 +9 9 011001010 0110010100000000 +10 10 1100101011 1100101011000000 +11 11 10010101101 1001010110100000 +12 12 001010110110 0010101101100000 +13 13 0101011011000 0101011011000000 +14 14 10101101100011 1010110110001100 +15 15 010110110001101 0101101100011010 +0 0 +1 -1 010010000110010101101100011011000110111 0100100001100101011011000110110001101110 +2 -2 1001000011001010110110001101100011011 1001000011001010110110001101100011011000 +3 -3 00100001100101011011000110110001101 0010000110010101101100011011000110100000 +4 -4 010000110010101101100011011000110 0100001100101011011000110110001100000000 +5 -5 1000011001010110110001101100011 10000110010101101100011011000110 +6 -6 00001100101011011000110110001 00001100101011011000110110001000 +7 -7 000110010101101100011011000 00011001010110110001101100000000 +8 -8 0011001010110110001101100 00110010101101100011011000000000 +9 -9 01100101011011000110110 011001010110110001101100 +10 -10 110010101101100011011 110010101101100011011000 +11 -11 1001010110110001101 100101011011000110100000 +12 -12 00101011011000110 001010110110001100000000 +13 -13 010101101100011 0101011011000110 +14 -14 1010110110001 1010110110001000 +15 -15 01011011000 0101101100000000 +0 -16 +-1 -15 +-2 -14 +-3 -13 +-4 -12 +-5 -11 +-6 -10 +-7 -9 +-8 -8 +-9 -7 00 00000000 +-10 -6 0001 00010000 +-11 -5 100011 10001100 +-12 -4 11000110 11000110 +-13 -3 0110001101 0110001101000000 +-14 -2 101100011011 1011000110110000 +-15 -1 11011000110111 1101100011011100 +0 0 +-1 1 1 10000000 +-2 2 11 11000000 +-3 3 111 11100000 +-4 4 1111 11110000 +-5 5 01111 01111000 +-6 6 101111 10111100 +-7 7 1101111 11011110 +-8 8 01101111 01101111 +-9 9 001101111 0011011110000000 +-10 10 0001101111 0001101111000000 +-11 11 10001101111 1000110111100000 +-12 12 110001101111 1100011011110000 +-13 13 0110001101111 0110001101111000 +-14 14 10110001101111 1011000110111100 +-15 15 110110001101111 1101100011011110 diff --git a/tests/queries/0_stateless/02154_bit_slice_for_string.sql b/tests/queries/0_stateless/02154_bit_slice_for_string.sql new file mode 100644 index 00000000000..3a73744f44b --- /dev/null +++ b/tests/queries/0_stateless/02154_bit_slice_for_string.sql @@ -0,0 +1,111 @@ +SELECT 'Const Offset'; +select 1, subString(bin('Hello'), 1), bin(bitSlice('Hello', 1)); +select 2, subString(bin('Hello'), 2), bin(bitSlice('Hello', 2)); +select 3, subString(bin('Hello'), 3), bin(bitSlice('Hello', 3)); +select 4, subString(bin('Hello'), 4), bin(bitSlice('Hello', 4)); +select 5, subString(bin('Hello'), 5), bin(bitSlice('Hello', 5)); +select 6, subString(bin('Hello'), 6), bin(bitSlice('Hello', 6)); +select 7, subString(bin('Hello'), 7), bin(bitSlice('Hello', 7)); +select 8, subString(bin('Hello'), 8), bin(bitSlice('Hello', 8)); +select 9, subString(bin('Hello'), 9), bin(bitSlice('Hello', 9)); +select 10, subString(bin('Hello'), 10), bin(bitSlice('Hello', 10)); +select 11, subString(bin('Hello'), 11), bin(bitSlice('Hello', 11)); +select 12, subString(bin('Hello'), 12), bin(bitSlice('Hello', 12)); +select 13, subString(bin('Hello'), 13), bin(bitSlice('Hello', 13)); +select 14, subString(bin('Hello'), 14), bin(bitSlice('Hello', 14)); +select 15, subString(bin('Hello'), 15), bin(bitSlice('Hello', 15)); +select 16, subString(bin('Hello'), 16), bin(bitSlice('Hello', 16)); + +select -1, subString(bin('Hello'), -1), bin(bitSlice('Hello', -1)); +select -2, subString(bin('Hello'), -2), bin(bitSlice('Hello', -2)); +select -3, subString(bin('Hello'), -3), bin(bitSlice('Hello', -3)); +select -4, subString(bin('Hello'), -4), bin(bitSlice('Hello', -4)); +select -5, subString(bin('Hello'), -5), bin(bitSlice('Hello', -5)); +select -6, subString(bin('Hello'), -6), bin(bitSlice('Hello', -6)); +select -7, subString(bin('Hello'), -7), bin(bitSlice('Hello', -7)); +select -8, subString(bin('Hello'), -8), bin(bitSlice('Hello', -8)); +select -9, subString(bin('Hello'), -9), bin(bitSlice('Hello', -9)); +select -10, subString(bin('Hello'), -10), bin(bitSlice('Hello', -10)); +select -11, subString(bin('Hello'), -11), bin(bitSlice('Hello', -11)); +select -12, subString(bin('Hello'), -12), bin(bitSlice('Hello', -12)); +select -13, subString(bin('Hello'), -13), bin(bitSlice('Hello', -13)); +select -14, subString(bin('Hello'), -14), bin(bitSlice('Hello', -14)); +select -15, subString(bin('Hello'), -15), bin(bitSlice('Hello', -15)); +select -16, subString(bin('Hello'), -16), bin(bitSlice('Hello', -16)); + + +SELECT 'Const Offset, Const Length'; +select 1, 1, subString(bin('Hello'), 1, 1), bin(bitSlice('Hello', 1, 1)); +select 2, 2, subString(bin('Hello'), 2, 2), bin(bitSlice('Hello', 2, 2)); +select 3, 3, subString(bin('Hello'), 3, 3), bin(bitSlice('Hello', 3, 3)); +select 4, 4, subString(bin('Hello'), 4, 4), bin(bitSlice('Hello', 4, 4)); +select 5, 5, subString(bin('Hello'), 5, 5), bin(bitSlice('Hello', 5, 5)); +select 6, 6, subString(bin('Hello'), 6, 6), bin(bitSlice('Hello', 6, 6)); +select 7, 7, subString(bin('Hello'), 7, 7), bin(bitSlice('Hello', 7, 7)); +select 8, 8, subString(bin('Hello'), 8, 8), bin(bitSlice('Hello', 8, 8)); +select 9, 9, subString(bin('Hello'), 9, 9), bin(bitSlice('Hello', 9, 9)); +select 10, 10, subString(bin('Hello'), 10, 10), bin(bitSlice('Hello', 10, 10)); +select 11, 11, subString(bin('Hello'), 11, 11), bin(bitSlice('Hello', 11, 11)); +select 12, 12, subString(bin('Hello'), 12, 12), bin(bitSlice('Hello', 12, 12)); +select 13, 13, subString(bin('Hello'), 13, 13), bin(bitSlice('Hello', 13, 13)); +select 14, 14, subString(bin('Hello'), 14, 14), bin(bitSlice('Hello', 14, 14)); +select 15, 15, subString(bin('Hello'), 15, 15), bin(bitSlice('Hello', 15, 15)); +select 16, 16, subString(bin('Hello'), 16, 16), bin(bitSlice('Hello', 16, 16)); + +select 1, -1, subString(bin('Hello'), 1, -1), bin(bitSlice('Hello', 1, -1)); +select 2, -2, subString(bin('Hello'), 2, -2), bin(bitSlice('Hello', 2, -2)); +select 3, -3, subString(bin('Hello'), 3, -3), bin(bitSlice('Hello', 3, -3)); +select 4, -4, subString(bin('Hello'), 4, -4), bin(bitSlice('Hello', 4, -4)); +select 5, -5, subString(bin('Hello'), 5, -5), bin(bitSlice('Hello', 5, -5)); +select 6, -6, subString(bin('Hello'), 6, -6), bin(bitSlice('Hello', 6, -6)); +select 7, -7, subString(bin('Hello'), 7, -7), bin(bitSlice('Hello', 7, -7)); +select 8, -8, subString(bin('Hello'), 8, -8), bin(bitSlice('Hello', 8, -8)); +select 9, -9, subString(bin('Hello'), 9, -9), bin(bitSlice('Hello', 9, -9)); +select 10, -10, subString(bin('Hello'), 10, -10), bin(bitSlice('Hello', 10, -10)); +select 11, -11, subString(bin('Hello'), 11, -11), bin(bitSlice('Hello', 11, -11)); +select 12, -12, subString(bin('Hello'), 12, -12), bin(bitSlice('Hello', 12, -12)); +select 13, -13, subString(bin('Hello'), 13, -13), bin(bitSlice('Hello', 13, -13)); +select 14, -14, subString(bin('Hello'), 14, -14), bin(bitSlice('Hello', 14, -14)); +select 15, -15, subString(bin('Hello'), 15, -15), bin(bitSlice('Hello', 15, -15)); +select 16, -16, subString(bin('Hello'), 16, -16), bin(bitSlice('Hello', 16, -16)); + +select -1, 1, subString(bin('Hello'), -1, 1), bin(bitSlice('Hello', -1, 1)); +select -2, 2, subString(bin('Hello'), -2, 2), bin(bitSlice('Hello', -2, 2)); +select -3, 3, subString(bin('Hello'), -3, 3), bin(bitSlice('Hello', -3, 3)); +select -4, 4, subString(bin('Hello'), -4, 4), bin(bitSlice('Hello', -4, 4)); +select -5, 5, subString(bin('Hello'), -5, 5), bin(bitSlice('Hello', -5, 5)); +select -6, 6, subString(bin('Hello'), -6, 6), bin(bitSlice('Hello', -6, 6)); +select -7, 7, subString(bin('Hello'), -7, 7), bin(bitSlice('Hello', -7, 7)); +select -8, 8, subString(bin('Hello'), -8, 8), bin(bitSlice('Hello', -8, 8)); +select -9, 9, subString(bin('Hello'), -9, 9), bin(bitSlice('Hello', -9, 9)); +select -10, 10, subString(bin('Hello'), -10, 10), bin(bitSlice('Hello', -10, 10)); +select -11, 11, subString(bin('Hello'), -11, 11), bin(bitSlice('Hello', -11, 11)); +select -12, 12, subString(bin('Hello'), -12, 12), bin(bitSlice('Hello', -12, 12)); +select -13, 13, subString(bin('Hello'), -13, 13), bin(bitSlice('Hello', -13, 13)); +select -14, 14, subString(bin('Hello'), -14, 14), bin(bitSlice('Hello', -14, 14)); +select -15, 15, subString(bin('Hello'), -15, 15), bin(bitSlice('Hello', -15, 15)); +select -16, 16, subString(bin('Hello'), -16, 16), bin(bitSlice('Hello', -16, 16)); + +select -1, -16, subString(bin('Hello'), -1, -16), bin(bitSlice('Hello', -1, -16)); +select -2, -15, subString(bin('Hello'), -2, -15), bin(bitSlice('Hello', -2, -15)); +select -3, -14, subString(bin('Hello'), -3, -14), bin(bitSlice('Hello', -3, -14)); +select -4, -13, subString(bin('Hello'), -4, -13), bin(bitSlice('Hello', -4, -13)); +select -5, -12, subString(bin('Hello'), -5, -12), bin(bitSlice('Hello', -5, -12)); +select -6, -11, subString(bin('Hello'), -6, -11), bin(bitSlice('Hello', -6, -11)); +select -7, -10, subString(bin('Hello'), -7, -10), bin(bitSlice('Hello', -7, -10)); +select -8, -9, subString(bin('Hello'), -8, -9), bin(bitSlice('Hello', -8, -9)); +select -9, -8, subString(bin('Hello'), -9, -8), bin(bitSlice('Hello', -9, -8)); +select -10, -7, subString(bin('Hello'), -10, -7), bin(bitSlice('Hello', -10, -7)); +select -11, -6, subString(bin('Hello'), -11, -6), bin(bitSlice('Hello', -11, -6)); +select -12, -5, subString(bin('Hello'), -12, -5), bin(bitSlice('Hello', -12, -5)); +select -13, -4, subString(bin('Hello'), -13, -4), bin(bitSlice('Hello', -13, -4)); +select -14, -3, subString(bin('Hello'), -14, -3), bin(bitSlice('Hello', -14, -3)); +select -15, -2, subString(bin('Hello'), -15, -2), bin(bitSlice('Hello', -15, -2)); +select -16, -1, subString(bin('Hello'), -16, -1), bin(bitSlice('Hello', -16, -1)); + +select 'Dynamic Offset, Dynamic Length'; + +select number, number, subString(bin('Hello'), number, number), bin(bitSlice('Hello', number, number)) from numbers(16); +select number, -number, subString(bin('Hello'), number, -number), bin(bitSlice('Hello', number, -number)) from numbers(16); +select -number, -16+number, subString(bin('Hello'), -number, -16+number), bin(bitSlice('Hello', -number, -16+number)) from numbers(16); +select -number, number, subString(bin('Hello'), -number, number), bin(bitSlice('Hello', -number, number)) from numbers(16); From 86db17e70fb8fc7ebbe20d6f631343759f0b9e90 Mon Sep 17 00:00:00 2001 From: RogerYK Date: Mon, 3 Jan 2022 16:31:11 +0800 Subject: [PATCH 014/403] Add comment and docs --- .../sql-reference/functions/bit-functions.md | 40 +++++++++++++++++++ src/Functions/bitSlice.cpp | 26 +++++------- 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/docs/en/sql-reference/functions/bit-functions.md b/docs/en/sql-reference/functions/bit-functions.md index 899b6385a3e..e0ac80087c2 100644 --- a/docs/en/sql-reference/functions/bit-functions.md +++ b/docs/en/sql-reference/functions/bit-functions.md @@ -117,6 +117,46 @@ Result: ## bitRotateRight(a, b) {#bitrotaterighta-b} +## bitSlice(s, offset, length) + +Returns a substring starting with the bit from the ‘offset’ index that is ‘length’ bits long. bits indexing starts from +1 + +**Syntax** + +``` sql +bitSlice(s, offset[, length]) +``` + +**Arguments** + +- `s` — s is [String](../../sql-reference/data-types/string.md) + or [FixedString](../../sql-reference/data-types/fixedstring.md). +- `offset` — The start index with bit, A positive value indicates an offset on the left, and a negative value is an + indent on the right. Numbering of the bits begins with 1. +- `length` — The length of substring with bit. If you specify a negative value, the function returns an open substring [ + offset, array_length - length). If you omit the value, the function returns the substring [offset, the_end_string] + +**Returned value** + +- The substring. [String](../../sql-reference/data-types/string.md) + +**Example** + +Query: + +``` sql +select bin('Hello'), bin(bitSlice('Hello', 1, 8)) +``` + +Result: + +``` text +┌─bin('Hello')─────────────────────────────┬─bin(bitSlice('Hello', 1, 8))─┐ +│ 0100100001100101011011000110110001101111 │ 01001000 │ +└──────────────────────────────────────────┴──────────────────────────────┘ +``` + ## bitTest {#bittest} Takes any integer and converts it into [binary form](https://en.wikipedia.org/wiki/Binary_number), returns the value of a bit at specified position. The countdown starts from 0 from the right to the left. diff --git a/src/Functions/bitSlice.cpp b/src/Functions/bitSlice.cpp index ce6f0187a67..0ed8bec2b1c 100644 --- a/src/Functions/bitSlice.cpp +++ b/src/Functions/bitSlice.cpp @@ -200,7 +200,7 @@ public: if (!size) return; bool abandon_last_byte = abandon_last_bit + shift_bit >= word_size; - if (abandon_last_byte) + if (abandon_last_byte) // shift may eliminate last byte size--; sink.elements.resize(sink.current_offset + size); UInt8 * out = &sink.elements[sink.current_offset]; @@ -245,7 +245,7 @@ public: void bitSliceFromRightConstantOffsetUnbounded(Source && src, StringSink && sink, size_t offset) const { size_t offset_byte = offset / word_size; - size_t offset_bit = (word_size - (offset % word_size)) % word_size; + size_t offset_bit = (word_size - (offset % word_size)) % word_size; // offset_bit always represent left offset bit if (offset_bit) offset_byte++; while (!src.isEnd()) @@ -292,7 +292,7 @@ public: { UInt64 offset = -static_cast(start); size_t offset_byte = offset / word_size; - size_t offset_bit = (word_size - (offset % word_size)) % word_size; + size_t offset_bit = (word_size - (offset % word_size)) % word_size; // offset_bit always represent left offset bit if (offset_bit) offset_byte++; size_t size = src.getElementSize(); @@ -321,7 +321,7 @@ public: { length_byte = (length + offset_bit) / word_size; over_bit = (length + offset_bit) % word_size; - if (over_bit && (length_byte || over_bit > offset_bit)) + if (over_bit && (length_byte || over_bit > offset_bit)) // begin and end are not in same byte OR there are gaps length_byte++; } @@ -332,9 +332,8 @@ public: { length_byte = std::max(remain_byte + (length / word_size), static_cast(0)); over_bit = word_size + (length % word_size); - if (length_byte == 1 && over_bit <= offset_bit) { + if (length_byte == 1 && over_bit <= offset_bit) // begin and end are in same byte AND there are no gaps length_byte = 0; - } } bool right_truncate = static_cast(length_byte) > remain_byte; size_t abandon_last_bit = (over_bit && !right_truncate) ? word_size - over_bit : 0; @@ -352,7 +351,7 @@ public: void bitSliceFromRightConstantOffsetBounded(Source && src, StringSink && sink, size_t offset, ssize_t length) const { size_t offset_byte = offset / word_size; - size_t offset_bit = (word_size - (offset % word_size)) % word_size; + size_t offset_bit = (word_size - (offset % word_size)) % word_size; // offset_bit always represent left offset bit if (offset_bit) offset_byte++; size_t length_byte = 0; @@ -361,7 +360,7 @@ public: { length_byte = (length + offset_bit) / word_size; over_bit = (length + offset_bit) % word_size; - if (over_bit && (length_byte || over_bit > offset_bit)) + if (over_bit && (length_byte || over_bit > offset_bit)) // begin and end are not in same byte OR there are gaps length_byte++; } @@ -372,9 +371,8 @@ public: { length_byte = std::max(static_cast(offset_byte) + (length / word_size), static_cast(0)); over_bit = word_size + (length % word_size); - if (length_byte == 1 && over_bit <= offset_bit) { + if (length_byte == 1 && over_bit <= offset_bit) // begin and end are in same byte AND there are no gaps length_byte = 0; - } } bool left_truncate = offset_byte > size; bool right_truncate = length_byte > offset_byte; @@ -431,7 +429,7 @@ public: else { offset_byte = offset / word_size; - offset_bit = (word_size - (offset % word_size)) % word_size; + offset_bit = (word_size - (offset % word_size)) % word_size; // offset_bit always represent left offset bit if (offset_bit) offset_byte++; bool left_truncate = offset_byte > size; @@ -443,16 +441,15 @@ public: { length_byte = (length + offset_bit) / word_size; over_bit = (length + offset_bit) % word_size; - if (over_bit && (length_byte || (over_bit > offset_bit))) + if (over_bit && (length_byte || (over_bit > offset_bit))) // begin and end are not in same byte OR there are gaps length_byte++; } else { length_byte = std::max(remain_byte + (static_cast(length) / word_size), static_cast(0)); over_bit = word_size + (length % word_size); - if (length_byte == 1 && over_bit <= offset_bit) { + if (length_byte == 1 && over_bit <= offset_bit) // begin and end are in same byte AND there are no gaps length_byte = 0; - } } bool right_truncate = static_cast(length_byte) > remain_byte; @@ -467,7 +464,6 @@ public: src.next(); } } - }; From 8707e3bea12a9c745abdf8f1bedb3b88c81ba30d Mon Sep 17 00:00:00 2001 From: RogerYK Date: Tue, 11 Jan 2022 23:22:16 +0800 Subject: [PATCH 015/403] make implementation clear and add more test --- .../sql-reference/functions/bit-functions.md | 15 +- src/Functions/bitSlice.cpp | 102 ++--- .../02154_bit_slice_for_fixedstring.reference | 401 +++++++++++------- .../02154_bit_slice_for_fixedstring.sql | 232 +++++----- .../02154_bit_slice_for_string.reference | 401 +++++++++++------- .../02154_bit_slice_for_string.sql | 233 +++++----- 6 files changed, 795 insertions(+), 589 deletions(-) diff --git a/docs/en/sql-reference/functions/bit-functions.md b/docs/en/sql-reference/functions/bit-functions.md index e0ac80087c2..24adb362c98 100644 --- a/docs/en/sql-reference/functions/bit-functions.md +++ b/docs/en/sql-reference/functions/bit-functions.md @@ -135,7 +135,8 @@ bitSlice(s, offset[, length]) - `offset` — The start index with bit, A positive value indicates an offset on the left, and a negative value is an indent on the right. Numbering of the bits begins with 1. - `length` — The length of substring with bit. If you specify a negative value, the function returns an open substring [ - offset, array_length - length). If you omit the value, the function returns the substring [offset, the_end_string] + offset, array_length - length). If you omit the value, the function returns the substring [offset, the_end_string]. + If length exceeds s, it will be truncate.If length isn't multiple of 8, will fill 0 on the right. **Returned value** @@ -147,6 +148,9 @@ Query: ``` sql select bin('Hello'), bin(bitSlice('Hello', 1, 8)) +select bin('Hello'), bin(bitSlice('Hello', 1, 2)) +select bin('Hello'), bin(bitSlice('Hello', 1, 9)) +select bin('Hello'), bin(bitSlice('Hello', -4, 8)) ``` Result: @@ -155,6 +159,15 @@ Result: ┌─bin('Hello')─────────────────────────────┬─bin(bitSlice('Hello', 1, 8))─┐ │ 0100100001100101011011000110110001101111 │ 01001000 │ └──────────────────────────────────────────┴──────────────────────────────┘ +┌─bin('Hello')─────────────────────────────┬─bin(bitSlice('Hello', 1, 2))─┐ +│ 0100100001100101011011000110110001101111 │ 01000000 │ +└──────────────────────────────────────────┴──────────────────────────────┘ +┌─bin('Hello')─────────────────────────────┬─bin(bitSlice('Hello', 1, 9))─┐ +│ 0100100001100101011011000110110001101111 │ 0100100000000000 │ +└──────────────────────────────────────────┴──────────────────────────────┘ +┌─bin('Hello')─────────────────────────────┬─bin(bitSlice('Hello', -4, 8))─┐ +│ 0100100001100101011011000110110001101111 │ 11110000 │ +└──────────────────────────────────────────┴───────────────────────────────┘ ``` ## bitTest {#bittest} diff --git a/src/Functions/bitSlice.cpp b/src/Functions/bitSlice.cpp index 0ed8bec2b1c..5e607e37187 100644 --- a/src/Functions/bitSlice.cpp +++ b/src/Functions/bitSlice.cpp @@ -39,6 +39,8 @@ public: bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + bool useDefaultImplementationForConstants() const override { return true; } + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { const size_t number_of_arguments = arguments.size(); @@ -76,62 +78,44 @@ public: ColumnPtr column_start = arguments[1].column; ColumnPtr column_length; + std::optional start_const; + std::optional length_const; + + if (const auto *column_start_const = checkAndGetColumn(column_start.get())) { + start_const = column_start_const->getInt(0); + } + if (number_of_arguments == 3) + { column_length = arguments[2].column; + if (const auto *column_length_const = checkAndGetColumn(column_length.get())) + length_const = column_length_const->getInt(0); + } - const ColumnConst * column_start_const = checkAndGetColumn(column_start.get()); - const ColumnConst * column_length_const = nullptr; - - if (number_of_arguments == 3) - column_length_const = checkAndGetColumn(column_length.get()); - - Int64 start_value = 0; - Int64 length_value = 0; - - if (column_start_const) - start_value = column_start_const->getInt(0); - if (column_length_const) - length_value = column_length_const->getInt(0); if (const ColumnString * col = checkAndGetColumn(column_string.get())) return executeForSource( column_start, - column_length, - column_start_const, - column_length_const, - start_value, - length_value, + column_length, start_const, length_const, StringSource(*col), input_rows_count); else if (const ColumnFixedString * col_fixed = checkAndGetColumn(column_string.get())) return executeForSource( column_start, - column_length, - column_start_const, - column_length_const, - start_value, - length_value, + column_length, start_const, length_const, FixedStringSource(*col_fixed), input_rows_count); else if (const ColumnConst * col_const = checkAndGetColumnConst(column_string.get())) return executeForSource( column_start, - column_length, - column_start_const, - column_length_const, - start_value, - length_value, + column_length, start_const, length_const, ConstSource(*col_const), input_rows_count); else if (const ColumnConst * col_const_fixed = checkAndGetColumnConst(column_string.get())) return executeForSource( column_start, - column_length, - column_start_const, - column_length_const, - start_value, - length_value, + column_length, start_const, length_const, ConstSource(*col_const_fixed), input_rows_count); else @@ -144,10 +128,8 @@ public: ColumnPtr executeForSource( const ColumnPtr & column_start, const ColumnPtr & column_length, - const ColumnConst * column_start_const, - const ColumnConst * column_length_const, - Int64 start_value, - Int64 length_value, + std::optional start_const, + std::optional length_const, Source && source, size_t input_rows_count) const { @@ -155,14 +137,15 @@ public: if (!column_length) { - if (column_start_const) + if (start_const) { + Int64 start_value = start_const.value(); if (start_value > 0) bitSliceFromLeftConstantOffsetUnbounded( source, StringSink(*col_res, input_rows_count), static_cast(start_value - 1)); else if (start_value < 0) bitSliceFromRightConstantOffsetUnbounded( - source, StringSink(*col_res, input_rows_count), -static_cast(start_value)); + source, StringSink(*col_res, input_rows_count), static_cast(-start_value)); else throw Exception("Indices in strings are 1-based", ErrorCodes::ZERO_ARRAY_OR_TUPLE_INDEX); } @@ -171,14 +154,16 @@ public: } else { - if (column_start_const && column_length_const) + if (start_const && length_const) { + Int64 start_value = start_const.value(); + Int64 length_value = length_const.value(); if (start_value > 0) bitSliceFromLeftConstantOffsetBounded( source, StringSink(*col_res, input_rows_count), static_cast(start_value - 1), length_value); else if (start_value < 0) bitSliceFromRightConstantOffsetBounded( - source, StringSink(*col_res, input_rows_count), -static_cast(start_value), length_value); + source, StringSink(*col_res, input_rows_count), static_cast(-start_value), length_value); else throw Exception("Indices in strings are 1-based", ErrorCodes::ZERO_ARRAY_OR_TUPLE_INDEX); } @@ -265,16 +250,10 @@ public: template void bitSliceDynamicOffsetUnbounded(Source && src, StringSink && sink, const IColumn & offset_column) const { - const bool is_null = offset_column.onlyNull(); - const auto * nullable = typeid_cast(&offset_column); - const ColumnUInt8::Container * null_map = nullable ? &nullable->getNullMapData() : nullptr; - const IColumn * nested_column = nullable ? &nullable->getNestedColumn() : &offset_column; - while (!src.isEnd()) { auto row_num = src.rowNum(); - bool has_offset = !is_null && (null_map && (*null_map)[row_num]); - Int64 start = has_offset ? nested_column->getInt(row_num) : 1; + Int64 start = offset_column.getInt(row_num); if (start != 0) { typename std::decay_t::Slice slice; @@ -390,36 +369,21 @@ public: template void bitSliceDynamicOffsetBounded(Source && src, StringSink && sink, const IColumn & offset_column, const IColumn & length_column) const { - const bool is_offset_null = offset_column.onlyNull(); - const auto * offset_nullable = typeid_cast(&offset_column); - const ColumnUInt8::Container * offset_null_map = offset_nullable ? &offset_nullable->getNullMapData() : nullptr; - const IColumn * offset_nested_column = offset_nullable ? &offset_nullable->getNestedColumn() : &offset_column; - - const bool is_length_null = length_column.onlyNull(); - const auto * length_nullable = typeid_cast(&length_column); - const ColumnUInt8::Container * length_null_map = length_nullable ? &length_nullable->getNullMapData() : nullptr; - const IColumn * length_nested_column = length_nullable ? &length_nullable->getNestedColumn() : &length_column; - - while (!src.isEnd()) { size_t row_num = src.rowNum(); - bool has_offset = !is_offset_null && !(offset_null_map && (*offset_null_map)[row_num]); - bool has_length = !is_length_null && !(length_null_map && (*length_null_map)[row_num]); - Int64 start = has_offset ? offset_nested_column->getInt(row_num) : 1; - Int64 length = has_length ? length_nested_column->getInt(row_num) : static_cast(src.getElementSize()); + Int64 start = offset_column.getInt(row_num); + Int64 length = length_column.getInt(row_num); if (start && length) { bool left_offset = start > 0; size_t offset = left_offset ? start - 1 : -start; + size_t size = src.getElementSize(); + size_t offset_byte; size_t offset_bit; size_t shift_bit; - size_t length_byte; - size_t over_bit; - size_t size = src.getElementSize(); - if (left_offset) { offset_byte = offset / word_size; @@ -437,6 +401,9 @@ public: } ssize_t remain_byte = left_offset ? size - offset_byte : offset_byte; + + size_t length_byte; + size_t over_bit; if (length > 0) { length_byte = (length + offset_bit) / word_size; @@ -459,7 +426,6 @@ public: writeSliceWithLeftShift(slice, sink, shift_bit, abandon_last_bit); } - sink.next(); src.next(); } diff --git a/tests/queries/0_stateless/02154_bit_slice_for_fixedstring.reference b/tests/queries/0_stateless/02154_bit_slice_for_fixedstring.reference index d42a3c76ac7..1f9e0f10266 100644 --- a/tests/queries/0_stateless/02154_bit_slice_for_fixedstring.reference +++ b/tests/queries/0_stateless/02154_bit_slice_for_fixedstring.reference @@ -1,163 +1,244 @@ Const Offset -1 01001000011001010110110001101100011011110000000000000000000000000000000000000000 01001000011001010110110001101100011011110000000000000000000000000000000000000000 -2 1001000011001010110110001101100011011110000000000000000000000000000000000000000 10010000110010101101100011011000110111100000000000000000000000000000000000000000 -3 001000011001010110110001101100011011110000000000000000000000000000000000000000 00100001100101011011000110110001101111000000000000000000000000000000000000000000 -4 01000011001010110110001101100011011110000000000000000000000000000000000000000 01000011001010110110001101100011011110000000000000000000000000000000000000000000 -5 1000011001010110110001101100011011110000000000000000000000000000000000000000 10000110010101101100011011000110111100000000000000000000000000000000000000000000 -6 000011001010110110001101100011011110000000000000000000000000000000000000000 00001100101011011000110110001101111000000000000000000000000000000000000000000000 -7 00011001010110110001101100011011110000000000000000000000000000000000000000 00011001010110110001101100011011110000000000000000000000000000000000000000000000 -8 0011001010110110001101100011011110000000000000000000000000000000000000000 00110010101101100011011000110111100000000000000000000000000000000000000000000000 -9 011001010110110001101100011011110000000000000000000000000000000000000000 011001010110110001101100011011110000000000000000000000000000000000000000 -10 11001010110110001101100011011110000000000000000000000000000000000000000 110010101101100011011000110111100000000000000000000000000000000000000000 -11 1001010110110001101100011011110000000000000000000000000000000000000000 100101011011000110110001101111000000000000000000000000000000000000000000 -12 001010110110001101100011011110000000000000000000000000000000000000000 001010110110001101100011011110000000000000000000000000000000000000000000 -13 01010110110001101100011011110000000000000000000000000000000000000000 010101101100011011000110111100000000000000000000000000000000000000000000 -14 1010110110001101100011011110000000000000000000000000000000000000000 101011011000110110001101111000000000000000000000000000000000000000000000 -15 010110110001101100011011110000000000000000000000000000000000000000 010110110001101100011011110000000000000000000000000000000000000000000000 -16 10110110001101100011011110000000000000000000000000000000000000000 101101100011011000110111100000000000000000000000000000000000000000000000 --1 0 00000000 --2 00 00000000 --3 000 00000000 --4 0000 00000000 --5 00000 00000000 --6 000000 00000000 --7 0000000 00000000 --8 00000000 00000000 --9 000000000 0000000000000000 --10 0000000000 0000000000000000 --11 00000000000 0000000000000000 --12 000000000000 0000000000000000 --13 0000000000000 0000000000000000 --14 00000000000000 0000000000000000 --15 000000000000000 0000000000000000 --16 0000000000000000 0000000000000000 +1 Hello\0 010010000110010101101100011011000110111100000000 010010000110010101101100011011000110111100000000 +2 Hello\0 10010000110010101101100011011000110111100000000 100100001100101011011000110110001101111000000000 +3 Hello\0 0010000110010101101100011011000110111100000000 001000011001010110110001101100011011110000000000 +4 Hello\0 010000110010101101100011011000110111100000000 010000110010101101100011011000110111100000000000 +5 Hello\0 10000110010101101100011011000110111100000000 100001100101011011000110110001101111000000000000 +6 Hello\0 0000110010101101100011011000110111100000000 000011001010110110001101100011011110000000000000 +7 Hello\0 000110010101101100011011000110111100000000 000110010101101100011011000110111100000000000000 +8 Hello\0 00110010101101100011011000110111100000000 001100101011011000110110001101111000000000000000 +9 Hello\0 0110010101101100011011000110111100000000 0110010101101100011011000110111100000000 +10 Hello\0 110010101101100011011000110111100000000 1100101011011000110110001101111000000000 +11 Hello\0 10010101101100011011000110111100000000 1001010110110001101100011011110000000000 +12 Hello\0 0010101101100011011000110111100000000 0010101101100011011000110111100000000000 +13 Hello\0 010101101100011011000110111100000000 0101011011000110110001101111000000000000 +14 Hello\0 10101101100011011000110111100000000 1010110110001101100011011110000000000000 +15 Hello\0 0101101100011011000110111100000000 0101101100011011000110111100000000000000 +16 Hello\0 101101100011011000110111100000000 1011011000110110001101111000000000000000 +-1 Hello\0 0 00000000 +-2 Hello\0 00 00000000 +-3 Hello\0 000 00000000 +-4 Hello\0 0000 00000000 +-5 Hello\0 00000 00000000 +-6 Hello\0 000000 00000000 +-7 Hello\0 0000000 00000000 +-8 Hello\0 00000000 00000000 +-9 Hello\0 100000000 1000000000000000 +-10 Hello\0 1100000000 1100000000000000 +-11 Hello\0 11100000000 1110000000000000 +-12 Hello\0 111100000000 1111000000000000 +-13 Hello\0 0111100000000 0111100000000000 +-14 Hello\0 10111100000000 1011110000000000 +-15 Hello\0 110111100000000 1101111000000000 +-16 Hello\0 0110111100000000 0110111100000000 +Const Truncate Offset +49 Hello\0 +-49 Hello\0 010010000110010101101100011011000110111100000000 010010000110010101101100011011000110111100000000 +Const Nullable Offset +1 \N \N \N +\N Hello\0 \N \N +\N \N \N \N Const Offset, Const Length -1 1 0 00000000 -2 2 10 10000000 -3 3 001 00100000 -4 4 0100 01000000 -5 5 10000 10000000 -6 6 000011 00001100 -7 7 0001100 00011000 -8 8 00110010 00110010 -9 9 011001010 0110010100000000 -10 10 1100101011 1100101011000000 -11 11 10010101101 1001010110100000 -12 12 001010110110 0010101101100000 -13 13 0101011011000 0101011011000000 -14 14 10101101100011 1010110110001100 -15 15 010110110001101 0101101100011010 -16 16 1011011000110110 1011011000110110 -1 -1 0100100001100101011011000110110001101111000000000000000000000000000000000000000 01001000011001010110110001101100011011110000000000000000000000000000000000000000 -2 -2 10010000110010101101100011011000110111100000000000000000000000000000000000000 10010000110010101101100011011000110111100000000000000000000000000000000000000000 -3 -3 001000011001010110110001101100011011110000000000000000000000000000000000000 00100001100101011011000110110001101111000000000000000000000000000000000000000000 -4 -4 0100001100101011011000110110001101111000000000000000000000000000000000000 01000011001010110110001101100011011110000000000000000000000000000000000000000000 -5 -5 10000110010101101100011011000110111100000000000000000000000000000000000 100001100101011011000110110001101111000000000000000000000000000000000000 -6 -6 000011001010110110001101100011011110000000000000000000000000000000000 000011001010110110001101100011011110000000000000000000000000000000000000 -7 -7 0001100101011011000110110001101111000000000000000000000000000000000 000110010101101100011011000110111100000000000000000000000000000000000000 -8 -8 00110010101101100011011000110111100000000000000000000000000000000 001100101011011000110110001101111000000000000000000000000000000000000000 -9 -9 011001010110110001101100011011110000000000000000000000000000000 0110010101101100011011000110111100000000000000000000000000000000 -10 -10 1100101011011000110110001101111000000000000000000000000000000 1100101011011000110110001101111000000000000000000000000000000000 -11 -11 10010101101100011011000110111100000000000000000000000000000 1001010110110001101100011011110000000000000000000000000000000000 -12 -12 001010110110001101100011011110000000000000000000000000000 0010101101100011011000110111100000000000000000000000000000000000 -13 -13 0101011011000110110001101111000000000000000000000000000 01010110110001101100011011110000000000000000000000000000 -14 -14 10101101100011011000110111100000000000000000000000000 10101101100011011000110111100000000000000000000000000000 -15 -15 010110110001101100011011110000000000000000000000000 01011011000110110001101111000000000000000000000000000000 -16 -16 1011011000110110001101111000000000000000000000000 10110110001101100011011110000000000000000000000000000000 --1 1 0 00000000 --2 2 00 00000000 --3 3 000 00000000 --4 4 0000 00000000 --5 5 00000 00000000 --6 6 000000 00000000 --7 7 0000000 00000000 --8 8 00000000 00000000 --9 9 000000000 0000000000000000 --10 10 0000000000 0000000000000000 --11 11 00000000000 0000000000000000 --12 12 000000000000 0000000000000000 --13 13 0000000000000 0000000000000000 --14 14 00000000000000 0000000000000000 --15 15 000000000000000 0000000000000000 --16 16 0000000000000000 0000000000000000 --1 -16 --2 -15 --3 -14 --4 -13 --5 -12 --6 -11 --7 -10 --8 -9 --9 -8 0 00000000 --10 -7 000 00000000 --11 -6 00000 00000000 --12 -5 0000000 00000000 --13 -4 000000000 0000000000000000 --14 -3 00000000000 0000000000000000 --15 -2 0000000000000 0000000000000000 --16 -1 000000000000000 0000000000000000 +1 1 Hello\0 0 00000000 +2 2 Hello\0 10 10000000 +3 3 Hello\0 001 00100000 +4 4 Hello\0 0100 01000000 +5 5 Hello\0 10000 10000000 +6 6 Hello\0 000011 00001100 +7 7 Hello\0 0001100 00011000 +8 8 Hello\0 00110010 00110010 +9 9 Hello\0 011001010 0110010100000000 +10 10 Hello\0 1100101011 1100101011000000 +11 11 Hello\0 10010101101 1001010110100000 +12 12 Hello\0 001010110110 0010101101100000 +13 13 Hello\0 0101011011000 0101011011000000 +14 14 Hello\0 10101101100011 1010110110001100 +15 15 Hello\0 010110110001101 0101101100011010 +16 16 Hello\0 1011011000110110 1011011000110110 +1 -1 Hello\0 01001000011001010110110001101100011011110000000 010010000110010101101100011011000110111100000000 +2 -2 Hello\0 100100001100101011011000110110001101111000000 100100001100101011011000110110001101111000000000 +3 -3 Hello\0 0010000110010101101100011011000110111100000 001000011001010110110001101100011011110000000000 +4 -4 Hello\0 01000011001010110110001101100011011110000 010000110010101101100011011000110111100000000000 +5 -5 Hello\0 100001100101011011000110110001101111000 1000011001010110110001101100011011110000 +6 -6 Hello\0 0000110010101101100011011000110111100 0000110010101101100011011000110111100000 +7 -7 Hello\0 00011001010110110001101100011011110 0001100101011011000110110001101111000000 +8 -8 Hello\0 001100101011011000110110001101111 0011001010110110001101100011011110000000 +9 -9 Hello\0 0110010101101100011011000110111 01100101011011000110110001101110 +10 -10 Hello\0 11001010110110001101100011011 11001010110110001101100011011000 +11 -11 Hello\0 100101011011000110110001101 10010101101100011011000110100000 +12 -12 Hello\0 0010101101100011011000110 00101011011000110110001100000000 +13 -13 Hello\0 01010110110001101100011 010101101100011011000110 +14 -14 Hello\0 101011011000110110001 101011011000110110001000 +15 -15 Hello\0 0101101100011011000 010110110001101100000000 +16 -16 Hello\0 10110110001101100 101101100011011000000000 +-1 1 Hello\0 0 00000000 +-2 2 Hello\0 00 00000000 +-3 3 Hello\0 000 00000000 +-4 4 Hello\0 0000 00000000 +-5 5 Hello\0 00000 00000000 +-6 6 Hello\0 000000 00000000 +-7 7 Hello\0 0000000 00000000 +-8 8 Hello\0 00000000 00000000 +-9 9 Hello\0 100000000 1000000000000000 +-10 10 Hello\0 1100000000 1100000000000000 +-11 11 Hello\0 11100000000 1110000000000000 +-12 12 Hello\0 111100000000 1111000000000000 +-13 13 Hello\0 0111100000000 0111100000000000 +-14 14 Hello\0 10111100000000 1011110000000000 +-15 15 Hello\0 110111100000000 1101111000000000 +-16 16 Hello\0 0110111100000000 0110111100000000 +-1 -16 Hello\0 +-2 -15 Hello\0 +-3 -14 Hello\0 +-4 -13 Hello\0 +-5 -12 Hello\0 +-6 -11 Hello\0 +-7 -10 Hello\0 +-8 -9 Hello\0 +-9 -8 Hello\0 1 10000000 +-10 -7 Hello\0 110 11000000 +-11 -6 Hello\0 11100 11100000 +-12 -5 Hello\0 1111000 11110000 +-13 -4 Hello\0 011110000 0111100000000000 +-14 -3 Hello\0 10111100000 1011110000000000 +-15 -2 Hello\0 1101111000000 1101111000000000 +-16 -1 Hello\0 011011110000000 0110111100000000 +Const Truncate Offset, Const Truncate Length +36 16 Hello\0 0111100000000 0111100000000000 +49 1 Hello\0 +-52 -44 Hello\0 0100 01000000 +-49 -48 Hello\0 +-49 49 Hello\0 010010000110010101101100011011000110111100000000 010010000110010101101100011011000110111100000000 +Const Nullable Offset, Const Nullable Length +1 1 \N \N \N +\N 1 Hello\0 \N \N +1 \N Hello\0 \N \N +\N \N \N \N \N Dynamic Offset, Dynamic Length -0 0 -1 1 0 00000000 -2 2 10 10000000 -3 3 001 00100000 -4 4 0100 01000000 -5 5 10000 10000000 -6 6 000011 00001100 -7 7 0001100 00011000 -8 8 00110010 00110010 -9 9 011001010 0110010100000000 -10 10 1100101011 1100101011000000 -11 11 10010101101 1001010110100000 -12 12 001010110110 0010101101100000 -13 13 0101011011000 0101011011000000 -14 14 10101101100011 1010110110001100 -15 15 010110110001101 0101101100011010 -0 0 -1 -1 0100100001100101011011000110110001101111000000000000000000000000000000000000000 01001000011001010110110001101100011011110000000000000000000000000000000000000000 -2 -2 10010000110010101101100011011000110111100000000000000000000000000000000000000 10010000110010101101100011011000110111100000000000000000000000000000000000000000 -3 -3 001000011001010110110001101100011011110000000000000000000000000000000000000 00100001100101011011000110110001101111000000000000000000000000000000000000000000 -4 -4 0100001100101011011000110110001101111000000000000000000000000000000000000 01000011001010110110001101100011011110000000000000000000000000000000000000000000 -5 -5 10000110010101101100011011000110111100000000000000000000000000000000000 100001100101011011000110110001101111000000000000000000000000000000000000 -6 -6 000011001010110110001101100011011110000000000000000000000000000000000 000011001010110110001101100011011110000000000000000000000000000000000000 -7 -7 0001100101011011000110110001101111000000000000000000000000000000000 000110010101101100011011000110111100000000000000000000000000000000000000 -8 -8 00110010101101100011011000110111100000000000000000000000000000000 001100101011011000110110001101111000000000000000000000000000000000000000 -9 -9 011001010110110001101100011011110000000000000000000000000000000 0110010101101100011011000110111100000000000000000000000000000000 -10 -10 1100101011011000110110001101111000000000000000000000000000000 1100101011011000110110001101111000000000000000000000000000000000 -11 -11 10010101101100011011000110111100000000000000000000000000000 1001010110110001101100011011110000000000000000000000000000000000 -12 -12 001010110110001101100011011110000000000000000000000000000 0010101101100011011000110111100000000000000000000000000000000000 -13 -13 0101011011000110110001101111000000000000000000000000000 01010110110001101100011011110000000000000000000000000000 -14 -14 10101101100011011000110111100000000000000000000000000 10101101100011011000110111100000000000000000000000000000 -15 -15 010110110001101100011011110000000000000000000000000 01011011000110110001101111000000000000000000000000000000 -0 -16 --1 -15 --2 -14 --3 -13 --4 -12 --5 -11 --6 -10 --7 -9 --8 -8 --9 -7 00 00000000 --10 -6 0000 00000000 --11 -5 000000 00000000 --12 -4 00000000 00000000 --13 -3 0000000000 0000000000000000 --14 -2 000000000000 0000000000000000 --15 -1 00000000000000 0000000000000000 -0 0 --1 1 0 00000000 --2 2 00 00000000 --3 3 000 00000000 --4 4 0000 00000000 --5 5 00000 00000000 --6 6 000000 00000000 --7 7 0000000 00000000 --8 8 00000000 00000000 --9 9 000000000 0000000000000000 --10 10 0000000000 0000000000000000 --11 11 00000000000 0000000000000000 --12 12 000000000000 0000000000000000 --13 13 0000000000000 0000000000000000 --14 14 00000000000000 0000000000000000 --15 15 000000000000000 0000000000000000 +0 0 Hello\0 +1 1 Hello\0 0 00000000 +2 2 Hello\0 10 10000000 +3 3 Hello\0 001 00100000 +4 4 Hello\0 0100 01000000 +5 5 Hello\0 10000 10000000 +6 6 Hello\0 000011 00001100 +7 7 Hello\0 0001100 00011000 +8 8 Hello\0 00110010 00110010 +9 9 Hello\0 011001010 0110010100000000 +10 10 Hello\0 1100101011 1100101011000000 +11 11 Hello\0 10010101101 1001010110100000 +12 12 Hello\0 001010110110 0010101101100000 +13 13 Hello\0 0101011011000 0101011011000000 +14 14 Hello\0 10101101100011 1010110110001100 +15 15 Hello\0 010110110001101 0101101100011010 +0 0 Hello\0 +1 -1 Hello\0 01001000011001010110110001101100011011110000000 010010000110010101101100011011000110111100000000 +2 -2 Hello\0 100100001100101011011000110110001101111000000 100100001100101011011000110110001101111000000000 +3 -3 Hello\0 0010000110010101101100011011000110111100000 001000011001010110110001101100011011110000000000 +4 -4 Hello\0 01000011001010110110001101100011011110000 010000110010101101100011011000110111100000000000 +5 -5 Hello\0 100001100101011011000110110001101111000 1000011001010110110001101100011011110000 +6 -6 Hello\0 0000110010101101100011011000110111100 0000110010101101100011011000110111100000 +7 -7 Hello\0 00011001010110110001101100011011110 0001100101011011000110110001101111000000 +8 -8 Hello\0 001100101011011000110110001101111 0011001010110110001101100011011110000000 +9 -9 Hello\0 0110010101101100011011000110111 01100101011011000110110001101110 +10 -10 Hello\0 11001010110110001101100011011 11001010110110001101100011011000 +11 -11 Hello\0 100101011011000110110001101 10010101101100011011000110100000 +12 -12 Hello\0 0010101101100011011000110 00101011011000110110001100000000 +13 -13 Hello\0 01010110110001101100011 010101101100011011000110 +14 -14 Hello\0 101011011000110110001 101011011000110110001000 +15 -15 Hello\0 0101101100011011000 010110110001101100000000 +0 -16 Hello\0 +-1 -15 Hello\0 +-2 -14 Hello\0 +-3 -13 Hello\0 +-4 -12 Hello\0 +-5 -11 Hello\0 +-6 -10 Hello\0 +-7 -9 Hello\0 +-8 -8 Hello\0 +-9 -7 Hello\0 10 10000000 +-10 -6 Hello\0 1100 11000000 +-11 -5 Hello\0 111000 11100000 +-12 -4 Hello\0 11110000 11110000 +-13 -3 Hello\0 0111100000 0111100000000000 +-14 -2 Hello\0 101111000000 1011110000000000 +-15 -1 Hello\0 11011110000000 1101111000000000 +0 0 Hello\0 +-1 1 Hello\0 0 00000000 +-2 2 Hello\0 00 00000000 +-3 3 Hello\0 000 00000000 +-4 4 Hello\0 0000 00000000 +-5 5 Hello\0 00000 00000000 +-6 6 Hello\0 000000 00000000 +-7 7 Hello\0 0000000 00000000 +-8 8 Hello\0 00000000 00000000 +-9 9 Hello\0 100000000 1000000000000000 +-10 10 Hello\0 1100000000 1100000000000000 +-11 11 Hello\0 11100000000 1110000000000000 +-12 12 Hello\0 111100000000 1111000000000000 +-13 13 Hello\0 0111100000000 0111100000000000 +-14 14 Hello\0 10111100000000 1011110000000000 +-15 15 Hello\0 110111100000000 1101111000000000 +Dynamic Truncate Offset, Dynamic Truncate Length +-8 8 Hello\0 00000000 00000000 +-7 8 Hello\0 0000000 00000000 +-6 8 Hello\0 000000 00000000 +-5 8 Hello\0 00000 00000000 +-4 8 Hello\0 0000 00000000 +-3 8 Hello\0 000 00000000 +-2 8 Hello\0 00 00000000 +-1 8 Hello\0 0 00000000 +0 8 Hello\0 +-4 0 Hello\0 +-4 1 Hello\0 0 00000000 +-4 2 Hello\0 00 00000000 +-4 3 Hello\0 000 00000000 +-4 4 Hello\0 0000 00000000 +-4 5 Hello\0 0000 00000000 +-4 6 Hello\0 0000 00000000 +-4 7 Hello\0 0000 00000000 +-4 8 Hello\0 0000 00000000 +-44 8 Hello\0 10000110 10000110 +-45 8 Hello\0 01000011 01000011 +-46 8 Hello\0 00100001 00100001 +-47 8 Hello\0 10010000 10010000 +-48 8 Hello\0 01001000 01001000 +-49 8 Hello\0 0100100 01001000 +-50 8 Hello\0 010010 01001000 +-51 8 Hello\0 01001 01001000 +-52 8 Hello\0 0100 01000000 +-52 0 Hello\0 +-52 1 Hello\0 +-52 2 Hello\0 +-52 3 Hello\0 +-52 4 Hello\0 +-52 5 Hello\0 0 00000000 +-52 6 Hello\0 01 01000000 +-52 7 Hello\0 010 01000000 +-52 8 Hello\0 0100 01000000 +-52 48 Hello\0 01001000011001010110110001101100011011110000 010010000110010101101100011011000110111100000000 +-52 49 Hello\0 010010000110010101101100011011000110111100000 010010000110010101101100011011000110111100000000 +-52 50 Hello\0 0100100001100101011011000110110001101111000000 010010000110010101101100011011000110111100000000 +-52 51 Hello\0 01001000011001010110110001101100011011110000000 010010000110010101101100011011000110111100000000 +-52 52 Hello\0 010010000110010101101100011011000110111100000000 010010000110010101101100011011000110111100000000 +-52 53 Hello\0 010010000110010101101100011011000110111100000000 010010000110010101101100011011000110111100000000 +-52 54 Hello\0 010010000110010101101100011011000110111100000000 010010000110010101101100011011000110111100000000 +-52 55 Hello\0 010010000110010101101100011011000110111100000000 010010000110010101101100011011000110111100000000 +-52 56 Hello\0 010010000110010101101100011011000110111100000000 010010000110010101101100011011000110111100000000 +Dynamic Nullable Offset, Dynamic Nullable Length +0 0 Hello\0 +\N 1 Hello\0 \N \N +2 \N Hello\0 \N \N +3 3 \N \N \N +4 4 Hello\0 0100 01000000 +\N 5 Hello\0 \N \N +6 \N Hello\0 \N \N +\N \N \N \N \N +8 8 Hello\0 00110010 00110010 +\N 9 Hello\0 \N \N +10 \N Hello\0 \N \N +11 11 \N \N \N +12 12 Hello\0 001010110110 0010101101100000 +\N 13 Hello\0 \N \N +14 \N Hello\0 \N \N +\N \N \N \N \N diff --git a/tests/queries/0_stateless/02154_bit_slice_for_fixedstring.sql b/tests/queries/0_stateless/02154_bit_slice_for_fixedstring.sql index 2d49a1ff1d6..f3e803396e9 100644 --- a/tests/queries/0_stateless/02154_bit_slice_for_fixedstring.sql +++ b/tests/queries/0_stateless/02154_bit_slice_for_fixedstring.sql @@ -1,111 +1,143 @@ SELECT 'Const Offset'; -select 1, subString(bin(toFixedString('Hello', 10)), 1), bin(bitSlice(toFixedString('Hello', 10), 1)); -select 2, subString(bin(toFixedString('Hello', 10)), 2), bin(bitSlice(toFixedString('Hello', 10), 2)); -select 3, subString(bin(toFixedString('Hello', 10)), 3), bin(bitSlice(toFixedString('Hello', 10), 3)); -select 4, subString(bin(toFixedString('Hello', 10)), 4), bin(bitSlice(toFixedString('Hello', 10), 4)); -select 5, subString(bin(toFixedString('Hello', 10)), 5), bin(bitSlice(toFixedString('Hello', 10), 5)); -select 6, subString(bin(toFixedString('Hello', 10)), 6), bin(bitSlice(toFixedString('Hello', 10), 6)); -select 7, subString(bin(toFixedString('Hello', 10)), 7), bin(bitSlice(toFixedString('Hello', 10), 7)); -select 8, subString(bin(toFixedString('Hello', 10)), 8), bin(bitSlice(toFixedString('Hello', 10), 8)); -select 9, subString(bin(toFixedString('Hello', 10)), 9), bin(bitSlice(toFixedString('Hello', 10), 9)); -select 10, subString(bin(toFixedString('Hello', 10)), 10), bin(bitSlice(toFixedString('Hello', 10), 10)); -select 11, subString(bin(toFixedString('Hello', 10)), 11), bin(bitSlice(toFixedString('Hello', 10), 11)); -select 12, subString(bin(toFixedString('Hello', 10)), 12), bin(bitSlice(toFixedString('Hello', 10), 12)); -select 13, subString(bin(toFixedString('Hello', 10)), 13), bin(bitSlice(toFixedString('Hello', 10), 13)); -select 14, subString(bin(toFixedString('Hello', 10)), 14), bin(bitSlice(toFixedString('Hello', 10), 14)); -select 15, subString(bin(toFixedString('Hello', 10)), 15), bin(bitSlice(toFixedString('Hello', 10), 15)); -select 16, subString(bin(toFixedString('Hello', 10)), 16), bin(bitSlice(toFixedString('Hello', 10), 16)); +select 1 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 2 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 3 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 4 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 5 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 6 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 7 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 8 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 9 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 10 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 11 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 12 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 13 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 14 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 15 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 16 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); -select -1, subString(bin(toFixedString('Hello', 10)), -1), bin(bitSlice(toFixedString('Hello', 10), -1)); -select -2, subString(bin(toFixedString('Hello', 10)), -2), bin(bitSlice(toFixedString('Hello', 10), -2)); -select -3, subString(bin(toFixedString('Hello', 10)), -3), bin(bitSlice(toFixedString('Hello', 10), -3)); -select -4, subString(bin(toFixedString('Hello', 10)), -4), bin(bitSlice(toFixedString('Hello', 10), -4)); -select -5, subString(bin(toFixedString('Hello', 10)), -5), bin(bitSlice(toFixedString('Hello', 10), -5)); -select -6, subString(bin(toFixedString('Hello', 10)), -6), bin(bitSlice(toFixedString('Hello', 10), -6)); -select -7, subString(bin(toFixedString('Hello', 10)), -7), bin(bitSlice(toFixedString('Hello', 10), -7)); -select -8, subString(bin(toFixedString('Hello', 10)), -8), bin(bitSlice(toFixedString('Hello', 10), -8)); -select -9, subString(bin(toFixedString('Hello', 10)), -9), bin(bitSlice(toFixedString('Hello', 10), -9)); -select -10, subString(bin(toFixedString('Hello', 10)), -10), bin(bitSlice(toFixedString('Hello', 10), -10)); -select -11, subString(bin(toFixedString('Hello', 10)), -11), bin(bitSlice(toFixedString('Hello', 10), -11)); -select -12, subString(bin(toFixedString('Hello', 10)), -12), bin(bitSlice(toFixedString('Hello', 10), -12)); -select -13, subString(bin(toFixedString('Hello', 10)), -13), bin(bitSlice(toFixedString('Hello', 10), -13)); -select -14, subString(bin(toFixedString('Hello', 10)), -14), bin(bitSlice(toFixedString('Hello', 10), -14)); -select -15, subString(bin(toFixedString('Hello', 10)), -15), bin(bitSlice(toFixedString('Hello', 10), -15)); -select -16, subString(bin(toFixedString('Hello', 10)), -16), bin(bitSlice(toFixedString('Hello', 10), -16)); +select -1 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -2 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -3 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -4 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -5 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -6 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -7 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -8 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -9 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -10 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -11 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -12 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -13 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -14 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -15 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -16 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +SELECT 'Const Truncate Offset'; +select 49 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -49 as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); + +SELECT 'Const Nullable Offset'; +select 1 as offset, null as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select null as offset, toFixedString('Hello', 6) as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select null as offset, null as s, subString(bin(s), offset), bin(bitSlice(s, offset)); SELECT 'Const Offset, Const Length'; -select 1, 1, subString(bin(toFixedString('Hello', 10)), 1, 1), bin(bitSlice(toFixedString('Hello', 10), 1, 1)); -select 2, 2, subString(bin(toFixedString('Hello', 10)), 2, 2), bin(bitSlice(toFixedString('Hello', 10), 2, 2)); -select 3, 3, subString(bin(toFixedString('Hello', 10)), 3, 3), bin(bitSlice(toFixedString('Hello', 10), 3, 3)); -select 4, 4, subString(bin(toFixedString('Hello', 10)), 4, 4), bin(bitSlice(toFixedString('Hello', 10), 4, 4)); -select 5, 5, subString(bin(toFixedString('Hello', 10)), 5, 5), bin(bitSlice(toFixedString('Hello', 10), 5, 5)); -select 6, 6, subString(bin(toFixedString('Hello', 10)), 6, 6), bin(bitSlice(toFixedString('Hello', 10), 6, 6)); -select 7, 7, subString(bin(toFixedString('Hello', 10)), 7, 7), bin(bitSlice(toFixedString('Hello', 10), 7, 7)); -select 8, 8, subString(bin(toFixedString('Hello', 10)), 8, 8), bin(bitSlice(toFixedString('Hello', 10), 8, 8)); -select 9, 9, subString(bin(toFixedString('Hello', 10)), 9, 9), bin(bitSlice(toFixedString('Hello', 10), 9, 9)); -select 10, 10, subString(bin(toFixedString('Hello', 10)), 10, 10), bin(bitSlice(toFixedString('Hello', 10), 10, 10)); -select 11, 11, subString(bin(toFixedString('Hello', 10)), 11, 11), bin(bitSlice(toFixedString('Hello', 10), 11, 11)); -select 12, 12, subString(bin(toFixedString('Hello', 10)), 12, 12), bin(bitSlice(toFixedString('Hello', 10), 12, 12)); -select 13, 13, subString(bin(toFixedString('Hello', 10)), 13, 13), bin(bitSlice(toFixedString('Hello', 10), 13, 13)); -select 14, 14, subString(bin(toFixedString('Hello', 10)), 14, 14), bin(bitSlice(toFixedString('Hello', 10), 14, 14)); -select 15, 15, subString(bin(toFixedString('Hello', 10)), 15, 15), bin(bitSlice(toFixedString('Hello', 10), 15, 15)); -select 16, 16, subString(bin(toFixedString('Hello', 10)), 16, 16), bin(bitSlice(toFixedString('Hello', 10), 16, 16)); +select 1 as offset, 1 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 2 as offset, 2 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 3 as offset, 3 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 4 as offset, 4 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 5 as offset, 5 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 6 as offset, 6 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 7 as offset, 7 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 8 as offset, 8 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 9 as offset, 9 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 10 as offset, 10 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 11 as offset, 11 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 12 as offset, 12 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 13 as offset, 13 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 14 as offset, 14 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 15 as offset, 15 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 16 as offset, 16 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); -select 1, -1, subString(bin(toFixedString('Hello', 10)), 1, -1), bin(bitSlice(toFixedString('Hello', 10), 1, -1)); -select 2, -2, subString(bin(toFixedString('Hello', 10)), 2, -2), bin(bitSlice(toFixedString('Hello', 10), 2, -2)); -select 3, -3, subString(bin(toFixedString('Hello', 10)), 3, -3), bin(bitSlice(toFixedString('Hello', 10), 3, -3)); -select 4, -4, subString(bin(toFixedString('Hello', 10)), 4, -4), bin(bitSlice(toFixedString('Hello', 10), 4, -4)); -select 5, -5, subString(bin(toFixedString('Hello', 10)), 5, -5), bin(bitSlice(toFixedString('Hello', 10), 5, -5)); -select 6, -6, subString(bin(toFixedString('Hello', 10)), 6, -6), bin(bitSlice(toFixedString('Hello', 10), 6, -6)); -select 7, -7, subString(bin(toFixedString('Hello', 10)), 7, -7), bin(bitSlice(toFixedString('Hello', 10), 7, -7)); -select 8, -8, subString(bin(toFixedString('Hello', 10)), 8, -8), bin(bitSlice(toFixedString('Hello', 10), 8, -8)); -select 9, -9, subString(bin(toFixedString('Hello', 10)), 9, -9), bin(bitSlice(toFixedString('Hello', 10), 9, -9)); -select 10, -10, subString(bin(toFixedString('Hello', 10)), 10, -10), bin(bitSlice(toFixedString('Hello', 10), 10, -10)); -select 11, -11, subString(bin(toFixedString('Hello', 10)), 11, -11), bin(bitSlice(toFixedString('Hello', 10), 11, -11)); -select 12, -12, subString(bin(toFixedString('Hello', 10)), 12, -12), bin(bitSlice(toFixedString('Hello', 10), 12, -12)); -select 13, -13, subString(bin(toFixedString('Hello', 10)), 13, -13), bin(bitSlice(toFixedString('Hello', 10), 13, -13)); -select 14, -14, subString(bin(toFixedString('Hello', 10)), 14, -14), bin(bitSlice(toFixedString('Hello', 10), 14, -14)); -select 15, -15, subString(bin(toFixedString('Hello', 10)), 15, -15), bin(bitSlice(toFixedString('Hello', 10), 15, -15)); -select 16, -16, subString(bin(toFixedString('Hello', 10)), 16, -16), bin(bitSlice(toFixedString('Hello', 10), 16, -16)); +select 1 as offset, -1 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 2 as offset, -2 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 3 as offset, -3 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 4 as offset, -4 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 5 as offset, -5 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 6 as offset, -6 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 7 as offset, -7 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 8 as offset, -8 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 9 as offset, -9 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 10 as offset, -10 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 11 as offset, -11 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 12 as offset, -12 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 13 as offset, -13 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 14 as offset, -14 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 15 as offset, -15 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 16 as offset, -16 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); -select -1, 1, subString(bin(toFixedString('Hello', 10)), -1, 1), bin(bitSlice(toFixedString('Hello', 10), -1, 1)); -select -2, 2, subString(bin(toFixedString('Hello', 10)), -2, 2), bin(bitSlice(toFixedString('Hello', 10), -2, 2)); -select -3, 3, subString(bin(toFixedString('Hello', 10)), -3, 3), bin(bitSlice(toFixedString('Hello', 10), -3, 3)); -select -4, 4, subString(bin(toFixedString('Hello', 10)), -4, 4), bin(bitSlice(toFixedString('Hello', 10), -4, 4)); -select -5, 5, subString(bin(toFixedString('Hello', 10)), -5, 5), bin(bitSlice(toFixedString('Hello', 10), -5, 5)); -select -6, 6, subString(bin(toFixedString('Hello', 10)), -6, 6), bin(bitSlice(toFixedString('Hello', 10), -6, 6)); -select -7, 7, subString(bin(toFixedString('Hello', 10)), -7, 7), bin(bitSlice(toFixedString('Hello', 10), -7, 7)); -select -8, 8, subString(bin(toFixedString('Hello', 10)), -8, 8), bin(bitSlice(toFixedString('Hello', 10), -8, 8)); -select -9, 9, subString(bin(toFixedString('Hello', 10)), -9, 9), bin(bitSlice(toFixedString('Hello', 10), -9, 9)); -select -10, 10, subString(bin(toFixedString('Hello', 10)), -10, 10), bin(bitSlice(toFixedString('Hello', 10), -10, 10)); -select -11, 11, subString(bin(toFixedString('Hello', 10)), -11, 11), bin(bitSlice(toFixedString('Hello', 10), -11, 11)); -select -12, 12, subString(bin(toFixedString('Hello', 10)), -12, 12), bin(bitSlice(toFixedString('Hello', 10), -12, 12)); -select -13, 13, subString(bin(toFixedString('Hello', 10)), -13, 13), bin(bitSlice(toFixedString('Hello', 10), -13, 13)); -select -14, 14, subString(bin(toFixedString('Hello', 10)), -14, 14), bin(bitSlice(toFixedString('Hello', 10), -14, 14)); -select -15, 15, subString(bin(toFixedString('Hello', 10)), -15, 15), bin(bitSlice(toFixedString('Hello', 10), -15, 15)); -select -16, 16, subString(bin(toFixedString('Hello', 10)), -16, 16), bin(bitSlice(toFixedString('Hello', 10), -16, 16)); +select -1 as offset, 1 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -2 as offset, 2 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -3 as offset, 3 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -4 as offset, 4 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -5 as offset, 5 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -6 as offset, 6 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -7 as offset, 7 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -8 as offset, 8 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -9 as offset, 9 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -10 as offset, 10 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -11 as offset, 11 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -12 as offset, 12 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -13 as offset, 13 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -14 as offset, 14 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -15 as offset, 15 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -16 as offset, 16 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); -select -1, -16, subString(bin(toFixedString('Hello', 10)), -1, -16), bin(bitSlice(toFixedString('Hello', 10), -1, -16)); -select -2, -15, subString(bin(toFixedString('Hello', 10)), -2, -15), bin(bitSlice(toFixedString('Hello', 10), -2, -15)); -select -3, -14, subString(bin(toFixedString('Hello', 10)), -3, -14), bin(bitSlice(toFixedString('Hello', 10), -3, -14)); -select -4, -13, subString(bin(toFixedString('Hello', 10)), -4, -13), bin(bitSlice(toFixedString('Hello', 10), -4, -13)); -select -5, -12, subString(bin(toFixedString('Hello', 10)), -5, -12), bin(bitSlice(toFixedString('Hello', 10), -5, -12)); -select -6, -11, subString(bin(toFixedString('Hello', 10)), -6, -11), bin(bitSlice(toFixedString('Hello', 10), -6, -11)); -select -7, -10, subString(bin(toFixedString('Hello', 10)), -7, -10), bin(bitSlice(toFixedString('Hello', 10), -7, -10)); -select -8, -9, subString(bin(toFixedString('Hello', 10)), -8, -9), bin(bitSlice(toFixedString('Hello', 10), -8, -9)); -select -9, -8, subString(bin(toFixedString('Hello', 10)), -9, -8), bin(bitSlice(toFixedString('Hello', 10), -9, -8)); -select -10, -7, subString(bin(toFixedString('Hello', 10)), -10, -7), bin(bitSlice(toFixedString('Hello', 10), -10, -7)); -select -11, -6, subString(bin(toFixedString('Hello', 10)), -11, -6), bin(bitSlice(toFixedString('Hello', 10), -11, -6)); -select -12, -5, subString(bin(toFixedString('Hello', 10)), -12, -5), bin(bitSlice(toFixedString('Hello', 10), -12, -5)); -select -13, -4, subString(bin(toFixedString('Hello', 10)), -13, -4), bin(bitSlice(toFixedString('Hello', 10), -13, -4)); -select -14, -3, subString(bin(toFixedString('Hello', 10)), -14, -3), bin(bitSlice(toFixedString('Hello', 10), -14, -3)); -select -15, -2, subString(bin(toFixedString('Hello', 10)), -15, -2), bin(bitSlice(toFixedString('Hello', 10), -15, -2)); -select -16, -1, subString(bin(toFixedString('Hello', 10)), -16, -1), bin(bitSlice(toFixedString('Hello', 10), -16, -1)); +select -1 as offset, -16 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -2 as offset, -15 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -3 as offset, -14 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -4 as offset, -13 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -5 as offset, -12 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -6 as offset, -11 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -7 as offset, -10 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -8 as offset, -9 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -9 as offset, -8 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -10 as offset, -7 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -11 as offset, -6 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -12 as offset, -5 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -13 as offset, -4 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -14 as offset, -3 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -15 as offset, -2 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -16 as offset, -1 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); + +select 'Const Truncate Offset, Const Truncate Length'; +select 36 as offset, 16 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 49 as offset, 1 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -52 as offset, -44 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -49 as offset, -48 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -49 as offset, 49 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); + +select 'Const Nullable Offset, Const Nullable Length'; +select 1 as offset, 1 as length, null as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)); +select null as offset, 1 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 1 as offset, null as length, toFixedString('Hello', 6) as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)); +select null as offset, null as length, null as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)); select 'Dynamic Offset, Dynamic Length'; +select number as offset, number as length, toFixedString('Hello', 6) as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)) from numbers(16); +select number as offset, -number as length, toFixedString('Hello', 6) as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)) from numbers(16); +select -number as offset, -16+number as length, toFixedString('Hello', 6) as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)) from numbers(16); +select -number as offset, number as length, toFixedString('Hello', 6) as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)) from numbers(16); -select number, number, subString(bin(toFixedString('Hello', 10)), number, number), bin(bitSlice(toFixedString('Hello', 10), number, number)) from numbers(16); -select number, -number, subString(bin(toFixedString('Hello', 10)), number, -number), bin(bitSlice(toFixedString('Hello', 10), number, -number)) from numbers(16); -select -number, -16+number, subString(bin(toFixedString('Hello', 10)), -number, -16+number), bin(bitSlice(toFixedString('Hello', 10), -number, -16+number)) from numbers(16); -select -number, number, subString(bin(toFixedString('Hello', 10)), -number, number), bin(bitSlice(toFixedString('Hello', 10), -number, number)) from numbers(16); +select 'Dynamic Truncate Offset, Dynamic Truncate Length'; +select number-8 as offset, 8 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)) from numbers(9); +select -4 as offset, number as length, toFixedString('Hello', 6) as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)) from numbers(9); +select -44-number as offset, 8 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)) from numbers(9); +select -52 as offset, number as length, toFixedString('Hello', 6) as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)) from numbers(9); +select -52 as offset, number + 48 as length, toFixedString('Hello', 6) as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)) from numbers(9); + +select 'Dynamic Nullable Offset, Dynamic Nullable Length'; +select if(number%4 ==1 or number%8==7, null, number) as offset, if(number%4==2 or number%8==7, null, number) as length,if(number%4 ==3, null, toFixedString('Hello', 6)) as s, + subString(bin(s), offset, length), bin(bitSlice(s, offset , length)) +from numbers(16); diff --git a/tests/queries/0_stateless/02154_bit_slice_for_string.reference b/tests/queries/0_stateless/02154_bit_slice_for_string.reference index 272f415a021..71a80c7abf3 100644 --- a/tests/queries/0_stateless/02154_bit_slice_for_string.reference +++ b/tests/queries/0_stateless/02154_bit_slice_for_string.reference @@ -1,163 +1,244 @@ Const Offset -1 0100100001100101011011000110110001101111 0100100001100101011011000110110001101111 -2 100100001100101011011000110110001101111 1001000011001010110110001101100011011110 -3 00100001100101011011000110110001101111 0010000110010101101100011011000110111100 -4 0100001100101011011000110110001101111 0100001100101011011000110110001101111000 -5 100001100101011011000110110001101111 1000011001010110110001101100011011110000 -6 00001100101011011000110110001101111 0000110010101101100011011000110111100000 -7 0001100101011011000110110001101111 0001100101011011000110110001101111000000 -8 001100101011011000110110001101111 0011001010110110001101100011011110000000 -9 01100101011011000110110001101111 01100101011011000110110001101111 -10 1100101011011000110110001101111 11001010110110001101100011011110 -11 100101011011000110110001101111 10010101101100011011000110111100 -12 00101011011000110110001101111 00101011011000110110001101111000 -13 0101011011000110110001101111 01010110110001101100011011110000 -14 101011011000110110001101111 10101101100011011000110111100000 -15 01011011000110110001101111 01011011000110110001101111000000 -16 1011011000110110001101111 10110110001101100011011110000000 --1 1 10000000 --2 11 11000000 --3 111 11100000 --4 1111 11110000 --5 01111 01111000 --6 101111 10111100 --7 1101111 11011110 --8 01101111 01101111 --9 001101111 0011011110000000 --10 0001101111 0001101111000000 --11 10001101111 1000110111100000 --12 110001101111 1100011011110000 --13 0110001101111 0110001101111000 --14 10110001101111 1011000110111100 --15 110110001101111 1101100011011110 --16 0110110001101111 0110110001101111 +1 Hello 0100100001100101011011000110110001101111 0100100001100101011011000110110001101111 +2 Hello 100100001100101011011000110110001101111 1001000011001010110110001101100011011110 +3 Hello 00100001100101011011000110110001101111 0010000110010101101100011011000110111100 +4 Hello 0100001100101011011000110110001101111 0100001100101011011000110110001101111000 +5 Hello 100001100101011011000110110001101111 1000011001010110110001101100011011110000 +6 Hello 00001100101011011000110110001101111 0000110010101101100011011000110111100000 +7 Hello 0001100101011011000110110001101111 0001100101011011000110110001101111000000 +8 Hello 001100101011011000110110001101111 0011001010110110001101100011011110000000 +9 Hello 01100101011011000110110001101111 01100101011011000110110001101111 +10 Hello 1100101011011000110110001101111 11001010110110001101100011011110 +11 Hello 100101011011000110110001101111 10010101101100011011000110111100 +12 Hello 00101011011000110110001101111 00101011011000110110001101111000 +13 Hello 0101011011000110110001101111 01010110110001101100011011110000 +14 Hello 101011011000110110001101111 10101101100011011000110111100000 +15 Hello 01011011000110110001101111 01011011000110110001101111000000 +16 Hello 1011011000110110001101111 10110110001101100011011110000000 +-1 Hello 1 10000000 +-2 Hello 11 11000000 +-3 Hello 111 11100000 +-4 Hello 1111 11110000 +-5 Hello 01111 01111000 +-6 Hello 101111 10111100 +-7 Hello 1101111 11011110 +-8 Hello 01101111 01101111 +-9 Hello 001101111 0011011110000000 +-10 Hello 0001101111 0001101111000000 +-11 Hello 10001101111 1000110111100000 +-12 Hello 110001101111 1100011011110000 +-13 Hello 0110001101111 0110001101111000 +-14 Hello 10110001101111 1011000110111100 +-15 Hello 110110001101111 1101100011011110 +-16 Hello 0110110001101111 0110110001101111 +Const Truncate Offset +41 Hello +-41 Hello 0100100001100101011011000110110001101111 0100100001100101011011000110110001101111 +Const Nullable Offset +1 \N \N \N +\N Hello \N \N +\N \N \N \N Const Offset, Const Length -1 1 0 00000000 -2 2 10 10000000 -3 3 001 00100000 -4 4 0100 01000000 -5 5 10000 10000000 -6 6 000011 00001100 -7 7 0001100 00011000 -8 8 00110010 00110010 -9 9 011001010 0110010100000000 -10 10 1100101011 1100101011000000 -11 11 10010101101 1001010110100000 -12 12 001010110110 0010101101100000 -13 13 0101011011000 0101011011000000 -14 14 10101101100011 1010110110001100 -15 15 010110110001101 0101101100011010 -16 16 1011011000110110 1011011000110110 -1 -1 010010000110010101101100011011000110111 0100100001100101011011000110110001101110 -2 -2 1001000011001010110110001101100011011 1001000011001010110110001101100011011000 -3 -3 00100001100101011011000110110001101 0010000110010101101100011011000110100000 -4 -4 010000110010101101100011011000110 0100001100101011011000110110001100000000 -5 -5 1000011001010110110001101100011 10000110010101101100011011000110 -6 -6 00001100101011011000110110001 00001100101011011000110110001000 -7 -7 000110010101101100011011000 00011001010110110001101100000000 -8 -8 0011001010110110001101100 00110010101101100011011000000000 -9 -9 01100101011011000110110 011001010110110001101100 -10 -10 110010101101100011011 110010101101100011011000 -11 -11 1001010110110001101 100101011011000110100000 -12 -12 00101011011000110 001010110110001100000000 -13 -13 010101101100011 0101011011000110 -14 -14 1010110110001 1010110110001000 -15 -15 01011011000 0101101100000000 -16 -16 101101100 1011011000000000 --1 1 1 10000000 --2 2 11 11000000 --3 3 111 11100000 --4 4 1111 11110000 --5 5 01111 01111000 --6 6 101111 10111100 --7 7 1101111 11011110 --8 8 01101111 01101111 --9 9 001101111 0011011110000000 --10 10 0001101111 0001101111000000 --11 11 10001101111 1000110111100000 --12 12 110001101111 1100011011110000 --13 13 0110001101111 0110001101111000 --14 14 10110001101111 1011000110111100 --15 15 110110001101111 1101100011011110 --16 16 0110110001101111 0110110001101111 --1 -16 --2 -15 --3 -14 --4 -13 --5 -12 --6 -11 --7 -10 --8 -9 --9 -8 0 00000000 --10 -7 000 00000000 --11 -6 10001 10001000 --12 -5 1100011 11000110 --13 -4 011000110 0110001100000000 --14 -3 10110001101 1011000110100000 --15 -2 1101100011011 1101100011011000 --16 -1 011011000110111 0110110001101110 +1 1 Hello 0 00000000 +2 2 Hello 10 10000000 +3 3 Hello 001 00100000 +4 4 Hello 0100 01000000 +5 5 Hello 10000 10000000 +6 6 Hello 000011 00001100 +7 7 Hello 0001100 00011000 +8 8 Hello 00110010 00110010 +9 9 Hello 011001010 0110010100000000 +10 10 Hello 1100101011 1100101011000000 +11 11 Hello 10010101101 1001010110100000 +12 12 Hello 001010110110 0010101101100000 +13 13 Hello 0101011011000 0101011011000000 +14 14 Hello 10101101100011 1010110110001100 +15 15 Hello 010110110001101 0101101100011010 +16 16 Hello 1011011000110110 1011011000110110 +1 -1 Hello 010010000110010101101100011011000110111 0100100001100101011011000110110001101110 +2 -2 Hello 1001000011001010110110001101100011011 1001000011001010110110001101100011011000 +3 -3 Hello 00100001100101011011000110110001101 0010000110010101101100011011000110100000 +4 -4 Hello 010000110010101101100011011000110 0100001100101011011000110110001100000000 +5 -5 Hello 1000011001010110110001101100011 10000110010101101100011011000110 +6 -6 Hello 00001100101011011000110110001 00001100101011011000110110001000 +7 -7 Hello 000110010101101100011011000 00011001010110110001101100000000 +8 -8 Hello 0011001010110110001101100 00110010101101100011011000000000 +9 -9 Hello 01100101011011000110110 011001010110110001101100 +10 -10 Hello 110010101101100011011 110010101101100011011000 +11 -11 Hello 1001010110110001101 100101011011000110100000 +12 -12 Hello 00101011011000110 001010110110001100000000 +13 -13 Hello 010101101100011 0101011011000110 +14 -14 Hello 1010110110001 1010110110001000 +15 -15 Hello 01011011000 0101101100000000 +16 -16 Hello 101101100 1011011000000000 +-1 1 Hello 1 10000000 +-2 2 Hello 11 11000000 +-3 3 Hello 111 11100000 +-4 4 Hello 1111 11110000 +-5 5 Hello 01111 01111000 +-6 6 Hello 101111 10111100 +-7 7 Hello 1101111 11011110 +-8 8 Hello 01101111 01101111 +-9 9 Hello 001101111 0011011110000000 +-10 10 Hello 0001101111 0001101111000000 +-11 11 Hello 10001101111 1000110111100000 +-12 12 Hello 110001101111 1100011011110000 +-13 13 Hello 0110001101111 0110001101111000 +-14 14 Hello 10110001101111 1011000110111100 +-15 15 Hello 110110001101111 1101100011011110 +-16 16 Hello 0110110001101111 0110110001101111 +-1 -16 Hello +-2 -15 Hello +-3 -14 Hello +-4 -13 Hello +-5 -12 Hello +-6 -11 Hello +-7 -10 Hello +-8 -9 Hello +-9 -8 Hello 0 00000000 +-10 -7 Hello 000 00000000 +-11 -6 Hello 10001 10001000 +-12 -5 Hello 1100011 11000110 +-13 -4 Hello 011000110 0110001100000000 +-14 -3 Hello 10110001101 1011000110100000 +-15 -2 Hello 1101100011011 1101100011011000 +-16 -1 Hello 011011000110111 0110110001101110 +Const Truncate Offset, Const Truncate Length +36 8 Hello 01111 01111000 +41 1 Hello +-44 -36 Hello 0100 01000000 +-41 -40 Hello +-41 41 Hello 0100100001100101011011000110110001101111 0100100001100101011011000110110001101111 +Const Nullable Offset, Const Nullable Length +1 1 \N \N \N +\N 1 Hello \N \N +1 \N Hello \N \N +\N \N \N \N \N Dynamic Offset, Dynamic Length -0 0 -1 1 0 00000000 -2 2 10 10000000 -3 3 001 00100000 -4 4 0100 01000000 -5 5 10000 10000000 -6 6 000011 00001100 -7 7 0001100 00011000 -8 8 00110010 00110010 -9 9 011001010 0110010100000000 -10 10 1100101011 1100101011000000 -11 11 10010101101 1001010110100000 -12 12 001010110110 0010101101100000 -13 13 0101011011000 0101011011000000 -14 14 10101101100011 1010110110001100 -15 15 010110110001101 0101101100011010 -0 0 -1 -1 010010000110010101101100011011000110111 0100100001100101011011000110110001101110 -2 -2 1001000011001010110110001101100011011 1001000011001010110110001101100011011000 -3 -3 00100001100101011011000110110001101 0010000110010101101100011011000110100000 -4 -4 010000110010101101100011011000110 0100001100101011011000110110001100000000 -5 -5 1000011001010110110001101100011 10000110010101101100011011000110 -6 -6 00001100101011011000110110001 00001100101011011000110110001000 -7 -7 000110010101101100011011000 00011001010110110001101100000000 -8 -8 0011001010110110001101100 00110010101101100011011000000000 -9 -9 01100101011011000110110 011001010110110001101100 -10 -10 110010101101100011011 110010101101100011011000 -11 -11 1001010110110001101 100101011011000110100000 -12 -12 00101011011000110 001010110110001100000000 -13 -13 010101101100011 0101011011000110 -14 -14 1010110110001 1010110110001000 -15 -15 01011011000 0101101100000000 -0 -16 --1 -15 --2 -14 --3 -13 --4 -12 --5 -11 --6 -10 --7 -9 --8 -8 --9 -7 00 00000000 --10 -6 0001 00010000 --11 -5 100011 10001100 --12 -4 11000110 11000110 --13 -3 0110001101 0110001101000000 --14 -2 101100011011 1011000110110000 --15 -1 11011000110111 1101100011011100 -0 0 --1 1 1 10000000 --2 2 11 11000000 --3 3 111 11100000 --4 4 1111 11110000 --5 5 01111 01111000 --6 6 101111 10111100 --7 7 1101111 11011110 --8 8 01101111 01101111 --9 9 001101111 0011011110000000 --10 10 0001101111 0001101111000000 --11 11 10001101111 1000110111100000 --12 12 110001101111 1100011011110000 --13 13 0110001101111 0110001101111000 --14 14 10110001101111 1011000110111100 --15 15 110110001101111 1101100011011110 +0 0 Hello +1 1 Hello 0 00000000 +2 2 Hello 10 10000000 +3 3 Hello 001 00100000 +4 4 Hello 0100 01000000 +5 5 Hello 10000 10000000 +6 6 Hello 000011 00001100 +7 7 Hello 0001100 00011000 +8 8 Hello 00110010 00110010 +9 9 Hello 011001010 0110010100000000 +10 10 Hello 1100101011 1100101011000000 +11 11 Hello 10010101101 1001010110100000 +12 12 Hello 001010110110 0010101101100000 +13 13 Hello 0101011011000 0101011011000000 +14 14 Hello 10101101100011 1010110110001100 +15 15 Hello 010110110001101 0101101100011010 +0 0 Hello +1 -1 Hello 010010000110010101101100011011000110111 0100100001100101011011000110110001101110 +2 -2 Hello 1001000011001010110110001101100011011 1001000011001010110110001101100011011000 +3 -3 Hello 00100001100101011011000110110001101 0010000110010101101100011011000110100000 +4 -4 Hello 010000110010101101100011011000110 0100001100101011011000110110001100000000 +5 -5 Hello 1000011001010110110001101100011 10000110010101101100011011000110 +6 -6 Hello 00001100101011011000110110001 00001100101011011000110110001000 +7 -7 Hello 000110010101101100011011000 00011001010110110001101100000000 +8 -8 Hello 0011001010110110001101100 00110010101101100011011000000000 +9 -9 Hello 01100101011011000110110 011001010110110001101100 +10 -10 Hello 110010101101100011011 110010101101100011011000 +11 -11 Hello 1001010110110001101 100101011011000110100000 +12 -12 Hello 00101011011000110 001010110110001100000000 +13 -13 Hello 010101101100011 0101011011000110 +14 -14 Hello 1010110110001 1010110110001000 +15 -15 Hello 01011011000 0101101100000000 +0 -16 Hello +-1 -15 Hello +-2 -14 Hello +-3 -13 Hello +-4 -12 Hello +-5 -11 Hello +-6 -10 Hello +-7 -9 Hello +-8 -8 Hello +-9 -7 Hello 00 00000000 +-10 -6 Hello 0001 00010000 +-11 -5 Hello 100011 10001100 +-12 -4 Hello 11000110 11000110 +-13 -3 Hello 0110001101 0110001101000000 +-14 -2 Hello 101100011011 1011000110110000 +-15 -1 Hello 11011000110111 1101100011011100 +0 0 Hello +-1 1 Hello 1 10000000 +-2 2 Hello 11 11000000 +-3 3 Hello 111 11100000 +-4 4 Hello 1111 11110000 +-5 5 Hello 01111 01111000 +-6 6 Hello 101111 10111100 +-7 7 Hello 1101111 11011110 +-8 8 Hello 01101111 01101111 +-9 9 Hello 001101111 0011011110000000 +-10 10 Hello 0001101111 0001101111000000 +-11 11 Hello 10001101111 1000110111100000 +-12 12 Hello 110001101111 1100011011110000 +-13 13 Hello 0110001101111 0110001101111000 +-14 14 Hello 10110001101111 1011000110111100 +-15 15 Hello 110110001101111 1101100011011110 +Dynamic Truncate Offset, Dynamic Truncate Length +-8 8 Hello 01101111 01101111 +-7 8 Hello 1101111 11011110 +-6 8 Hello 101111 10111100 +-5 8 Hello 01111 01111000 +-4 8 Hello 1111 11110000 +-3 8 Hello 111 11100000 +-2 8 Hello 11 11000000 +-1 8 Hello 1 10000000 +0 8 Hello +-4 0 Hello +-4 1 Hello 1 10000000 +-4 2 Hello 11 11000000 +-4 3 Hello 111 11100000 +-4 4 Hello 1111 11110000 +-4 5 Hello 1111 11110000 +-4 6 Hello 1111 11110000 +-4 7 Hello 1111 11110000 +-4 8 Hello 1111 11110000 +-36 8 Hello 10000110 10000110 +-37 8 Hello 01000011 01000011 +-38 8 Hello 00100001 00100001 +-39 8 Hello 10010000 10010000 +-40 8 Hello 01001000 01001000 +-41 8 Hello 0100100 01001000 +-42 8 Hello 010010 01001000 +-43 8 Hello 01001 01001000 +-44 8 Hello 0100 01000000 +-44 0 Hello +-44 1 Hello +-44 2 Hello +-44 3 Hello +-44 4 Hello +-44 5 Hello 0 00000000 +-44 6 Hello 01 01000000 +-44 7 Hello 010 01000000 +-44 8 Hello 0100 01000000 +-44 40 Hello 010010000110010101101100011011000110 0100100001100101011011000110110001100000 +-44 41 Hello 0100100001100101011011000110110001101 0100100001100101011011000110110001101000 +-44 42 Hello 01001000011001010110110001101100011011 0100100001100101011011000110110001101100 +-44 43 Hello 010010000110010101101100011011000110111 0100100001100101011011000110110001101110 +-44 44 Hello 0100100001100101011011000110110001101111 0100100001100101011011000110110001101111 +-44 45 Hello 0100100001100101011011000110110001101111 0100100001100101011011000110110001101111 +-44 46 Hello 0100100001100101011011000110110001101111 0100100001100101011011000110110001101111 +-44 47 Hello 0100100001100101011011000110110001101111 0100100001100101011011000110110001101111 +-44 48 Hello 0100100001100101011011000110110001101111 0100100001100101011011000110110001101111 +Dynamic Nullable Offset, Dynamic Nullable Length +0 0 Hello +\N 1 Hello \N \N +2 \N Hello \N \N +3 3 \N \N \N +4 4 Hello 0100 01000000 +\N 5 Hello \N \N +6 \N Hello \N \N +\N \N \N \N \N +8 8 Hello 00110010 00110010 +\N 9 Hello \N \N +10 \N Hello \N \N +11 11 \N \N \N +12 12 Hello 001010110110 0010101101100000 +\N 13 Hello \N \N +14 \N Hello \N \N +\N \N \N \N \N diff --git a/tests/queries/0_stateless/02154_bit_slice_for_string.sql b/tests/queries/0_stateless/02154_bit_slice_for_string.sql index 3a73744f44b..f192301a7c3 100644 --- a/tests/queries/0_stateless/02154_bit_slice_for_string.sql +++ b/tests/queries/0_stateless/02154_bit_slice_for_string.sql @@ -1,111 +1,144 @@ SELECT 'Const Offset'; -select 1, subString(bin('Hello'), 1), bin(bitSlice('Hello', 1)); -select 2, subString(bin('Hello'), 2), bin(bitSlice('Hello', 2)); -select 3, subString(bin('Hello'), 3), bin(bitSlice('Hello', 3)); -select 4, subString(bin('Hello'), 4), bin(bitSlice('Hello', 4)); -select 5, subString(bin('Hello'), 5), bin(bitSlice('Hello', 5)); -select 6, subString(bin('Hello'), 6), bin(bitSlice('Hello', 6)); -select 7, subString(bin('Hello'), 7), bin(bitSlice('Hello', 7)); -select 8, subString(bin('Hello'), 8), bin(bitSlice('Hello', 8)); -select 9, subString(bin('Hello'), 9), bin(bitSlice('Hello', 9)); -select 10, subString(bin('Hello'), 10), bin(bitSlice('Hello', 10)); -select 11, subString(bin('Hello'), 11), bin(bitSlice('Hello', 11)); -select 12, subString(bin('Hello'), 12), bin(bitSlice('Hello', 12)); -select 13, subString(bin('Hello'), 13), bin(bitSlice('Hello', 13)); -select 14, subString(bin('Hello'), 14), bin(bitSlice('Hello', 14)); -select 15, subString(bin('Hello'), 15), bin(bitSlice('Hello', 15)); -select 16, subString(bin('Hello'), 16), bin(bitSlice('Hello', 16)); +select 1 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 2 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 3 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 4 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 5 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 6 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 7 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 8 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 9 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 10 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 11 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 12 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 13 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 14 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 15 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select 16 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); -select -1, subString(bin('Hello'), -1), bin(bitSlice('Hello', -1)); -select -2, subString(bin('Hello'), -2), bin(bitSlice('Hello', -2)); -select -3, subString(bin('Hello'), -3), bin(bitSlice('Hello', -3)); -select -4, subString(bin('Hello'), -4), bin(bitSlice('Hello', -4)); -select -5, subString(bin('Hello'), -5), bin(bitSlice('Hello', -5)); -select -6, subString(bin('Hello'), -6), bin(bitSlice('Hello', -6)); -select -7, subString(bin('Hello'), -7), bin(bitSlice('Hello', -7)); -select -8, subString(bin('Hello'), -8), bin(bitSlice('Hello', -8)); -select -9, subString(bin('Hello'), -9), bin(bitSlice('Hello', -9)); -select -10, subString(bin('Hello'), -10), bin(bitSlice('Hello', -10)); -select -11, subString(bin('Hello'), -11), bin(bitSlice('Hello', -11)); -select -12, subString(bin('Hello'), -12), bin(bitSlice('Hello', -12)); -select -13, subString(bin('Hello'), -13), bin(bitSlice('Hello', -13)); -select -14, subString(bin('Hello'), -14), bin(bitSlice('Hello', -14)); -select -15, subString(bin('Hello'), -15), bin(bitSlice('Hello', -15)); -select -16, subString(bin('Hello'), -16), bin(bitSlice('Hello', -16)); +select -1 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -2 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -3 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -4 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -5 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -6 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -7 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -8 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -9 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -10 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -11 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -12 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -13 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -14 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -15 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -16 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +SELECT 'Const Truncate Offset'; +select 41 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select -41 as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); + +SELECT 'Const Nullable Offset'; +select 1 as offset, null as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select null as offset, 'Hello' as s, subString(bin(s), offset), bin(bitSlice(s, offset)); +select null as offset, null as s, subString(bin(s), offset), bin(bitSlice(s, offset)); + SELECT 'Const Offset, Const Length'; -select 1, 1, subString(bin('Hello'), 1, 1), bin(bitSlice('Hello', 1, 1)); -select 2, 2, subString(bin('Hello'), 2, 2), bin(bitSlice('Hello', 2, 2)); -select 3, 3, subString(bin('Hello'), 3, 3), bin(bitSlice('Hello', 3, 3)); -select 4, 4, subString(bin('Hello'), 4, 4), bin(bitSlice('Hello', 4, 4)); -select 5, 5, subString(bin('Hello'), 5, 5), bin(bitSlice('Hello', 5, 5)); -select 6, 6, subString(bin('Hello'), 6, 6), bin(bitSlice('Hello', 6, 6)); -select 7, 7, subString(bin('Hello'), 7, 7), bin(bitSlice('Hello', 7, 7)); -select 8, 8, subString(bin('Hello'), 8, 8), bin(bitSlice('Hello', 8, 8)); -select 9, 9, subString(bin('Hello'), 9, 9), bin(bitSlice('Hello', 9, 9)); -select 10, 10, subString(bin('Hello'), 10, 10), bin(bitSlice('Hello', 10, 10)); -select 11, 11, subString(bin('Hello'), 11, 11), bin(bitSlice('Hello', 11, 11)); -select 12, 12, subString(bin('Hello'), 12, 12), bin(bitSlice('Hello', 12, 12)); -select 13, 13, subString(bin('Hello'), 13, 13), bin(bitSlice('Hello', 13, 13)); -select 14, 14, subString(bin('Hello'), 14, 14), bin(bitSlice('Hello', 14, 14)); -select 15, 15, subString(bin('Hello'), 15, 15), bin(bitSlice('Hello', 15, 15)); -select 16, 16, subString(bin('Hello'), 16, 16), bin(bitSlice('Hello', 16, 16)); +select 1 as offset, 1 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 2 as offset, 2 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 3 as offset, 3 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 4 as offset, 4 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 5 as offset, 5 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 6 as offset, 6 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 7 as offset, 7 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 8 as offset, 8 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 9 as offset, 9 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 10 as offset, 10 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 11 as offset, 11 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 12 as offset, 12 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 13 as offset, 13 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 14 as offset, 14 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 15 as offset, 15 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 16 as offset, 16 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); -select 1, -1, subString(bin('Hello'), 1, -1), bin(bitSlice('Hello', 1, -1)); -select 2, -2, subString(bin('Hello'), 2, -2), bin(bitSlice('Hello', 2, -2)); -select 3, -3, subString(bin('Hello'), 3, -3), bin(bitSlice('Hello', 3, -3)); -select 4, -4, subString(bin('Hello'), 4, -4), bin(bitSlice('Hello', 4, -4)); -select 5, -5, subString(bin('Hello'), 5, -5), bin(bitSlice('Hello', 5, -5)); -select 6, -6, subString(bin('Hello'), 6, -6), bin(bitSlice('Hello', 6, -6)); -select 7, -7, subString(bin('Hello'), 7, -7), bin(bitSlice('Hello', 7, -7)); -select 8, -8, subString(bin('Hello'), 8, -8), bin(bitSlice('Hello', 8, -8)); -select 9, -9, subString(bin('Hello'), 9, -9), bin(bitSlice('Hello', 9, -9)); -select 10, -10, subString(bin('Hello'), 10, -10), bin(bitSlice('Hello', 10, -10)); -select 11, -11, subString(bin('Hello'), 11, -11), bin(bitSlice('Hello', 11, -11)); -select 12, -12, subString(bin('Hello'), 12, -12), bin(bitSlice('Hello', 12, -12)); -select 13, -13, subString(bin('Hello'), 13, -13), bin(bitSlice('Hello', 13, -13)); -select 14, -14, subString(bin('Hello'), 14, -14), bin(bitSlice('Hello', 14, -14)); -select 15, -15, subString(bin('Hello'), 15, -15), bin(bitSlice('Hello', 15, -15)); -select 16, -16, subString(bin('Hello'), 16, -16), bin(bitSlice('Hello', 16, -16)); +select 1 as offset, -1 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 2 as offset, -2 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 3 as offset, -3 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 4 as offset, -4 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 5 as offset, -5 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 6 as offset, -6 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 7 as offset, -7 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 8 as offset, -8 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 9 as offset, -9 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 10 as offset, -10 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 11 as offset, -11 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 12 as offset, -12 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 13 as offset, -13 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 14 as offset, -14 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 15 as offset, -15 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 16 as offset, -16 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); -select -1, 1, subString(bin('Hello'), -1, 1), bin(bitSlice('Hello', -1, 1)); -select -2, 2, subString(bin('Hello'), -2, 2), bin(bitSlice('Hello', -2, 2)); -select -3, 3, subString(bin('Hello'), -3, 3), bin(bitSlice('Hello', -3, 3)); -select -4, 4, subString(bin('Hello'), -4, 4), bin(bitSlice('Hello', -4, 4)); -select -5, 5, subString(bin('Hello'), -5, 5), bin(bitSlice('Hello', -5, 5)); -select -6, 6, subString(bin('Hello'), -6, 6), bin(bitSlice('Hello', -6, 6)); -select -7, 7, subString(bin('Hello'), -7, 7), bin(bitSlice('Hello', -7, 7)); -select -8, 8, subString(bin('Hello'), -8, 8), bin(bitSlice('Hello', -8, 8)); -select -9, 9, subString(bin('Hello'), -9, 9), bin(bitSlice('Hello', -9, 9)); -select -10, 10, subString(bin('Hello'), -10, 10), bin(bitSlice('Hello', -10, 10)); -select -11, 11, subString(bin('Hello'), -11, 11), bin(bitSlice('Hello', -11, 11)); -select -12, 12, subString(bin('Hello'), -12, 12), bin(bitSlice('Hello', -12, 12)); -select -13, 13, subString(bin('Hello'), -13, 13), bin(bitSlice('Hello', -13, 13)); -select -14, 14, subString(bin('Hello'), -14, 14), bin(bitSlice('Hello', -14, 14)); -select -15, 15, subString(bin('Hello'), -15, 15), bin(bitSlice('Hello', -15, 15)); -select -16, 16, subString(bin('Hello'), -16, 16), bin(bitSlice('Hello', -16, 16)); +select -1 as offset, 1 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -2 as offset, 2 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -3 as offset, 3 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -4 as offset, 4 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -5 as offset, 5 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -6 as offset, 6 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -7 as offset, 7 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -8 as offset, 8 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -9 as offset, 9 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -10 as offset, 10 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -11 as offset, 11 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -12 as offset, 12 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -13 as offset, 13 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -14 as offset, 14 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -15 as offset, 15 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -16 as offset, 16 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); -select -1, -16, subString(bin('Hello'), -1, -16), bin(bitSlice('Hello', -1, -16)); -select -2, -15, subString(bin('Hello'), -2, -15), bin(bitSlice('Hello', -2, -15)); -select -3, -14, subString(bin('Hello'), -3, -14), bin(bitSlice('Hello', -3, -14)); -select -4, -13, subString(bin('Hello'), -4, -13), bin(bitSlice('Hello', -4, -13)); -select -5, -12, subString(bin('Hello'), -5, -12), bin(bitSlice('Hello', -5, -12)); -select -6, -11, subString(bin('Hello'), -6, -11), bin(bitSlice('Hello', -6, -11)); -select -7, -10, subString(bin('Hello'), -7, -10), bin(bitSlice('Hello', -7, -10)); -select -8, -9, subString(bin('Hello'), -8, -9), bin(bitSlice('Hello', -8, -9)); -select -9, -8, subString(bin('Hello'), -9, -8), bin(bitSlice('Hello', -9, -8)); -select -10, -7, subString(bin('Hello'), -10, -7), bin(bitSlice('Hello', -10, -7)); -select -11, -6, subString(bin('Hello'), -11, -6), bin(bitSlice('Hello', -11, -6)); -select -12, -5, subString(bin('Hello'), -12, -5), bin(bitSlice('Hello', -12, -5)); -select -13, -4, subString(bin('Hello'), -13, -4), bin(bitSlice('Hello', -13, -4)); -select -14, -3, subString(bin('Hello'), -14, -3), bin(bitSlice('Hello', -14, -3)); -select -15, -2, subString(bin('Hello'), -15, -2), bin(bitSlice('Hello', -15, -2)); -select -16, -1, subString(bin('Hello'), -16, -1), bin(bitSlice('Hello', -16, -1)); +select -1 as offset, -16 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -2 as offset, -15 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -3 as offset, -14 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -4 as offset, -13 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -5 as offset, -12 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -6 as offset, -11 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -7 as offset, -10 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -8 as offset, -9 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -9 as offset, -8 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -10 as offset, -7 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -11 as offset, -6 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -12 as offset, -5 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -13 as offset, -4 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -14 as offset, -3 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -15 as offset, -2 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -16 as offset, -1 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); + +select 'Const Truncate Offset, Const Truncate Length'; +select 36 as offset, 8 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 41 as offset, 1 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -44 as offset, -36 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -41 as offset, -40 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select -41 as offset, 41 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); + +select 'Const Nullable Offset, Const Nullable Length'; +select 1 as offset, 1 as length, null as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)); +select null as offset, 1 as length, 'Hello' as s, subString(bin(s), offset, length), bin(bitSlice(s, offset, length)); +select 1 as offset, null as length, 'Hello' as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)); +select null as offset, null as length, null as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)); select 'Dynamic Offset, Dynamic Length'; +select number as offset, number as length, 'Hello' as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)) from numbers(16); +select number as offset, -number as length, 'Hello' as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)) from numbers(16); +select -number as offset, -16+number as length, 'Hello' as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)) from numbers(16); +select -number as offset, number as length, 'Hello' as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)) from numbers(16); -select number, number, subString(bin('Hello'), number, number), bin(bitSlice('Hello', number, number)) from numbers(16); -select number, -number, subString(bin('Hello'), number, -number), bin(bitSlice('Hello', number, -number)) from numbers(16); -select -number, -16+number, subString(bin('Hello'), -number, -16+number), bin(bitSlice('Hello', -number, -16+number)) from numbers(16); -select -number, number, subString(bin('Hello'), -number, number), bin(bitSlice('Hello', -number, number)) from numbers(16); +select 'Dynamic Truncate Offset, Dynamic Truncate Length'; +select number-8 as offset, 8 as length, 'Hello' as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)) from numbers(9); +select -4 as offset, number as length, 'Hello' as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)) from numbers(9); +select -36-number as offset, 8 as length, 'Hello' as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)) from numbers(9); +select -44 as offset, number as length, 'Hello' as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)) from numbers(9); +select -44 as offset, number + 40 as length, 'Hello' as s, subString(bin(s), offset , length), bin(bitSlice(s, offset, length)) from numbers(9); + +select 'Dynamic Nullable Offset, Dynamic Nullable Length'; +select if(number%4 ==1 or number%8==7, null, number) as offset, if(number%4==2 or number%8==7, null, number) as length,if(number%4 ==3, null, 'Hello') as s, + subString(bin(s), offset, length), bin(bitSlice(s, offset , length)) +from numbers(16); From b6e40427a0e160fa3cd461ae7dbfb4b58654a9ae Mon Sep 17 00:00:00 2001 From: RogerYK Date: Tue, 11 Jan 2022 23:35:34 +0800 Subject: [PATCH 016/403] fix style --- src/Functions/bitSlice.cpp | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/src/Functions/bitSlice.cpp b/src/Functions/bitSlice.cpp index 5e607e37187..685147f7bd5 100644 --- a/src/Functions/bitSlice.cpp +++ b/src/Functions/bitSlice.cpp @@ -81,43 +81,30 @@ public: std::optional start_const; std::optional length_const; - if (const auto *column_start_const = checkAndGetColumn(column_start.get())) { + if (const auto * column_start_const = checkAndGetColumn(column_start.get())) + { start_const = column_start_const->getInt(0); } if (number_of_arguments == 3) { column_length = arguments[2].column; - if (const auto *column_length_const = checkAndGetColumn(column_length.get())) + if (const auto * column_length_const = checkAndGetColumn(column_length.get())) length_const = column_length_const->getInt(0); } - if (const ColumnString * col = checkAndGetColumn(column_string.get())) - return executeForSource( - column_start, - column_length, start_const, length_const, - StringSource(*col), - input_rows_count); + return executeForSource(column_start, column_length, start_const, length_const, StringSource(*col), input_rows_count); else if (const ColumnFixedString * col_fixed = checkAndGetColumn(column_string.get())) return executeForSource( - column_start, - column_length, start_const, length_const, - FixedStringSource(*col_fixed), - input_rows_count); + column_start, column_length, start_const, length_const, FixedStringSource(*col_fixed), input_rows_count); else if (const ColumnConst * col_const = checkAndGetColumnConst(column_string.get())) return executeForSource( - column_start, - column_length, start_const, length_const, - ConstSource(*col_const), - input_rows_count); + column_start, column_length, start_const, length_const, ConstSource(*col_const), input_rows_count); else if (const ColumnConst * col_const_fixed = checkAndGetColumnConst(column_string.get())) return executeForSource( - column_start, - column_length, start_const, length_const, - ConstSource(*col_const_fixed), - input_rows_count); + column_start, column_length, start_const, length_const, ConstSource(*col_const_fixed), input_rows_count); else throw Exception( "Illegal column " + arguments[0].column->getName() + " of first argument of function " + getName(), @@ -372,8 +359,8 @@ public: while (!src.isEnd()) { size_t row_num = src.rowNum(); - Int64 start = offset_column.getInt(row_num); - Int64 length = length_column.getInt(row_num); + Int64 start = offset_column.getInt(row_num); + Int64 length = length_column.getInt(row_num); if (start && length) { From 1dd903f28da617ef6e8c437c9bf6c2c8d1e10d9c Mon Sep 17 00:00:00 2001 From: RogerYK Date: Wed, 12 Jan 2022 19:13:08 +0800 Subject: [PATCH 017/403] fix undefined behavior --- src/Functions/bitSlice.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Functions/bitSlice.cpp b/src/Functions/bitSlice.cpp index 685147f7bd5..7d9b48acc89 100644 --- a/src/Functions/bitSlice.cpp +++ b/src/Functions/bitSlice.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -132,7 +131,7 @@ public: source, StringSink(*col_res, input_rows_count), static_cast(start_value - 1)); else if (start_value < 0) bitSliceFromRightConstantOffsetUnbounded( - source, StringSink(*col_res, input_rows_count), static_cast(-start_value)); + source, StringSink(*col_res, input_rows_count), -static_cast(start_value)); else throw Exception("Indices in strings are 1-based", ErrorCodes::ZERO_ARRAY_OR_TUPLE_INDEX); } @@ -150,7 +149,7 @@ public: source, StringSink(*col_res, input_rows_count), static_cast(start_value - 1), length_value); else if (start_value < 0) bitSliceFromRightConstantOffsetBounded( - source, StringSink(*col_res, input_rows_count), static_cast(-start_value), length_value); + source, StringSink(*col_res, input_rows_count), -static_cast(start_value), length_value); else throw Exception("Indices in strings are 1-based", ErrorCodes::ZERO_ARRAY_OR_TUPLE_INDEX); } @@ -365,7 +364,7 @@ public: if (start && length) { bool left_offset = start > 0; - size_t offset = left_offset ? start - 1 : -start; + size_t offset = left_offset ? static_cast(start - 1) : -static_cast(start); size_t size = src.getElementSize(); size_t offset_byte; From ad86dad2d1ea52fc4499d73a1cec8cc0aa25ce0d Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 25 Dec 2021 10:24:21 +0300 Subject: [PATCH 018/403] Fix status of SKIPPED integration tests (was incorrectly marked as ERROR) Like here [1]. - test_jemalloc_percpu_arena/test.py::test_jemalloc_percpu_arena -- ERROR - test_input_format_parallel_parsing_memory_tracking/test.py::test_memory_tracking_total -- FAIL But test_jemalloc_percpu_arena is SKIPPED. [1]: https://s3.amazonaws.com/clickhouse-test-reports/33140/b4420cfa3e834ab9026914978697ded3180122a0/integration_tests__thread__actions__[3/4].html v2: tests/integration/ci-runner: include SKIPPED tests into "short test summary info" Otherwise they cannot be distinguished from ERRORS. v3: tests/integration/ci-runner: improve report parser v4: rewrite get_counters() to parse tests verbosity log instead of test short summary, since later does not provide test name for skipped tests --- tests/integration/ci-runner.py | 70 ++++++++++++++++++++-------------- 1 file changed, 42 insertions(+), 28 deletions(-) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 6058a332c29..40cb2c6fdd7 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -64,38 +64,45 @@ def chunks(lst, n): for i in range(0, len(lst), n): yield lst[i:i + n] -def parse_test_results_output(fname): - read = False - description_output = [] - with open(fname, 'r') as out: - for line in out: - if read and line.strip() and not line.startswith('=='): - description_output.append(line.strip()) - if 'short test summary info' in line: - read = True - return description_output - - -def get_counters(output): +def get_counters(fname): counters = { - "ERROR": set([]), - "PASSED": set([]), - "FAILED": set([]), + "ERROR": set([]), + "PASSED": set([]), + "FAILED": set([]), + "SKIPPED": set([]), } - for line in output: - if '.py' in line: + with open(fname, 'r') as out: + for line in out: + line = line.strip() + # Example of log: + # + # test_mysql_protocol/test.py::test_golang_client + # [gw0] [ 7%] ERROR test_mysql_protocol/test.py::test_golang_client + # + # And only the line with test status should be matched + if not('.py::' in line and ' ' in line): + continue + line_arr = line.strip().split(' ') - state = line_arr[0] - test_name = ' '.join(line_arr[1:]) - if ' - ' in test_name: - test_name = test_name[:test_name.find(' - ')] + if len(line_arr) < 2: + logging.debug("Strange line %s", line) + continue + + # Lines like: + # [gw0] [ 7%] ERROR test_mysql_protocol/test.py::test_golang_client + state = line_arr[-2] + test_name = line_arr[-1] + if state in counters: counters[state].add(test_name) else: - logging.info("Strange line %s", line) - else: - logging.info("Strange line %s", line) + # will skip lines line: + # 30.76s call test_host_ip_change/test.py::test_ip_change_drop_dns_cache + # 5.71s teardown test_host_ip_change/test.py::test_user_access_ip_change[node1] + # and similar + logging.debug("Strange state in line %s", line) + return {k: list(v) for k, v in counters.items()} @@ -459,7 +466,12 @@ class ClickhouseIntegrationTestsRunner: test_cmd = ' '.join([test for test in sorted(test_names)]) parallel_cmd = " --parallel {} ".format(num_workers) if num_workers > 0 else "" - cmd = "cd {}/tests/integration && timeout -s 9 1h ./runner {} {} -t {} {} '-rfEp --run-id={} --color=no --durations=0 {}' | tee {}".format( + # -r -- show extra test summary: + # -f -- (f)ailed + # -E -- (E)rror + # -p -- (p)assed + # -s -- (s)kipped + cmd = "cd {}/tests/integration && timeout -s 9 1h ./runner {} {} -t {} {} '-rfEps --run-id={} --color=no --durations=0 {}' | tee {}".format( repo_path, self._get_runner_opts(), image_cmd, test_cmd, parallel_cmd, i, _get_deselect_option(self.should_skip_tests()), info_path) log_basename = test_group_str + "_" + str(i) + ".log" @@ -490,8 +502,9 @@ class ClickhouseIntegrationTestsRunner: if os.path.exists(info_path): extra_logs_names.append(info_basename) - lines = parse_test_results_output(info_path) - new_counters = get_counters(lines) + new_counters = get_counters(info_path) + for state, tests in new_counters.items(): + logging.info("Tests with %s state (%s): %s", state, len(tests), tests) times_lines = parse_test_times(info_path) new_tests_times = get_test_times(times_lines) self._update_counters(counters, new_counters) @@ -521,6 +534,7 @@ class ClickhouseIntegrationTestsRunner: for test in tests_in_group: if (test not in counters["PASSED"] and test not in counters["ERROR"] and + test not in counters["SKIPPED"] and test not in counters["FAILED"] and '::' in test): counters["ERROR"].append(test) From a037d286fb8269058032d61f53c8ef8ceee6faa6 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Mon, 10 Jan 2022 19:22:37 -0800 Subject: [PATCH 019/403] add func h3EdgeLengthKm --- src/Functions/h3EdgeLengthKm.cpp | 87 ++++++++++++++++++++++++++ src/Functions/registerFunctionsGeo.cpp | 2 + 2 files changed, 89 insertions(+) create mode 100644 src/Functions/h3EdgeLengthKm.cpp diff --git a/src/Functions/h3EdgeLengthKm.cpp b/src/Functions/h3EdgeLengthKm.cpp new file mode 100644 index 00000000000..215f31abc00 --- /dev/null +++ b/src/Functions/h3EdgeLengthKm.cpp @@ -0,0 +1,87 @@ +#include "config_functions.h" + +#if USE_H3 + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int ARGUMENT_OUT_OF_BOUND; +} + +namespace +{ + +class FunctionH3EdgeLengthKm : public IFunction +{ +public: + static constexpr auto name = "h3EdgeLengthKm"; + + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + std::string getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 1; } + bool useDefaultImplementationForConstants() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + const auto * arg = arguments[0].get(); + if (!WhichDataType(arg).isUInt8()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt8", + arg->getName(), 1, getName()); + + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto * column = checkAndGetColumn(arguments[0].column.get()); + const auto & data = column->getData(); + + auto dst = ColumnVector::create(); + auto & dst_data = dst->getData(); + dst_data.resize(input_rows_count); + + for (size_t row = 0; row < input_rows_count; ++row) + { + const UInt64 resolution = data[row]; + if (resolution > MAX_H3_RES) + throw Exception( + ErrorCodes::ARGUMENT_OUT_OF_BOUND, + "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ", + resolution, getName(), MAX_H3_RES); + Float64 res = getHexagonEdgeLengthAvgKm(resolution); + dst_data[row] = res; + } + + return dst; + } +}; + +} + +void registerFunctionH3EdgeLengthKm(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + +#endif diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index d0bb47ea3d7..dfa6098b411 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -30,6 +30,7 @@ void registerFunctionH3ToGeo(FunctionFactory &); void registerFunctionH3ToGeoBoundary(FunctionFactory &); void registerFunctionH3EdgeAngle(FunctionFactory &); void registerFunctionH3EdgeLengthM(FunctionFactory &); +void registerFunctionH3EdgeLengthKm(FunctionFactory &); void registerFunctionH3GetResolution(FunctionFactory &); void registerFunctionH3IsValid(FunctionFactory &); void registerFunctionH3KRing(FunctionFactory &); @@ -92,6 +93,7 @@ void registerFunctionsGeo(FunctionFactory & factory) registerFunctionH3ToGeoBoundary(factory); registerFunctionH3EdgeAngle(factory); registerFunctionH3EdgeLengthM(factory); + registerFunctionH3EdgeLengthKm(factory); registerFunctionH3GetResolution(factory); registerFunctionH3IsValid(factory); registerFunctionH3KRing(factory); From 54016388f080b6421e3209f73396fa2e14b83809 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Mon, 10 Jan 2022 20:06:00 -0800 Subject: [PATCH 020/403] tests for h3EdgeLengthKm --- .../02165_h3_edge_length_km.reference | 16 ++++++++++++++++ .../0_stateless/02165_h3_edge_length_km.sql | 18 ++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 tests/queries/0_stateless/02165_h3_edge_length_km.reference create mode 100644 tests/queries/0_stateless/02165_h3_edge_length_km.sql diff --git a/tests/queries/0_stateless/02165_h3_edge_length_km.reference b/tests/queries/0_stateless/02165_h3_edge_length_km.reference new file mode 100644 index 00000000000..95380a2a80e --- /dev/null +++ b/tests/queries/0_stateless/02165_h3_edge_length_km.reference @@ -0,0 +1,16 @@ +1107.712591 +418.6760055 +158.2446558 +59.81085794 +22.6063794 +8.544408276 +3.229482772 +1.220629759 +0.461354684 +0.174375668 +0.065907807 +0.024910561 +0.009415526 +0.003559893 +0.001348575 +0.000509713 diff --git a/tests/queries/0_stateless/02165_h3_edge_length_km.sql b/tests/queries/0_stateless/02165_h3_edge_length_km.sql new file mode 100644 index 00000000000..e67b691ef66 --- /dev/null +++ b/tests/queries/0_stateless/02165_h3_edge_length_km.sql @@ -0,0 +1,18 @@ +-- Tags: no-fasttest + +SELECT h3EdgeLengthKm(0); +SELECT h3EdgeLengthKm(1); +SELECT h3EdgeLengthKm(2); +SELECT h3EdgeLengthKm(3); +SELECT h3EdgeLengthKm(4); +SELECT h3EdgeLengthKm(5); +SELECT h3EdgeLengthKm(6); +SELECT h3EdgeLengthKm(7); +SELECT h3EdgeLengthKm(8); +SELECT h3EdgeLengthKm(9); +SELECT h3EdgeLengthKm(10); +SELECT h3EdgeLengthKm(11); +SELECT h3EdgeLengthKm(12); +SELECT h3EdgeLengthKm(13); +SELECT h3EdgeLengthKm(14); +SELECT h3EdgeLengthKm(15); From 76e059b8eac8e8e17268f6b7361d46fe3b452053 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Mon, 10 Jan 2022 20:17:28 -0800 Subject: [PATCH 021/403] docs for h3EdgeLengthKm --- docs/en/sql-reference/functions/geo/h3.md | 34 +++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docs/en/sql-reference/functions/geo/h3.md b/docs/en/sql-reference/functions/geo/h3.md index 2efe980a4cf..917a07f7e37 100644 --- a/docs/en/sql-reference/functions/geo/h3.md +++ b/docs/en/sql-reference/functions/geo/h3.md @@ -156,6 +156,40 @@ Result: └─────────────┘ ``` +## h3EdgeLengthKm {#h3edgelengthkm} + +Calculates the average length of the [H3](#h3index) hexagon edge in kilometers. + +**Syntax** + +``` sql +h3EdgeLengthKm(resolution) +``` + +**Parameter** + +- `resolution` — Index resolution. Type: [UInt8](../../../sql-reference/data-types/int-uint.md). Range: `[0, 15]`. + +**Returned values** + +- The average length of the [H3](#h3index) hexagon edge in kilometers. Type: [Float64](../../../sql-reference/data-types/float.md). + +**Example** + +Query: + +``` sql +SELECT h3EdgeLengthKm(15) AS edgeLengthKm; +``` + +Result: + +``` text +┌─edgeLengthKm─┐ +│ 0.000509713 │ +└──────────────┘ +``` + ## geoToH3 {#geotoh3} Returns [H3](#h3index) point index `(lon, lat)` with specified resolution. From 2d6801aa95c01693628549dad4f706710365fc9e Mon Sep 17 00:00:00 2001 From: bharatnc Date: Mon, 10 Jan 2022 22:36:39 -0800 Subject: [PATCH 022/403] add h3ExactEdgeLengthM func --- src/Functions/h3ExactEdgeLengthM.cpp | 90 ++++++++++++++++++++++++++ src/Functions/registerFunctionsGeo.cpp | 2 + 2 files changed, 92 insertions(+) create mode 100644 src/Functions/h3ExactEdgeLengthM.cpp diff --git a/src/Functions/h3ExactEdgeLengthM.cpp b/src/Functions/h3ExactEdgeLengthM.cpp new file mode 100644 index 00000000000..5b7cb91e427 --- /dev/null +++ b/src/Functions/h3ExactEdgeLengthM.cpp @@ -0,0 +1,90 @@ +#include "config_functions.h" + +#if USE_H3 + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int ILLEGAL_COLUMN; +} + +namespace +{ + +class FunctionH3ExactEdgeLengthM : public IFunction +{ +public: + static constexpr auto name = "h3ExactEdgeLengthM"; + + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + std::string getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 1; } + bool useDefaultImplementationForConstants() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + const auto * arg = arguments[0].get(); + if (!WhichDataType(arg).isUInt64()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arg->getName(), 1, getName()); + + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto * column = checkAndGetColumn(arguments[0].column.get()); + if (!column) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[0].type->getName(), + 1, + getName()); + + const auto & data = column->getData(); + + auto dst = ColumnVector::create(); + auto & dst_data = dst->getData(); + dst_data.resize(input_rows_count); + + for (size_t row = 0; row < input_rows_count; ++row) + { + const UInt64 index = data[row]; + Float64 res = exactEdgeLengthM(index); + dst_data[row] = res; + } + + return dst; + } +}; + +} + +void registerFunctionH3ExactEdgeLengthM(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + +#endif diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index dfa6098b411..19bcbdc2d4d 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -31,6 +31,7 @@ void registerFunctionH3ToGeoBoundary(FunctionFactory &); void registerFunctionH3EdgeAngle(FunctionFactory &); void registerFunctionH3EdgeLengthM(FunctionFactory &); void registerFunctionH3EdgeLengthKm(FunctionFactory &); +void registerFunctionH3ExactEdgeLengthM(FunctionFactory &); void registerFunctionH3GetResolution(FunctionFactory &); void registerFunctionH3IsValid(FunctionFactory &); void registerFunctionH3KRing(FunctionFactory &); @@ -94,6 +95,7 @@ void registerFunctionsGeo(FunctionFactory & factory) registerFunctionH3EdgeAngle(factory); registerFunctionH3EdgeLengthM(factory); registerFunctionH3EdgeLengthKm(factory); + registerFunctionH3ExactEdgeLengthM(factory); registerFunctionH3GetResolution(factory); registerFunctionH3IsValid(factory); registerFunctionH3KRing(factory); From 43eafc93166d5edc6d9b2570074b1ca5d928a3d4 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Wed, 12 Jan 2022 19:31:54 -0800 Subject: [PATCH 023/403] tests for h3ExactEdgeLengthM --- .../02165_h3_exact_edge_length_m.reference | 16 ++++++++++ .../02165_h3_exact_edge_length_m.sql | 29 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 tests/queries/0_stateless/02165_h3_exact_edge_length_m.reference create mode 100644 tests/queries/0_stateless/02165_h3_exact_edge_length_m.sql diff --git a/tests/queries/0_stateless/02165_h3_exact_edge_length_m.reference b/tests/queries/0_stateless/02165_h3_exact_edge_length_m.reference new file mode 100644 index 00000000000..52dcaaf8548 --- /dev/null +++ b/tests/queries/0_stateless/02165_h3_exact_edge_length_m.reference @@ -0,0 +1,16 @@ +489555.59989912313 +192390.78306095628 +66919.13220366438 +1263609.6633631135 +480744.0319163875 +195449.63163407316 +1263609.663363112 +461806.9719440694 +190087.69842412468 +1263609.6633631124 +465419.72260404145 +64819.70466298482 +1263609.6633631117 +69636.41611246637 +195627.4718146093 +67660.85681290775 diff --git a/tests/queries/0_stateless/02165_h3_exact_edge_length_m.sql b/tests/queries/0_stateless/02165_h3_exact_edge_length_m.sql new file mode 100644 index 00000000000..093ab1dd2d2 --- /dev/null +++ b/tests/queries/0_stateless/02165_h3_exact_edge_length_m.sql @@ -0,0 +1,29 @@ +-- Tags: no-fasttest + +DROP TABLE IF EXISTS h3_indexes; + +CREATE TABLE h3_indexes (h3_index UInt64) ENGINE = Memory; + +-- Test h3 indices selected from original test fixture: https://github.com/uber/h3/blob/master/src/apps/testapps/testH3CellAreaExhaustive.c + +INSERT INTO h3_indexes VALUES (1298057039473278975); +INSERT INTO h3_indexes VALUES (1370114633511206911); +INSERT INTO h3_indexes VALUES (1442172227549134847); +INSERT INTO h3_indexes VALUES (1514229821587062783); +INSERT INTO h3_indexes VALUES (1232301846085763071); +INSERT INTO h3_indexes VALUES (1304359440123691007); +INSERT INTO h3_indexes VALUES (1376417034161618943); +INSERT INTO h3_indexes VALUES (1448474628199546879); +INSERT INTO h3_indexes VALUES (1598506838100279295); +INSERT INTO h3_indexes VALUES (1238219417666453503); +INSERT INTO h3_indexes VALUES (1310277011704381439); +INSERT INTO h3_indexes VALUES (1382334605742309375); +INSERT INTO h3_indexes VALUES (1458182628678041599); +INSERT INTO h3_indexes VALUES (1530240222715969535); +INSERT INTO h3_indexes VALUES (1602297816753897471); +INSERT INTO h3_indexes VALUES (1242009915283734527); + +SELECT h3ExactEdgeLengthM(h3_index) FROM h3_indexes ORDER BY h3_index; + +DROP TABLE h3_indexes; + From 01630aafeec1a058ad0e801c59600e69b08f0b38 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Wed, 12 Jan 2022 19:32:19 -0800 Subject: [PATCH 024/403] docs for h3ExactEdgeLengthM --- docs/en/sql-reference/functions/geo/h3.md | 36 +++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/en/sql-reference/functions/geo/h3.md b/docs/en/sql-reference/functions/geo/h3.md index 917a07f7e37..ff36ce917f5 100644 --- a/docs/en/sql-reference/functions/geo/h3.md +++ b/docs/en/sql-reference/functions/geo/h3.md @@ -914,4 +914,40 @@ Result: └─────────────────────┘ ``` +## h3ExactEdgeLengthM {#h3exactedgelengthM} + +Returns the exact edge length of the unidirectional edge represented by the input h3 index in meters. + +**Syntax** + +``` sql +h3ExactEdgeLengthM(index) +``` + +**Parameter** + +- `index` — Hexagon index number. Type: [UInt64](../../../sql-reference/data-types/int-uint.md). + +**Returned value** + +- Edge length in meters. + +Type: [Float64](../../../sql-reference/data-types/float.md). + +**Example** + +Query: + +``` sql +SELECT h3ExactEdgeLengthM(1310277011704381439) AS exactEdgeLengthM;; +``` + +Result: + +``` text +┌───exactEdgeLengthM─┐ +│ 195449.63163407316 │ +└────────────────────┘ +``` + [Original article](https://clickhouse.com/docs/en/sql-reference/functions/geo/h3) From e4d8b607453a598610bd3849179bff6e27f731e4 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Thu, 13 Jan 2022 22:24:57 -0800 Subject: [PATCH 025/403] add column check for h3EdgeLengthKm --- src/Functions/h3EdgeLengthKm.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Functions/h3EdgeLengthKm.cpp b/src/Functions/h3EdgeLengthKm.cpp index 215f31abc00..b4324ad817e 100644 --- a/src/Functions/h3EdgeLengthKm.cpp +++ b/src/Functions/h3EdgeLengthKm.cpp @@ -20,6 +20,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ARGUMENT_OUT_OF_BOUND; +extern const int ILLEGAL_COLUMN; } namespace @@ -53,6 +54,14 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { const auto * column = checkAndGetColumn(arguments[0].column.get()); + if (!column) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt8", + arguments[0].type->getName(), + 1, + getName()); + const auto & data = column->getData(); auto dst = ColumnVector::create(); From 905d4abdd6b78b9f7a26f5b4bf58a7536de290c5 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Thu, 13 Jan 2022 22:51:08 -0800 Subject: [PATCH 026/403] add h3ExactEdgeLengthKm func --- src/Functions/h3ExactEdgeLengthKm.cpp | 90 ++++++++++++++++++++++++++ src/Functions/registerFunctionsGeo.cpp | 2 + 2 files changed, 92 insertions(+) create mode 100644 src/Functions/h3ExactEdgeLengthKm.cpp diff --git a/src/Functions/h3ExactEdgeLengthKm.cpp b/src/Functions/h3ExactEdgeLengthKm.cpp new file mode 100644 index 00000000000..7aa9e573bed --- /dev/null +++ b/src/Functions/h3ExactEdgeLengthKm.cpp @@ -0,0 +1,90 @@ +#include "config_functions.h" + +#if USE_H3 + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int ILLEGAL_COLUMN; +} + +namespace +{ + +class FunctionH3ExactEdgeLengthKm : public IFunction +{ +public: + static constexpr auto name = "h3ExactEdgeLengthKm"; + + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + std::string getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 1; } + bool useDefaultImplementationForConstants() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + const auto * arg = arguments[0].get(); + if (!WhichDataType(arg).isUInt64()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arg->getName(), 1, getName()); + + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto * column = checkAndGetColumn(arguments[0].column.get()); + if (!column) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[0].type->getName(), + 1, + getName()); + + const auto & data = column->getData(); + + auto dst = ColumnVector::create(); + auto & dst_data = dst->getData(); + dst_data.resize(input_rows_count); + + for (size_t row = 0; row < input_rows_count; ++row) + { + const UInt64 index = data[row]; + Float64 res = exactEdgeLengthKm(index); + dst_data[row] = res; + } + + return dst; + } +}; + +} + +void registerFunctionH3ExactEdgeLengthKm(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + +#endif diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index 19bcbdc2d4d..6eabe5adc49 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -32,6 +32,7 @@ void registerFunctionH3EdgeAngle(FunctionFactory &); void registerFunctionH3EdgeLengthM(FunctionFactory &); void registerFunctionH3EdgeLengthKm(FunctionFactory &); void registerFunctionH3ExactEdgeLengthM(FunctionFactory &); +void registerFunctionH3ExactEdgeLengthKm(FunctionFactory &); void registerFunctionH3GetResolution(FunctionFactory &); void registerFunctionH3IsValid(FunctionFactory &); void registerFunctionH3KRing(FunctionFactory &); @@ -96,6 +97,7 @@ void registerFunctionsGeo(FunctionFactory & factory) registerFunctionH3EdgeLengthM(factory); registerFunctionH3EdgeLengthKm(factory); registerFunctionH3ExactEdgeLengthM(factory); + registerFunctionH3ExactEdgeLengthKm(factory); registerFunctionH3GetResolution(factory); registerFunctionH3IsValid(factory); registerFunctionH3KRing(factory); From b8b9b16adba4f64feef11029f875f96c65681f2a Mon Sep 17 00:00:00 2001 From: bharatnc Date: Thu, 13 Jan 2022 23:40:46 -0800 Subject: [PATCH 027/403] tests for h3ExactEdgeLengthKm --- .../02165_h3_exact_edge_length_Km.reference | 16 ++++++++++ .../02165_h3_exact_edge_length_Km.sql | 29 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 tests/queries/0_stateless/02165_h3_exact_edge_length_Km.reference create mode 100644 tests/queries/0_stateless/02165_h3_exact_edge_length_Km.sql diff --git a/tests/queries/0_stateless/02165_h3_exact_edge_length_Km.reference b/tests/queries/0_stateless/02165_h3_exact_edge_length_Km.reference new file mode 100644 index 00000000000..1e44981f9eb --- /dev/null +++ b/tests/queries/0_stateless/02165_h3_exact_edge_length_Km.reference @@ -0,0 +1,16 @@ +489.55559989912314 +192.39078306095627 +66.91913220366439 +1263.6096633631134 +480.7440319163875 +195.44963163407317 +1263.6096633631118 +461.80697194406935 +190.08769842412468 +1263.6096633631123 +465.41972260404145 +64.81970466298482 +1263.6096633631116 +69.63641611246636 +195.6274718146093 +67.66085681290775 diff --git a/tests/queries/0_stateless/02165_h3_exact_edge_length_Km.sql b/tests/queries/0_stateless/02165_h3_exact_edge_length_Km.sql new file mode 100644 index 00000000000..26607227484 --- /dev/null +++ b/tests/queries/0_stateless/02165_h3_exact_edge_length_Km.sql @@ -0,0 +1,29 @@ +-- Tags: no-fasttest + +DROP TABLE IF EXISTS h3_indexes; + +CREATE TABLE h3_indexes (h3_index UInt64) ENGINE = Memory; + +-- Test h3 indices selected from original test fixture: https://github.com/uber/h3/blob/master/src/apps/testapps/testH3CellAreaExhaustive.c + +INSERT INTO h3_indexes VALUES (1298057039473278975); +INSERT INTO h3_indexes VALUES (1370114633511206911); +INSERT INTO h3_indexes VALUES (1442172227549134847); +INSERT INTO h3_indexes VALUES (1514229821587062783); +INSERT INTO h3_indexes VALUES (1232301846085763071); +INSERT INTO h3_indexes VALUES (1304359440123691007); +INSERT INTO h3_indexes VALUES (1376417034161618943); +INSERT INTO h3_indexes VALUES (1448474628199546879); +INSERT INTO h3_indexes VALUES (1598506838100279295); +INSERT INTO h3_indexes VALUES (1238219417666453503); +INSERT INTO h3_indexes VALUES (1310277011704381439); +INSERT INTO h3_indexes VALUES (1382334605742309375); +INSERT INTO h3_indexes VALUES (1458182628678041599); +INSERT INTO h3_indexes VALUES (1530240222715969535); +INSERT INTO h3_indexes VALUES (1602297816753897471); +INSERT INTO h3_indexes VALUES (1242009915283734527); + +SELECT h3ExactEdgeLengthKm(h3_index) FROM h3_indexes ORDER BY h3_index; + +DROP TABLE h3_indexes; + From 4a3aef2ecf0f3ea41f37fd83032a15a5ef67d58f Mon Sep 17 00:00:00 2001 From: bharatnc Date: Thu, 13 Jan 2022 23:41:32 -0800 Subject: [PATCH 028/403] docs h3ExactEdgeLengthKm --- docs/en/sql-reference/functions/geo/h3.md | 38 ++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/geo/h3.md b/docs/en/sql-reference/functions/geo/h3.md index ff36ce917f5..896b0226259 100644 --- a/docs/en/sql-reference/functions/geo/h3.md +++ b/docs/en/sql-reference/functions/geo/h3.md @@ -914,7 +914,7 @@ Result: └─────────────────────┘ ``` -## h3ExactEdgeLengthM {#h3exactedgelengthM} +## h3ExactEdgeLengthM {#h3exactedgelengthm} Returns the exact edge length of the unidirectional edge represented by the input h3 index in meters. @@ -950,4 +950,40 @@ Result: └────────────────────┘ ``` +## h3ExactEdgeLengthKm {#h3exactedgelengthkm} + +Returns the exact edge length of the unidirectional edge represented by the input h3 index in kilometers. + +**Syntax** + +``` sql +h3ExactEdgeLengthKm(index) +``` + +**Parameter** + +- `index` — Hexagon index number. Type: [UInt64](../../../sql-reference/data-types/int-uint.md). + +**Returned value** + +- Edge length in kilometers. + +Type: [Float64](../../../sql-reference/data-types/float.md). + +**Example** + +Query: + +``` sql +SELECT h3ExactEdgeLengthKm(1310277011704381439) AS exactEdgeLengthKm;; +``` + +Result: + +``` text +┌──exactEdgeLengthKm─┐ +│ 195.44963163407317 │ +└────────────────────┘ +``` + [Original article](https://clickhouse.com/docs/en/sql-reference/functions/geo/h3) From c9aca0d7c6973447b2c174307bc492c576575eb5 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Thu, 13 Jan 2022 23:42:56 -0800 Subject: [PATCH 029/403] add h3ExactEdgeLengthRads func --- src/Functions/h3ExactEdgeLengthRads.cpp | 90 +++++++++++++++++++++++++ src/Functions/registerFunctionsGeo.cpp | 2 + 2 files changed, 92 insertions(+) create mode 100644 src/Functions/h3ExactEdgeLengthRads.cpp diff --git a/src/Functions/h3ExactEdgeLengthRads.cpp b/src/Functions/h3ExactEdgeLengthRads.cpp new file mode 100644 index 00000000000..d2b9345c989 --- /dev/null +++ b/src/Functions/h3ExactEdgeLengthRads.cpp @@ -0,0 +1,90 @@ +#include "config_functions.h" + +#if USE_H3 + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int ILLEGAL_COLUMN; +} + +namespace +{ + +class FunctionH3ExactEdgeLengthRads : public IFunction +{ +public: + static constexpr auto name = "h3ExactEdgeLengthRads"; + + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + std::string getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 1; } + bool useDefaultImplementationForConstants() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + const auto * arg = arguments[0].get(); + if (!WhichDataType(arg).isUInt64()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arg->getName(), 1, getName()); + + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto * column = checkAndGetColumn(arguments[0].column.get()); + if (!column) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[0].type->getName(), + 1, + getName()); + + const auto & data = column->getData(); + + auto dst = ColumnVector::create(); + auto & dst_data = dst->getData(); + dst_data.resize(input_rows_count); + + for (size_t row = 0; row < input_rows_count; ++row) + { + const UInt64 index = data[row]; + Float64 res = exactEdgeLengthRads(index); + dst_data[row] = res; + } + + return dst; + } +}; + +} + +void registerFunctionH3ExactEdgeLengthRads(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + +#endif diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index 6eabe5adc49..ee407926ffa 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -33,6 +33,7 @@ void registerFunctionH3EdgeLengthM(FunctionFactory &); void registerFunctionH3EdgeLengthKm(FunctionFactory &); void registerFunctionH3ExactEdgeLengthM(FunctionFactory &); void registerFunctionH3ExactEdgeLengthKm(FunctionFactory &); +void registerFunctionH3ExactEdgeLengthRads(FunctionFactory &); void registerFunctionH3GetResolution(FunctionFactory &); void registerFunctionH3IsValid(FunctionFactory &); void registerFunctionH3KRing(FunctionFactory &); @@ -98,6 +99,7 @@ void registerFunctionsGeo(FunctionFactory & factory) registerFunctionH3EdgeLengthKm(factory); registerFunctionH3ExactEdgeLengthM(factory); registerFunctionH3ExactEdgeLengthKm(factory); + registerFunctionH3ExactEdgeLengthRads(factory); registerFunctionH3GetResolution(factory); registerFunctionH3IsValid(factory); registerFunctionH3KRing(factory); From 8830a9f52ae98dcb62783ab5394b9d376dda3f57 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Thu, 13 Jan 2022 23:49:56 -0800 Subject: [PATCH 030/403] tests for h3ExactEdgeLengthRads --- .../02165_h3_exact_edge_length_rads.reference | 16 ++++++++++ .../02165_h3_exact_edge_length_rads.sql | 29 +++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 tests/queries/0_stateless/02165_h3_exact_edge_length_rads.reference create mode 100644 tests/queries/0_stateless/02165_h3_exact_edge_length_rads.sql diff --git a/tests/queries/0_stateless/02165_h3_exact_edge_length_rads.reference b/tests/queries/0_stateless/02165_h3_exact_edge_length_rads.reference new file mode 100644 index 00000000000..cec63f72b07 --- /dev/null +++ b/tests/queries/0_stateless/02165_h3_exact_edge_length_rads.reference @@ -0,0 +1,16 @@ +0.07684116278590451 +0.03019786002394998 +0.010503697500779932 +0.19833750417794152 +0.07545808979092708 +0.030677980118976447 +0.19833750417794127 +0.0724857089044268 +0.029836365432681984 +0.19833750417794133 +0.07305277005463119 +0.010174169141909536 +0.19833750417794122 +0.010930205246202099 +0.030705894101096694 +0.010620119376973209 diff --git a/tests/queries/0_stateless/02165_h3_exact_edge_length_rads.sql b/tests/queries/0_stateless/02165_h3_exact_edge_length_rads.sql new file mode 100644 index 00000000000..d618e69f032 --- /dev/null +++ b/tests/queries/0_stateless/02165_h3_exact_edge_length_rads.sql @@ -0,0 +1,29 @@ +-- Tags: no-fasttest + +DROP TABLE IF EXISTS h3_indexes; + +CREATE TABLE h3_indexes (h3_index UInt64) ENGINE = Memory; + +-- Test h3 indices selected from original test fixture: https://github.com/uber/h3/blob/master/src/apps/testapps/testH3CellAreaExhaustive.c + +INSERT INTO h3_indexes VALUES (1298057039473278975); +INSERT INTO h3_indexes VALUES (1370114633511206911); +INSERT INTO h3_indexes VALUES (1442172227549134847); +INSERT INTO h3_indexes VALUES (1514229821587062783); +INSERT INTO h3_indexes VALUES (1232301846085763071); +INSERT INTO h3_indexes VALUES (1304359440123691007); +INSERT INTO h3_indexes VALUES (1376417034161618943); +INSERT INTO h3_indexes VALUES (1448474628199546879); +INSERT INTO h3_indexes VALUES (1598506838100279295); +INSERT INTO h3_indexes VALUES (1238219417666453503); +INSERT INTO h3_indexes VALUES (1310277011704381439); +INSERT INTO h3_indexes VALUES (1382334605742309375); +INSERT INTO h3_indexes VALUES (1458182628678041599); +INSERT INTO h3_indexes VALUES (1530240222715969535); +INSERT INTO h3_indexes VALUES (1602297816753897471); +INSERT INTO h3_indexes VALUES (1242009915283734527); + +SELECT h3ExactEdgeLengthRads(h3_index) FROM h3_indexes ORDER BY h3_index; + +DROP TABLE h3_indexes; + From 768a6c47b1d5e2d71ccf3ba12f39994a29b6c42d Mon Sep 17 00:00:00 2001 From: bharatnc Date: Thu, 13 Jan 2022 23:50:16 -0800 Subject: [PATCH 031/403] docs h3ExactEdgeLengthRads --- docs/en/sql-reference/functions/geo/h3.md | 36 +++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/docs/en/sql-reference/functions/geo/h3.md b/docs/en/sql-reference/functions/geo/h3.md index 896b0226259..e8f91a0cbbb 100644 --- a/docs/en/sql-reference/functions/geo/h3.md +++ b/docs/en/sql-reference/functions/geo/h3.md @@ -986,4 +986,40 @@ Result: └────────────────────┘ ``` +## h3ExactEdgeLengthRads {#h3exactedgelengthrads} + +Returns the exact edge length of the unidirectional edge represented by the input h3 index in radians. + +**Syntax** + +``` sql +h3ExactEdgeLengthRads(index) +``` + +**Parameter** + +- `index` — Hexagon index number. Type: [UInt64](../../../sql-reference/data-types/int-uint.md). + +**Returned value** + +- Edge length in radians. + +Type: [Float64](../../../sql-reference/data-types/float.md). + +**Example** + +Query: + +``` sql +SELECT h3ExactEdgeLengthRads(1310277011704381439) AS exactEdgeLengthRads;; +``` + +Result: + +``` text +┌──exactEdgeLengthRads─┐ +│ 0.030677980118976447 │ +└──────────────────────┘ +``` + [Original article](https://clickhouse.com/docs/en/sql-reference/functions/geo/h3) From a175425651f55b43705cdbfd67a326027f73e9de Mon Sep 17 00:00:00 2001 From: bharatnc Date: Thu, 13 Jan 2022 23:51:09 -0800 Subject: [PATCH 032/403] minor fixes to docs --- docs/en/sql-reference/functions/geo/h3.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/functions/geo/h3.md b/docs/en/sql-reference/functions/geo/h3.md index e8f91a0cbbb..afcf1f6bccc 100644 --- a/docs/en/sql-reference/functions/geo/h3.md +++ b/docs/en/sql-reference/functions/geo/h3.md @@ -930,7 +930,7 @@ h3ExactEdgeLengthM(index) **Returned value** -- Edge length in meters. +- Exact edge length in meters. Type: [Float64](../../../sql-reference/data-types/float.md). @@ -966,7 +966,7 @@ h3ExactEdgeLengthKm(index) **Returned value** -- Edge length in kilometers. +- Exact edge length in kilometers. Type: [Float64](../../../sql-reference/data-types/float.md). @@ -1002,7 +1002,7 @@ h3ExactEdgeLengthRads(index) **Returned value** -- Edge length in radians. +- Exact edge length in radians. Type: [Float64](../../../sql-reference/data-types/float.md). From a6806043352f1662c30d736f61eae3491e93f3cb Mon Sep 17 00:00:00 2001 From: bharatnc Date: Fri, 14 Jan 2022 00:02:41 -0800 Subject: [PATCH 033/403] add h3NumHexagons func --- src/Functions/h3NumHexagons.cpp | 90 ++++++++++++++++++++++++++ src/Functions/registerFunctionsGeo.cpp | 2 + 2 files changed, 92 insertions(+) create mode 100644 src/Functions/h3NumHexagons.cpp diff --git a/src/Functions/h3NumHexagons.cpp b/src/Functions/h3NumHexagons.cpp new file mode 100644 index 00000000000..643b62730d4 --- /dev/null +++ b/src/Functions/h3NumHexagons.cpp @@ -0,0 +1,90 @@ +#include "config_functions.h" + +#if USE_H3 + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ +extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int ILLEGAL_COLUMN; +} + +namespace +{ + +class FunctionH3NumHexagons : public IFunction +{ +public: + static constexpr auto name = "h3NumHexagons"; + + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + std::string getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 1; } + bool useDefaultImplementationForConstants() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + const auto * arg = arguments[0].get(); + if (!WhichDataType(arg).isUInt8()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt8", + arg->getName(), 1, getName()); + + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto * column = checkAndGetColumn(arguments[0].column.get()); + if (!column) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt8", + arguments[0].type->getName(), + 1, + getName()); + + const auto & data = column->getData(); + + auto dst = ColumnVector::create(); + auto & dst_data = dst->getData(); + dst_data.resize(input_rows_count); + + for (size_t row = 0; row < input_rows_count; ++row) + { + const UInt64 resolution = data[row]; + Float64 res = getNumCells(resolution); + dst_data[row] = res; + } + + return dst; + } +}; + +} + +void registerFunctionH3NumHexagons(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + +#endif diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index ee407926ffa..74fb429e03b 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -52,6 +52,7 @@ void registerFunctionH3RadsToDegs(FunctionFactory &); void registerFunctionH3HexAreaKm2(FunctionFactory &); void registerFunctionH3CellAreaM2(FunctionFactory &); void registerFunctionH3CellAreaRads2(FunctionFactory &); +void registerFunctionH3NumHexagons(FunctionFactory &); #endif @@ -118,6 +119,7 @@ void registerFunctionsGeo(FunctionFactory & factory) registerFunctionH3HexAreaKm2(factory); registerFunctionH3CellAreaM2(factory); registerFunctionH3CellAreaRads2(factory); + registerFunctionH3NumHexagons(factory); #endif #if USE_S2_GEOMETRY From f0bffbfee586968c0273c03f657f050496360494 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Fri, 14 Jan 2022 00:10:18 -0800 Subject: [PATCH 034/403] check max H3 resolution --- src/Functions/h3NumHexagons.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Functions/h3NumHexagons.cpp b/src/Functions/h3NumHexagons.cpp index 643b62730d4..ea880dae554 100644 --- a/src/Functions/h3NumHexagons.cpp +++ b/src/Functions/h3NumHexagons.cpp @@ -69,7 +69,12 @@ public: for (size_t row = 0; row < input_rows_count; ++row) { - const UInt64 resolution = data[row]; + const int resolution = data[row]; + if (resolution > MAX_H3_RES) + throw Exception( + ErrorCodes::ARGUMENT_OUT_OF_BOUND, + "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ", + resolution, getName(), MAX_H3_RES); Float64 res = getNumCells(resolution); dst_data[row] = res; } From 165ce544ee75e7c4ba04dd1724f516ec3fb4ae71 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Fri, 14 Jan 2022 00:17:06 -0800 Subject: [PATCH 035/403] tests for numHexagons --- .../02165_h3_num_hexagons.reference | 16 ++++++++++++++++ .../0_stateless/02165_h3_num_hexagons.sql | 19 +++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 tests/queries/0_stateless/02165_h3_num_hexagons.reference create mode 100644 tests/queries/0_stateless/02165_h3_num_hexagons.sql diff --git a/tests/queries/0_stateless/02165_h3_num_hexagons.reference b/tests/queries/0_stateless/02165_h3_num_hexagons.reference new file mode 100644 index 00000000000..b6cfe94c218 --- /dev/null +++ b/tests/queries/0_stateless/02165_h3_num_hexagons.reference @@ -0,0 +1,16 @@ +122 +842 +5882 +41162 +288122 +2016842 +14117882 +98825162 +691776122 +4842432842 +33897029882 +237279209162 +1660954464122 +11626681248842 +81386768741882 +569707381193162 diff --git a/tests/queries/0_stateless/02165_h3_num_hexagons.sql b/tests/queries/0_stateless/02165_h3_num_hexagons.sql new file mode 100644 index 00000000000..7ab48b3738b --- /dev/null +++ b/tests/queries/0_stateless/02165_h3_num_hexagons.sql @@ -0,0 +1,19 @@ +-- Tags: no-fasttest + +SELECT h3NumHexagons(0); +SELECT h3NumHexagons(1); +SELECT h3NumHexagons(2); +SELECT h3NumHexagons(3); +SELECT h3NumHexagons(4); +SELECT h3NumHexagons(5); +SELECT h3NumHexagons(6); +SELECT h3NumHexagons(7); +SELECT h3NumHexagons(8); +SELECT h3NumHexagons(9); +SELECT h3NumHexagons(10); +SELECT h3NumHexagons(11); +SELECT h3NumHexagons(12); +SELECT h3NumHexagons(13); +SELECT h3NumHexagons(14); +SELECT h3NumHexagons(15); +SELECT h3NumHexagons(16); -- { serverError 69 } From f303e0b23b91404980906b5cc58f646ca59fedd7 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Fri, 14 Jan 2022 00:20:34 -0800 Subject: [PATCH 036/403] fix return type --- src/Functions/h3NumHexagons.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Functions/h3NumHexagons.cpp b/src/Functions/h3NumHexagons.cpp index ea880dae554..0ee8e45813e 100644 --- a/src/Functions/h3NumHexagons.cpp +++ b/src/Functions/h3NumHexagons.cpp @@ -47,7 +47,7 @@ public: "Illegal type {} of argument {} of function {}. Must be UInt8", arg->getName(), 1, getName()); - return std::make_shared(); + return std::make_shared(); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override @@ -63,7 +63,7 @@ public: const auto & data = column->getData(); - auto dst = ColumnVector::create(); + auto dst = ColumnVector::create(); auto & dst_data = dst->getData(); dst_data.resize(input_rows_count); @@ -75,7 +75,7 @@ public: ErrorCodes::ARGUMENT_OUT_OF_BOUND, "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ", resolution, getName(), MAX_H3_RES); - Float64 res = getNumCells(resolution); + Int64 res = getNumCells(resolution); dst_data[row] = res; } From 03d9eeec8b70dfbddb07079fe1da9d32c63ce9c4 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Fri, 14 Jan 2022 00:26:06 -0800 Subject: [PATCH 037/403] docs h3NumHexagons --- docs/en/sql-reference/functions/geo/h3.md | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/docs/en/sql-reference/functions/geo/h3.md b/docs/en/sql-reference/functions/geo/h3.md index afcf1f6bccc..c466b082ce6 100644 --- a/docs/en/sql-reference/functions/geo/h3.md +++ b/docs/en/sql-reference/functions/geo/h3.md @@ -1022,4 +1022,39 @@ Result: └──────────────────────┘ ``` +## h3NumHexagons {#h3numhexagons} + +Returns the number of unique H3 indices at the given resolution. + +**Syntax** + +``` sql +h3NumHexagons(resolution) +``` + +**Parameter** + +- `resolution` — Index resolution. Range: `[0, 15]`. Type: [UInt8](../../../sql-reference/data-types/int-uint.md). + +**Returned value** + +- Number of H3 indices. + +Type: [Int64](../../../sql-reference/data-types/int-uint.md). + +**Example** + +Query: + +``` sql +SELECT h3NumHexagons(3) AS numHexagons; +``` + +Result: + +``` text +┌─numHexagons─┐ +│ 41162 │ +└─────────────┘ +``` [Original article](https://clickhouse.com/docs/en/sql-reference/functions/geo/h3) From 9f35c0affce6f948370bf34e3439c330bf58f18d Mon Sep 17 00:00:00 2001 From: bharatnc Date: Fri, 14 Jan 2022 00:28:31 -0800 Subject: [PATCH 038/403] fix res type in various funcs --- src/Functions/h3EdgeLengthKm.cpp | 2 +- src/Functions/h3EdgeLengthM.cpp | 2 +- src/Functions/h3HexAreaKm2.cpp | 2 +- src/Functions/h3HexAreaM2.cpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Functions/h3EdgeLengthKm.cpp b/src/Functions/h3EdgeLengthKm.cpp index b4324ad817e..7259bf66040 100644 --- a/src/Functions/h3EdgeLengthKm.cpp +++ b/src/Functions/h3EdgeLengthKm.cpp @@ -70,7 +70,7 @@ public: for (size_t row = 0; row < input_rows_count; ++row) { - const UInt64 resolution = data[row]; + const int resolution = data[row]; if (resolution > MAX_H3_RES) throw Exception( ErrorCodes::ARGUMENT_OUT_OF_BOUND, diff --git a/src/Functions/h3EdgeLengthM.cpp b/src/Functions/h3EdgeLengthM.cpp index 3eef9be9345..192632d66f5 100644 --- a/src/Functions/h3EdgeLengthM.cpp +++ b/src/Functions/h3EdgeLengthM.cpp @@ -65,7 +65,7 @@ public: for (size_t row = 0; row < input_rows_count; ++row) { - const UInt64 resolution = col_hindex->getUInt(row); + const int resolution = col_hindex->getUInt(row); if (resolution > MAX_H3_RES) throw Exception( ErrorCodes::ARGUMENT_OUT_OF_BOUND, diff --git a/src/Functions/h3HexAreaKm2.cpp b/src/Functions/h3HexAreaKm2.cpp index 933fcf21424..10e8340dbb7 100644 --- a/src/Functions/h3HexAreaKm2.cpp +++ b/src/Functions/h3HexAreaKm2.cpp @@ -70,7 +70,7 @@ public: for (size_t row = 0; row < input_rows_count; ++row) { - const UInt64 resolution = data[row]; + const int resolution = data[row]; if (resolution > MAX_H3_RES) throw Exception( ErrorCodes::ARGUMENT_OUT_OF_BOUND, diff --git a/src/Functions/h3HexAreaM2.cpp b/src/Functions/h3HexAreaM2.cpp index 96b301806a5..fc8a4de831b 100644 --- a/src/Functions/h3HexAreaM2.cpp +++ b/src/Functions/h3HexAreaM2.cpp @@ -60,7 +60,7 @@ public: for (size_t row = 0; row < input_rows_count; ++row) { - const UInt64 resolution = col_hindex->getUInt(row); + const int resolution = col_hindex->getUInt(row); if (resolution > MAX_H3_RES) throw Exception( ErrorCodes::ARGUMENT_OUT_OF_BOUND, From 8b10bd92219dfadab01ca295b23d7a6c33d10426 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Fri, 14 Jan 2022 00:52:02 -0800 Subject: [PATCH 039/403] fix style check --- src/Functions/h3NumHexagons.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Functions/h3NumHexagons.cpp b/src/Functions/h3NumHexagons.cpp index 0ee8e45813e..18dba3d9aec 100644 --- a/src/Functions/h3NumHexagons.cpp +++ b/src/Functions/h3NumHexagons.cpp @@ -20,6 +20,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int ILLEGAL_COLUMN; +extern const int ARGUMENT_OUT_OF_BOUND; } namespace From 46f3a699dfdaf69af05a4e8d4ec63503c5891d99 Mon Sep 17 00:00:00 2001 From: hanqf-git Date: Fri, 14 Jan 2022 20:21:22 +0800 Subject: [PATCH 040/403] Add x86 feature avx512 support for memcmpSmall --- src/Common/memcmpSmall.h | 234 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 233 insertions(+), 1 deletion(-) diff --git a/src/Common/memcmpSmall.h b/src/Common/memcmpSmall.h index db8641cb44d..57b9c731897 100644 --- a/src/Common/memcmpSmall.h +++ b/src/Common/memcmpSmall.h @@ -25,8 +25,240 @@ inline int cmp(T a, T b) /// We can process uninitialized memory in the functions below. /// Results don't depend on the values inside uninitialized memory but Memory Sanitizer cannot see it. /// Disable optimized functions if compile with Memory Sanitizer. +#if defined(__AVX512BW__) && defined(__AVX512VL__) && !defined(MEMORY_SANITIZER) +#include -#if defined(__SSE2__) && !defined(MEMORY_SANITIZER) + +/** All functions works under the following assumptions: + * - it's possible to read up to 15 excessive bytes after end of 'a' and 'b' region; + * - memory regions are relatively small and extra loop unrolling is not worth to do. + */ + +/** Variant when memory regions may have different sizes. + */ +template +inline int memcmpSmallAllowOverflow15(const Char * a, size_t a_size, const Char * b, size_t b_size) +{ + size_t min_size = std::min(a_size, b_size); + + for (size_t offset = 0; offset < min_size; offset += 16) + { + uint16_t mask = _mm_cmp_epi8_mask( + _mm_loadu_si128(reinterpret_cast(a + offset)), + _mm_loadu_si128(reinterpret_cast(b + offset)), _MM_CMPINT_NE); + + if (mask) + { + offset += __builtin_ctz(mask); + + if (offset >= min_size) + break; + + return detail::cmp(a[offset], b[offset]); + } + } + + return detail::cmp(a_size, b_size); +} + + +/** Variant when memory regions may have different sizes. + * But compare the regions as the smaller one is padded with zero bytes up to the size of the larger. + * It's needed to hold that: toFixedString('abc', 5) = 'abc' + * for compatibility with SQL standard. + */ +template +inline int memcmpSmallLikeZeroPaddedAllowOverflow15(const Char * a, size_t a_size, const Char * b, size_t b_size) +{ + size_t min_size = std::min(a_size, b_size); + + for (size_t offset = 0; offset < min_size; offset += 16) + { + uint16_t mask = _mm_cmp_epi8_mask( + _mm_loadu_si128(reinterpret_cast(a + offset)), + _mm_loadu_si128(reinterpret_cast(b + offset)), _MM_CMPINT_NE); + + if (mask) + { + offset += __builtin_ctz(mask); + + if (offset >= min_size) + break; + + return detail::cmp(a[offset], b[offset]); + } + } + + /// The strings are equal up to min_size. + /// If the rest of the larger string is zero bytes then the strings are considered equal. + + size_t max_size; + const Char * longest; + int cmp; + + if (a_size == b_size) + { + return 0; + } + else if (a_size > b_size) + { + max_size = a_size; + longest = a; + cmp = 1; + } + else + { + max_size = b_size; + longest = b; + cmp = -1; + } + + const __m128i zero16 = _mm_setzero_si128(); + + for (size_t offset = min_size; offset < max_size; offset += 16) + { + uint16_t mask = _mm_cmpneq_epi8_mask( + _mm_loadu_si128(reinterpret_cast(longest + offset)), + zero16); + + if (mask) + { + offset += __builtin_ctz(mask); + + if (offset >= max_size) + return 0; + return cmp; + } + } + + return 0; +} + + +/** Variant when memory regions have same size. + * TODO Check if the compiler can optimize previous function when the caller pass identical sizes. + */ +template +inline int memcmpSmallAllowOverflow15(const Char * a, const Char * b, size_t size) +{ + for (size_t offset = 0; offset < size; offset += 16) + { + uint16_t mask = _mm_cmp_epi8_mask( + _mm_loadu_si128(reinterpret_cast(a + offset)), + _mm_loadu_si128(reinterpret_cast(b + offset)), _MM_CMPINT_NE); + + if (mask) + { + offset += __builtin_ctz(mask); + + if (offset >= size) + return 0; + + return detail::cmp(a[offset], b[offset]); + } + } + + return 0; +} + + +/** Compare memory regions for equality. + */ +template +inline bool memequalSmallAllowOverflow15(const Char * a, size_t a_size, const Char * b, size_t b_size) +{ + if (a_size != b_size) + return false; + + for (size_t offset = 0; offset < a_size; offset += 16) + { + uint16_t mask = _mm_cmp_epi8_mask( + _mm_loadu_si128(reinterpret_cast(a + offset)), + _mm_loadu_si128(reinterpret_cast(b + offset)), _MM_CMPINT_NE); + + if (mask) + { + offset += __builtin_ctz(mask); + return offset >= a_size; + } + } + + return true; +} + + +/** Variant when the caller know in advance that the size is a multiple of 16. + */ +template +inline int memcmpSmallMultipleOf16(const Char * a, const Char * b, size_t size) +{ + for (size_t offset = 0; offset < size; offset += 16) + { + uint16_t mask = _mm_cmp_epi8_mask( + _mm_loadu_si128(reinterpret_cast(a + offset)), + _mm_loadu_si128(reinterpret_cast(b + offset)), _MM_CMPINT_NE); + + if (mask) + { + offset += __builtin_ctz(mask); + return detail::cmp(a[offset], b[offset]); + } + } + + return 0; +} + + +/** Variant when the size is 16 exactly. + */ +template +inline int memcmp16(const Char * a, const Char * b) +{ + uint16_t mask = _mm_cmp_epi8_mask( + _mm_loadu_si128(reinterpret_cast(a)), + _mm_loadu_si128(reinterpret_cast(b)), _MM_CMPINT_NE); + + if (mask) + { + auto offset = __builtin_ctz(mask); + return detail::cmp(a[offset], b[offset]); + } + + return 0; +} + + +/** Variant when the size is 16 exactly. + */ +inline bool memequal16(const void * a, const void * b) +{ + return 0xFFFF == _mm_cmp_epi8_mask( + _mm_loadu_si128(reinterpret_cast(a)), + _mm_loadu_si128(reinterpret_cast(b)), _MM_CMPINT_EQ); +} + + +/** Compare memory region to zero */ +inline bool memoryIsZeroSmallAllowOverflow15(const void * data, size_t size) +{ + const __m128i zero16 = _mm_setzero_si128(); + + for (size_t offset = 0; offset < size; offset += 16) + { + uint16_t mask = _mm_cmp_epi8_mask(zero16, + _mm_loadu_si128(reinterpret_cast(reinterpret_cast(data) + offset)), _MM_CMPINT_NE); + + if (mask) + { + offset += __builtin_ctz(mask); + return offset >= size; + } + } + + return true; +} + +#elif defined(__SSE2__) && !defined(MEMORY_SANITIZER) #include From d9a64d1f8675e81304b881eeae3ca2957e8d4cb0 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 14 Jan 2022 15:47:26 +0300 Subject: [PATCH 041/403] Fix optimize_aggregation_in_order in case of empty result set Note, that this is not complete fix, see the next two patches. Signed-off-by: Azat Khuzhin --- .../Transforms/MergingAggregatedTransform.cpp | 6 +++++- ...6_optimize_aggregation_in_order_empty.reference | 8 ++++++++ .../02176_optimize_aggregation_in_order_empty.sql | 14 ++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.reference create mode 100644 tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.sql diff --git a/src/Processors/Transforms/MergingAggregatedTransform.cpp b/src/Processors/Transforms/MergingAggregatedTransform.cpp index ddc58d830da..37419f55aae 100644 --- a/src/Processors/Transforms/MergingAggregatedTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedTransform.cpp @@ -23,7 +23,11 @@ void MergingAggregatedTransform::consume(Chunk chunk) LOG_TRACE(log, "Reading blocks of partially aggregated data."); } - total_input_rows += chunk.getNumRows(); + size_t input_rows = chunk.getNumRows(); + if (!input_rows) + return; + + total_input_rows += input_rows; ++total_input_blocks; const auto & info = chunk.getChunkInfo(); diff --git a/tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.reference b/tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.reference new file mode 100644 index 00000000000..645cec31b47 --- /dev/null +++ b/tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.reference @@ -0,0 +1,8 @@ +-- { echoOn } + +-- regression for optimize_aggregation_in_order with empty result set +-- that cause at first +-- "Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform" +-- at first and after +-- "Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform" +select count() from remote('127.{1,2}', currentDatabase(), data_02176) where key = 0 group by key settings optimize_aggregation_in_order=1; diff --git a/tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.sql b/tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.sql new file mode 100644 index 00000000000..a86fd4357c8 --- /dev/null +++ b/tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.sql @@ -0,0 +1,14 @@ +drop table if exists data_02176; +create table data_02176 (key Int) Engine=MergeTree() order by key; + +-- { echoOn } + +-- regression for optimize_aggregation_in_order with empty result set +-- that cause at first +-- "Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform" +-- at first and after +-- "Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform" +select count() from remote('127.{1,2}', currentDatabase(), data_02176) where key = 0 group by key settings optimize_aggregation_in_order=1; + +-- { echoOff } +drop table data_02176; From 06402386eb12954121500a8bce1d2ca56e306f64 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 14 Jan 2022 16:49:04 +0300 Subject: [PATCH 042/403] Do not generate empty chunks in AggregatingInOrderTransform This is just a micro optimization and it should not affect anything, real fixes are in separate patches (previous and next). Signed-off-by: Azat Khuzhin --- src/Processors/Transforms/AggregatingInOrderTransform.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Processors/Transforms/AggregatingInOrderTransform.cpp b/src/Processors/Transforms/AggregatingInOrderTransform.cpp index fae1ede1f9c..857f362c4be 100644 --- a/src/Processors/Transforms/AggregatingInOrderTransform.cpp +++ b/src/Processors/Transforms/AggregatingInOrderTransform.cpp @@ -255,6 +255,8 @@ void AggregatingInOrderTransform::generate() res.getByPosition(i + res_key_columns.size()).column = std::move(res_aggregate_columns[i]); to_push_chunk = convertToChunk(res); + if (!to_push_chunk.getNumRows()) + return; /// Clear arenas to allow to free them, when chunk will reach the end of pipeline. /// It's safe clear them here, because columns with aggregate functions already holds them. From a4c2f23b07f6b0e4810350424d34b217368c7daf Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Fri, 14 Jan 2022 16:25:34 +0300 Subject: [PATCH 043/403] Fix "Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform" In case of optimize_aggregation_in_order there will be ChunkInfoWithAllocatedBytes. Signed-off-by: Azat Khuzhin --- ...gingAggregatedMemoryEfficientTransform.cpp | 72 ++++++++++++------- ...ge_optimize_aggregation_in_order.reference | 6 ++ ...77_merge_optimize_aggregation_in_order.sql | 12 ++++ 3 files changed, 65 insertions(+), 25 deletions(-) create mode 100644 tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.reference create mode 100644 tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.sql diff --git a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp index e1fa965f025..d01a809e666 100644 --- a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp @@ -1,9 +1,9 @@ #include - -#include #include #include +#include #include +#include namespace DB { @@ -250,22 +250,30 @@ void GroupingAggregatedTransform::addChunk(Chunk chunk, size_t input) if (!info) throw Exception("Chunk info was not set for chunk in GroupingAggregatedTransform.", ErrorCodes::LOGICAL_ERROR); - const auto * agg_info = typeid_cast(info.get()); - if (!agg_info) - throw Exception("Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform.", ErrorCodes::LOGICAL_ERROR); + if (const auto * agg_info = typeid_cast(info.get())) + { + Int32 bucket = agg_info->bucket_num; + bool is_overflows = agg_info->is_overflows; - Int32 bucket = agg_info->bucket_num; - bool is_overflows = agg_info->is_overflows; - - if (is_overflows) - overflow_chunks.emplace_back(std::move(chunk)); - else if (bucket < 0) + if (is_overflows) + overflow_chunks.emplace_back(std::move(chunk)); + else if (bucket < 0) + single_level_chunks.emplace_back(std::move(chunk)); + else + { + chunks_map[bucket].emplace_back(std::move(chunk)); + has_two_level = true; + last_bucket_number[input] = bucket; + } + } + else if (const auto * in_order_info = typeid_cast(info.get())) + { single_level_chunks.emplace_back(std::move(chunk)); + } else { - chunks_map[bucket].emplace_back(std::move(chunk)); - has_two_level = true; - last_bucket_number[input] = bucket; + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Chunk should have AggregatedChunkInfo/ChunkInfoWithAllocatedBytes in GroupingAggregatedTransform."); } } @@ -318,16 +326,27 @@ void MergingAggregatedBucketTransform::transform(Chunk & chunk) throw Exception("Chunk info was not set for chunk in MergingAggregatedBucketTransform.", ErrorCodes::LOGICAL_ERROR); - const auto * agg_info = typeid_cast(cur_info.get()); - if (!agg_info) - throw Exception("Chunk should have AggregatedChunkInfo in MergingAggregatedBucketTransform.", - ErrorCodes::LOGICAL_ERROR); + if (const auto * agg_info = typeid_cast(cur_info.get())) + { + Block block = header.cloneWithColumns(cur_chunk.detachColumns()); + block.info.is_overflows = agg_info->is_overflows; + block.info.bucket_num = agg_info->bucket_num; - Block block = header.cloneWithColumns(cur_chunk.detachColumns()); - block.info.is_overflows = agg_info->is_overflows; - block.info.bucket_num = agg_info->bucket_num; + blocks_list.emplace_back(std::move(block)); + } + else if (const auto * in_order_info = typeid_cast(cur_info.get())) + { + Block block = header.cloneWithColumns(cur_chunk.detachColumns()); + block.info.is_overflows = false; + block.info.bucket_num = -1; - blocks_list.emplace_back(std::move(block)); + blocks_list.emplace_back(std::move(block)); + } + else + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Chunk should have AggregatedChunkInfo/ChunkInfoWithAllocatedBytes in MergingAggregatedBucketTransform."); + } } auto res_info = std::make_shared(); @@ -379,7 +398,8 @@ void SortingAggregatedTransform::addChunk(Chunk chunk, size_t from_input) const auto * agg_info = typeid_cast(info.get()); if (!agg_info) - throw Exception("Chunk should have AggregatedChunkInfo in SortingAggregatedTransform.", ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Chunk should have AggregatedChunkInfo in SortingAggregatedTransform."); Int32 bucket = agg_info->bucket_num; bool is_overflows = agg_info->is_overflows; @@ -389,8 +409,10 @@ void SortingAggregatedTransform::addChunk(Chunk chunk, size_t from_input) else { if (chunks[bucket]) - throw Exception("SortingAggregatedTransform already got bucket with number " + toString(bucket), - ErrorCodes::LOGICAL_ERROR); + { + throw Exception(ErrorCodes::LOGICAL_ERROR, + "SortingAggregatedTransform already got bucket with number {}", bucket); + } chunks[bucket] = std::move(chunk); last_bucket_number[from_input] = bucket; diff --git a/tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.reference b/tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.reference new file mode 100644 index 00000000000..00e893213c0 --- /dev/null +++ b/tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.reference @@ -0,0 +1,6 @@ +-- { echoOn } + +-- regression for optimize_aggregation_in_order +-- that cause "Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform" error +select count() from remote('127.{1,2}', currentDatabase(), data_02177) group by key settings optimize_aggregation_in_order=1; +2 diff --git a/tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.sql b/tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.sql new file mode 100644 index 00000000000..17c4a1dba29 --- /dev/null +++ b/tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.sql @@ -0,0 +1,12 @@ +drop table if exists data_02177; +create table data_02177 (key Int) Engine=MergeTree() order by key; +insert into data_02177 values (1); + +-- { echoOn } + +-- regression for optimize_aggregation_in_order +-- that cause "Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform" error +select count() from remote('127.{1,2}', currentDatabase(), data_02177) group by key settings optimize_aggregation_in_order=1; + +-- { echoOff } +drop table data_02177; From ac1665cdcb67afc01b45906f4396bd1f10337e3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Fri, 14 Jan 2022 15:46:03 +0100 Subject: [PATCH 044/403] AsynchronousMetrics: Ignore inaccessible sensors --- src/Interpreters/AsynchronousMetrics.cpp | 34 ++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index d1c5fbebbc7..72e49505b54 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -109,6 +109,23 @@ void AsynchronousMetrics::openSensors() else break; } + + file->rewind(); + Int64 temperature = 0; + try + { + readText(temperature, *file); + } + catch (const ErrnoException & e) + { + LOG_WARNING( + &Poco::Logger::get("AsynchronousMetrics"), + "Thermal monitor '{}' exists but could not be read, error {}.", + thermal_device_index, + e.getErrno()); + continue; + } + thermal.emplace_back(std::move(file)); } } @@ -222,6 +239,23 @@ void AsynchronousMetrics::openSensorsChips() std::replace(sensor_name.begin(), sensor_name.end(), ' ', '_'); } + file->rewind(); + Int64 temperature = 0; + try + { + readText(temperature, *file); + } + catch (const ErrnoException & e) + { + LOG_WARNING( + &Poco::Logger::get("AsynchronousMetrics"), + "Hardware monitor '{}', sensor '{}' exists but could not be read, error {}.", + hwmon_name, + sensor_name, + e.getErrno()); + continue; + } + hwmon_devices[hwmon_name][sensor_name] = std::move(file); } } From 89a181bd195ac9a761bd79375c75f4efee6e442c Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 14 Jan 2022 18:16:18 +0300 Subject: [PATCH 045/403] Make better --- src/Formats/FormatFactory.cpp | 17 +++++++++-------- src/Formats/FormatFactory.h | 18 +++++++++--------- .../Formats/Impl/ArrowBlockOutputFormat.cpp | 4 ++-- .../Impl/CustomSeparatedRowOutputFormat.cpp | 2 +- .../Formats/Impl/JSONRowOutputFormat.cpp | 4 ++-- .../Formats/Impl/ORCBlockOutputFormat.cpp | 2 +- .../Formats/Impl/ParquetBlockOutputFormat.cpp | 2 +- .../Formats/Impl/TemplateBlockOutputFormat.cpp | 2 +- .../Formats/Impl/XMLRowOutputFormat.cpp | 2 +- src/Storages/HDFS/StorageHDFS.cpp | 1 + src/Storages/HDFS/WriteBufferFromHDFS.cpp | 1 - src/Storages/StorageFile.cpp | 2 +- src/Storages/StorageS3.cpp | 7 +++---- 13 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index ad1db85386c..9548bb754fa 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -392,24 +392,25 @@ void FormatFactory::registerNonTrivialPrefixAndSuffixChecker(const String & name target = std::move(non_trivial_prefix_and_suffix_checker); } -void FormatFactory::registerSuffixChecker(const String & name, SuffixChecker suffix_checker) +void FormatFactory::registerAppendSupportChecker(const String & name, AppendSupportChecker append_support_checker) { - auto & target = dict[name].suffix_checker; + auto & target = dict[name].append_support_checker; if (target) throw Exception("FormatFactory: Suffix checker " + name + " is already registered", ErrorCodes::LOGICAL_ERROR); - target = std::move(suffix_checker); + target = std::move(append_support_checker); } -void FormatFactory::markFormatWithSuffix(const String & name) +void FormatFactory::markFormatDoesntSupportAppend(const String & name) { - registerSuffixChecker(name, [](const FormatSettings &){ return true; }); + registerAppendSupportChecker(name, [](const FormatSettings &){ return true; }); } -bool FormatFactory::checkIfFormatHasSuffix(const String & name, ContextPtr context, const std::optional & format_settings_) +bool FormatFactory::checkIfFormatSupportAppend(const String & name, ContextPtr context, const std::optional & format_settings_) { auto format_settings = format_settings_ ? *format_settings_ : getFormatSettings(context); - auto & suffix_checker = dict[name].suffix_checker; - return suffix_checker && suffix_checker(format_settings); + auto & append_support_checker = dict[name].append_support_checker; + /// By default we consider that format supports append + return !append_support_checker || append_support_checker(format_settings); } void FormatFactory::registerOutputFormat(const String & name, OutputCreator output_creator) diff --git a/src/Formats/FormatFactory.h b/src/Formats/FormatFactory.h index 5acf79f2047..8a671d5a365 100644 --- a/src/Formats/FormatFactory.h +++ b/src/Formats/FormatFactory.h @@ -93,9 +93,9 @@ private: /// The checker should return true if parallel parsing should be disabled. using NonTrivialPrefixAndSuffixChecker = std::function; - /// Some formats can have suffix after data depending on settings. - /// The checker should return true if format will write some suffix after data. - using SuffixChecker = std::function; + /// Some formats can support append depending on settings. + /// The checker should return true if format support append. + using AppendSupportChecker = std::function; using SchemaReaderCreator = std::function; using ExternalSchemaReaderCreator = std::function; @@ -110,7 +110,7 @@ private: bool supports_parallel_formatting{false}; bool is_column_oriented{false}; NonTrivialPrefixAndSuffixChecker non_trivial_prefix_and_suffix_checker; - SuffixChecker suffix_checker; + AppendSupportChecker append_support_checker; }; using FormatsDictionary = std::unordered_map; @@ -172,13 +172,13 @@ public: void registerNonTrivialPrefixAndSuffixChecker(const String & name, NonTrivialPrefixAndSuffixChecker non_trivial_prefix_and_suffix_checker); - void registerSuffixChecker(const String & name, SuffixChecker suffix_checker); + void registerAppendSupportChecker(const String & name, AppendSupportChecker append_support_checker); - /// If format always contains suffix, you an use this method instead of - /// registerSuffixChecker with suffix_checker that always returns true. - void markFormatWithSuffix(const String & name); + /// If format always doesn't support append, you can use this method instead of + /// registerAppendSupportChecker with append_support_checker that always returns true. + void markFormatDoesntSupportAppend(const String & name); - bool checkIfFormatHasSuffix(const String & name, ContextPtr context, const std::optional & format_settings_ = std::nullopt); + bool checkIfFormatSupportAppend(const String & name, ContextPtr context, const std::optional & format_settings_ = std::nullopt); /// Register format by its name. void registerInputFormat(const String & name, InputCreator input_creator); diff --git a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp index b60d38f317c..c87191d0a12 100644 --- a/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowBlockOutputFormat.cpp @@ -93,7 +93,7 @@ void registerOutputFormatArrow(FormatFactory & factory) { return std::make_shared(buf, sample, false, format_settings); }); - factory.markFormatWithSuffix("Arrow"); + factory.markFormatDoesntSupportAppend("Arrow"); factory.registerOutputFormat( "ArrowStream", @@ -104,7 +104,7 @@ void registerOutputFormatArrow(FormatFactory & factory) { return std::make_shared(buf, sample, true, format_settings); }); - factory.markFormatWithSuffix("ArrowStream"); + factory.markFormatDoesntSupportAppend("ArrowStream"); } } diff --git a/src/Processors/Formats/Impl/CustomSeparatedRowOutputFormat.cpp b/src/Processors/Formats/Impl/CustomSeparatedRowOutputFormat.cpp index 3fb7c0a36a3..88b9226fd8a 100644 --- a/src/Processors/Formats/Impl/CustomSeparatedRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/CustomSeparatedRowOutputFormat.cpp @@ -92,7 +92,7 @@ void registerOutputFormatCustomSeparated(FormatFactory & factory) factory.markOutputFormatSupportsParallelFormatting(format_name); - factory.registerSuffixChecker(format_name, [](const FormatSettings & settings) + factory.registerAppendSupportChecker(format_name, [](const FormatSettings & settings) { return !settings.custom.result_after_delimiter.empty(); }); diff --git a/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp b/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp index 577efdb1a21..7ac7d45d26d 100644 --- a/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONRowOutputFormat.cpp @@ -284,7 +284,7 @@ void registerOutputFormatJSON(FormatFactory & factory) }); factory.markOutputFormatSupportsParallelFormatting("JSON"); - factory.markFormatWithSuffix("JSON"); + factory.markFormatDoesntSupportAppend("JSON"); factory.registerOutputFormat("JSONStrings", []( WriteBuffer & buf, @@ -296,7 +296,7 @@ void registerOutputFormatJSON(FormatFactory & factory) }); factory.markOutputFormatSupportsParallelFormatting("JSONStrings"); - factory.markFormatWithSuffix("JSONStrings"); + factory.markFormatDoesntSupportAppend("JSONStrings"); } } diff --git a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp index 33d4b3e568d..67287c8a661 100644 --- a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp @@ -526,7 +526,7 @@ void registerOutputFormatORC(FormatFactory & factory) { return std::make_shared(buf, sample, format_settings); }); - factory.markFormatWithSuffix("ORC"); + factory.markFormatDoesntSupportAppend("ORC"); } } diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index 96c3a80b732..c3ddc632641 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -85,7 +85,7 @@ void registerOutputFormatParquet(FormatFactory & factory) { return std::make_shared(buf, sample, format_settings); }); - factory.markFormatWithSuffix("Parquet"); + factory.markFormatDoesntSupportAppend("Parquet"); } } diff --git a/src/Processors/Formats/Impl/TemplateBlockOutputFormat.cpp b/src/Processors/Formats/Impl/TemplateBlockOutputFormat.cpp index eded88298e8..6babac98102 100644 --- a/src/Processors/Formats/Impl/TemplateBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/TemplateBlockOutputFormat.cpp @@ -236,7 +236,7 @@ void registerOutputFormatTemplate(FormatFactory & factory) return std::make_shared(sample, buf, settings, resultset_format, row_format, settings.template_settings.row_between_delimiter); }); - factory.registerSuffixChecker("Template", [](const FormatSettings & settings) + factory.registerAppendSupportChecker("Template", [](const FormatSettings & settings) { if (settings.template_settings.resultset_format.empty()) return false; diff --git a/src/Processors/Formats/Impl/XMLRowOutputFormat.cpp b/src/Processors/Formats/Impl/XMLRowOutputFormat.cpp index 56bbbc0673c..6a80131a65e 100644 --- a/src/Processors/Formats/Impl/XMLRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/XMLRowOutputFormat.cpp @@ -256,7 +256,7 @@ void registerOutputFormatXML(FormatFactory & factory) }); factory.markOutputFormatSupportsParallelFormatting("XML"); - factory.markFormatWithSuffix("XML"); + factory.markFormatDoesntSupportAppend("XML"); } } diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index 36287b92855..c088ad07a34 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -52,6 +52,7 @@ namespace ErrorCodes extern const int ACCESS_DENIED; extern const int DATABASE_ACCESS_DENIED; extern const int CANNOT_EXTRACT_TABLE_STRUCTURE; + extern const int BAD_ARGUMENTS; } namespace { diff --git a/src/Storages/HDFS/WriteBufferFromHDFS.cpp b/src/Storages/HDFS/WriteBufferFromHDFS.cpp index 2950046b633..2addfc0069f 100644 --- a/src/Storages/HDFS/WriteBufferFromHDFS.cpp +++ b/src/Storages/HDFS/WriteBufferFromHDFS.cpp @@ -15,7 +15,6 @@ namespace ErrorCodes extern const int NETWORK_ERROR; extern const int CANNOT_OPEN_FILE; extern const int CANNOT_FSYNC; -extern const int BAD_ARGUMENTS; } diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 586b8d97875..48e0e82821a 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -833,7 +833,7 @@ SinkToStoragePtr StorageFile::write( throw Exception("Table '" + getStorageID().getNameForLogs() + "' is in readonly mode because of globs in filepath", ErrorCodes::DATABASE_ACCESS_DENIED); if (!context->getSettingsRef().engine_file_truncate_on_insert && !is_path_with_globs - && FormatFactory::instance().checkIfFormatHasSuffix(format_name, context, format_settings) && fs::exists(paths.back()) + && !FormatFactory::instance().checkIfFormatSupportAppend(format_name, context, format_settings) && fs::exists(paths.back()) && fs::file_size(paths.back()) != 0) { if (context->getSettingsRef().engine_file_allow_create_multiple_files) diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index ea563937ca6..e15878cf893 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -555,7 +555,6 @@ StorageS3::StorageS3( std::shared_ptr StorageS3::createFileIterator(const ClientAuthentication & client_auth, const std::vector & keys, bool is_key_with_globs, bool distributed_processing, ContextPtr local_context) { - std::shared_ptr iterator_wrapper{nullptr}; if (distributed_processing) { return std::make_shared( @@ -567,7 +566,7 @@ std::shared_ptr StorageS3::createFileIterator( { /// Iterate through disclosed globs and make a source for each file auto glob_iterator = std::make_shared(*client_auth.client, client_auth.uri); - iterator_wrapper = std::make_shared([glob_iterator]() + return std::make_shared([glob_iterator]() { return glob_iterator->next(); }); @@ -575,7 +574,7 @@ std::shared_ptr StorageS3::createFileIterator( else { auto keys_iterator = std::make_shared(keys); - iterator_wrapper = std::make_shared([keys_iterator]() + return std::make_shared([keys_iterator]() { return keys_iterator->next(); }); @@ -662,7 +661,7 @@ SinkToStoragePtr StorageS3::write(const ASTPtr & query, const StorageMetadataPtr client_auth.uri.bucket, keys.back()); } - + auto sample_block = metadata_snapshot->getSampleBlock(); auto chosen_compression_method = chooseCompressionMethod(keys.back(), compression_method); bool has_wildcards = client_auth.uri.bucket.find(PARTITION_ID_WILDCARD) != String::npos || keys.back().find(PARTITION_ID_WILDCARD) != String::npos; From 253035a5df2b900f77dfbf0a1cf774123bf36efb Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 14 Jan 2022 19:17:06 +0300 Subject: [PATCH 046/403] Fix --- src/Formats/FormatFactory.cpp | 2 +- src/Processors/Formats/Impl/AvroRowOutputFormat.cpp | 1 + .../Formats/Impl/CustomSeparatedRowOutputFormat.cpp | 2 +- src/Processors/Formats/Impl/TemplateBlockOutputFormat.cpp | 4 ++-- src/Storages/StorageFile.cpp | 4 ++-- tests/queries/0_stateless/02168_avro_bug.sql | 6 +++--- 6 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 9548bb754fa..53f13bf97a7 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -402,7 +402,7 @@ void FormatFactory::registerAppendSupportChecker(const String & name, AppendSupp void FormatFactory::markFormatDoesntSupportAppend(const String & name) { - registerAppendSupportChecker(name, [](const FormatSettings &){ return true; }); + registerAppendSupportChecker(name, [](const FormatSettings &){ return false; }); } bool FormatFactory::checkIfFormatSupportAppend(const String & name, ContextPtr context, const std::optional & format_settings_) diff --git a/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp b/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp index fb3389475ac..ae5ce6099a9 100644 --- a/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowOutputFormat.cpp @@ -479,6 +479,7 @@ void registerOutputFormatAvro(FormatFactory & factory) { return std::make_shared(buf, sample, params, settings); }); + factory.markFormatDoesntSupportAppend("Avro"); } } diff --git a/src/Processors/Formats/Impl/CustomSeparatedRowOutputFormat.cpp b/src/Processors/Formats/Impl/CustomSeparatedRowOutputFormat.cpp index 88b9226fd8a..4c8cf19b923 100644 --- a/src/Processors/Formats/Impl/CustomSeparatedRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/CustomSeparatedRowOutputFormat.cpp @@ -94,7 +94,7 @@ void registerOutputFormatCustomSeparated(FormatFactory & factory) factory.registerAppendSupportChecker(format_name, [](const FormatSettings & settings) { - return !settings.custom.result_after_delimiter.empty(); + return settings.custom.result_after_delimiter.empty(); }); }; diff --git a/src/Processors/Formats/Impl/TemplateBlockOutputFormat.cpp b/src/Processors/Formats/Impl/TemplateBlockOutputFormat.cpp index 6babac98102..5c5b99f61da 100644 --- a/src/Processors/Formats/Impl/TemplateBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/TemplateBlockOutputFormat.cpp @@ -239,7 +239,7 @@ void registerOutputFormatTemplate(FormatFactory & factory) factory.registerAppendSupportChecker("Template", [](const FormatSettings & settings) { if (settings.template_settings.resultset_format.empty()) - return false; + return true; auto resultset_format = ParsedTemplateFormatString( FormatSchemaInfo(settings.template_settings.resultset_format, "Template", false, settings.schema.is_server, settings.schema.format_schema_path), @@ -247,7 +247,7 @@ void registerOutputFormatTemplate(FormatFactory & factory) { return static_cast(TemplateBlockOutputFormat::stringToResultsetPart(partName)); }); - return !resultset_format.delimiters.empty() && !resultset_format.delimiters.back().empty(); + return resultset_format.delimiters.empty() || resultset_format.delimiters.back().empty(); }); } } diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 48e0e82821a..dae162c9e1f 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -852,8 +852,8 @@ SinkToStoragePtr StorageFile::write( else throw Exception( ErrorCodes::CANNOT_APPEND_TO_FILE, - "Cannot append data in format {} to file, because this format contains suffix and " - "data can be written to a file only once. You can allow to create a new file " + "Cannot append data in format {} to file, because this format doesn't support appends." + " You can allow to create a new file " "on each insert by enabling setting engine_file_allow_create_multiple_files", format_name); } diff --git a/tests/queries/0_stateless/02168_avro_bug.sql b/tests/queries/0_stateless/02168_avro_bug.sql index 78eedf3258e..e50c78fb0b8 100644 --- a/tests/queries/0_stateless/02168_avro_bug.sql +++ b/tests/queries/0_stateless/02168_avro_bug.sql @@ -1,5 +1,5 @@ -- Tags: no-fasttest -insert into table function file('data.avro', 'Avro', 'x UInt64') select * from numbers(10); -insert into table function file('data.avro', 'Avro', 'x UInt64') select * from numbers(10); -insert into table function file('data.avro', 'Avro', 'x UInt64') select * from numbers(10); +insert into table function file('data.avro', 'Parquet', 'x UInt64') select * from numbers(10); +insert into table function file('data.avro', 'Parquet', 'x UInt64') select * from numbers(10); -- { serverError CANNOT_APPEND_TO_FILE } +insert into table function file('data.avro', 'Parquet', 'x UInt64') select * from numbers(10); -- { serverError CANNOT_APPEND_TO_FILE } select 'OK'; From af54ca4866c642fb63deb73fb89c62fd283aa52a Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 14 Jan 2022 21:18:16 +0300 Subject: [PATCH 047/403] Fix partition write --- src/Storages/HDFS/StorageHDFS.cpp | 66 +++++++++++++++---------------- src/Storages/StorageFile.cpp | 29 +++++++------- src/Storages/StorageS3.cpp | 58 +++++++++++++-------------- 3 files changed, 76 insertions(+), 77 deletions(-) diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index 6e975190ad1..a5d01bb1428 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -536,41 +536,8 @@ Pipe StorageHDFS::read( SinkToStoragePtr StorageHDFS::write(const ASTPtr & query, const StorageMetadataPtr & metadata_snapshot, ContextPtr context_) { - if (is_path_with_globs) - throw Exception(ErrorCodes::DATABASE_ACCESS_DENIED, "URI '{}' contains globs, so the table is in readonly mode", uris.back()); - String current_uri = uris.back(); - const auto [path_from_uri, uri_without_path] = getPathFromUriAndUriWithoutPath(current_uri); - - HDFSBuilderWrapper builder = createHDFSBuilder(uri_without_path + "/", context_->getGlobalContext()->getConfigRef()); - HDFSFSPtr fs = createHDFSFS(builder.get()); - - bool truncate_on_insert = context_->getSettingsRef().hdfs_truncate_on_insert; - if (!truncate_on_insert && !hdfsExists(fs.get(), path_from_uri.c_str())) - { - if (context_->getSettingsRef().hdfs_create_new_file_on_insert) - { - auto pos = uris[0].find_first_of('.', uris[0].find_last_of('/')); - size_t index = uris.size(); - String new_uri; - do - { - new_uri = uris[0].substr(0, pos) + "." + std::to_string(index) + (pos == std::string::npos ? "" : uris[0].substr(pos)); - ++index; - } - while (!hdfsExists(fs.get(), new_uri.c_str())); - uris.push_back(new_uri); - current_uri = new_uri; - } - else - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "File with path {} already exists. If you want to overwrite it, enable setting hdfs_truncate_on_insert, " - "if you want to create new file on each insert, enable setting hdfs_create_new_file_on_insert", - path_from_uri); - } - bool has_wildcards = current_uri.find(PartitionedSink::PARTITION_ID_WILDCARD) != String::npos; const auto * insert_query = dynamic_cast(query.get()); auto partition_by_ast = insert_query ? (insert_query->partition_by ? insert_query->partition_by : partition_by) : nullptr; @@ -588,6 +555,39 @@ SinkToStoragePtr StorageHDFS::write(const ASTPtr & query, const StorageMetadataP } else { + if (is_path_with_globs) + throw Exception(ErrorCodes::DATABASE_ACCESS_DENIED, "URI '{}' contains globs, so the table is in readonly mode", uris.back()); + + const auto [path_from_uri, uri_without_path] = getPathFromUriAndUriWithoutPath(current_uri); + + HDFSBuilderWrapper builder = createHDFSBuilder(uri_without_path + "/", context_->getGlobalContext()->getConfigRef()); + HDFSFSPtr fs = createHDFSFS(builder.get()); + + bool truncate_on_insert = context_->getSettingsRef().hdfs_truncate_on_insert; + if (!truncate_on_insert && !hdfsExists(fs.get(), path_from_uri.c_str())) + { + if (context_->getSettingsRef().hdfs_create_new_file_on_insert) + { + auto pos = uris[0].find_first_of('.', uris[0].find_last_of('/')); + size_t index = uris.size(); + String new_uri; + do + { + new_uri = uris[0].substr(0, pos) + "." + std::to_string(index) + (pos == std::string::npos ? "" : uris[0].substr(pos)); + ++index; + } + while (!hdfsExists(fs.get(), new_uri.c_str())); + uris.push_back(new_uri); + current_uri = new_uri; + } + else + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "File with path {} already exists. If you want to overwrite it, enable setting hdfs_truncate_on_insert, " + "if you want to create new file on each insert, enable setting hdfs_create_new_file_on_insert", + path_from_uri); + } + return std::make_shared(current_uri, format_name, metadata_snapshot->getSampleBlock(), diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index dae162c9e1f..e6c5f25dd57 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -605,7 +605,7 @@ public: int table_fd_, bool use_table_fd_, std::string base_path_, - std::vector paths_, + std::string path_, const CompressionMethod compression_method_, const std::optional & format_settings_, const String format_name_, @@ -617,7 +617,7 @@ public: , table_fd(table_fd_) , use_table_fd(use_table_fd_) , base_path(base_path_) - , paths(paths_) + , path(path_) , compression_method(compression_method_) , format_name(format_name_) , format_settings(format_settings_) @@ -634,7 +634,7 @@ public: int table_fd_, bool use_table_fd_, std::string base_path_, - std::vector paths_, + const std::string & path_, const CompressionMethod compression_method_, const std::optional & format_settings_, const String format_name_, @@ -646,7 +646,7 @@ public: , table_fd(table_fd_) , use_table_fd(use_table_fd_) , base_path(base_path_) - , paths(paths_) + , path(path_) , compression_method(compression_method_) , format_name(format_name_) , format_settings(format_settings_) @@ -668,9 +668,8 @@ public: } else { - assert(!paths.empty()); flags |= O_WRONLY | O_APPEND | O_CREAT; - naked_buffer = std::make_unique(paths.back(), DBMS_DEFAULT_BUFFER_SIZE, flags); + naked_buffer = std::make_unique(path, DBMS_DEFAULT_BUFFER_SIZE, flags); } /// In case of formats with prefixes if file is not empty we have already written prefix. @@ -710,7 +709,7 @@ private: int table_fd; bool use_table_fd; std::string base_path; - std::vector paths; + std::string path; CompressionMethod compression_method; std::string format_name; std::optional format_settings; @@ -753,7 +752,6 @@ public: { auto partition_path = PartitionedSink::replaceWildcards(path, partition_id); PartitionedSink::validatePartitionKey(partition_path, true); - Strings result_paths = {partition_path}; checkCreationIsAllowed(context, context->getUserFilesPath(), partition_path); return std::make_shared( metadata_snapshot, @@ -761,7 +759,7 @@ public: -1, /* use_table_fd */false, base_path, - result_paths, + partition_path, compression_method, format_settings, format_name, @@ -795,7 +793,6 @@ SinkToStoragePtr StorageFile::write( int flags = 0; - std::string path; if (context->getSettingsRef().engine_file_truncate_on_insert) flags |= O_TRUNC; @@ -816,7 +813,7 @@ SinkToStoragePtr StorageFile::write( std::unique_lock{rwlock, getLockTimeout(context)}, base_path, path_for_partitioned_write, - chooseCompressionMethod(path, compression_method), + chooseCompressionMethod(path_for_partitioned_write, compression_method), format_settings, format_name, context, @@ -824,14 +821,15 @@ SinkToStoragePtr StorageFile::write( } else { + String path; if (!paths.empty()) { - path = paths[0]; - fs::create_directories(fs::path(path).parent_path()); - if (is_path_with_globs) throw Exception("Table '" + getStorageID().getNameForLogs() + "' is in readonly mode because of globs in filepath", ErrorCodes::DATABASE_ACCESS_DENIED); + path = paths.back(); + fs::create_directories(fs::path(path).parent_path()); + if (!context->getSettingsRef().engine_file_truncate_on_insert && !is_path_with_globs && !FormatFactory::instance().checkIfFormatSupportAppend(format_name, context, format_settings) && fs::exists(paths.back()) && fs::file_size(paths.back()) != 0) @@ -848,6 +846,7 @@ SinkToStoragePtr StorageFile::write( } while (fs::exists(new_path)); paths.push_back(new_path); + path = new_path; } else throw Exception( @@ -866,7 +865,7 @@ SinkToStoragePtr StorageFile::write( table_fd, use_table_fd, base_path, - paths, + path, chooseCompressionMethod(path, compression_method), format_settings, format_name, diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 44fe7c3d4bf..2fa5ee0fa64 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -633,35 +633,6 @@ SinkToStoragePtr StorageS3::write(const ASTPtr & query, const StorageMetadataPtr { updateClientAndAuthSettings(local_context, client_auth); - if (is_key_with_globs) - throw Exception(ErrorCodes::DATABASE_ACCESS_DENIED, "S3 key '{}' contains globs, so the table is in readonly mode", client_auth.uri.key); - - bool truncate_in_insert = local_context->getSettingsRef().s3_truncate_on_insert; - - if (!truncate_in_insert && checkIfObjectExists(client_auth.client, client_auth.uri.bucket, keys.back())) - { - if (local_context->getSettingsRef().s3_create_new_file_on_insert) - { - size_t index = keys.size(); - auto pos = keys[0].find_first_of('.'); - String new_key; - do - { - new_key = keys[0].substr(0, pos) + "." + std::to_string(index) + (pos == std::string::npos ? "" : keys[0].substr(pos)); - ++index; - } - while (checkIfObjectExists(client_auth.client, client_auth.uri.bucket, new_key)); - keys.push_back(new_key); - } - else - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Object in bucket {} with key {} already exists. If you want to overwrite it, enable setting s3_truncate_on_insert, if you " - "want to create a new file on each insert, enable setting s3_create_new_file_on_insert", - client_auth.uri.bucket, - keys.back()); - } - auto sample_block = metadata_snapshot->getSampleBlock(); auto chosen_compression_method = chooseCompressionMethod(keys.back(), compression_method); bool has_wildcards = client_auth.uri.bucket.find(PARTITION_ID_WILDCARD) != String::npos || keys.back().find(PARTITION_ID_WILDCARD) != String::npos; @@ -687,6 +658,35 @@ SinkToStoragePtr StorageS3::write(const ASTPtr & query, const StorageMetadataPtr } else { + if (is_key_with_globs) + throw Exception(ErrorCodes::DATABASE_ACCESS_DENIED, "S3 key '{}' contains globs, so the table is in readonly mode", client_auth.uri.key); + + bool truncate_in_insert = local_context->getSettingsRef().s3_truncate_on_insert; + + if (!truncate_in_insert && checkIfObjectExists(client_auth.client, client_auth.uri.bucket, keys.back())) + { + if (local_context->getSettingsRef().s3_create_new_file_on_insert) + { + size_t index = keys.size(); + auto pos = keys[0].find_first_of('.'); + String new_key; + do + { + new_key = keys[0].substr(0, pos) + "." + std::to_string(index) + (pos == std::string::npos ? "" : keys[0].substr(pos)); + ++index; + } + while (checkIfObjectExists(client_auth.client, client_auth.uri.bucket, new_key)); + keys.push_back(new_key); + } + else + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Object in bucket {} with key {} already exists. If you want to overwrite it, enable setting s3_truncate_on_insert, if you " + "want to create a new file on each insert, enable setting s3_create_new_file_on_insert", + client_auth.uri.bucket, + keys.back()); + } + return std::make_shared( format_name, sample_block, From 8d2791a1c3b232aa930daa437d4bbbe966d4aba1 Mon Sep 17 00:00:00 2001 From: Tiaonmmn Date: Sun, 16 Jan 2022 00:30:37 +0800 Subject: [PATCH 048/403] Update apply-catboost-model.md --- docs/zh/guides/apply-catboost-model.md | 32 +++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/zh/guides/apply-catboost-model.md b/docs/zh/guides/apply-catboost-model.md index 72f5fa38e84..adc5b48eb55 100644 --- a/docs/zh/guides/apply-catboost-model.md +++ b/docs/zh/guides/apply-catboost-model.md @@ -1,6 +1,4 @@ --- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 41 toc_title: "\u5E94\u7528CatBoost\u6A21\u578B" --- @@ -10,10 +8,10 @@ toc_title: "\u5E94\u7528CatBoost\u6A21\u578B" [CatBoost](https://catboost.ai) 是一个由[Yandex](https://yandex.com/company/)开发的开源免费机器学习库。 -通过这篇指导，您将学会如何用SQL建模，使用ClickHouse预先训练好的模型来推断数据。 +通过本篇文档，您将学会如何用SQL语句调用已经存放在Clickhouse中的预训练模型来预测数据。 -在ClickHouse中应用CatBoost模型的一般过程: +为了在ClickHouse中应用CatBoost模型，需要进行如下步骤： 1. [创建数据表](#create-table). 2. [将数据插入到表中](#insert-data-to-table). @@ -22,24 +20,26 @@ toc_title: "\u5E94\u7528CatBoost\u6A21\u578B" 有关训练CatBoost模型的详细信息，请参阅 [训练和模型应用](https://catboost.ai/docs/features/training.html#training). +您可以通过[RELOAD MODEL](https://clickhouse.com/docs/en/sql-reference/statements/system/#query_language-system-reload-model)与[RELOAD MODELS](https://clickhouse.com/docs/en/sql-reference/statements/system/#query_language-system-reload-models)语句来重载CatBoost模型。 + ## 先决条件 {#prerequisites} 请先安装 [Docker](https://docs.docker.com/install/)。 !!! note "注" - [Docker](https://www.docker.com) 是一个软件平台，用户可以用来创建独立于其余系统、集成CatBoost和ClickHouse的容器。 + [Docker](https://www.docker.com) 是一个软件平台，用户可以用Docker来创建独立于已有系统并集成了CatBoost和ClickHouse的容器。 在应用CatBoost模型之前: -**1.** 从容器仓库拉取docker映像 (https://hub.docker.com/r/yandex/tutorial-catboost-clickhouse) : +**1.** 从容器仓库拉取示例docker镜像 (https://hub.docker.com/r/yandex/tutorial-catboost-clickhouse) : ``` bash $ docker pull yandex/tutorial-catboost-clickhouse ``` -此Docker映像包含运行CatBoost和ClickHouse所需的所有内容：代码、运行环境、库、环境变量和配置文件。 +此示例Docker镜像包含运行CatBoost和ClickHouse所需的所有内容：代码、运行时、库、环境变量和配置文件。 -**2.** 确保已成功拉取Docker映像: +**2.** 确保已成功拉取Docker镜像: ``` bash $ docker image ls @@ -47,7 +47,7 @@ REPOSITORY TAG IMAGE ID CR yandex/tutorial-catboost-clickhouse latest 622e4d17945b 22 hours ago 1.37GB ``` -**3.** 基于此映像启动一个Docker容器: +**3.** 基于此镜像启动一个Docker容器: ``` bash $ docker run -it -p 8888:8888 yandex/tutorial-catboost-clickhouse @@ -124,9 +124,9 @@ FROM amazon_train ## 3. 将CatBoost集成到ClickHouse中 {#integrate-catboost-into-clickhouse} !!! note "注" - **可跳过。** Docker映像包含运行CatBoost和ClickHouse所需的所有内容。 + **可跳过。** 示例Docker映像已经包含了运行CatBoost和ClickHouse所需的所有内容。 -CatBoost集成到ClickHouse步骤: +为了将CatBoost集成进ClickHouse，需要进行如下步骤： **1.** 构建评估库。 @@ -134,13 +134,13 @@ CatBoost集成到ClickHouse步骤: 有关如何构建库文件的详细信息，请参阅 [CatBoost文件](https://catboost.ai/docs/concepts/c-plus-plus-api_dynamic-c-pluplus-wrapper.html). -**2.** 创建一个新目录（位置与名称可随意指定）, 如 `data` 并将创建的库文件放入其中。 Docker映像已经包含了库 `data/libcatboostmodel.so`. +**2.** 创建一个新目录（位置与名称可随意指定）, 如 `data` 并将创建的库文件放入其中。 示例Docker镜像已经包含了库 `data/libcatboostmodel.so`. **3.** 创建一个新目录来放配置模型, 如 `models`. **4.** 创建一个模型配置文件，如 `models/amazon_model.xml`. -**5.** 描述模型配置: +**5.** 修改模型配置: ``` xml @@ -165,9 +165,9 @@ CatBoost集成到ClickHouse步骤: /home/catboost/models/*_model.xml ``` -## 4. 运行从SQL推断的模型 {#run-model-inference} +## 4. 使用SQL调用预测模型 {#run-model-inference} -测试模型是否正常，运行ClickHouse客户端 `$ clickhouse client`. +为了测试模型是否正常，可以使用ClickHouse客户端 `$ clickhouse client`. 让我们确保模型能正常工作: @@ -189,7 +189,7 @@ LIMIT 10 ``` !!! note "注" - 函数 [modelEvaluate](../sql-reference/functions/other-functions.md#function-modelevaluate) 返回带有多类模型的每类原始预测的元组。 + 函数 [modelEvaluate](../sql-reference/functions/other-functions.md#function-modelevaluate) 会对多类别模型返回一个元组，其中包含每一类别的原始预测值。 执行预测: From 1cdadbd250a37821a8f3ce176e22f05acaa176e9 Mon Sep 17 00:00:00 2001 From: feng lv Date: Sun, 16 Jan 2022 06:25:11 +0000 Subject: [PATCH 049/403] Fix sumIf rewrite --- .../RewriteSumIfFunctionVisitor.cpp | 3 +- .../02177_sum_if_not_found.reference | 3 ++ .../0_stateless/02177_sum_if_not_found.sql | 35 +++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/02177_sum_if_not_found.reference create mode 100644 tests/queries/0_stateless/02177_sum_if_not_found.sql diff --git a/src/Interpreters/RewriteSumIfFunctionVisitor.cpp b/src/Interpreters/RewriteSumIfFunctionVisitor.cpp index 7f725c1d8a5..50e6bec28f3 100644 --- a/src/Interpreters/RewriteSumIfFunctionVisitor.cpp +++ b/src/Interpreters/RewriteSumIfFunctionVisitor.cpp @@ -25,7 +25,8 @@ void RewriteSumIfFunctionMatcher::visit(const ASTFunction & func, ASTPtr & ast, auto lower_name = Poco::toLower(func.name); - if (lower_name != "sum" && lower_name != "sumif") + /// sumIf, SumIf or sUMIf are valid function names, but sumIF or sumiF are not + if (lower_name != "sum" && (lower_name != "sumif" || !endsWith(func.name, "If"))) return; const auto & func_arguments = func.arguments->children; diff --git a/tests/queries/0_stateless/02177_sum_if_not_found.reference b/tests/queries/0_stateless/02177_sum_if_not_found.reference new file mode 100644 index 00000000000..bb0b1cf658d --- /dev/null +++ b/tests/queries/0_stateless/02177_sum_if_not_found.reference @@ -0,0 +1,3 @@ +0 +0 +0 diff --git a/tests/queries/0_stateless/02177_sum_if_not_found.sql b/tests/queries/0_stateless/02177_sum_if_not_found.sql new file mode 100644 index 00000000000..c888f8b39aa --- /dev/null +++ b/tests/queries/0_stateless/02177_sum_if_not_found.sql @@ -0,0 +1,35 @@ +SELECT sumIf(1, 0); +SELECT SumIf(1, 0); +SELECT sUmIf(1, 0); +SELECT sumIF(1, 0); -- { serverError 46 } + +DROP TABLE IF EXISTS data; +DROP TABLE IF EXISTS agg; + +CREATE TABLE data +( + `n` UInt32, + `t` DateTime +) +ENGINE = Null; + +CREATE TABLE agg +ENGINE = AggregatingMergeTree +ORDER BY tuple() AS +SELECT + t, + sumIF(n, 0) +FROM data +GROUP BY t; -- { serverError 46} + +CREATE TABLE agg +ENGINE = AggregatingMergeTree +ORDER BY tuple() AS +SELECT + t, + sumIf(n, 0) +FROM data +GROUP BY t; + +DROP TABLE data; +DROP TABLE agg; From 06267208930f27d540ad7fe24addf36acf93a0ab Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 11 Jan 2022 13:14:42 +0300 Subject: [PATCH 050/403] Added IntervalTree --- src/Common/IntervalTree.h | 566 +++++++++++++++++++++++++++++ src/Common/examples/CMakeLists.txt | 3 + 2 files changed, 569 insertions(+) create mode 100644 src/Common/IntervalTree.h diff --git a/src/Common/IntervalTree.h b/src/Common/IntervalTree.h new file mode 100644 index 00000000000..f521c4d098c --- /dev/null +++ b/src/Common/IntervalTree.h @@ -0,0 +1,566 @@ +#pragma once + +#include + +#include + + +namespace DB +{ + +template +struct Interval +{ + using IntervalStorageType = TIntervalStorageType; + IntervalStorageType left; + IntervalStorageType right; + + Interval(IntervalStorageType left_, IntervalStorageType right_) + : left(left_) + , right(right_) + {} + + inline bool contains(IntervalStorageType point) const + { + return left <= point && point <= right; + } +}; + +template +bool operator<(const Interval & lhs, const Interval & rhs) +{ + return std::tie(lhs.left, lhs.right) < std::tie(rhs.left, rhs.right); +} + +template +bool operator==(const Interval & lhs, const Interval & rhs) +{ + return std::tie(lhs.left, lhs.right) == std::tie(lhs.left, rhs.right); +} + +template +bool operator>(const Interval & lhs, const Interval & rhs) +{ + return std::tie(lhs.left, lhs.right) > std::tie(rhs.left, rhs.right); +} + +struct IntervalTreeVoidValue {}; + +template +class IntervalTree +{ +public: + using IntervalStorageType = typename Interval::IntervalStorageType; + + static constexpr bool is_empty_value = std::is_same_v; + static constexpr bool is_nonempty_value = !std::is_same_v; + + template , bool> = true> + void emplace(Interval interval) + { + ++intervals_size; + sorted_intervals.emplace_back(interval); + } + + template , bool> = true, typename ...Args> + void emplace(Interval interval, Args && ... args) + { + ++intervals_size; + sorted_intervals.emplace_back(interval, Value(std::forward(args)...)); + } + + template , bool> = true> + void insert(Interval interval) + { + ++intervals_size; + sorted_intervals.emplace_back(interval); + } + + template , bool> = true> + void insert(Interval interval, const Value & value) + { + ++intervals_size; + sorted_intervals.emplace_back(interval, value); + } + + template , bool> = true> + void insert(Interval interval, Value && value) + { + ++intervals_size; + sorted_intervals.emplace_back(interval, std::move(value)); + } + + void construct() + { + nodes.resize(sorted_intervals.size()); + buildTree(); + } + + template + void find(IntervalStorageType point, IntervalCallback && callback) const + { + findIntervalsImpl(point, callback); + } + + bool has(IntervalStorageType point) const + { + bool has_intervals = false; + + if constexpr (is_empty_value) + { + find(point, [&](auto &) + { + has_intervals = true; + return false; + }); + } + else + { + find(point, [&](auto &, auto &) + { + has_intervals = true; + return false; + }); + } + + return has_intervals; + } + + + struct Node + { + size_t sorted_intervals_range_start_index; + size_t sorted_intervals_range_size; + + IntervalStorageType middle_element; + + inline bool hasValue() const + { + return sorted_intervals_range_size != 0; + } + }; + + using IntervalWithEmptyValue = Interval; + using IntervalWithNonEmptyValue = std::pair; + + using IntervalWithValue = std::conditional_t; + + class Iterator + { + public: + + bool operator==(const Iterator & rhs) const + { + return node_index == rhs.node_index & tree == rhs.tree && current_interval_index == rhs.current_interval_index; + } + + bool operator!=(const Iterator & rhs) const + { + return !(*this == rhs); + } + + const IntervalWithValue & operator*() + { + return getCurrentValue(); + } + + const IntervalWithValue & operator*() const + { + return getCurrentValue(); + } + + const IntervalWithValue *operator->() + { + return &getCurrentValue(); + } + + const IntervalWithValue *operator->() const + { + return &getCurrentValue(); + } + + Iterator &operator++() { + iterateToNext(); + return *this; + } + + // Iterator operator++(int) { + // Iterator copy(*this); + // iterateToNext(); + // return copy; + // } + + // Iterator &operator--() { + // iterateToPrevious(); + // return *this; + // } + + // Iterator operator--(int) { + // Iterator copy(*this); + // iterateToNext(); + // return copy; + // } + + private: + friend class IntervalTree; + + Iterator(size_t node_index_, size_t current_interval_index_, const IntervalTree * tree_) + : node_index(node_index_) + , current_interval_index(current_interval_index_) + , tree(tree_) + {} + + size_t node_index; + size_t current_interval_index; + const IntervalTree * tree; + + void iterateToNext() + { + size_t nodes_size = tree->nodes.size(); + auto & current_node = tree->nodes[node_index]; + + ++current_interval_index; + + if (current_interval_index < current_node.sorted_intervals_range_size) + return; + + size_t node_index_copy = node_index + 1; + for (; node_index_copy < nodes_size; ++node_index_copy) + { + auto & node = tree->nodes[node_index_copy]; + + if (node.hasValue()) + { + node_index = node_index_copy; + current_interval_index = 0; + break; + } + } + } + + void iterateToPrevious() + { + if (current_interval_index > 0) + { + --current_interval_index; + return; + } + + while (node_index > 0) + { + auto & node = tree->nodes[node_index - 1]; + if (node.hasValue()) + { + current_interval_index = node.sorted_intervals_range_size - 1; + break; + } + + --node_index; + } + } + + const IntervalWithValue & getCurrentValue() + { + auto & current_node = tree->nodes[node_index]; + size_t interval_index = current_node.sorted_intervals_range_start_index + current_interval_index; + return tree->sorted_intervals[interval_index]; + } + }; + + using iterator = Iterator; + using const_iterator = Iterator; + + iterator begin() + { + size_t start_index = findFirstIteratorNodeIndex(); + return Iterator(start_index, 0, this); + } + + iterator end() + { + size_t end_index = findLastIteratorNodeIndex(); + size_t last_interval_index = nodes[end_index].sorted_intervals_range_size; + return Iterator(end_index, last_interval_index, this); + } + + const_iterator begin() const + { + size_t start_index = findFirstIteratorNodeIndex(); + return Iterator(start_index, 0, this); + } + + const_iterator end() const + { + size_t end_index = findLastIteratorNodeIndex(); + size_t last_interval_index = nodes[end_index].sorted_intervals_range_size; + return Iterator(end_index, last_interval_index, this); + } + + const_iterator cbegin() const + { + return begin(); + } + + const_iterator cend() const + { + return end(); + } + + size_t getIntervalsSize() const + { + return intervals_size; + } + +private: + + void buildTree() + { + std::vector temporary_points_storage; + temporary_points_storage.reserve(sorted_intervals.size() * 2); + + std::vector left_intervals; + std::vector right_intervals; + std::vector intervals_sorted_by_left_asc; + std::vector intervals_sorted_by_right_desc; + + struct StackFrame + { + size_t index; + std::vector intervals; + }; + + std::vector stack; + stack.emplace_back(StackFrame{0, std::move(sorted_intervals)}); + sorted_intervals.clear(); + + while (!stack.empty()) + { + auto frame = std::move(stack.back()); + stack.pop_back(); + + size_t current_index = frame.index; + auto & current_intervals = frame.intervals; + + if (current_intervals.empty()) + continue; + + if (current_index >= nodes.size()) + nodes.resize(current_index + 1); + + temporary_points_storage.clear(); + intervalsToPoints(current_intervals, temporary_points_storage); + auto median = pointsMedian(temporary_points_storage); + + left_intervals.clear(); + right_intervals.clear(); + intervals_sorted_by_left_asc.clear(); + intervals_sorted_by_right_desc.clear(); + + for (const auto & interval_with_value : current_intervals) + { + auto & interval = getInterval(interval_with_value); + + if (interval.right < median) + { + left_intervals.emplace_back(interval_with_value); + } + else if (interval.left > median) + { + right_intervals.emplace_back(interval_with_value); + } + else + { + intervals_sorted_by_left_asc.emplace_back(interval_with_value); + intervals_sorted_by_right_desc.emplace_back(interval_with_value); + } + } + + std::sort(intervals_sorted_by_left_asc.begin(), intervals_sorted_by_left_asc.end(), [](auto & lhs, auto & rhs) + { + auto & lhs_interval = getInterval(lhs); + auto & rhs_interval = getInterval(rhs); + return lhs_interval < rhs_interval; + }); + + std::sort(intervals_sorted_by_right_desc.begin(), intervals_sorted_by_right_desc.end(), [](auto & lhs, auto & rhs) + { + auto & lhs_interval = getInterval(lhs); + auto & rhs_interval = getInterval(rhs); + return lhs_interval > rhs_interval; + }); + + size_t sorted_intervals_range_start_index = sorted_intervals.size(); + + sorted_intervals.insert(sorted_intervals.end(), intervals_sorted_by_left_asc.begin(), intervals_sorted_by_left_asc.end()); + sorted_intervals.insert(sorted_intervals.end(), intervals_sorted_by_right_desc.begin(), intervals_sorted_by_right_desc.end()); + + auto & node = nodes[current_index]; + node.middle_element = median; + node.sorted_intervals_range_start_index = sorted_intervals_range_start_index; + node.sorted_intervals_range_size = intervals_sorted_by_left_asc.size(); + + size_t left_child_index = current_index * 2 + 1; + stack.emplace_back(StackFrame{left_child_index, std::move(left_intervals)}); + + size_t right_child_index = current_index * 2 + 2; + stack.emplace_back(StackFrame{right_child_index, std::move(right_intervals)}); + } + } + + template + void findIntervalsImpl(IntervalStorageType point, IntervalCallback && callback) const + { + size_t current_index = 0; + + while (true) + { + if (current_index >= nodes.size()) + break; + + auto & node = nodes[current_index]; + if (!node.hasValue()) + break; + + auto middle_element = node.middle_element; + + if (point < middle_element) + { + size_t start = node.sorted_intervals_range_start_index; + size_t end = start + node.sorted_intervals_range_size; + + for (; start != end; ++start) + { + auto & interval_with_value_left_sorted_asc = sorted_intervals[start]; + auto & interval_left_sorted_asc = getInterval(interval_with_value_left_sorted_asc); + if (interval_left_sorted_asc.left > point) + break; + + bool should_continue = callCallback(interval_with_value_left_sorted_asc, callback); + if (unlikely(!should_continue)) + return; + } + + size_t left_child_index = current_index * 2 + 1; + current_index = left_child_index; + } + else + { + size_t start = node.sorted_intervals_range_start_index + node.sorted_intervals_range_size; + size_t end = start + node.sorted_intervals_range_size; + + for (; start != end; ++start) + { + auto & interval_with_value_right_sorted_desc = sorted_intervals[start]; + auto & interval_right_sorted_desc = getInterval(interval_with_value_right_sorted_desc); + if (interval_right_sorted_desc.right < point) + break; + + bool should_continue = callCallback(interval_with_value_right_sorted_desc, callback); + if (unlikely(!should_continue)) + return; + } + + if (likely(point > middle_element)) + { + size_t right_child_index = current_index * 2 + 2; + current_index = right_child_index; + } + else + { + break; + } + } + } + } + + size_t findFirstIteratorNodeIndex() const + { + if (nodes.empty()) + return 0; + + size_t nodes_size = nodes.size(); + size_t result_index = 0; + + for (; result_index < nodes_size; ++result_index) + { + if (nodes[result_index].hasValue()) + break; + } + + return result_index; + } + + size_t findLastIteratorNodeIndex() const + { + if (nodes.empty()) + return 0; + + size_t nodes_size = nodes.size(); + int64_t result_index = static_cast(nodes_size - 1); + for (; result_index >= 0; --result_index) + { + if (nodes[result_index].hasValue()) + break; + } + + size_t result = static_cast(result_index); + return result; + } + + std::vector nodes; + std::vector sorted_intervals; + size_t intervals_size = 0; + + static const Interval & getInterval(const IntervalWithValue & interval_with_value) + { + if constexpr (is_empty_value) + return interval_with_value; + else + return interval_with_value.first; + } + + template + static bool callCallback(const IntervalWithValue & interval, IntervalCallback && callback) + { + if constexpr (is_empty_value) + return callback(interval); + else + return callback(interval.first, interval.second); + } + + static void intervalsToPoints(const std::vector & intervals, std::vector & temporary_points_storage) + { + for (const auto & interval_with_value : intervals) + { + auto & interval = getInterval(interval_with_value); + temporary_points_storage.emplace_back(interval.left); + temporary_points_storage.emplace_back(interval.right); + } + } + + static IntervalStorageType pointsMedian(std::vector & points) + { + size_t size = points.size(); + size_t middle_element_index = size / 2; + + std::nth_element(points.begin(), points.begin() + middle_element_index, points.end()); + + if (size % 2 == 0) + { + return (points[middle_element_index] + points[middle_element_index - 1]) / 2; + } + else + { + return points[middle_element_index]; + } + } + +}; + +template +using IntervalSet = IntervalTree; + +template +using IntervalMap = IntervalTree; + +} diff --git a/src/Common/examples/CMakeLists.txt b/src/Common/examples/CMakeLists.txt index be91101ef40..7b21591f83e 100644 --- a/src/Common/examples/CMakeLists.txt +++ b/src/Common/examples/CMakeLists.txt @@ -83,3 +83,6 @@ target_link_libraries (executable_udf PRIVATE dbms) add_executable(hive_metastore_client hive_metastore_client.cpp) target_link_libraries (hive_metastore_client PUBLIC hivemetastore ${THRIFT_LIBRARY}) + +add_executable (interval_tree interval_tree.cpp) +target_link_libraries (interval_tree PRIVATE dbms) From 1076ec55393764024d5c8bb500ea701e359a9d89 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 11 Jan 2022 13:15:00 +0300 Subject: [PATCH 051/403] Added IntervalTree example --- src/Common/examples/interval_tree.cpp | 120 ++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 src/Common/examples/interval_tree.cpp diff --git a/src/Common/examples/interval_tree.cpp b/src/Common/examples/interval_tree.cpp new file mode 100644 index 00000000000..ee6459e3345 --- /dev/null +++ b/src/Common/examples/interval_tree.cpp @@ -0,0 +1,120 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +using namespace DB; +using Int64Interval = Interval; + +struct CollectIntervalsCallback +{ + explicit CollectIntervalsCallback(std::set & result_intervals_) + : result_intervals(result_intervals_) + { + } + + bool operator()(Int64Interval interval) + { + result_intervals.insert(interval); + return true; + } + + std::set & result_intervals; +}; + +void findNecessaryIntervals(const std::vector & intervals, Int64 point, std::set & result_intervals) +{ + for (const auto & interval : intervals) + { + if (interval.contains(point)) + result_intervals.insert(interval); + } +} + +int main(int, char **) +{ + { + IntervalSet tree; + + tree.emplace(Int64Interval(0, 5)); + tree.emplace(Int64Interval(10, 15)); + + tree.construct(); + + for (const auto & interval : tree) + { + std::cout << "Interval left " << interval.left << " right " << interval.right << std::endl; + } + } + { + IntervalMap tree; + + tree.emplace(Int64Interval(0, 5), "value1"); + tree.emplace(Int64Interval(10, 15), "value2"); + + tree.construct(); + + for (const auto & [interval, value] : tree) + { + std::cout << "Interval left " << interval.left << " right " << interval.right; + std::cout << " value " << value << std::endl; + } + } + { + IntervalSet tree; + for (size_t i = 0; i < 5; ++i) + { + tree.emplace(Int64Interval(0, i)); + } + + tree.construct(); + + for (const auto & interval : tree) + { + std::cout << "Interval left " << interval.left << " right " << interval.right << std::endl; + } + + for (Int64 i = 0; i < 5; ++i) + { + tree.find(i, [](auto & interval) + { + std::cout << "Interval left " << interval.left << " right " << interval.right << std::endl; + return true; + }); + } + } + { + IntervalMap tree; + for (size_t i = 0; i < 5; ++i) + { + tree.emplace(Int64Interval(0, i), "Value " + std::to_string(i)); + } + + tree.construct(); + + for (const auto & [interval, value] : tree) + { + std::cout << "Interval left " << interval.left << " right " << interval.right; + std::cout << " value " << value << std::endl; + } + + for (Int64 i = 0; i < 5; ++i) + { + tree.find(i, [](auto & interval, auto & value) + { + std::cout << "Interval left " << interval.left << " right " << interval.right; + std::cout << " value " << value << std::endl; + + return true; + }); + } + } + + return 0; +} From 9c3cc7adab74658c957e6e1b8d7658182ba492c5 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 11 Jan 2022 13:15:12 +0300 Subject: [PATCH 052/403] RangeHashedDictionary use IntervalTree --- src/Dictionaries/RangeHashedDictionary.cpp | 150 ++++++++++++--------- src/Dictionaries/RangeHashedDictionary.h | 31 ++--- 2 files changed, 100 insertions(+), 81 deletions(-) diff --git a/src/Dictionaries/RangeHashedDictionary.cpp b/src/Dictionaries/RangeHashedDictionary.cpp index 9dcc38dc4b2..028818f2019 100644 --- a/src/Dictionaries/RangeHashedDictionary.cpp +++ b/src/Dictionaries/RangeHashedDictionary.cpp @@ -22,6 +22,11 @@ using RangeStorageType = DB::RangeStorageType; const RangeStorageType RANGE_MIN_NULL_VALUE = std::numeric_limits::max(); const RangeStorageType RANGE_MAX_NULL_VALUE = std::numeric_limits::lowest(); +bool isCorrectDate(const RangeStorageType & date) +{ + return 0 < date && date <= DATE_LUT_MAX_DAY_NUM; +} + // Handle both kinds of null values: explicit nulls of NullableColumn and 'implicit' nulls of Date type. RangeStorageType getColumnIntValueOrDefault(const DB::IColumn & column, size_t index, bool isDate, const RangeStorageType & default_value) { @@ -29,7 +34,7 @@ RangeStorageType getColumnIntValueOrDefault(const DB::IColumn & column, size_t i return default_value; const RangeStorageType result = static_cast(column.getInt(index)); - if (isDate && !DB::Range::isCorrectDate(result)) + if (isDate && !isCorrectDate(result)) return default_value; return result; @@ -57,20 +62,6 @@ namespace ErrorCodes extern const int UNSUPPORTED_METHOD; } -bool Range::isCorrectDate(const RangeStorageType & date) -{ - return 0 < date && date <= DATE_LUT_MAX_DAY_NUM; -} - -bool Range::contains(const RangeStorageType & value) const -{ - return left <= value && value <= right; -} - -static bool operator<(const Range & left, const Range & right) -{ - return std::tie(left.left, left.right) < std::tie(right.left, right.right); -} template RangeHashedDictionary::RangeHashedDictionary( @@ -260,16 +251,8 @@ ColumnUInt8::Ptr RangeHashedDictionary::hasKeys(const Colum if (it) { const auto date = dates[key_index]; - const auto & ranges_and_values = it->getMapped(); - const auto val_it = std::find_if( - std::begin(ranges_and_values), - std::end(ranges_and_values), - [date](const Value & v) - { - return v.range.contains(date); - }); - - out[key_index] = val_it != std::end(ranges_and_values); + const auto & interval_tree = it->getMapped(); + out[key_index] = interval_tree.has(date); keys_found += out[key_index]; } else @@ -324,6 +307,8 @@ void RangeHashedDictionary::loadData() updateData(); } + constructAttributeIntervalTrees(); + if (require_nonempty && 0 == element_count) throw Exception(ErrorCodes::DICTIONARY_IS_EMPTY, "{}: dictionary source is empty and 'require_nonempty' property is set."); @@ -399,6 +384,9 @@ void RangeHashedDictionary::getItemsImpl( DictionaryKeysExtractor keys_extractor(key_columns_copy, arena_holder.getComplexKeyArena()); const size_t keys_size = keys_extractor.getKeysSize(); + + static constexpr auto max_range_value = std::numeric_limits::min(); + for (size_t key_index = 0; key_index < keys_size; ++key_index) { auto key = keys_extractor.extractCurrentKey(); @@ -407,30 +395,57 @@ void RangeHashedDictionary::getItemsImpl( if (it) { const auto date = dates[key_index]; - const auto & ranges_and_values = it->getMapped(); - const auto val_it = std::find_if( - std::begin(ranges_and_values), - std::end(ranges_and_values), - [date](const Value & v) - { - return v.range.contains(date); - }); + const auto & interval_tree = it->getMapped(); - if (val_it != std::end(ranges_and_values)) + std::optional min_value; + RangeInterval min_range {max_range_value, max_range_value}; + bool has_interval = false; + + std::cout << "Date " << date << " intervals size " << interval_tree.getIntervalsSize() << std::endl; + for (auto & [range, _] : interval_tree) + { + std::cout << "Range left " << range.left << " range right " << range.right << std::endl; + } + + interval_tree.find(date, [&](auto & interval, auto & value) + { + std::cerr << "Find interval " << std::endl; + has_interval = true; + + if (min_range < interval) + min_value = value; + + return true; + }); + // auto value_it = std::lower_bound( + // std::begin(ranges_and_values), + // std::end(ranges_and_values), + // Range{date, date}, + // [](const Value & value, const Range & date_range) + // { + // return value.range < date_range; + // }); + + // if (value_it != std::end(ranges_and_values) && value_it != ranges_and_values.begin()) + // --value_it; + + std::cout << "Has interval " << has_interval << " min interval "; + std::cout << " left " << min_range.left << " right " << min_range.right << std::endl; + + if (min_value.has_value()) { ++keys_found; - auto & value = val_it->value; if constexpr (is_nullable) { - if (value.has_value()) - set_value(key_index, *value, false); + if (min_value.has_value()) + set_value(key_index, *min_value, false); else set_value(key_index, default_value_extractor[key_index], true); } else { - set_value(key_index, *value, false); + set_value(key_index, *min_value, false); } keys_extractor.rollbackCurrentKey(); @@ -542,7 +557,7 @@ void RangeHashedDictionary::blockToAttributes(const Block & if constexpr (std::is_same_v) key = copyStringInArena(string_arena, key); - setAttributeValue(attribute, key, Range{lower_bound, upper_bound}, attribute_column[key_index]); + setAttributeValue(attribute, key, RangeInterval{lower_bound, upper_bound}, attribute_column[key_index]); keys_extractor.rollbackCurrentKey(); } @@ -550,18 +565,38 @@ void RangeHashedDictionary::blockToAttributes(const Block & } } +template +void RangeHashedDictionary::constructAttributeIntervalTrees() +{ + for (auto & attribute : attributes) + { + auto type_call = [&](const auto & dictionary_attribute_type) + { + using Type = std::decay_t; + using AttributeType = typename Type::AttributeType; + using ValueType = DictionaryValueType; + + auto & collection = std::get>(attribute.maps); + for (auto & [_, ranges] : collection) + ranges.construct(); + }; + + callOnDictionaryAttributeType(attribute.type, type_call); + } +} + template template -void RangeHashedDictionary::setAttributeValueImpl(Attribute & attribute, KeyType key, const Range & range, const Field & value) +void RangeHashedDictionary::setAttributeValueImpl(Attribute & attribute, KeyType key, const RangeInterval & interval, const Field & value) { using ValueType = std::conditional_t, StringRef, T>; auto & collection = std::get>(attribute.maps); - Value value_to_insert; + std::optional value_to_insert; if (attribute.is_nullable && value.isNull()) { - value_to_insert = { range, {} }; + value_to_insert = std::nullopt; } else { @@ -569,11 +604,11 @@ void RangeHashedDictionary::setAttributeValueImpl(Attribute { const auto & string = value.get(); StringRef string_ref = copyStringInArena(string_arena, string); - value_to_insert = Value{ range, { string_ref }}; + value_to_insert = { string_ref }; } else { - value_to_insert = Value{ range, { value.get() }}; + value_to_insert = { value.get() }; } } @@ -582,26 +617,18 @@ void RangeHashedDictionary::setAttributeValueImpl(Attribute if (it) { auto & values = it->getMapped(); - - const auto insert_it = std::lower_bound( - std::begin(values), - std::end(values), - range, - [](const Value & lhs, const Range & rhs_range) - { - return lhs.range < rhs_range; - }); - - values.insert(insert_it, std::move(value_to_insert)); + values.emplace(interval, std::move(value_to_insert)); } else { - collection.insert({key, Values{std::move(value_to_insert)}}); + Values values; + values.emplace(interval, value_to_insert); + collection.insert({key, std::move(values)}); } } template -void RangeHashedDictionary::setAttributeValue(Attribute & attribute, KeyType key, const Range & range, const Field & value) +void RangeHashedDictionary::setAttributeValue(Attribute & attribute, KeyType key, const RangeInterval & range, const Field & value) { auto type_call = [&](const auto &dictionary_attribute_type) { @@ -650,14 +677,15 @@ void RangeHashedDictionary::getKeysAndDates( end_dates.reserve(collection.size()); const bool is_date = isDate(dict_struct.range_min->type); + (void)(is_date); for (const auto & key : collection) { - for (const auto & value : key.getMapped()) + for (const auto & [interval, _] : key.getMapped()) { keys.push_back(key.getKey()); - start_dates.push_back(value.range.left); - end_dates.push_back(value.range.right); + start_dates.push_back(interval.left); + end_dates.push_back(interval.right); if constexpr (std::numeric_limits::max() > DATE_LUT_MAX_DAY_NUM) /// Avoid warning about tautological comparison in next line. if (is_date && static_cast(end_dates.back()) > DATE_LUT_MAX_DAY_NUM) @@ -676,7 +704,7 @@ PaddedPODArray RangeHashedDictionary::makeDateKeys( for (size_t i = 0; i < keys.size(); ++i) { - if (Range::isCorrectDate(block_start_dates[i])) + if (isCorrectDate(block_start_dates[i])) keys[i] = block_start_dates[i]; // NOLINT else keys[i] = block_end_dates[i]; // NOLINT diff --git a/src/Dictionaries/RangeHashedDictionary.h b/src/Dictionaries/RangeHashedDictionary.h index a9b41a4c4d0..bf19a58f1fe 100644 --- a/src/Dictionaries/RangeHashedDictionary.h +++ b/src/Dictionaries/RangeHashedDictionary.h @@ -8,26 +8,19 @@ #include #include #include -#include +#include + #include #include #include #include + namespace DB { using RangeStorageType = Int64; -struct Range -{ - RangeStorageType left; - RangeStorageType right; - - static bool isCorrectDate(const RangeStorageType & date); - bool contains(const RangeStorageType & value) const; -}; - template class RangeHashedDictionary final : public IDictionary { @@ -94,15 +87,11 @@ public: Pipe read(const Names & column_names, size_t max_block_size, size_t num_streams) const override; private: - template - struct Value final - { - Range range; - std::optional value; - }; + + using RangeInterval = Interval; template - using Values = std::vector>; + using Values = IntervalMap>; template using CollectionType = std::conditional_t< @@ -160,10 +149,12 @@ private: void blockToAttributes(const Block & block); - template - void setAttributeValueImpl(Attribute & attribute, KeyType key, const Range & range, const Field & value); + void constructAttributeIntervalTrees(); - void setAttributeValue(Attribute & attribute, KeyType key, const Range & range, const Field & value); + template + void setAttributeValueImpl(Attribute & attribute, KeyType key, const RangeInterval & range, const Field & value); + + void setAttributeValue(Attribute & attribute, KeyType key, const RangeInterval & range, const Field & value); template void getKeysAndDates( From 12f352305aa88f288016a4f9d66d17a2d7cdca94 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 11 Jan 2022 18:44:30 +0300 Subject: [PATCH 053/403] Fix tests --- src/Dictionaries/RangeHashedDictionary.cpp | 23 +--------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/src/Dictionaries/RangeHashedDictionary.cpp b/src/Dictionaries/RangeHashedDictionary.cpp index 028818f2019..24d01d616fe 100644 --- a/src/Dictionaries/RangeHashedDictionary.cpp +++ b/src/Dictionaries/RangeHashedDictionary.cpp @@ -401,15 +401,8 @@ void RangeHashedDictionary::getItemsImpl( RangeInterval min_range {max_range_value, max_range_value}; bool has_interval = false; - std::cout << "Date " << date << " intervals size " << interval_tree.getIntervalsSize() << std::endl; - for (auto & [range, _] : interval_tree) - { - std::cout << "Range left " << range.left << " range right " << range.right << std::endl; - } - interval_tree.find(date, [&](auto & interval, auto & value) { - std::cerr << "Find interval " << std::endl; has_interval = true; if (min_range < interval) @@ -417,22 +410,8 @@ void RangeHashedDictionary::getItemsImpl( return true; }); - // auto value_it = std::lower_bound( - // std::begin(ranges_and_values), - // std::end(ranges_and_values), - // Range{date, date}, - // [](const Value & value, const Range & date_range) - // { - // return value.range < date_range; - // }); - // if (value_it != std::end(ranges_and_values) && value_it != ranges_and_values.begin()) - // --value_it; - - std::cout << "Has interval " << has_interval << " min interval "; - std::cout << " left " << min_range.left << " right " << min_range.right << std::endl; - - if (min_value.has_value()) + if (has_interval) { ++keys_found; From c3d0e8e6f161dcad727bce4489cd9a573cb9f5d9 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 11 Jan 2022 18:50:40 +0300 Subject: [PATCH 054/403] Fix style check --- src/Common/IntervalTree.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Common/IntervalTree.h b/src/Common/IntervalTree.h index f521c4d098c..b41372e22ef 100644 --- a/src/Common/IntervalTree.h +++ b/src/Common/IntervalTree.h @@ -179,7 +179,8 @@ public: return &getCurrentValue(); } - Iterator &operator++() { + Iterator &operator++() + { iterateToNext(); return *this; } From ea78c0b33c861966bcab400d2cbc4fb6ea6f26ba Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Sun, 16 Jan 2022 16:45:33 +0000 Subject: [PATCH 055/403] Fixed style --- src/Common/IntervalTree.h | 298 +++++++++++---------- src/Dictionaries/RangeHashedDictionary.cpp | 4 +- src/Dictionaries/RangeHashedDictionary.h | 4 +- 3 files changed, 158 insertions(+), 148 deletions(-) diff --git a/src/Common/IntervalTree.h b/src/Common/IntervalTree.h index b41372e22ef..1f257f71c73 100644 --- a/src/Common/IntervalTree.h +++ b/src/Common/IntervalTree.h @@ -3,6 +3,7 @@ #include #include +#include namespace DB @@ -15,36 +16,50 @@ struct Interval IntervalStorageType left; IntervalStorageType right; - Interval(IntervalStorageType left_, IntervalStorageType right_) - : left(left_) - , right(right_) - {} + Interval(IntervalStorageType left_, IntervalStorageType right_) : left(left_), right(right_) { } - inline bool contains(IntervalStorageType point) const - { - return left <= point && point <= right; - } + inline bool contains(IntervalStorageType point) const { return left <= point && point <= right; } }; -template +template bool operator<(const Interval & lhs, const Interval & rhs) { return std::tie(lhs.left, lhs.right) < std::tie(rhs.left, rhs.right); } -template -bool operator==(const Interval & lhs, const Interval & rhs) +template +bool operator<=(const Interval & lhs, const Interval & rhs) { - return std::tie(lhs.left, lhs.right) == std::tie(lhs.left, rhs.right); + return std::tie(lhs.left, lhs.right) <= std::tie(rhs.left, rhs.right); } -template +template +bool operator==(const Interval & lhs, const Interval & rhs) +{ + return std::tie(lhs.left, lhs.right) == std::tie(rhs.left, rhs.right); +} + +template +bool operator!=(const Interval & lhs, const Interval & rhs) +{ + return std::tie(lhs.left, lhs.right) != std::tie(rhs.left, rhs.right); +} + +template bool operator>(const Interval & lhs, const Interval & rhs) { return std::tie(lhs.left, lhs.right) > std::tie(rhs.left, rhs.right); } -struct IntervalTreeVoidValue {}; +template +bool operator>=(const Interval & lhs, const Interval & rhs) +{ + return std::tie(lhs.left, lhs.right) >= std::tie(rhs.left, rhs.right); +} + +struct IntervalTreeVoidValue +{ +}; template class IntervalTree @@ -53,52 +68,66 @@ public: using IntervalStorageType = typename Interval::IntervalStorageType; static constexpr bool is_empty_value = std::is_same_v; - static constexpr bool is_nonempty_value = !std::is_same_v; + + IntervalTree() { nodes.resize(1); } template , bool> = true> void emplace(Interval interval) { - ++intervals_size; + assert(!tree_constructed); sorted_intervals.emplace_back(interval); + increaseIntervalsSize(); } - template , bool> = true, typename ...Args> - void emplace(Interval interval, Args && ... args) + template , bool> = true, typename... Args> + void emplace(Interval interval, Args &&... args) { - ++intervals_size; - sorted_intervals.emplace_back(interval, Value(std::forward(args)...)); + assert(!tree_constructed); + sorted_intervals.emplace_back( + std::piecewise_construct, std::forward_as_tuple(interval), std::forward_as_tuple(std::forward(args)...)); + increaseIntervalsSize(); } template , bool> = true> void insert(Interval interval) { - ++intervals_size; + assert(!tree_constructed); sorted_intervals.emplace_back(interval); + increaseIntervalsSize(); } template , bool> = true> void insert(Interval interval, const Value & value) { - ++intervals_size; - sorted_intervals.emplace_back(interval, value); + assert(!tree_constructed); + sorted_intervals.emplace_back(std::piecewise_construct, interval, value); + increaseIntervalsSize(); } template , bool> = true> void insert(Interval interval, Value && value) { - ++intervals_size; - sorted_intervals.emplace_back(interval, std::move(value)); + assert(!tree_constructed); + sorted_intervals.emplace_back(std::piecewise_construct, interval, std::move(value)); + increaseIntervalsSize(); } void construct() { nodes.resize(sorted_intervals.size()); buildTree(); + tree_constructed = true; } template void find(IntervalStorageType point, IntervalCallback && callback) const { + if (unlikely(!tree_constructed)) + { + findIntervalsNonConstructedImpl(point, callback); + return; + } + findIntervalsImpl(point, callback); } @@ -108,16 +137,14 @@ public: if constexpr (is_empty_value) { - find(point, [&](auto &) - { + find(point, [&](auto &) { has_intervals = true; return false; }); } else { - find(point, [&](auto &, auto &) - { + find(point, [&](auto &, auto &) { has_intervals = true; return false; }); @@ -126,7 +153,43 @@ public: return has_intervals; } + class Iterator; + using iterator = Iterator; + using const_iterator = Iterator; + iterator begin() + { + size_t start_index = findFirstIteratorNodeIndex(); + return Iterator(start_index, 0, this); + } + + iterator end() + { + size_t end_index = findLastIteratorNodeIndex(); + size_t last_interval_index = nodes[end_index].sorted_intervals_range_size; + return Iterator(end_index, last_interval_index, this); + } + + const_iterator begin() const + { + size_t start_index = findFirstIteratorNodeIndex(); + return Iterator(start_index, 0, this); + } + + const_iterator end() const + { + size_t end_index = findLastIteratorNodeIndex(); + size_t last_interval_index = nodes[end_index].sorted_intervals_range_size; + return Iterator(end_index, last_interval_index, this); + } + + const_iterator cbegin() const { return begin(); } + + const_iterator cend() const { return end(); } + + size_t getIntervalsSize() const { return intervals_size; } + +private: struct Node { size_t sorted_intervals_range_start_index; @@ -134,10 +197,7 @@ public: IntervalStorageType middle_element; - inline bool hasValue() const - { - return sorted_intervals_range_size != 0; - } + inline bool hasValue() const { return sorted_intervals_range_size != 0; } }; using IntervalWithEmptyValue = Interval; @@ -145,71 +205,58 @@ public: using IntervalWithValue = std::conditional_t; +public: class Iterator { public: - bool operator==(const Iterator & rhs) const { return node_index == rhs.node_index & tree == rhs.tree && current_interval_index == rhs.current_interval_index; } - bool operator!=(const Iterator & rhs) const - { - return !(*this == rhs); - } + bool operator!=(const Iterator & rhs) const { return !(*this == rhs); } - const IntervalWithValue & operator*() - { - return getCurrentValue(); - } + const IntervalWithValue & operator*() { return getCurrentValue(); } - const IntervalWithValue & operator*() const - { - return getCurrentValue(); - } + const IntervalWithValue & operator*() const { return getCurrentValue(); } - const IntervalWithValue *operator->() - { - return &getCurrentValue(); - } + const IntervalWithValue * operator->() { return &getCurrentValue(); } - const IntervalWithValue *operator->() const - { - return &getCurrentValue(); - } + const IntervalWithValue * operator->() const { return &getCurrentValue(); } - Iterator &operator++() + Iterator & operator++() { iterateToNext(); return *this; } - // Iterator operator++(int) { - // Iterator copy(*this); - // iterateToNext(); - // return copy; - // } + Iterator operator++(int) // NOLINT + { + Iterator copy(*this); + iterateToNext(); + return copy; + } - // Iterator &operator--() { - // iterateToPrevious(); - // return *this; - // } + Iterator & operator--() + { + iterateToPrevious(); + return *this; + } - // Iterator operator--(int) { - // Iterator copy(*this); - // iterateToNext(); - // return copy; - // } + Iterator operator--(int) // NOLINT + { + Iterator copy(*this); + iterateToPrevious(); + return copy; + } private: friend class IntervalTree; Iterator(size_t node_index_, size_t current_interval_index_, const IntervalTree * tree_) - : node_index(node_index_) - , current_interval_index(current_interval_index_) - , tree(tree_) - {} + : node_index(node_index_), current_interval_index(current_interval_index_), tree(tree_) + { + } size_t node_index; size_t current_interval_index; @@ -268,52 +315,7 @@ public: } }; - using iterator = Iterator; - using const_iterator = Iterator; - - iterator begin() - { - size_t start_index = findFirstIteratorNodeIndex(); - return Iterator(start_index, 0, this); - } - - iterator end() - { - size_t end_index = findLastIteratorNodeIndex(); - size_t last_interval_index = nodes[end_index].sorted_intervals_range_size; - return Iterator(end_index, last_interval_index, this); - } - - const_iterator begin() const - { - size_t start_index = findFirstIteratorNodeIndex(); - return Iterator(start_index, 0, this); - } - - const_iterator end() const - { - size_t end_index = findLastIteratorNodeIndex(); - size_t last_interval_index = nodes[end_index].sorted_intervals_range_size; - return Iterator(end_index, last_interval_index, this); - } - - const_iterator cbegin() const - { - return begin(); - } - - const_iterator cend() const - { - return end(); - } - - size_t getIntervalsSize() const - { - return intervals_size; - } - private: - void buildTree() { std::vector temporary_points_storage; @@ -376,18 +378,16 @@ private: } } - std::sort(intervals_sorted_by_left_asc.begin(), intervals_sorted_by_left_asc.end(), [](auto & lhs, auto & rhs) - { + std::sort(intervals_sorted_by_left_asc.begin(), intervals_sorted_by_left_asc.end(), [](auto & lhs, auto & rhs) { auto & lhs_interval = getInterval(lhs); auto & rhs_interval = getInterval(rhs); - return lhs_interval < rhs_interval; + return lhs_interval.left < rhs_interval.left; }); - std::sort(intervals_sorted_by_right_desc.begin(), intervals_sorted_by_right_desc.end(), [](auto & lhs, auto & rhs) - { + std::sort(intervals_sorted_by_right_desc.begin(), intervals_sorted_by_right_desc.end(), [](auto & lhs, auto & rhs) { auto & lhs_interval = getInterval(lhs); auto & rhs_interval = getInterval(rhs); - return lhs_interval > rhs_interval; + return lhs_interval.right > rhs_interval.right; }); size_t sorted_intervals_range_start_index = sorted_intervals.size(); @@ -474,11 +474,20 @@ private: } } - size_t findFirstIteratorNodeIndex() const + template + void findIntervalsNonConstructedImpl(IntervalStorageType point, IntervalCallback & callback) const { - if (nodes.empty()) - return 0; + for (auto & interval_with_value : sorted_intervals) + { + auto & interval = getInterval(interval_with_value); + if (interval.contains(point)) + callCallback(interval_with_value, callback); + } + } + + inline size_t findFirstIteratorNodeIndex() const + { size_t nodes_size = nodes.size(); size_t result_index = 0; @@ -491,11 +500,8 @@ private: return result_index; } - size_t findLastIteratorNodeIndex() const + inline size_t findLastIteratorNodeIndex() const { - if (nodes.empty()) - return 0; - size_t nodes_size = nodes.size(); int64_t result_index = static_cast(nodes_size - 1); for (; result_index >= 0; --result_index) @@ -508,11 +514,18 @@ private: return result; } + inline void increaseIntervalsSize() + { + ++intervals_size; + nodes[0].sorted_intervals_range_size = intervals_size; + } + std::vector nodes; std::vector sorted_intervals; size_t intervals_size = 0; + bool tree_constructed = false; - static const Interval & getInterval(const IntervalWithValue & interval_with_value) + static inline const Interval & getInterval(const IntervalWithValue & interval_with_value) { if constexpr (is_empty_value) return interval_with_value; @@ -521,7 +534,7 @@ private: } template - static bool callCallback(const IntervalWithValue & interval, IntervalCallback && callback) + static inline bool callCallback(const IntervalWithValue & interval, IntervalCallback && callback) { if constexpr (is_empty_value) return callback(interval); @@ -529,7 +542,8 @@ private: return callback(interval.first, interval.second); } - static void intervalsToPoints(const std::vector & intervals, std::vector & temporary_points_storage) + static inline void + intervalsToPoints(const std::vector & intervals, std::vector & temporary_points_storage) { for (const auto & interval_with_value : intervals) { @@ -539,23 +553,19 @@ private: } } - static IntervalStorageType pointsMedian(std::vector & points) + static inline IntervalStorageType pointsMedian(std::vector & points) { size_t size = points.size(); size_t middle_element_index = size / 2; std::nth_element(points.begin(), points.begin() + middle_element_index, points.end()); - if (size % 2 == 0) - { - return (points[middle_element_index] + points[middle_element_index - 1]) / 2; - } - else - { - return points[middle_element_index]; - } + /** We should not get median as average of middle_element_index and middle_element_index - 1 + * because we want point in node to intersect some interval. + * Example: Intervals [1, 1], [3, 3]. If we choose 2 as average point, it does not intersect any interval. + */ + return points[middle_element_index]; } - }; template diff --git a/src/Dictionaries/RangeHashedDictionary.cpp b/src/Dictionaries/RangeHashedDictionary.cpp index 24d01d616fe..d4e7e43b2ac 100644 --- a/src/Dictionaries/RangeHashedDictionary.cpp +++ b/src/Dictionaries/RangeHashedDictionary.cpp @@ -607,14 +607,14 @@ void RangeHashedDictionary::setAttributeValueImpl(Attribute } template -void RangeHashedDictionary::setAttributeValue(Attribute & attribute, KeyType key, const RangeInterval & range, const Field & value) +void RangeHashedDictionary::setAttributeValue(Attribute & attribute, KeyType key, const RangeInterval & interval, const Field & value) { auto type_call = [&](const auto &dictionary_attribute_type) { using Type = std::decay_t; using AttributeType = typename Type::AttributeType; - setAttributeValueImpl(attribute, key, range, value); + setAttributeValueImpl(attribute, key, interval, value); }; callOnDictionaryAttributeType(attribute.type, type_call); diff --git a/src/Dictionaries/RangeHashedDictionary.h b/src/Dictionaries/RangeHashedDictionary.h index bf19a58f1fe..45d419996bc 100644 --- a/src/Dictionaries/RangeHashedDictionary.h +++ b/src/Dictionaries/RangeHashedDictionary.h @@ -152,9 +152,9 @@ private: void constructAttributeIntervalTrees(); template - void setAttributeValueImpl(Attribute & attribute, KeyType key, const RangeInterval & range, const Field & value); + void setAttributeValueImpl(Attribute & attribute, KeyType key, const RangeInterval & interval, const Field & value); - void setAttributeValue(Attribute & attribute, KeyType key, const RangeInterval & range, const Field & value); + void setAttributeValue(Attribute & attribute, KeyType key, const RangeInterval & interval, const Field & value); template void getKeysAndDates( From 3a10be3cb3066897306818027a96c5147e8f7702 Mon Sep 17 00:00:00 2001 From: Leonid Krylov Date: Sun, 16 Jan 2022 21:42:49 +0300 Subject: [PATCH 056/403] Update h3.md Edit h3ToGeo description. --- docs/ru/sql-reference/functions/geo/h3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/geo/h3.md b/docs/ru/sql-reference/functions/geo/h3.md index 8f7b98f0a45..15a6cc8ed2a 100644 --- a/docs/ru/sql-reference/functions/geo/h3.md +++ b/docs/ru/sql-reference/functions/geo/h3.md @@ -195,7 +195,7 @@ SELECT geoToH3(37.79506683, 55.71290588, 15) AS h3Index; ## h3ToGeo {#h3togeo} -Возвращает географические координаты долготы и широты, соответствующие указанному [H3](#h3index)-индексу. +Возвращает географические координаты долготы и широты центра шестигранника, соответствующие указанному [H3](#h3index)-индексу. **Синтаксис** From a10807306512c9dfa177beaff30d6ae4e1d818d7 Mon Sep 17 00:00:00 2001 From: Leonid Krylov Date: Sun, 16 Jan 2022 21:47:43 +0300 Subject: [PATCH 057/403] Update h3.md Edit h3ToGeo description. --- docs/en/sql-reference/functions/geo/h3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/geo/h3.md b/docs/en/sql-reference/functions/geo/h3.md index 2efe980a4cf..219b8cee1b4 100644 --- a/docs/en/sql-reference/functions/geo/h3.md +++ b/docs/en/sql-reference/functions/geo/h3.md @@ -197,7 +197,7 @@ Result: ## h3ToGeo {#h3togeo} -Returns the geographical coordinates of longitude and latitude corresponding to the provided [H3](#h3index) index. +Returns the centroid longitude and latitude corresponding to the provided [H3](#h3index) index. **Syntax** From 0288967538d9471da3ae8b74d029b9a3afb842f4 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Mon, 17 Jan 2022 00:30:03 +0400 Subject: [PATCH 058/403] Disable kerberos if parsing the config failed --- src/Access/ExternalAuthenticators.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Access/ExternalAuthenticators.cpp b/src/Access/ExternalAuthenticators.cpp index d4100c4e520..d5e56c2f438 100644 --- a/src/Access/ExternalAuthenticators.cpp +++ b/src/Access/ExternalAuthenticators.cpp @@ -283,10 +283,15 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur } } + kerberos_params.reset(); try { if (kerberos_keys_count > 0) - parseKerberosParams(kerberos_params.emplace(), config); + { + GSSAcceptorContext::Params kerberos_params_tmp; + parseKerberosParams(kerberos_params_tmp, config); + kerberos_params = std::move(kerberos_params_tmp); + } } catch (...) { From 28bc286d8bcb7ec64f6c6fe3ad970f1dccc48411 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Mon, 17 Jan 2022 00:31:10 +0400 Subject: [PATCH 059/403] Do not allow ldap servers with the same name Reset the set of ldap servers when config is re-read --- src/Access/ExternalAuthenticators.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/Access/ExternalAuthenticators.cpp b/src/Access/ExternalAuthenticators.cpp index d5e56c2f438..3e1c289b207 100644 --- a/src/Access/ExternalAuthenticators.cpp +++ b/src/Access/ExternalAuthenticators.cpp @@ -270,12 +270,21 @@ void ExternalAuthenticators::setConfiguration(const Poco::Util::AbstractConfigur Poco::Util::AbstractConfiguration::Keys ldap_server_names; config.keys("ldap_servers", ldap_server_names); - for (const auto & ldap_server_name : ldap_server_names) + ldap_client_params_blueprint.clear(); + for (auto ldap_server_name : ldap_server_names) { try { - ldap_client_params_blueprint.erase(ldap_server_name); - parseLDAPServer(ldap_client_params_blueprint.emplace(ldap_server_name, LDAPClient::Params{}).first->second, config, ldap_server_name); + const auto bracket_pos = ldap_server_name.find('['); + if (bracket_pos != std::string::npos) + ldap_server_name.resize(bracket_pos); + + if (ldap_client_params_blueprint.count(ldap_server_name) > 0) + throw Exception("Multiple LDAP servers with the same name are not allowed", ErrorCodes::BAD_ARGUMENTS); + + LDAPClient::Params ldap_client_params_tmp; + parseLDAPServer(ldap_client_params_tmp, config, ldap_server_name); + ldap_client_params_blueprint.emplace(std::move(ldap_server_name), std::move(ldap_client_params_tmp)); } catch (...) { From b25731e7da211f399134360a179d53b8278ff331 Mon Sep 17 00:00:00 2001 From: Denis Glazachev Date: Mon, 17 Jan 2022 00:31:46 +0400 Subject: [PATCH 060/403] Revert the statment about the disabled kerberos support, if config has errors --- docs/en/operations/external-authenticators/kerberos.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/en/operations/external-authenticators/kerberos.md b/docs/en/operations/external-authenticators/kerberos.md index f326762a610..da84c1f6a89 100644 --- a/docs/en/operations/external-authenticators/kerberos.md +++ b/docs/en/operations/external-authenticators/kerberos.md @@ -51,6 +51,9 @@ With filtering by realm: ``` +!!! warning "Note" + You can define only one `kerberos` section. The presence of multiple `kerberos` sections will force ClickHouse to disable Kerberos authentication. + !!! warning "Note" `principal` and `realm` sections cannot be specified at the same time. The presence of both `principal` and `realm` sections will force ClickHouse to disable Kerberos authentication. From d2bc77ac471c937856788f1b9ca01e2ec16195fe Mon Sep 17 00:00:00 2001 From: hanqf-git Date: Mon, 17 Jan 2022 12:03:12 +0800 Subject: [PATCH 061/403] Add x86 feature avx512 support for memcmpSmall --- cmake/cpu_features.cmake | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cmake/cpu_features.cmake b/cmake/cpu_features.cmake index 20c61ead3d2..94be0bf452a 100644 --- a/cmake/cpu_features.cmake +++ b/cmake/cpu_features.cmake @@ -150,6 +150,20 @@ else () set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}") endif () + set (TEST_FLAG "-mavx512bw -mavx512vl") + set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0") + check_cxx_source_compiles(" + #include + int main() { + auto mask = _mm_cmp_epi8_mask(__m128i(), __m128i(), 0); + (void)mask; + return 0; + } + " HAVE_AVX512_BW_VL) + if (HAVE_AVX512_BW_VL AND ENABLE_AVX512) + set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}") + endif () + set (TEST_FLAG "-mbmi") set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0") check_cxx_source_compiles(" @@ -183,6 +197,9 @@ else () if (HAVE_AVX512) set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mavx512f -mavx512bw -mprefer-vector-width=256") endif () + if (HAVE_AVX512_BW_VL) + set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mavx512bw -mavx512vl") + endif () endif () endif () From 71ce4b46d8d4b6ad93b945109e63273b637dfff7 Mon Sep 17 00:00:00 2001 From: hanqf-git Date: Mon, 17 Jan 2022 16:13:49 +0800 Subject: [PATCH 062/403] Add x86 feature avx512 support for memcmpSmall. Co-authored-by: @vesslanjin jun.i.jin@intel.com --- cmake/cpu_features.cmake | 23 ++++------------------- src/CMakeLists.txt | 1 + 2 files changed, 5 insertions(+), 19 deletions(-) diff --git a/cmake/cpu_features.cmake b/cmake/cpu_features.cmake index 94be0bf452a..535d1b3c93e 100644 --- a/cmake/cpu_features.cmake +++ b/cmake/cpu_features.cmake @@ -134,7 +134,7 @@ else () set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}") endif () - set (TEST_FLAG "-mavx512f -mavx512bw") + set (TEST_FLAG "-mavx512f -mavx512bw -mavx512vl") set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0") check_cxx_source_compiles(" #include @@ -143,6 +143,8 @@ else () (void)a; auto b = _mm512_add_epi16(__m512i(), __m512i()); (void)b; + auto c = _mm_cmp_epi8_mask(__m128i(), __m128i(), 0); + (void)c; return 0; } " HAVE_AVX512) @@ -150,20 +152,6 @@ else () set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}") endif () - set (TEST_FLAG "-mavx512bw -mavx512vl") - set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0") - check_cxx_source_compiles(" - #include - int main() { - auto mask = _mm_cmp_epi8_mask(__m128i(), __m128i(), 0); - (void)mask; - return 0; - } - " HAVE_AVX512_BW_VL) - if (HAVE_AVX512_BW_VL AND ENABLE_AVX512) - set (COMPILER_FLAGS "${COMPILER_FLAGS} ${TEST_FLAG}") - endif () - set (TEST_FLAG "-mbmi") set (CMAKE_REQUIRED_FLAGS "${TEST_FLAG} -O0") check_cxx_source_compiles(" @@ -195,10 +183,7 @@ else () set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mbmi") endif () if (HAVE_AVX512) - set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mavx512f -mavx512bw -mprefer-vector-width=256") - endif () - if (HAVE_AVX512_BW_VL) - set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mavx512bw -mavx512vl") + set (X86_INTRINSICS_FLAGS "${X86_INTRINSICS_FLAGS} -mavx512f -mavx512bw -mavx512vl -mprefer-vector-width=256") endif () endif () endif () diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c9e9f736e0d..dbb5ddf7fc5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -304,6 +304,7 @@ set_source_files_properties( Columns/ColumnsCommon.cpp Columns/ColumnVector.cpp Columns/ColumnDecimal.cpp + Columns/ColumnString.cpp PROPERTIES COMPILE_FLAGS "${X86_INTRINSICS_FLAGS}") if(RE2_LIBRARY) From f311960b000d03a4d58183c0bb3eb21022631c84 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 17 Jan 2022 14:22:01 +0000 Subject: [PATCH 063/403] Added tests --- src/Common/IntervalTree.h | 28 +- src/Common/examples/interval_tree.cpp | 25 -- src/Common/tests/gtest_interval_tree.cpp | 540 +++++++++++++++++++++++ 3 files changed, 558 insertions(+), 35 deletions(-) create mode 100644 src/Common/tests/gtest_interval_tree.cpp diff --git a/src/Common/IntervalTree.h b/src/Common/IntervalTree.h index 1f257f71c73..ec17ca11a9f 100644 --- a/src/Common/IntervalTree.h +++ b/src/Common/IntervalTree.h @@ -114,7 +114,9 @@ public: void construct() { - nodes.resize(sorted_intervals.size()); + assert(!tree_constructed); + nodes.clear(); + nodes.reserve(sorted_intervals.size()); buildTree(); tree_constructed = true; } @@ -166,7 +168,7 @@ public: iterator end() { size_t end_index = findLastIteratorNodeIndex(); - size_t last_interval_index = nodes[end_index].sorted_intervals_range_size; + size_t last_interval_index = end_index < nodes.size() ? nodes[end_index].sorted_intervals_range_size : 0; return Iterator(end_index, last_interval_index, this); } @@ -179,7 +181,8 @@ public: const_iterator end() const { size_t end_index = findLastIteratorNodeIndex(); - size_t last_interval_index = nodes[end_index].sorted_intervals_range_size; + size_t last_interval_index = end_index < nodes.size() ? nodes[end_index].sorted_intervals_range_size : 0; + return Iterator(end_index, last_interval_index, this); } @@ -211,7 +214,7 @@ public: public: bool operator==(const Iterator & rhs) const { - return node_index == rhs.node_index & tree == rhs.tree && current_interval_index == rhs.current_interval_index; + return node_index == rhs.node_index && current_interval_index == rhs.current_interval_index && tree == rhs.tree; } bool operator!=(const Iterator & rhs) const { return !(*this == rhs); } @@ -307,7 +310,7 @@ public: } } - const IntervalWithValue & getCurrentValue() + const IntervalWithValue & getCurrentValue() const { auto & current_node = tree->nodes[node_index]; size_t interval_index = current_node.sorted_intervals_range_start_index + current_interval_index; @@ -475,7 +478,7 @@ private: } template - void findIntervalsNonConstructedImpl(IntervalStorageType point, IntervalCallback & callback) const + void findIntervalsNonConstructedImpl(IntervalStorageType point, IntervalCallback && callback) const { for (auto & interval_with_value : sorted_intervals) { @@ -497,21 +500,26 @@ private: break; } + if (unlikely(result_index == nodes_size)) + result_index = 0; + return result_index; } inline size_t findLastIteratorNodeIndex() const { + if (unlikely(nodes.empty())) + return 0; + size_t nodes_size = nodes.size(); - int64_t result_index = static_cast(nodes_size - 1); - for (; result_index >= 0; --result_index) + size_t result_index = nodes_size - 1; + for (; result_index != 0; --result_index) { if (nodes[result_index].hasValue()) break; } - size_t result = static_cast(result_index); - return result; + return result_index; } inline void increaseIntervalsSize() diff --git a/src/Common/examples/interval_tree.cpp b/src/Common/examples/interval_tree.cpp index ee6459e3345..5ffc2f1ccd3 100644 --- a/src/Common/examples/interval_tree.cpp +++ b/src/Common/examples/interval_tree.cpp @@ -12,31 +12,6 @@ using namespace DB; using Int64Interval = Interval; -struct CollectIntervalsCallback -{ - explicit CollectIntervalsCallback(std::set & result_intervals_) - : result_intervals(result_intervals_) - { - } - - bool operator()(Int64Interval interval) - { - result_intervals.insert(interval); - return true; - } - - std::set & result_intervals; -}; - -void findNecessaryIntervals(const std::vector & intervals, Int64 point, std::set & result_intervals) -{ - for (const auto & interval : intervals) - { - if (interval.contains(point)) - result_intervals.insert(interval); - } -} - int main(int, char **) { { diff --git a/src/Common/tests/gtest_interval_tree.cpp b/src/Common/tests/gtest_interval_tree.cpp new file mode 100644 index 00000000000..9becb02d768 --- /dev/null +++ b/src/Common/tests/gtest_interval_tree.cpp @@ -0,0 +1,540 @@ +#include + +#include +#include + +#include +#include + + +using namespace DB; +using Int64Interval = Interval; + +template +std::set intervalSetToSet(const IntervalSet & interval_set) +{ + std::set result; + + for (const auto & interval : interval_set) + result.insert(interval); + + return result; +} + +template +std::map intervalMapToMap(const IntervalMap & interval_map) +{ + std::map result; + + for (const auto & [interval, value] : interval_map) + result.emplace(interval, value); + + return result; +} + +template +struct CollectIntervalsSetCallback +{ + explicit CollectIntervalsSetCallback(std::set & result_intervals_) + : result_intervals(result_intervals_) + { + } + + bool operator()(IntervalType interval) + { + result_intervals.insert(interval); + return true; + } + + std::set & result_intervals; +}; + +using CollectIntervalsSetInt64Callback = CollectIntervalsSetCallback; + +template +std::set intervalSetFindIntervals(const IntervalSet & interval_set, typename IntervalType::IntervalStorageType point) +{ + std::set result; + CollectIntervalsSetCallback callback(result); + + interval_set.find(point, callback); + + return result; +} + +template +struct CollectIntervalsMapCallback +{ + explicit CollectIntervalsMapCallback(std::map & result_intervals_) + : result_intervals(result_intervals_) + { + } + + bool operator()(IntervalType interval, const Value & value) + { + result_intervals.emplace(interval, value); + return true; + } + + std::map & result_intervals; +}; + + +template +std::map intervalMapFindIntervals(const IntervalMap & interval_set, typename IntervalType::IntervalStorageType point) +{ + std::map result; + CollectIntervalsMapCallback callback(result); + + interval_set.find(point, callback); + + return result; +} + +TEST(IntervalTree, IntervalSetBasic) +{ + for (size_t intervals_size = 0; intervals_size < 120; ++intervals_size) + { + std::set expected; + IntervalSet set; + + for (size_t interval_index = 0; interval_index < intervals_size; ++interval_index) + { + auto interval = Int64Interval(interval_index * 2, interval_index * 2 + 1); + expected.insert(interval); + set.insert(interval); + } + + ASSERT_TRUE(set.getIntervalsSize() == expected.size()); + ASSERT_TRUE(set.getIntervalsSize() == intervals_size); + ASSERT_TRUE(intervalSetToSet(set) == expected); + + for (const auto & expected_interval : expected) + { + std::set expected_intervals = {{expected_interval}}; + + auto actual_intervals = intervalSetFindIntervals(set, expected_interval.left); + ASSERT_TRUE(actual_intervals.size() == 1); + ASSERT_TRUE(actual_intervals == expected_intervals); + + actual_intervals = intervalSetFindIntervals(set, expected_interval.right); + ASSERT_TRUE(actual_intervals.size() == 1); + ASSERT_TRUE(actual_intervals == expected_intervals); + + ASSERT_TRUE(set.has(expected_interval.left)); + ASSERT_TRUE(set.has(expected_interval.right)); + } + + set.construct(); + + ASSERT_TRUE(intervalSetToSet(set) == expected); + + for (const auto & expected_interval : expected) + { + auto actual_interval = intervalSetFindIntervals(set, expected_interval.left); + ASSERT_TRUE(actual_interval.size() == 1); + ASSERT_TRUE(actual_interval == std::set{expected_interval}); + + actual_interval = intervalSetFindIntervals(set, expected_interval.right); + ASSERT_TRUE(actual_interval.size() == 1); + ASSERT_TRUE(actual_interval == std::set{expected_interval}); + + ASSERT_TRUE(set.has(expected_interval.left)); + ASSERT_TRUE(set.has(expected_interval.right)); + } + } +} + +TEST(IntervalTree, IntervalSetPoints) +{ + for (size_t intervals_size = 0; intervals_size < 120; ++intervals_size) + { + std::set expected; + IntervalSet set; + + for (size_t interval_index = 0; interval_index < intervals_size; ++interval_index) + { + auto interval = Int64Interval(interval_index, interval_index); + expected.insert(interval); + set.insert(interval); + } + + ASSERT_TRUE(set.getIntervalsSize() == expected.size()); + ASSERT_TRUE(set.getIntervalsSize() == intervals_size); + ASSERT_TRUE(intervalSetToSet(set) == expected); + + for (const auto & expected_interval : expected) + { + std::set expected_intervals = {{expected_interval}}; + + auto actual_intervals = intervalSetFindIntervals(set, expected_interval.left); + ASSERT_TRUE(actual_intervals.size() == 1); + ASSERT_TRUE(actual_intervals == expected_intervals); + + actual_intervals = intervalSetFindIntervals(set, expected_interval.right); + ASSERT_TRUE(actual_intervals.size() == 1); + ASSERT_TRUE(actual_intervals == expected_intervals); + + ASSERT_TRUE(set.has(expected_interval.left)); + ASSERT_TRUE(set.has(expected_interval.right)); + } + + set.construct(); + + ASSERT_TRUE(intervalSetToSet(set) == expected); + + for (const auto & expected_interval : expected) + { + auto actual_interval = intervalSetFindIntervals(set, expected_interval.left); + ASSERT_TRUE(actual_interval.size() == 1); + ASSERT_TRUE(actual_interval == std::set{expected_interval}); + + actual_interval = intervalSetFindIntervals(set, expected_interval.right); + ASSERT_TRUE(actual_interval.size() == 1); + ASSERT_TRUE(actual_interval == std::set{expected_interval}); + + ASSERT_TRUE(set.has(expected_interval.left)); + ASSERT_TRUE(set.has(expected_interval.right)); + } + } +} + +TEST(IntervalTree, IntervalSetIntersectingIntervals) +{ + for (size_t intervals_size = 0; intervals_size < 120; ++intervals_size) + { + std::set expected; + IntervalSet set; + + for (size_t interval_index = 0; interval_index < intervals_size; ++interval_index) + { + auto interval = Int64Interval(0, interval_index * 2 + 1); + expected.insert(interval); + set.insert(interval); + } + + ASSERT_TRUE(set.getIntervalsSize() == expected.size()); + ASSERT_TRUE(set.getIntervalsSize() == intervals_size); + ASSERT_TRUE(intervalSetToSet(set) == expected); + + for (const auto & expected_interval : expected) + { + auto actual_intervals = intervalSetFindIntervals(set, expected_interval.left); + ASSERT_TRUE(actual_intervals.size() == expected.size()); + ASSERT_TRUE(actual_intervals == expected); + + ASSERT_TRUE(set.has(expected_interval.left)); + ASSERT_TRUE(set.has(expected_interval.right)); + } + + set.construct(); + + ASSERT_TRUE(intervalSetToSet(set) == expected); + + for (const auto & expected_interval : expected) + { + auto actual_intervals = intervalSetFindIntervals(set, expected_interval.left); + ASSERT_TRUE(actual_intervals.size() == expected.size()); + ASSERT_TRUE(actual_intervals == expected); + + ASSERT_TRUE(set.has(expected_interval.left)); + ASSERT_TRUE(set.has(expected_interval.right)); + } + } +} + +TEST(IntervalTree, IntervalSetIterators) +{ + { + IntervalSet set; + ASSERT_TRUE(set.begin() == set.end()); + ASSERT_TRUE(set.cbegin() == set.cend()); + set.construct(); + ASSERT_TRUE(set.begin() == set.end()); + ASSERT_TRUE(set.cbegin() == set.cend()); + } + { + IntervalSet set; + set.emplace(Int64Interval(0, 5)); + ASSERT_TRUE(set.begin() != set.end()); + ASSERT_TRUE((*set.begin()).left == 0); + ASSERT_TRUE((*set.begin()).right == 5); + ASSERT_TRUE(set.begin()->left == 0); + ASSERT_TRUE(set.begin()->right == 5); + auto begin = set.begin(); + ++begin; + ASSERT_TRUE(begin == set.end()); + + begin = set.begin(); + begin++; + ASSERT_TRUE(begin == set.end()); + + auto end = set.end(); + --end; + ASSERT_TRUE(set.begin() == end); + + end = set.end(); + end--; + ASSERT_TRUE(set.begin() == end); + } + { + for (size_t intervals_size = 0; intervals_size < 120; ++intervals_size) + { + std::set expected; + IntervalSet set; + + for (size_t interval_index = 0; interval_index < intervals_size; ++interval_index) + { + auto interval = Int64Interval(interval_index * 2, interval_index * 2 + 1); + set.insert(interval); + expected.insert(interval); + } + + auto end = set.end(); + auto begin = set.begin(); + + std::set actual; + + while (end != begin) + { + --end; + actual.insert(*end); + } + + if (end != begin) + actual.insert(*end); + + ASSERT_TRUE(actual == expected); + } + } +} + +TEST(IntervalTree, IntervalMapBasic) +{ + for (size_t intervals_size = 0; intervals_size < 120; ++intervals_size) + { + std::map expected; + IntervalMap map; + + for (size_t interval_index = 0; interval_index < intervals_size; ++interval_index) + { + auto interval = Int64Interval(interval_index * 2, interval_index * 2 + 1); + auto value = std::to_string(interval.left); + expected.emplace(interval, value); + map.emplace(interval, value); + } + + ASSERT_TRUE(map.getIntervalsSize() == expected.size()); + ASSERT_TRUE(map.getIntervalsSize() == intervals_size); + ASSERT_TRUE(intervalMapToMap(map) == expected); + + for (const auto & [expected_interval, value] : expected) + { + std::map expected_intervals = {{expected_interval, std::to_string(expected_interval.left)}}; + + auto actual_intervals = intervalMapFindIntervals(map, expected_interval.left); + ASSERT_TRUE(actual_intervals.size() == 1); + ASSERT_TRUE(actual_intervals == expected_intervals); + + actual_intervals = intervalMapFindIntervals(map, expected_interval.right); + ASSERT_TRUE(actual_intervals.size() == 1); + ASSERT_TRUE(actual_intervals == expected_intervals); + + ASSERT_TRUE(map.has(expected_interval.left)); + ASSERT_TRUE(map.has(expected_interval.right)); + } + + map.construct(); + + ASSERT_TRUE(intervalMapToMap(map) == expected); + + for (const auto & [expected_interval, value] : expected) + { + std::map expected_intervals = {{expected_interval, std::to_string(expected_interval.left)}}; + + auto actual_intervals = intervalMapFindIntervals(map, expected_interval.left); + ASSERT_TRUE(actual_intervals.size() == 1); + ASSERT_TRUE(actual_intervals == expected_intervals); + + actual_intervals = intervalMapFindIntervals(map, expected_interval.right); + ASSERT_TRUE(actual_intervals.size() == 1); + ASSERT_TRUE(actual_intervals == expected_intervals); + + ASSERT_TRUE(map.has(expected_interval.left)); + ASSERT_TRUE(map.has(expected_interval.right)); + } + } +} + +TEST(IntervalTree, IntervalMapPoints) +{ + for (size_t intervals_size = 0; intervals_size < 120; ++intervals_size) + { + std::map expected; + IntervalMap map; + + for (size_t interval_index = 0; interval_index < intervals_size; ++interval_index) + { + auto interval = Int64Interval(interval_index, interval_index); + auto value = std::to_string(interval.left); + expected.emplace(interval, value); + map.emplace(interval, value); + } + + ASSERT_TRUE(map.getIntervalsSize() == expected.size()); + ASSERT_TRUE(map.getIntervalsSize() == intervals_size); + ASSERT_TRUE(intervalMapToMap(map) == expected); + + for (const auto & [expected_interval, value] : expected) + { + std::map expected_intervals = {{expected_interval, std::to_string(expected_interval.left)}}; + + auto actual_intervals = intervalMapFindIntervals(map, expected_interval.left); + ASSERT_TRUE(actual_intervals.size() == 1); + ASSERT_TRUE(actual_intervals == expected_intervals); + + actual_intervals = intervalMapFindIntervals(map, expected_interval.right); + ASSERT_TRUE(actual_intervals.size() == 1); + ASSERT_TRUE(actual_intervals == expected_intervals); + + ASSERT_TRUE(map.has(expected_interval.left)); + ASSERT_TRUE(map.has(expected_interval.right)); + } + + map.construct(); + + ASSERT_TRUE(intervalMapToMap(map) == expected); + + for (const auto & [expected_interval, value] : expected) + { + std::map expected_intervals = {{expected_interval, std::to_string(expected_interval.left)}}; + + auto actual_intervals = intervalMapFindIntervals(map, expected_interval.left); + ASSERT_TRUE(actual_intervals.size() == 1); + ASSERT_TRUE(actual_intervals == expected_intervals); + + actual_intervals = intervalMapFindIntervals(map, expected_interval.right); + ASSERT_TRUE(actual_intervals.size() == 1); + ASSERT_TRUE(actual_intervals == expected_intervals); + + ASSERT_TRUE(map.has(expected_interval.left)); + ASSERT_TRUE(map.has(expected_interval.right)); + } + } +} + +TEST(IntervalTree, IntervalMapIntersectingIntervals) +{ + for (size_t intervals_size = 0; intervals_size < 120; ++intervals_size) + { + std::map expected; + IntervalMap map; + + for (size_t interval_index = 0; interval_index < intervals_size; ++interval_index) + { + auto interval = Int64Interval(0, interval_index * 2 + 1); + auto value = std::to_string(interval.left); + expected.emplace(interval, value); + map.emplace(interval, value); + } + + ASSERT_TRUE(map.getIntervalsSize() == expected.size()); + ASSERT_TRUE(map.getIntervalsSize() == intervals_size); + ASSERT_TRUE(intervalMapToMap(map) == expected); + + for (const auto & [expected_interval, value] : expected) + { + auto actual_intervals = intervalMapFindIntervals(map, expected_interval.left); + + ASSERT_TRUE(actual_intervals.size() == expected.size()); + ASSERT_TRUE(actual_intervals == expected); + + ASSERT_TRUE(map.has(expected_interval.left)); + ASSERT_TRUE(map.has(expected_interval.right)); + } + + map.construct(); + + ASSERT_TRUE(intervalMapToMap(map) == expected); + + for (const auto & [expected_interval, value] : expected) + { + auto actual_intervals = intervalMapFindIntervals(map, expected_interval.left); + + ASSERT_TRUE(actual_intervals.size() == expected.size()); + ASSERT_TRUE(actual_intervals == expected); + + ASSERT_TRUE(map.has(expected_interval.left)); + ASSERT_TRUE(map.has(expected_interval.right)); + } + } +} + +TEST(IntervalTree, IntervalMapIterators) +{ + { + IntervalMap map; + ASSERT_TRUE(map.begin() == map.end()); + ASSERT_TRUE(map.cbegin() == map.cend()); + map.construct(); + ASSERT_TRUE(map.begin() == map.end()); + ASSERT_TRUE(map.cbegin() == map.cend()); + } + { + IntervalMap map; + map.emplace(Int64Interval(0, 5), "value"); + ASSERT_TRUE(map.begin() != map.end()); + ASSERT_TRUE((*map.begin()).first.left == 0); + ASSERT_TRUE((*map.begin()).first.right == 5); + ASSERT_TRUE((*map.begin()).second == "value"); + ASSERT_TRUE(map.begin()->first.left == 0); + ASSERT_TRUE(map.begin()->first.right == 5); + ASSERT_TRUE(map.begin()->second == "value"); + auto begin = map.begin(); + ++begin; + ASSERT_TRUE(begin == map.end()); + + begin = map.begin(); + begin++; + ASSERT_TRUE(begin == map.end()); + + auto end = map.end(); + --end; + ASSERT_TRUE(map.begin() == end); + + end = map.end(); + end--; + ASSERT_TRUE(map.begin() == end); + } + { + for (size_t intervals_size = 0; intervals_size < 120; ++intervals_size) + { + std::map expected; + IntervalMap map; + + for (size_t interval_index = 0; interval_index < intervals_size; ++interval_index) + { + auto interval = Int64Interval(interval_index * 2, interval_index * 2 + 1); + auto value = std::to_string(interval.left); + map.emplace(interval, value); + expected.emplace(interval, value); + } + + auto end = map.end(); + auto begin = map.begin(); + + std::map actual; + + while (end != begin) + { + --end; + actual.insert(*end); + } + + if (end != begin) + actual.insert(*end); + + ASSERT_TRUE(actual == expected); + } + } +} From 42ce3f2ae867815cebf02111d5d55fc5014a1461 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 17 Jan 2022 14:37:23 +0000 Subject: [PATCH 064/403] Fixed tests --- src/Dictionaries/RangeHashedDictionary.cpp | 13 +++++++------ .../01676_range_hashed_dictionary.sql | 16 ++++++++-------- ...02008_complex_key_range_hashed_dictionary.sql | 16 ++++++++-------- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/src/Dictionaries/RangeHashedDictionary.cpp b/src/Dictionaries/RangeHashedDictionary.cpp index d4e7e43b2ac..5b493c21357 100644 --- a/src/Dictionaries/RangeHashedDictionary.cpp +++ b/src/Dictionaries/RangeHashedDictionary.cpp @@ -384,9 +384,6 @@ void RangeHashedDictionary::getItemsImpl( DictionaryKeysExtractor keys_extractor(key_columns_copy, arena_holder.getComplexKeyArena()); const size_t keys_size = keys_extractor.getKeysSize(); - - static constexpr auto max_range_value = std::numeric_limits::min(); - for (size_t key_index = 0; key_index < keys_size; ++key_index) { auto key = keys_extractor.extractCurrentKey(); @@ -398,15 +395,19 @@ void RangeHashedDictionary::getItemsImpl( const auto & interval_tree = it->getMapped(); std::optional min_value; - RangeInterval min_range {max_range_value, max_range_value}; + std::optional min_range; bool has_interval = false; interval_tree.find(date, [&](auto & interval, auto & value) { has_interval = true; - if (min_range < interval) - min_value = value; + if (min_range && interval < *min_range) + min_range = interval; + else + min_range = interval; + + min_value = value; return true; }); diff --git a/tests/queries/0_stateless/01676_range_hashed_dictionary.sql b/tests/queries/0_stateless/01676_range_hashed_dictionary.sql index ff69d61b26b..7d1fc60e90d 100644 --- a/tests/queries/0_stateless/01676_range_hashed_dictionary.sql +++ b/tests/queries/0_stateless/01676_range_hashed_dictionary.sql @@ -45,13 +45,13 @@ SELECT dictHas('database_for_range_dict.range_dictionary', toUInt64(2), toDate(' SELECT dictHas('database_for_range_dict.range_dictionary', toUInt64(2), toDate('2019-05-31')); SELECT 'select columns from dictionary'; SELECT 'allColumns'; -SELECT * FROM database_for_range_dict.range_dictionary; +SELECT * FROM database_for_range_dict.range_dictionary ORDER BY CountryID, StartDate, EndDate; SELECT 'noColumns'; -SELECT 1 FROM database_for_range_dict.range_dictionary; +SELECT 1 FROM database_for_range_dict.range_dictionary ORDER BY CountryID, StartDate, EndDate; SELECT 'onlySpecificColumns'; -SELECT CountryID, StartDate, Tax FROM database_for_range_dict.range_dictionary; +SELECT CountryID, StartDate, Tax FROM database_for_range_dict.range_dictionary ORDER BY CountryID, StartDate, EndDate; SELECT 'onlySpecificColumn'; -SELECT Tax FROM database_for_range_dict.range_dictionary; +SELECT Tax FROM database_for_range_dict.range_dictionary ORDER BY CountryID, StartDate, EndDate; DROP DICTIONARY database_for_range_dict.range_dictionary; DROP TABLE database_for_range_dict.date_table; @@ -97,13 +97,13 @@ SELECT dictHas('database_for_range_dict.range_dictionary_nullable', toUInt64(2), SELECT dictHas('database_for_range_dict.range_dictionary_nullable', toUInt64(2), toDate('2019-05-31')); SELECT 'select columns from dictionary'; SELECT 'allColumns'; -SELECT * FROM database_for_range_dict.range_dictionary_nullable; +SELECT * FROM database_for_range_dict.range_dictionary_nullable ORDER BY CountryID, StartDate, EndDate; SELECT 'noColumns'; -SELECT 1 FROM database_for_range_dict.range_dictionary_nullable; +SELECT 1 FROM database_for_range_dict.range_dictionary_nullable ORDER BY CountryID, StartDate, EndDate; SELECT 'onlySpecificColumns'; -SELECT CountryID, StartDate, Tax FROM database_for_range_dict.range_dictionary_nullable; +SELECT CountryID, StartDate, Tax FROM database_for_range_dict.range_dictionary_nullable ORDER BY CountryID, StartDate, EndDate; SELECT 'onlySpecificColumn'; -SELECT Tax FROM database_for_range_dict.range_dictionary_nullable; +SELECT Tax FROM database_for_range_dict.range_dictionary_nullable ORDER BY CountryID, StartDate, EndDate; DROP DICTIONARY database_for_range_dict.range_dictionary_nullable; DROP TABLE database_for_range_dict.date_table; diff --git a/tests/queries/0_stateless/02008_complex_key_range_hashed_dictionary.sql b/tests/queries/0_stateless/02008_complex_key_range_hashed_dictionary.sql index 677879b1ebd..72cac481376 100644 --- a/tests/queries/0_stateless/02008_complex_key_range_hashed_dictionary.sql +++ b/tests/queries/0_stateless/02008_complex_key_range_hashed_dictionary.sql @@ -45,13 +45,13 @@ SELECT dictHas('range_dictionary', (toUInt64(2), '2'), toDate('2019-05-29')); SELECT dictHas('range_dictionary', (toUInt64(2), '2'), toDate('2019-05-31')); SELECT 'select columns from dictionary'; SELECT 'allColumns'; -SELECT * FROM range_dictionary; +SELECT * FROM range_dictionary ORDER BY CountryID, StartDate, EndDate; SELECT 'noColumns'; -SELECT 1 FROM range_dictionary; +SELECT 1 FROM range_dictionary ORDER BY CountryID, StartDate, EndDate; SELECT 'onlySpecificColumns'; -SELECT CountryID, StartDate, Tax FROM range_dictionary; +SELECT CountryID, StartDate, Tax FROM range_dictionary ORDER BY CountryID, StartDate, EndDate; SELECT 'onlySpecificColumn'; -SELECT Tax FROM range_dictionary; +SELECT Tax FROM range_dictionary ORDER BY CountryID, StartDate, EndDate; DROP TABLE date_table; DROP DICTIONARY range_dictionary; @@ -99,13 +99,13 @@ SELECT dictHas('range_dictionary_nullable', (toUInt64(2), '2'), toDate('2019-05- SELECT dictHas('range_dictionary_nullable', (toUInt64(2), '2'), toDate('2019-05-31')); SELECT 'select columns from dictionary'; SELECT 'allColumns'; -SELECT * FROM range_dictionary_nullable; +SELECT * FROM range_dictionary_nullable ORDER BY CountryID, StartDate, EndDate; SELECT 'noColumns'; -SELECT 1 FROM range_dictionary_nullable; +SELECT 1 FROM range_dictionary_nullable ORDER BY CountryID, StartDate, EndDate; SELECT 'onlySpecificColumns'; -SELECT CountryID, StartDate, Tax FROM range_dictionary_nullable; +SELECT CountryID, StartDate, Tax FROM range_dictionary_nullable ORDER BY CountryID, StartDate, EndDate; SELECT 'onlySpecificColumn'; -SELECT Tax FROM range_dictionary_nullable; +SELECT Tax FROM range_dictionary_nullable ORDER BY CountryID, StartDate, EndDate; DROP TABLE date_table; DROP DICTIONARY range_dictionary_nullable; From 07f3d08a1ad9ec1b0f7f0de65f0a62fae1e10756 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 17 Jan 2022 17:54:09 +0300 Subject: [PATCH 065/403] Fix stupid bug --- src/Coordination/SnapshotableHashTable.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Coordination/SnapshotableHashTable.h b/src/Coordination/SnapshotableHashTable.h index f400659bb0d..e4c7ed7a872 100644 --- a/src/Coordination/SnapshotableHashTable.h +++ b/src/Coordination/SnapshotableHashTable.h @@ -16,7 +16,8 @@ struct ListNode { StringRef key; V value; - bool active_in_map; + + bool active_in_map{true}; bool free_key{false}; }; @@ -202,8 +203,8 @@ public: else { map.erase(it->getKey()); - list.erase(list_itr); arena.free(const_cast(list_itr->key.data), list_itr->key.size); + list.erase(list_itr); } updateDataSize(ERASE, key.size(), 0, old_data_size); @@ -285,8 +286,10 @@ public: void clear() { - list.clear(); map.clear(); + for (auto itr = list.begin(); itr != list.end(); ++itr) + arena.free(const_cast(itr->key.data), itr->key.size); + list.clear(); updateDataSize(CLEAR, 0, 0, 0); } From 690590f7c302747ccbde9c4621b95138a8b0736b Mon Sep 17 00:00:00 2001 From: alexeypavlenko Date: Thu, 13 Jan 2022 11:22:25 +0200 Subject: [PATCH 066/403] Fix default value generation for columns with dot --- src/Interpreters/addMissingDefaults.cpp | 5 +++-- ...167_columns_with_dots_default_values.reference | 1 + .../02167_columns_with_dots_default_values.sql | 15 +++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/02167_columns_with_dots_default_values.reference create mode 100644 tests/queries/0_stateless/02167_columns_with_dots_default_values.sql diff --git a/src/Interpreters/addMissingDefaults.cpp b/src/Interpreters/addMissingDefaults.cpp index ffabb43ed42..7ff90dab767 100644 --- a/src/Interpreters/addMissingDefaults.cpp +++ b/src/Interpreters/addMissingDefaults.cpp @@ -58,10 +58,11 @@ ActionsDAGPtr addMissingDefaults( continue; String offsets_name = Nested::extractTableName(column.name); - if (nested_groups.count(offsets_name)) + const auto* array_type = typeid_cast(column.type.get()) + if (array_type && nested_groups.count(offsets_name)) { - DataTypePtr nested_type = typeid_cast(*column.type).getNestedType(); + const auto& nested_type = array_type->getNestedType(); ColumnPtr nested_column = nested_type->createColumnConstWithDefaultValue(0); const auto & constant = actions->addColumn({std::move(nested_column), nested_type, column.name}); diff --git a/tests/queries/0_stateless/02167_columns_with_dots_default_values.reference b/tests/queries/0_stateless/02167_columns_with_dots_default_values.reference new file mode 100644 index 00000000000..c3f983ab30e --- /dev/null +++ b/tests/queries/0_stateless/02167_columns_with_dots_default_values.reference @@ -0,0 +1 @@ +id [str1,str2] diff --git a/tests/queries/0_stateless/02167_columns_with_dots_default_values.sql b/tests/queries/0_stateless/02167_columns_with_dots_default_values.sql new file mode 100644 index 00000000000..a80b6f7b110 --- /dev/null +++ b/tests/queries/0_stateless/02167_columns_with_dots_default_values.sql @@ -0,0 +1,15 @@ +DROP TABLE IF EXISTS test_nested_default + +CREATE TABLE test_nested_default +( + `id` String, + `with_dot.str` String, + `with_dot.array.string` Array(String) +) +ENGINE = MergeTree() +ORDER BY id; + +INSERT INTO test_nested_default(`id`, `with_dot.array.string`) VALUES('id', ['str1', 'str2']); +SELECT * FROM test_nested_default; + +DROP TABLE test_nested_default; From 65519527ef50a3d748d66fbde3e6404eda8b7814 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 13 Jan 2022 12:44:25 +0300 Subject: [PATCH 067/403] Update addMissingDefaults.cpp Fix style --- src/Interpreters/addMissingDefaults.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/addMissingDefaults.cpp b/src/Interpreters/addMissingDefaults.cpp index 7ff90dab767..c3aa55173cb 100644 --- a/src/Interpreters/addMissingDefaults.cpp +++ b/src/Interpreters/addMissingDefaults.cpp @@ -58,11 +58,10 @@ ActionsDAGPtr addMissingDefaults( continue; String offsets_name = Nested::extractTableName(column.name); - const auto* array_type = typeid_cast(column.type.get()) + const auto * array_type = typeid_cast(column.type.get()) if (array_type && nested_groups.count(offsets_name)) { - - const auto& nested_type = array_type->getNestedType(); + const auto & nested_type = array_type->getNestedType(); ColumnPtr nested_column = nested_type->createColumnConstWithDefaultValue(0); const auto & constant = actions->addColumn({std::move(nested_column), nested_type, column.name}); From 96ba5b14c58915ab6eb66e4d30cd03590c086c06 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 13 Jan 2022 12:54:07 +0300 Subject: [PATCH 068/403] Update addMissingDefaults.cpp --- src/Interpreters/addMissingDefaults.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/addMissingDefaults.cpp b/src/Interpreters/addMissingDefaults.cpp index c3aa55173cb..f6d5f83fce3 100644 --- a/src/Interpreters/addMissingDefaults.cpp +++ b/src/Interpreters/addMissingDefaults.cpp @@ -58,7 +58,7 @@ ActionsDAGPtr addMissingDefaults( continue; String offsets_name = Nested::extractTableName(column.name); - const auto * array_type = typeid_cast(column.type.get()) + const auto * array_type = typeid_cast(column.type.get()); if (array_type && nested_groups.count(offsets_name)) { const auto & nested_type = array_type->getNestedType(); From 8778eda2e0dcd8b2963e389aa38038a2a1c92c02 Mon Sep 17 00:00:00 2001 From: alexeypavlenko Date: Thu, 13 Jan 2022 15:30:32 +0200 Subject: [PATCH 069/403] fix unit test (sorry, took a while to build it on my laptop) --- .../02167_columns_with_dots_default_values.reference | 2 +- .../0_stateless/02167_columns_with_dots_default_values.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02167_columns_with_dots_default_values.reference b/tests/queries/0_stateless/02167_columns_with_dots_default_values.reference index c3f983ab30e..9b962ccf476 100644 --- a/tests/queries/0_stateless/02167_columns_with_dots_default_values.reference +++ b/tests/queries/0_stateless/02167_columns_with_dots_default_values.reference @@ -1 +1 @@ -id [str1,str2] +id ['str1','str2'] diff --git a/tests/queries/0_stateless/02167_columns_with_dots_default_values.sql b/tests/queries/0_stateless/02167_columns_with_dots_default_values.sql index a80b6f7b110..90ecdd40623 100644 --- a/tests/queries/0_stateless/02167_columns_with_dots_default_values.sql +++ b/tests/queries/0_stateless/02167_columns_with_dots_default_values.sql @@ -1,4 +1,4 @@ -DROP TABLE IF EXISTS test_nested_default +DROP TABLE IF EXISTS test_nested_default; CREATE TABLE test_nested_default ( From 4cc1e25d45b29bb6d4d7852acca5bdf10aef7b04 Mon Sep 17 00:00:00 2001 From: alexeypavlenko Date: Fri, 14 Jan 2022 10:33:07 +0200 Subject: [PATCH 070/403] simplify test --- .../0_stateless/02167_columns_with_dots_default_values.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02167_columns_with_dots_default_values.sql b/tests/queries/0_stateless/02167_columns_with_dots_default_values.sql index 90ecdd40623..f110fae16c3 100644 --- a/tests/queries/0_stateless/02167_columns_with_dots_default_values.sql +++ b/tests/queries/0_stateless/02167_columns_with_dots_default_values.sql @@ -4,7 +4,7 @@ CREATE TABLE test_nested_default ( `id` String, `with_dot.str` String, - `with_dot.array.string` Array(String) + `with_dot.array` Array(String) ) ENGINE = MergeTree() ORDER BY id; From 06b08b0e3adfe6c3bc06a92fb6df09173edc0b99 Mon Sep 17 00:00:00 2001 From: alexeypavlenko Date: Mon, 17 Jan 2022 23:06:56 +0200 Subject: [PATCH 071/403] fix insert --- .../0_stateless/02167_columns_with_dots_default_values.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02167_columns_with_dots_default_values.sql b/tests/queries/0_stateless/02167_columns_with_dots_default_values.sql index f110fae16c3..81c83c2cecc 100644 --- a/tests/queries/0_stateless/02167_columns_with_dots_default_values.sql +++ b/tests/queries/0_stateless/02167_columns_with_dots_default_values.sql @@ -9,7 +9,7 @@ CREATE TABLE test_nested_default ENGINE = MergeTree() ORDER BY id; -INSERT INTO test_nested_default(`id`, `with_dot.array.string`) VALUES('id', ['str1', 'str2']); +INSERT INTO test_nested_default(`id`, `with_dot.array`) VALUES('id', ['str1', 'str2']); SELECT * FROM test_nested_default; DROP TABLE test_nested_default; From c6e205b2ad7a2e10f12915856066e37105804d49 Mon Sep 17 00:00:00 2001 From: liuneng1994 <1398775315@qq.com> Date: Tue, 18 Jan 2022 05:55:41 +0000 Subject: [PATCH 072/403] add error check --- src/Storages/HDFS/StorageHDFS.cpp | 3 +++ tests/integration/test_storage_hdfs/test.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index f22f6f66ced..429599a5d3d 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -72,6 +72,9 @@ namespace HDFSFileInfo ls; ls.file_info = hdfsListDirectory(fs.get(), prefix_without_globs.data(), &ls.length); + if (ls.file_info == nullptr) { + throw Exception(ErrorCodes::ACCESS_DENIED, "Cannot list directory {}: {}", prefix_without_globs, String(hdfsGetLastError())); + } Strings result; for (int i = 0; i < ls.length; ++i) { diff --git a/tests/integration/test_storage_hdfs/test.py b/tests/integration/test_storage_hdfs/test.py index f317fb5429a..fe1ea373e56 100644 --- a/tests/integration/test_storage_hdfs/test.py +++ b/tests/integration/test_storage_hdfs/test.py @@ -361,6 +361,10 @@ def test_hdfsCluster(started_cluster): assert actual == expected fs.delete(dir, recursive=True) +def testHdfsDirectoryNotExist(started_cluster): + ddl ="create table HDFSStorageWithNotExistDir (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/dir1/empty', 'TSV')"; + node1.query(ddl) + assert "Cannot list directory" in node1.query_and_get_error("select * from HDFSStorageWithNotExistDir") if __name__ == '__main__': cluster.start() From 3391f3f06e7effa422c6b370cf51a3c4eaa79bf2 Mon Sep 17 00:00:00 2001 From: frank chen Date: Tue, 18 Jan 2022 14:27:08 +0800 Subject: [PATCH 073/403] Change the order Signed-off-by: frank chen --- .../en/sql-reference/functions/encoding-functions.md | 12 ++++++------ src/Functions/FunctionsBinaryRepr.cpp | 7 ++++--- .../0_stateless/02128_hex_bin_on_uuid.reference | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/en/sql-reference/functions/encoding-functions.md b/docs/en/sql-reference/functions/encoding-functions.md index f3beddac1bd..ec1524f1fa3 100644 --- a/docs/en/sql-reference/functions/encoding-functions.md +++ b/docs/en/sql-reference/functions/encoding-functions.md @@ -152,14 +152,14 @@ Result: Query: ``` sql -SELECT lower(hex(toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0'))) as big_endian_hex +SELECT lower(hex(toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0'))) as uuid_hex ``` Result: ``` text -┌─big_endian_hex───────────────────┐ -│ 907ba6006ad3dba061f0c4045cb311e7 │ +┌─uuid_hex─────────────────────────┐ +│ 61f0c4045cb311e7907ba6006ad3dba0 │ └──────────────────────────────────┘ ``` @@ -302,14 +302,14 @@ Result: Query: ``` sql -SELECT lower(bin(toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0'))) as big_endian_bin +SELECT bin(toUUID('61f0c404-5cb3-11e7-907b-a6006ad3dba0')) as bin_uuid ``` Result: ``` text -┌─big_endian_bin───────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ -│ 10010000011110111010011000000000011010101101001111011011101000000110000111110000110001000000010001011100101100110001000111100111 │ +┌─bin_uuid─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ 01100001111100001100010000000100010111001011001100010001111001111001000001111011101001100000000001101010110100111101101110100000 │ └──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ ``` diff --git a/src/Functions/FunctionsBinaryRepr.cpp b/src/Functions/FunctionsBinaryRepr.cpp index 59ba722f3c0..7cc0fc324ab 100644 --- a/src/Functions/FunctionsBinaryRepr.cpp +++ b/src/Functions/FunctionsBinaryRepr.cpp @@ -519,9 +519,10 @@ public: char * begin = reinterpret_cast(&out_vec[pos]); char * end = begin; - // index 1 stores the higher 64 bit - Impl::executeOneUInt(uuid[i].toUnderType().items[1], end, false, false); - Impl::executeOneUInt(uuid[i].toUnderType().items[0], end, false, true); + // use executeOnUInt instead of using executeOneString + // because the latter one outputs the string in the memory order + Impl::executeOneUInt(uuid[i].toUnderType().items[0], end, false, false); + Impl::executeOneUInt(uuid[i].toUnderType().items[1], end, false, true); pos += end - begin; out_offsets[i] = pos; diff --git a/tests/queries/0_stateless/02128_hex_bin_on_uuid.reference b/tests/queries/0_stateless/02128_hex_bin_on_uuid.reference index 673bca894f5..d0b85f7c688 100644 --- a/tests/queries/0_stateless/02128_hex_bin_on_uuid.reference +++ b/tests/queries/0_stateless/02128_hex_bin_on_uuid.reference @@ -1,5 +1,5 @@ 32 1 -00009d773a2fd3190000000080e746f8 +0000000080e746f800009d773a2fd319 1 128 From f526ff2a5163550fe024fcfc321b4137acb21fe1 Mon Sep 17 00:00:00 2001 From: Vxider Date: Tue, 18 Jan 2022 09:03:16 +0000 Subject: [PATCH 074/403] remove unused variable --- src/Storages/WindowView/StorageWindowView.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index a81a5a9649a..d14ce7e1c52 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -200,7 +200,6 @@ namespace { String window_id_name; String window_id_alias; - Aliases * aliases; }; static bool needChildVisit(ASTPtr &, const ASTPtr &) { return true; } From ba0a7028075a3501d2d1d7c729ac871cdf022271 Mon Sep 17 00:00:00 2001 From: Vxider Date: Tue, 18 Jan 2022 09:20:15 +0000 Subject: [PATCH 075/403] enable stream to table join in windowview --- src/Storages/LiveView/StorageBlocks.h | 2 +- src/Storages/WindowView/StorageWindowView.cpp | 170 +++++++++--------- .../WindowView/WindowViewProxyStorage.h | 48 ----- ...7_window_view_parser_inner_table.reference | 10 +- .../01047_window_view_parser_inner_table.sql | 31 ++-- .../01048_window_view_parser.reference | 36 ++-- .../0_stateless/01048_window_view_parser.sql | 26 ++- ...indow_view_event_tumble_asc_join.reference | 3 + ...01071_window_view_event_tumble_asc_join.sh | 39 ++++ 9 files changed, 201 insertions(+), 164 deletions(-) delete mode 100644 src/Storages/WindowView/WindowViewProxyStorage.h create mode 100644 tests/queries/0_stateless/01071_window_view_event_tumble_asc_join.reference create mode 100755 tests/queries/0_stateless/01071_window_view_event_tumble_asc_join.sh diff --git a/src/Storages/LiveView/StorageBlocks.h b/src/Storages/LiveView/StorageBlocks.h index f010ef50ecd..c085e600d39 100644 --- a/src/Storages/LiveView/StorageBlocks.h +++ b/src/Storages/LiveView/StorageBlocks.h @@ -10,7 +10,7 @@ namespace DB class StorageBlocks : public IStorage { /* Storage based on the prepared streams that already contain data blocks. - * Used by Live Views to complete stored query based on the mergeable blocks. + * Used by Live and Window Views to complete stored query based on the mergeable blocks. */ public: StorageBlocks(const StorageID & table_id_, diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index d14ce7e1c52..be3cda1c980 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -40,9 +40,10 @@ #include #include +#include + #include #include -#include #include @@ -238,6 +239,23 @@ namespace } }; + struct DropTableIdentifierMatcher + { + using Visitor = InDepthNodeVisitor; + + struct Data{}; + + static bool needChildVisit(ASTPtr &, const ASTPtr &) { return true; } + + static void visit(ASTPtr & ast, Data &) + { + if (auto * t = ast->as()) + { + ast = std::make_shared(t->shortName()); + } + } + }; + IntervalKind strToIntervalKind(const String& interval_str) { if (interval_str == "Second") @@ -462,16 +480,26 @@ std::pair StorageWindowView::getNewBlocks(UInt32 watermark) current_header, getWindowIDColumnPosition(current_header), window_column_name_and_type, window_value); }); + Pipes pipes; auto pipe = QueryPipelineBuilder::getPipe(std::move(builder)); - auto parent_table_metadata = getParentStorage()->getInMemoryMetadataPtr(); - auto required_columns = parent_table_metadata->getColumns(); - required_columns.add(ColumnDescription("____timestamp", std::make_shared())); - auto proxy_storage = std::make_shared( - StorageID(getStorageID().database_name, "WindowViewProxyStorage"), required_columns, - std::move(pipe), QueryProcessingStage::WithMergeableState); + pipes.emplace_back(std::move(pipe)); + + auto creator = [&](const StorageID & blocks_id_global) { + auto parent_table_metadata = getParentStorage()->getInMemoryMetadataPtr(); + auto required_columns = parent_table_metadata->getColumns(); + required_columns.add(ColumnDescription("____timestamp", std::make_shared())); + return StorageBlocks::createStorage(blocks_id_global, required_columns, std::move(pipes), QueryProcessingStage::WithMergeableState); + }; + + TemporaryTableHolder blocks_storage(window_view_context, creator); InterpreterSelectQuery select( - getFinalQuery(), window_view_context, proxy_storage, nullptr, SelectQueryOptions(QueryProcessingStage::Complete)); + getFinalQuery(), + window_view_context, + blocks_storage.getTable(), + blocks_storage.getTable()->getInMemoryMetadataPtr(), + SelectQueryOptions(QueryProcessingStage::Complete)); + builder = select.buildQueryPipeline(); builder.addSimpleTransform([&](const Block & current_header) @@ -576,16 +604,16 @@ std::shared_ptr StorageWindowView::getInnerTableCreateQuery( auto columns_list = std::make_shared(); + String window_id_column_name; if (is_time_column_func_now) { auto column_window = std::make_shared(); column_window->name = window_id_name; column_window->type = std::make_shared("UInt32"); columns_list->children.push_back(column_window); + window_id_column_name = window_id_name; } - bool has_window_id = false; - for (const auto & column : t_sample_block.getColumnsWithTypeAndName()) { ParserIdentifierWithOptionalParameters parser; @@ -595,28 +623,17 @@ std::shared_ptr StorageWindowView::getInnerTableCreateQuery( column_dec->name = column.name; column_dec->type = ast; columns_list->children.push_back(column_dec); - if (!is_time_column_func_now && !has_window_id) + if (!is_time_column_func_now && window_id_column_name.empty() && startsWith(column.name, "windowID")) { - if (startsWith(column.name, "windowID")) - has_window_id = true; + window_id_column_name = column.name; } } - if (!is_time_column_func_now && !has_window_id) + if (window_id_column_name.empty()) throw Exception( "The first argument of time window function should not be a constant value.", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_WINDOW_VIEW); - ToIdentifierMatcher::Data query_data; - query_data.window_id_name = window_id_name; - query_data.window_id_alias = window_id_alias; - ToIdentifierMatcher::Visitor to_identifier_visitor(query_data); - - ReplaceFunctionNowData time_now_data; - ReplaceFunctionNowVisitor time_now_visitor(time_now_data); - ReplaceFunctionWindowMatcher::Data func_hop_data; - ReplaceFunctionWindowMatcher::Visitor func_window_visitor(func_hop_data); - auto new_storage = std::make_shared(); /// storage != nullptr in case create window view with ENGINE syntax if (storage) @@ -632,6 +649,19 @@ std::shared_ptr StorageWindowView::getInnerTableCreateQuery( "The ENGINE of WindowView must be MergeTree family of table engines " "including the engines with replication support"); + ToIdentifierMatcher::Data query_data; + query_data.window_id_name = window_id_name; + query_data.window_id_alias = window_id_alias; + ToIdentifierMatcher::Visitor to_identifier_visitor(query_data); + + ReplaceFunctionNowData time_now_data; + ReplaceFunctionNowVisitor time_now_visitor(time_now_data); + ReplaceFunctionWindowMatcher::Data func_hop_data; + ReplaceFunctionWindowMatcher::Visitor func_window_visitor(func_hop_data); + + DropTableIdentifierMatcher::Data drop_table_identifier_data; + DropTableIdentifierMatcher::Visitor drop_table_identifier_visitor(drop_table_identifier_data); + new_storage->set(new_storage->engine, storage->engine->clone()); auto visit = [&](const IAST * ast, IAST *& field) @@ -639,24 +669,22 @@ std::shared_ptr StorageWindowView::getInnerTableCreateQuery( if (ast) { auto node = ast->clone(); + QueryNormalizer(normalizer_data).visit(node); /// now() -> ____timestamp if (is_time_column_func_now) { time_now_visitor.visit(node); function_now_timezone = time_now_data.now_timezone; } + drop_table_identifier_visitor.visit(node); /// tumble/hop -> windowID func_window_visitor.visit(node); to_identifier_visitor.visit(node); - QueryNormalizer(normalizer_data).visit(node); node->setAlias(""); new_storage->set(field, node); } }; - for (auto & [alias_name, ast] : aliases) - ast = std::make_shared(ast->getColumnName()); - visit(storage->partition_by, new_storage->partition_by); visit(storage->primary_key, new_storage->primary_key); visit(storage->order_by, new_storage->order_by); @@ -669,35 +697,8 @@ std::shared_ptr StorageWindowView::getInnerTableCreateQuery( { new_storage->set(new_storage->engine, makeASTFunction("AggregatingMergeTree")); - for (auto & child : inner_select_query->groupBy()->children) - if (auto * ast_with_alias = dynamic_cast(child.get())) - ast_with_alias->setAlias(""); - - auto order_by = std::make_shared(); - order_by->name = "tuple"; - order_by->arguments = inner_select_query->groupBy(); - order_by->children.push_back(order_by->arguments); - - ASTPtr order_by_ptr = order_by; - if (is_time_column_func_now) - { - time_now_visitor.visit(order_by_ptr); - function_now_timezone = time_now_data.now_timezone; - } - to_identifier_visitor.visit(order_by_ptr); - - for (auto & child : order_by->arguments->children) - { - if (child->getColumnName() == window_id_name) - { - ASTPtr tmp = child; - child = order_by->arguments->children[0]; - order_by->arguments->children[0] = tmp; - break; - } - } - new_storage->set(new_storage->order_by, order_by_ptr); - new_storage->set(new_storage->primary_key, std::make_shared(window_id_name)); + new_storage->set(new_storage->order_by, std::make_shared(window_id_column_name)); + new_storage->set(new_storage->primary_key, std::make_shared(window_id_column_name)); } auto new_columns = std::make_shared(); @@ -1235,35 +1236,36 @@ void StorageWindowView::writeIntoWindowView( return std::make_shared(stream_header, adding_column_actions); }); } - - InterpreterSelectQuery select_block( - window_view.getMergeableQuery(), local_context, {std::move(pipe)}, - QueryProcessingStage::WithMergeableState); - - builder = select_block.buildQueryPipeline(); - builder.addSimpleTransform([&](const Block & current_header) - { - return std::make_shared( - current_header, - local_context->getSettingsRef().min_insert_block_size_rows, - local_context->getSettingsRef().min_insert_block_size_bytes); - }); } - else + + Pipes pipes; + pipes.emplace_back(std::move(pipe)); + + auto creator = [&](const StorageID & blocks_id_global) { + auto parent_metadata = window_view.getParentStorage()->getInMemoryMetadataPtr(); + auto required_columns = parent_metadata->getColumns(); + required_columns.add(ColumnDescription("____timestamp", std::make_shared())); + return StorageBlocks::createStorage(blocks_id_global, required_columns, std::move(pipes), QueryProcessingStage::FetchColumns); + }; + TemporaryTableHolder blocks_storage(local_context, creator); + + InterpreterSelectQuery select_block( + window_view.getMergeableQuery(), + local_context, + blocks_storage.getTable(), + blocks_storage.getTable()->getInMemoryMetadataPtr(), + QueryProcessingStage::WithMergeableState); + + builder = select_block.buildQueryPipeline(); + builder.addSimpleTransform([&](const Block & current_header) { + return std::make_shared( + current_header, + local_context->getSettingsRef().min_insert_block_size_rows, + local_context->getSettingsRef().min_insert_block_size_bytes); + }); + + if (!window_view.is_proctime) { - InterpreterSelectQuery select_block( - window_view.getMergeableQuery(), local_context, {std::move(pipe)}, - QueryProcessingStage::WithMergeableState); - - builder = select_block.buildQueryPipeline(); - builder.addSimpleTransform([&](const Block & current_header) - { - return std::make_shared( - current_header, - local_context->getSettingsRef().min_insert_block_size_rows, - local_context->getSettingsRef().min_insert_block_size_bytes); - }); - UInt32 block_max_timestamp = 0; if (window_view.is_watermark_bounded || window_view.allowed_lateness) { diff --git a/src/Storages/WindowView/WindowViewProxyStorage.h b/src/Storages/WindowView/WindowViewProxyStorage.h deleted file mode 100644 index 55426bddd5d..00000000000 --- a/src/Storages/WindowView/WindowViewProxyStorage.h +++ /dev/null @@ -1,48 +0,0 @@ -#pragma once - -#include -#include -#include - -namespace DB -{ - -class WindowViewProxyStorage : public IStorage -{ -public: - WindowViewProxyStorage(const StorageID & table_id_, ColumnsDescription columns_, Pipe pipe_, QueryProcessingStage::Enum to_stage_) - : IStorage(table_id_) - , pipe(std::move(pipe_)) - , to_stage(to_stage_) - { - StorageInMemoryMetadata storage_metadata; - storage_metadata.setColumns(columns_); - setInMemoryMetadata(storage_metadata); - } - -public: - std::string getName() const override { return "WindowViewProxy"; } - - QueryProcessingStage::Enum - getQueryProcessingStage(ContextPtr, QueryProcessingStage::Enum, const StorageMetadataPtr &, SelectQueryInfo &) const override - { - return to_stage; - } - - Pipe read( - const Names &, - const StorageMetadataPtr & /*metadata_snapshot*/, - SelectQueryInfo & /*query_info*/, - ContextPtr /*context*/, - QueryProcessingStage::Enum /*processed_stage*/, - size_t /*max_block_size*/, - unsigned /*num_streams*/) override - { - return std::move(pipe); - } - -private: - Pipe pipe; - QueryProcessingStage::Enum to_stage; -}; -} diff --git a/tests/queries/0_stateless/01047_window_view_parser_inner_table.reference b/tests/queries/0_stateless/01047_window_view_parser_inner_table.reference index 96f7cbb1d69..319cbef0914 100644 --- a/tests/queries/0_stateless/01047_window_view_parser_inner_table.reference +++ b/tests/queries/0_stateless/01047_window_view_parser_inner_table.reference @@ -1,6 +1,4 @@ ---TUMBLE--- -||---DEFAULT ENGINE WITH DATA COLUMN ALIAS--- -CREATE TABLE test_01047.`.inner.wv`\n(\n `b` Int32,\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'))`\nORDER BY (`windowID(timestamp, toIntervalSecond(\'1\'))`, b)\nSETTINGS index_granularity = 8192 ||---WINDOW COLUMN NAME--- CREATE TABLE test_01047.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 ||---WINDOW COLUMN ALIAS--- @@ -13,9 +11,10 @@ CREATE TABLE test_01047.`.inner.wv`\n(\n `b` Int32,\n `windowID(timestamp, CREATE TABLE test_01047.`.inner.wv`\n(\n `plus(a, b)` Int64,\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'))`\nORDER BY (`windowID(timestamp, toIntervalSecond(\'1\'))`, `plus(a, b)`)\nSETTINGS index_granularity = 8192 ||---PARTITION--- CREATE TABLE test_01047.`.inner.wv`\n(\n `windowID(____timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY `windowID(____timestamp, toIntervalSecond(\'1\'))`\nORDER BY `windowID(____timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 +||---JOIN--- +CREATE TABLE test_01047.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32),\n `count(mt_2.b)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01047.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32),\n `count(mt_2.b)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 ---HOP--- -||---DEFAULT ENGINE WITH DATA COLUMN ALIAS--- -CREATE TABLE test_01047.`.inner.wv`\n(\n `b` Int32,\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY (`windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`, b)\nSETTINGS index_granularity = 8192 ||---WINDOW COLUMN NAME--- CREATE TABLE test_01047.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 ||---WINDOW COLUMN ALIAS--- @@ -28,3 +27,6 @@ CREATE TABLE test_01047.`.inner.wv`\n(\n `b` Int32,\n `windowID(timestamp, CREATE TABLE test_01047.`.inner.wv`\n(\n `plus(a, b)` Int64,\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY (`windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`, `plus(a, b)`)\nSETTINGS index_granularity = 8192 ||---PARTITION--- CREATE TABLE test_01047.`.inner.wv`\n(\n `windowID(____timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY `windowID(____timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY `windowID(____timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 +||---JOIN--- +CREATE TABLE test_01047.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32),\n `count(mt_2.b)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01047.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32),\n `count(mt_2.b)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01047_window_view_parser_inner_table.sql b/tests/queries/0_stateless/01047_window_view_parser_inner_table.sql index 595d93e0771..3e4f95d098e 100644 --- a/tests/queries/0_stateless/01047_window_view_parser_inner_table.sql +++ b/tests/queries/0_stateless/01047_window_view_parser_inner_table.sql @@ -5,16 +5,12 @@ DROP DATABASE IF EXISTS test_01047; CREATE DATABASE test_01047 ENGINE=Ordinary; DROP TABLE IF EXISTS test_01047.mt; +DROP TABLE IF EXISTS test_01047.mt_2; CREATE TABLE test_01047.mt(a Int32, b Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple(); +CREATE TABLE test_01047.mt_2(a Int32, b Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple(); SELECT '---TUMBLE---'; -SELECT '||---DEFAULT ENGINE WITH DATA COLUMN ALIAS---'; -DROP TABLE IF EXISTS test_01047.wv; -DROP TABLE IF EXISTS test_01047.`.inner.wv`; -CREATE WINDOW VIEW test_01047.wv AS SELECT count(a) AS count, b as id FROM test_01047.mt GROUP BY id, tumble(timestamp, INTERVAL '1' SECOND); -SHOW CREATE TABLE test_01047.`.inner.wv`; - SELECT '||---WINDOW COLUMN NAME---'; DROP TABLE IF EXISTS test_01047.wv; DROP TABLE IF EXISTS test_01047.`.inner.wv`; @@ -51,14 +47,17 @@ DROP TABLE IF EXISTS test_01047.`.inner.wv`; CREATE WINDOW VIEW test_01047.wv ENGINE AggregatingMergeTree ORDER BY wid PARTITION BY wid AS SELECT count(a) AS count, tumble(now(), INTERVAL '1' SECOND) AS wid FROM test_01047.mt GROUP BY wid; SHOW CREATE TABLE test_01047.`.inner.wv`; - -SELECT '---HOP---'; -SELECT '||---DEFAULT ENGINE WITH DATA COLUMN ALIAS---'; +SELECT '||---JOIN---'; DROP TABLE IF EXISTS test_01047.wv; -DROP TABLE IF EXISTS test_01047.`.inner.wv`; -CREATE WINDOW VIEW test_01047.wv AS SELECT count(a) AS count, b as id FROM test_01047.mt GROUP BY id, hop(timestamp, INTERVAL '1' SECOND, INTERVAL '3' SECOND); +CREATE WINDOW VIEW test_01047.wv ENGINE AggregatingMergeTree ORDER BY tumble(test_01047.mt.timestamp, INTERVAL '1' SECOND) AS SELECT count(test_01047.mt.a), count(test_01047.mt_2.b), wid FROM test_01047.mt JOIN test_01047.mt_2 ON test_01047.mt.timestamp = test_01047.mt_2.timestamp GROUP BY tumble(test_01047.mt.timestamp, INTERVAL '1' SECOND) AS wid; SHOW CREATE TABLE test_01047.`.inner.wv`; +DROP TABLE IF EXISTS test_01047.wv; +CREATE WINDOW VIEW test_01047.wv ENGINE AggregatingMergeTree ORDER BY wid AS SELECT count(test_01047.mt.a), count(test_01047.mt_2.b), wid FROM test_01047.mt JOIN test_01047.mt_2 ON test_01047.mt.timestamp = test_01047.mt_2.timestamp GROUP BY tumble(test_01047.mt.timestamp, INTERVAL '1' SECOND) AS wid; +SHOW CREATE TABLE test_01047.`.inner.wv`; + + +SELECT '---HOP---'; SELECT '||---WINDOW COLUMN NAME---'; DROP TABLE IF EXISTS test_01047.wv; DROP TABLE IF EXISTS test_01047.`.inner.wv`; @@ -95,5 +94,15 @@ DROP TABLE IF EXISTS test_01047.`.inner.wv`; CREATE WINDOW VIEW test_01047.wv ENGINE AggregatingMergeTree ORDER BY wid PARTITION BY wid AS SELECT count(a) AS count, hopEnd(wid) FROM test_01047.mt GROUP BY hop(now(), INTERVAL '1' SECOND, INTERVAL '3' SECOND) as wid; SHOW CREATE TABLE test_01047.`.inner.wv`; +SELECT '||---JOIN---'; +DROP TABLE IF EXISTS test_01047.wv; +CREATE WINDOW VIEW test_01047.wv ENGINE AggregatingMergeTree ORDER BY hop(test_01047.mt.timestamp, INTERVAL '1' SECOND, INTERVAL '3' SECOND) AS SELECT count(test_01047.mt.a), count(test_01047.mt_2.b), wid FROM test_01047.mt JOIN test_01047.mt_2 ON test_01047.mt.timestamp = test_01047.mt_2.timestamp GROUP BY hop(test_01047.mt.timestamp, INTERVAL '1' SECOND, INTERVAL '3' SECOND) AS wid; +SHOW CREATE TABLE test_01047.`.inner.wv`; + +DROP TABLE IF EXISTS test_01047.wv; +CREATE WINDOW VIEW test_01047.wv ENGINE AggregatingMergeTree ORDER BY wid AS SELECT count(test_01047.mt.a), count(test_01047.mt_2.b), wid FROM test_01047.mt JOIN test_01047.mt_2 ON test_01047.mt.timestamp = test_01047.mt_2.timestamp GROUP BY hop(test_01047.mt.timestamp, INTERVAL '1' SECOND, INTERVAL '3' SECOND) AS wid; +SHOW CREATE TABLE test_01047.`.inner.wv`; + DROP TABLE test_01047.wv; DROP TABLE test_01047.mt; +DROP TABLE test_01047.mt_2; diff --git a/tests/queries/0_stateless/01048_window_view_parser.reference b/tests/queries/0_stateless/01048_window_view_parser.reference index 6625313f572..c055971bef3 100644 --- a/tests/queries/0_stateless/01048_window_view_parser.reference +++ b/tests/queries/0_stateless/01048_window_view_parser.reference @@ -1,26 +1,34 @@ ---TUMBLE--- ||---WINDOW COLUMN NAME--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(1))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(1))`\nORDER BY tuple(`windowID(timestamp, toIntervalSecond(1))`)\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(1))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(1))`\nORDER BY `windowID(timestamp, toIntervalSecond(1))`\nSETTINGS index_granularity = 8192 ||---WINDOW COLUMN ALIAS--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'))`\nORDER BY tuple(`windowID(timestamp, toIntervalSecond(\'1\'))`)\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 ||---IDENTIFIER--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `b` Int32,\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'))`\nORDER BY (`windowID(timestamp, toIntervalSecond(\'1\'))`, b)\nSETTINGS index_granularity = 8192 -CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `b` Int32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'))`\nORDER BY (`windowID(timestamp, toIntervalSecond(\'1\'))`, b)\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `b` Int32,\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `b` Int32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 ||---FUNCTION--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `plus(a, b)` Int64,\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'))`\nORDER BY (`windowID(timestamp, toIntervalSecond(\'1\'))`, `plus(a, b)`)\nSETTINGS index_granularity = 8192 -CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `plus(a, b)` Int64,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'))`\nORDER BY (`windowID(timestamp, toIntervalSecond(\'1\'))`, `plus(a, b)`)\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `plus(a, b)` Int64,\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `plus(a, b)` Int64,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 ||---TimeZone--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), \'Asia/Shanghai\')` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), \'Asia/Shanghai\')`\nORDER BY tuple(`windowID(timestamp, toIntervalSecond(\'1\'), \'Asia/Shanghai\')`)\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), \'Asia/Shanghai\')` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), \'Asia/Shanghai\')`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), \'Asia/Shanghai\')`\nSETTINGS index_granularity = 8192 +||---DATA COLUMN ALIAS--- +CREATE TABLE test_01048.`.inner.wv`\n(\n `b` Int32,\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 +||---JOIN--- +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32),\n `count(mt_2.b)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'))`\nSETTINGS index_granularity = 8192 ---HOP--- ||---WINDOW COLUMN NAME--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(1), toIntervalSecond(3))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(1), toIntervalSecond(3))`\nORDER BY tuple(`windowID(timestamp, toIntervalSecond(1), toIntervalSecond(3))`)\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(1), toIntervalSecond(3))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(1), toIntervalSecond(3))`\nORDER BY `windowID(timestamp, toIntervalSecond(1), toIntervalSecond(3))`\nSETTINGS index_granularity = 8192 ||---WINDOW COLUMN ALIAS--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY tuple(`windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`)\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 ||---IDENTIFIER--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `b` Int32,\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY (`windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`, b)\nSETTINGS index_granularity = 8192 -CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `b` Int32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY (`windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`, b)\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `b` Int32,\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `b` Int32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 ||---FUNCTION--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `plus(a, b)` Int64,\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY (`windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`, `plus(a, b)`)\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `plus(a, b)` Int64,\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `plus(a, b)` Int64,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 ||---TimeZone--- -CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(1), toIntervalSecond(3), \'Asia/Shanghai\')` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(1), toIntervalSecond(3), \'Asia/Shanghai\')`\nORDER BY tuple(`windowID(timestamp, toIntervalSecond(1), toIntervalSecond(3), \'Asia/Shanghai\')`)\nSETTINGS index_granularity = 8192 -CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `plus(a, b)` Int64,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY (`windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`, `plus(a, b)`)\nSETTINGS index_granularity = 8192 +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(1), toIntervalSecond(3), \'Asia/Shanghai\')` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(1), toIntervalSecond(3), \'Asia/Shanghai\')`\nORDER BY `windowID(timestamp, toIntervalSecond(1), toIntervalSecond(3), \'Asia/Shanghai\')`\nSETTINGS index_granularity = 8192 +||---DATA COLUMN ALIAS--- +CREATE TABLE test_01048.`.inner.wv`\n(\n `b` Int32,\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 +||---JOIN--- +CREATE TABLE test_01048.`.inner.wv`\n(\n `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))` UInt32,\n `count(a)` AggregateFunction(count, Int32),\n `count(mt_2.b)` AggregateFunction(count, Int32)\n)\nENGINE = AggregatingMergeTree\nPRIMARY KEY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nORDER BY `windowID(timestamp, toIntervalSecond(\'1\'), toIntervalSecond(\'3\'))`\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/01048_window_view_parser.sql b/tests/queries/0_stateless/01048_window_view_parser.sql index 3f57f6fbd91..e17352205e3 100644 --- a/tests/queries/0_stateless/01048_window_view_parser.sql +++ b/tests/queries/0_stateless/01048_window_view_parser.sql @@ -5,8 +5,10 @@ DROP DATABASE IF EXISTS test_01048; CREATE DATABASE test_01048 ENGINE=Ordinary; DROP TABLE IF EXISTS test_01048.mt; +DROP TABLE IF EXISTS test_01048.mt_2; CREATE TABLE test_01048.mt(a Int32, b Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple(); +CREATE TABLE test_01048.mt_2(a Int32, b Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple(); SELECT '---TUMBLE---'; SELECT '||---WINDOW COLUMN NAME---'; @@ -42,6 +44,16 @@ DROP TABLE IF EXISTS test_01048.wv; CREATE WINDOW VIEW test_01048.wv AS SELECT count(a) AS count, tumble(timestamp, INTERVAL '1' SECOND, 'Asia/Shanghai') AS wid FROM test_01048.mt GROUP BY wid; SHOW CREATE TABLE test_01048.`.inner.wv`; +SELECT '||---DATA COLUMN ALIAS---'; +DROP TABLE IF EXISTS test_01048.wv; +CREATE WINDOW VIEW test_01048.wv AS SELECT count(a) AS count, b as id FROM test_01048.mt GROUP BY id, tumble(timestamp, INTERVAL '1' SECOND); +SHOW CREATE TABLE test_01048.`.inner.wv`; + +SELECT '||---JOIN---'; +DROP TABLE IF EXISTS test_01048.wv; +CREATE WINDOW VIEW test_01048.wv AS SELECT count(test_01048.mt.a), count(test_01048.mt_2.b), wid FROM test_01048.mt JOIN test_01048.mt_2 ON test_01048.mt.timestamp = test_01048.mt_2.timestamp GROUP BY tumble(test_01048.mt.timestamp, INTERVAL '1' SECOND) AS wid; +SHOW CREATE TABLE test_01048.`.inner.wv`; + SELECT '---HOP---'; SELECT '||---WINDOW COLUMN NAME---'; @@ -68,15 +80,25 @@ DROP TABLE IF EXISTS test_01048.wv; CREATE WINDOW VIEW test_01048.wv AS SELECT count(a) AS count FROM test_01048.mt GROUP BY plus(a, b) as _type, hop(timestamp, INTERVAL '1' SECOND, INTERVAL '3' SECOND) AS wid; SHOW CREATE TABLE test_01048.`.inner.wv`; +DROP TABLE IF EXISTS test_01048.wv; +CREATE WINDOW VIEW test_01048.wv AS SELECT count(a) AS count FROM test_01048.mt GROUP BY hop(timestamp, INTERVAL '1' SECOND, INTERVAL '3' SECOND) AS wid, plus(a, b); +SHOW CREATE TABLE test_01048.`.inner.wv`; + SELECT '||---TimeZone---'; DROP TABLE IF EXISTS test_01048.wv; CREATE WINDOW VIEW test_01048.wv AS SELECT count(a) AS count, hopEnd(wid) as wend FROM test_01048.mt GROUP BY hop(timestamp, INTERVAL 1 SECOND, INTERVAL 3 SECOND, 'Asia/Shanghai') as wid; SHOW CREATE TABLE test_01048.`.inner.wv`; - +SELECT '||---DATA COLUMN ALIAS---'; DROP TABLE IF EXISTS test_01048.wv; -CREATE WINDOW VIEW test_01048.wv AS SELECT count(a) AS count FROM test_01048.mt GROUP BY hop(timestamp, INTERVAL '1' SECOND, INTERVAL '3' SECOND) AS wid, plus(a, b); +CREATE WINDOW VIEW test_01048.wv AS SELECT count(a) AS count, b as id FROM test_01048.mt GROUP BY id, hop(timestamp, INTERVAL '1' SECOND, INTERVAL '3' SECOND); +SHOW CREATE TABLE test_01048.`.inner.wv`; + +SELECT '||---JOIN---'; +DROP TABLE IF EXISTS test_01048.wv; +CREATE WINDOW VIEW test_01048.wv AS SELECT count(test_01048.mt.a), count(test_01048.mt_2.b), wid FROM test_01048.mt JOIN test_01048.mt_2 ON test_01048.mt.timestamp = test_01048.mt_2.timestamp GROUP BY hop(test_01048.mt.timestamp, INTERVAL '1' SECOND, INTERVAL '3' SECOND) AS wid; SHOW CREATE TABLE test_01048.`.inner.wv`; DROP TABLE test_01048.wv; DROP TABLE test_01048.mt; +DROP TABLE test_01048.mt_2; diff --git a/tests/queries/0_stateless/01071_window_view_event_tumble_asc_join.reference b/tests/queries/0_stateless/01071_window_view_event_tumble_asc_join.reference new file mode 100644 index 00000000000..e433cdedd69 --- /dev/null +++ b/tests/queries/0_stateless/01071_window_view_event_tumble_asc_join.reference @@ -0,0 +1,3 @@ +3 6 1990-01-01 12:00:05 +2 4 1990-01-01 12:00:10 +2 4 1990-01-01 12:00:15 diff --git a/tests/queries/0_stateless/01071_window_view_event_tumble_asc_join.sh b/tests/queries/0_stateless/01071_window_view_event_tumble_asc_join.sh new file mode 100755 index 00000000000..263dd5597c4 --- /dev/null +++ b/tests/queries/0_stateless/01071_window_view_event_tumble_asc_join.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --multiquery < Date: Tue, 18 Jan 2022 12:31:31 +0300 Subject: [PATCH 076/403] Fix restore S3 disk without detached directories --- src/Disks/S3/DiskS3.cpp | 1 + .../test_merge_tree_s3_restore/test.py | 111 ++++++++++++++---- 2 files changed, 88 insertions(+), 24 deletions(-) diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index 0bebf91df97..ed960528abe 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -999,6 +999,7 @@ void DiskS3::restoreFileOperations(const RestoreInformation & restore_informatio if (metadata_disk->exists(to_path)) metadata_disk->removeRecursive(to_path); + createDirectories(directoryPath(to_path)); metadata_disk->moveDirectory(from_path, to_path); } } diff --git a/tests/integration/test_merge_tree_s3_restore/test.py b/tests/integration/test_merge_tree_s3_restore/test.py index e12b69cdf17..acbcd8c04cf 100644 --- a/tests/integration/test_merge_tree_s3_restore/test.py +++ b/tests/integration/test_merge_tree_s3_restore/test.py @@ -63,11 +63,11 @@ def generate_values(date_str, count, sign=1): return ",".join(["('{}',{},'{}',{})".format(x, y, z, 0) for x, y, z in data]) -def create_table(node, table_name, attach=False, replicated=False): - node.query("CREATE DATABASE IF NOT EXISTS s3 ENGINE = Ordinary") +def create_table(node, table_name, attach=False, replicated=False, db_atomic=False, uuid=""): + node.query("CREATE DATABASE IF NOT EXISTS s3 ENGINE = {engine}".format(engine="Atomic" if db_atomic else "Ordinary")) create_table_statement = """ - {create} TABLE s3.{table_name} {on_cluster} ( + {create} TABLE s3.{table_name} {uuid} {on_cluster} ( dt Date, id Int64, data String, @@ -81,9 +81,10 @@ def create_table(node, table_name, attach=False, replicated=False): old_parts_lifetime=600, index_granularity=512 """.format(create="ATTACH" if attach else "CREATE", - table_name=table_name, - on_cluster="ON CLUSTER '{}'".format(node.name) if replicated else "", - engine="ReplicatedMergeTree('/clickhouse/tables/{cluster}/test', '{replica}')" if replicated else "MergeTree()") + table_name=table_name, + uuid="UUID '{uuid}'".format(uuid=uuid) if db_atomic and uuid else "", + on_cluster="ON CLUSTER '{}'".format(node.name) if replicated else "", + engine="ReplicatedMergeTree('/clickhouse/tables/{cluster}/test', '{replica}')" if replicated else "MergeTree()") node.query(create_table_statement) @@ -124,6 +125,13 @@ def get_revision_counter(node, backup_number): ['bash', '-c', 'cat /var/lib/clickhouse/disks/s3/shadow/{}/revision.txt'.format(backup_number)], user='root')) +def get_table_uuid(node, db_atomic, table): + uuid = "" + if db_atomic: + uuid = node.query("SELECT uuid FROM system.tables WHERE database='s3' AND table='{}' FORMAT TabSeparated".format(table)).strip() + return uuid + + @pytest.fixture(autouse=True) def drop_table(cluster): yield @@ -146,10 +154,13 @@ def drop_table(cluster): @pytest.mark.parametrize( "replicated", [False, True] ) -def test_full_restore(cluster, replicated): +@pytest.mark.parametrize( + "db_atomic", [False, True] +) +def test_full_restore(cluster, replicated, db_atomic): node = cluster.instances["node"] - create_table(node, "test", attach=False, replicated=replicated) + create_table(node, "test", attach=False, replicated=replicated, db_atomic=db_atomic) node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-03', 4096))) node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-04', 4096, -1))) @@ -166,10 +177,14 @@ def test_full_restore(cluster, replicated): assert node.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0) -def test_restore_another_bucket_path(cluster): +@pytest.mark.parametrize( + "db_atomic", [False, True] +) +def test_restore_another_bucket_path(cluster, db_atomic): node = cluster.instances["node"] - create_table(node, "test") + create_table(node, "test", db_atomic=db_atomic) + uuid = get_table_uuid(node, db_atomic, "test") node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-03', 4096))) node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-04', 4096, -1))) @@ -186,7 +201,7 @@ def test_restore_another_bucket_path(cluster): create_restore_file(node_another_bucket, bucket="root") node_another_bucket.query("SYSTEM RESTART DISK s3") - create_table(node_another_bucket, "test", attach=True) + create_table(node_another_bucket, "test", attach=True, db_atomic=db_atomic, uuid=uuid) assert node_another_bucket.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(4096 * 4) assert node_another_bucket.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0) @@ -195,16 +210,20 @@ def test_restore_another_bucket_path(cluster): create_restore_file(node_another_bucket_path, bucket="root2", path="data") node_another_bucket_path.query("SYSTEM RESTART DISK s3") - create_table(node_another_bucket_path, "test", attach=True) + create_table(node_another_bucket_path, "test", attach=True, db_atomic=db_atomic, uuid=uuid) assert node_another_bucket_path.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(4096 * 4) assert node_another_bucket_path.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0) -def test_restore_different_revisions(cluster): +@pytest.mark.parametrize( + "db_atomic", [False, True] +) +def test_restore_different_revisions(cluster, db_atomic): node = cluster.instances["node"] - create_table(node, "test") + create_table(node, "test", db_atomic=db_atomic) + uuid = get_table_uuid(node, db_atomic, "test") node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-03', 4096))) node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-04', 4096, -1))) @@ -233,7 +252,7 @@ def test_restore_different_revisions(cluster): # Restore to revision 1 (2 parts). create_restore_file(node_another_bucket, revision=revision1, bucket="root") node_another_bucket.query("SYSTEM RESTART DISK s3") - create_table(node_another_bucket, "test", attach=True) + create_table(node_another_bucket, "test", attach=True, db_atomic=db_atomic, uuid=uuid) assert node_another_bucket.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(4096 * 2) assert node_another_bucket.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0) @@ -260,10 +279,14 @@ def test_restore_different_revisions(cluster): assert node_another_bucket.query("SELECT count(*) from system.parts where table = 'test'") == '5\n' -def test_restore_mutations(cluster): +@pytest.mark.parametrize( + "db_atomic", [False, True] +) +def test_restore_mutations(cluster, db_atomic): node = cluster.instances["node"] - create_table(node, "test") + create_table(node, "test", db_atomic=db_atomic) + uuid = get_table_uuid(node, db_atomic, "test") node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-03', 4096))) node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-03', 4096, -1))) @@ -281,7 +304,7 @@ def test_restore_mutations(cluster): # Restore to revision before mutation. create_restore_file(node_another_bucket, revision=revision_before_mutation, bucket="root") node_another_bucket.query("SYSTEM RESTART DISK s3") - create_table(node_another_bucket, "test", attach=True) + create_table(node_another_bucket, "test", attach=True, db_atomic=db_atomic, uuid=uuid) assert node_another_bucket.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(4096 * 2) assert node_another_bucket.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0) @@ -315,10 +338,14 @@ def test_restore_mutations(cluster): assert node_another_bucket.query("SELECT sum(counter) FROM s3.test WHERE id > 0 FORMAT Values") == "({})".format(4096) -def test_migrate_to_restorable_schema(cluster): +@pytest.mark.parametrize( + "db_atomic", [False, True] +) +def test_migrate_to_restorable_schema(cluster, db_atomic): node = cluster.instances["node_not_restorable"] - create_table(node, "test") + create_table(node, "test", db_atomic=db_atomic) + uuid = get_table_uuid(node, db_atomic, "test") node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-03', 4096))) node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-04', 4096, -1))) @@ -341,7 +368,7 @@ def test_migrate_to_restorable_schema(cluster): # Restore to revision before mutation. create_restore_file(node_another_bucket, revision=revision, bucket="root", path="another_data") node_another_bucket.query("SYSTEM RESTART DISK s3") - create_table(node_another_bucket, "test", attach=True) + create_table(node_another_bucket, "test", attach=True, db_atomic=db_atomic, uuid=uuid) assert node_another_bucket.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(4096 * 6) assert node_another_bucket.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0) @@ -350,10 +377,14 @@ def test_migrate_to_restorable_schema(cluster): @pytest.mark.parametrize( "replicated", [False, True] ) -def test_restore_to_detached(cluster, replicated): +@pytest.mark.parametrize( + "db_atomic", [False, True] +) +def test_restore_to_detached(cluster, replicated, db_atomic): node = cluster.instances["node"] - create_table(node, "test", attach=False, replicated=replicated) + create_table(node, "test", attach=False, replicated=replicated, db_atomic=db_atomic) + uuid = get_table_uuid(node, db_atomic, "test") node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-03', 4096))) node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-04', 4096, -1))) @@ -374,7 +405,7 @@ def test_restore_to_detached(cluster, replicated): create_restore_file(node_another_bucket, revision=revision, bucket="root", path="data", detached=True) node_another_bucket.query("SYSTEM RESTART DISK s3") - create_table(node_another_bucket, "test", replicated=replicated) + create_table(node_another_bucket, "test", replicated=replicated, db_atomic=db_atomic, uuid=uuid) assert node_another_bucket.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(0) @@ -393,3 +424,35 @@ def test_restore_to_detached(cluster, replicated): assert node_another_bucket.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(4096 * 5) assert node_another_bucket.query("SELECT sum(id) FROM s3.test FORMAT Values") == "({})".format(0) assert node_another_bucket.query("SELECT sum(counter) FROM s3.test FORMAT Values") == "({})".format(4096 * 5) + + +@pytest.mark.parametrize( + "replicated", [False, True] +) +@pytest.mark.parametrize( + "db_atomic", [False, True] +) +def test_restore_without_detached(cluster, replicated, db_atomic): + node = cluster.instances["node"] + + create_table(node, "test", attach=False, replicated=replicated, db_atomic=db_atomic) + uuid = get_table_uuid(node, db_atomic, "test") + + node.query("INSERT INTO s3.test VALUES {}".format(generate_values('2020-01-03', 1))) + + assert node.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(1) + + node.query("ALTER TABLE s3.test FREEZE") + revision = get_revision_counter(node, 1) + + node_another_bucket = cluster.instances["node_another_bucket"] + + create_restore_file(node_another_bucket, revision=revision, bucket="root", path="data", detached=True) + node_another_bucket.query("SYSTEM RESTART DISK s3") + create_table(node_another_bucket, "test", replicated=replicated, db_atomic=db_atomic, uuid=uuid) + + assert node_another_bucket.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(0) + + node_another_bucket.query("ALTER TABLE s3.test ATTACH PARTITION '2020-01-03'") + + assert node_another_bucket.query("SELECT count(*) FROM s3.test FORMAT Values") == "({})".format(1) From 15ea37c4aed8358a31e14c676c668255e5ecdfd7 Mon Sep 17 00:00:00 2001 From: liuneng1994 <1398775315@qq.com> Date: Tue, 18 Jan 2022 12:47:04 +0000 Subject: [PATCH 077/403] ignore file not found exception Signed-off-by: neng.liu --- src/Storages/HDFS/StorageHDFS.cpp | 6 ++++-- tests/integration/test_storage_hdfs/test.py | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index 82a7df665ee..bf11623f3c5 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -72,8 +72,10 @@ namespace HDFSFileInfo ls; ls.file_info = hdfsListDirectory(fs.get(), prefix_without_globs.data(), &ls.length); - if (ls.file_info == nullptr) { - throw Exception(ErrorCodes::ACCESS_DENIED, "Cannot list directory {}: {}", prefix_without_globs, String(hdfsGetLastError())); + if (ls.file_info == nullptr && errno != ENOENT) { + // ignore file not found exception, keep throw other exception + throw Exception( + ErrorCodes::ACCESS_DENIED, "Cannot list directory {}: {}", prefix_without_globs, String(hdfsGetLastError())); } Strings result; for (int i = 0; i < ls.length; ++i) diff --git a/tests/integration/test_storage_hdfs/test.py b/tests/integration/test_storage_hdfs/test.py index 5d3a5ce00d7..3c7104749a9 100644 --- a/tests/integration/test_storage_hdfs/test.py +++ b/tests/integration/test_storage_hdfs/test.py @@ -361,10 +361,10 @@ def test_hdfsCluster(started_cluster): assert actual == expected fs.delete(dir, recursive=True) -def testHdfsDirectoryNotExist(started_cluster): - ddl ="create table HDFSStorageWithNotExistDir (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/dir1/empty', 'TSV')"; +def test_hdfs_directory_not_exist(started_cluster): + ddl ="create table HDFSStorageWithNotExistDir (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/data/not_eixst', 'TSV')"; node1.query(ddl) - assert "Cannot list directory" in node1.query_and_get_error("select * from HDFSStorageWithNotExistDir") + assert "" == node1.query("select * from HDFSStorageWithNotExistDir") def test_format_detection(started_cluster): node1.query(f"create table arrow_table (x UInt64) engine=HDFS('hdfs://hdfs1:9000/data.arrow')") From 60bcf8822843b2d06480772011058ecef1f71fd7 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 18 Jan 2022 13:20:43 +0000 Subject: [PATCH 078/403] Added IntervalTree documentation --- src/Common/IntervalTree.h | 111 ++++++++++++++++++--- src/Common/examples/interval_tree.cpp | 8 +- src/Common/tests/gtest_interval_tree.cpp | 16 +-- src/Dictionaries/RangeHashedDictionary.cpp | 6 +- src/Dictionaries/RangeHashedDictionary.h | 2 +- 5 files changed, 115 insertions(+), 28 deletions(-) diff --git a/src/Common/IntervalTree.h b/src/Common/IntervalTree.h index ec17ca11a9f..2668f11762a 100644 --- a/src/Common/IntervalTree.h +++ b/src/Common/IntervalTree.h @@ -9,6 +9,9 @@ namespace DB { +/** Structure that holds closed interval with left and right. + * Example: [1, 1] is valid interval, that contain point 1. + */ template struct Interval { @@ -61,6 +64,56 @@ struct IntervalTreeVoidValue { }; +/** Tree structure that allow to efficiently retrieve all intervals that intersect specific point. + * https://en.wikipedia.org/wiki/Interval_tree + * + * Search for all intervals intersecting point has complexity O(log(n) + k), k is count of intervals that intersect point. + * If we need to only check if there are some interval intersecting point such operation has complexity O(log(n)). + * + * Explanation: + * + * IntervalTree structure is balanced tree. Each node contains: + * 1. Point + * 2. Intervals sorted by left ascending that intersect that point. + * 3. Intervals sorted by right descending that intersect that point. + * + * Build: + * + * To keep tree relatively balanced we can use median of all segment points. + * On each step build tree node with intervals. For root node input intervals are all intervals. + * First split intervals in 4 groups. + * 1. Intervals that lie that are less than median point. Interval right is less than median point. + * 2. Intervals that lie that are greater than median point. Interval right is less than median point. + * 3. Intervals that intersect node sorted by left ascending. + * 4. Intervals that intersect node sorted by right ascending. + * + * If intervals in 1 group are not empty. Continue build left child recursively with intervals from 1 group. + * If intervals in 2 group are not empty. Continue build right child recursively with intervals from 2 group. + * + * Search: + * + * Search for intervals intersecting point is started from root node. + * If search point is less than point in node, then we check intervals sorted by left ascending + * until left is greater than search point. + * If there is left child, continue search recursively in left child. + * + * If search point is greater than point in node, then we check intervals sorted by right descending + * until right is less than search point. + * If there is right child, continue search recursively in right child. + * + * If search point is equal to point in node, then we can emit all intervals that intersect current tree node + * and stop searching. + * + * Additional details: + * 1. To improve cache locality tree is stored implicitly in array, after build method is called + * other intervals cannot be added to the tree. + * 2. Additionaly to improve cache locality in tree node we store sorted intervals for all nodes in separate + * array. In node we store only start of its sorted intervals, and also size of intersecting intervals. + * If we need to retrieve intervals sorted by left ascending they will be stored in indexes + * [sorted_intervals_start_index, sorted_intervals_start_index + intersecting_intervals_size). + * If we need to retrieve intervals sorted by right descending they will be store in indexes + * [sorted_intervals_start_index + intersecting_intervals_size, sorted_intervals_start_index + intersecting_intervals_size * 2). + */ template class IntervalTree { @@ -74,7 +127,7 @@ public: template , bool> = true> void emplace(Interval interval) { - assert(!tree_constructed); + assert(!tree_is_build); sorted_intervals.emplace_back(interval); increaseIntervalsSize(); } @@ -82,7 +135,7 @@ public: template , bool> = true, typename... Args> void emplace(Interval interval, Args &&... args) { - assert(!tree_constructed); + assert(!tree_is_build); sorted_intervals.emplace_back( std::piecewise_construct, std::forward_as_tuple(interval), std::forward_as_tuple(std::forward(args)...)); increaseIntervalsSize(); @@ -91,7 +144,7 @@ public: template , bool> = true> void insert(Interval interval) { - assert(!tree_constructed); + assert(!tree_is_build); sorted_intervals.emplace_back(interval); increaseIntervalsSize(); } @@ -99,7 +152,7 @@ public: template , bool> = true> void insert(Interval interval, const Value & value) { - assert(!tree_constructed); + assert(!tree_is_build); sorted_intervals.emplace_back(std::piecewise_construct, interval, value); increaseIntervalsSize(); } @@ -107,24 +160,52 @@ public: template , bool> = true> void insert(Interval interval, Value && value) { - assert(!tree_constructed); + assert(!tree_is_build); sorted_intervals.emplace_back(std::piecewise_construct, interval, std::move(value)); increaseIntervalsSize(); } - void construct() + /// Build tree, after that intervals cannot be inserted, and only search or iteration can be performed. + void build() { - assert(!tree_constructed); + assert(!tree_is_build); nodes.clear(); nodes.reserve(sorted_intervals.size()); buildTree(); - tree_constructed = true; + tree_is_build = true; } + /** Find all intervals intersecting point. + * + * Callback interface for IntervalSet: + * + * template + * struct IntervalSetCallback + * { + * bool operator()(const IntervalType & interval) + * { + * bool should_continue_interval_iteration = false; + * return should_continue_interval_iteration; + * } + * }; + * + * Callback interface for IntervalMap: + * + * template + * struct IntervalMapCallback + * { + * bool operator()(const IntervalType & interval, const Value & value) + * { + * bool should_continue_interval_iteration = false; + * return should_continue_interval_iteration; + * } + * }; + */ + template void find(IntervalStorageType point, IntervalCallback && callback) const { - if (unlikely(!tree_constructed)) + if (unlikely(!tree_is_build)) { findIntervalsNonConstructedImpl(point, callback); return; @@ -133,6 +214,7 @@ public: findIntervalsImpl(point, callback); } + /// Check if there is an interval intersecting point bool has(IntervalStorageType point) const { bool has_intervals = false; @@ -395,8 +477,11 @@ private: size_t sorted_intervals_range_start_index = sorted_intervals.size(); - sorted_intervals.insert(sorted_intervals.end(), intervals_sorted_by_left_asc.begin(), intervals_sorted_by_left_asc.end()); - sorted_intervals.insert(sorted_intervals.end(), intervals_sorted_by_right_desc.begin(), intervals_sorted_by_right_desc.end()); + for (auto && interval_sorted_by_left_asc : intervals_sorted_by_left_asc) + sorted_intervals.emplace_back(std::move(interval_sorted_by_left_asc)); + + for (auto && interval_sorted_by_right_desc : intervals_sorted_by_right_desc) + sorted_intervals.emplace_back(std::move(interval_sorted_by_right_desc)); auto & node = nodes[current_index]; node.middle_element = median; @@ -471,6 +556,7 @@ private: } else { + /// This is case when point == middle_element. break; } } @@ -524,6 +610,7 @@ private: inline void increaseIntervalsSize() { + /// Before tree is build we store all intervals size in our first node to allow tree iteration. ++intervals_size; nodes[0].sorted_intervals_range_size = intervals_size; } @@ -531,7 +618,7 @@ private: std::vector nodes; std::vector sorted_intervals; size_t intervals_size = 0; - bool tree_constructed = false; + bool tree_is_build = false; static inline const Interval & getInterval(const IntervalWithValue & interval_with_value) { diff --git a/src/Common/examples/interval_tree.cpp b/src/Common/examples/interval_tree.cpp index 5ffc2f1ccd3..086fab37bbe 100644 --- a/src/Common/examples/interval_tree.cpp +++ b/src/Common/examples/interval_tree.cpp @@ -20,7 +20,7 @@ int main(int, char **) tree.emplace(Int64Interval(0, 5)); tree.emplace(Int64Interval(10, 15)); - tree.construct(); + tree.build(); for (const auto & interval : tree) { @@ -33,7 +33,7 @@ int main(int, char **) tree.emplace(Int64Interval(0, 5), "value1"); tree.emplace(Int64Interval(10, 15), "value2"); - tree.construct(); + tree.build(); for (const auto & [interval, value] : tree) { @@ -48,7 +48,7 @@ int main(int, char **) tree.emplace(Int64Interval(0, i)); } - tree.construct(); + tree.build(); for (const auto & interval : tree) { @@ -71,7 +71,7 @@ int main(int, char **) tree.emplace(Int64Interval(0, i), "Value " + std::to_string(i)); } - tree.construct(); + tree.build(); for (const auto & [interval, value] : tree) { diff --git a/src/Common/tests/gtest_interval_tree.cpp b/src/Common/tests/gtest_interval_tree.cpp index 9becb02d768..d9f19841b66 100644 --- a/src/Common/tests/gtest_interval_tree.cpp +++ b/src/Common/tests/gtest_interval_tree.cpp @@ -125,7 +125,7 @@ TEST(IntervalTree, IntervalSetBasic) ASSERT_TRUE(set.has(expected_interval.right)); } - set.construct(); + set.build(); ASSERT_TRUE(intervalSetToSet(set) == expected); @@ -179,7 +179,7 @@ TEST(IntervalTree, IntervalSetPoints) ASSERT_TRUE(set.has(expected_interval.right)); } - set.construct(); + set.build(); ASSERT_TRUE(intervalSetToSet(set) == expected); @@ -227,7 +227,7 @@ TEST(IntervalTree, IntervalSetIntersectingIntervals) ASSERT_TRUE(set.has(expected_interval.right)); } - set.construct(); + set.build(); ASSERT_TRUE(intervalSetToSet(set) == expected); @@ -249,7 +249,7 @@ TEST(IntervalTree, IntervalSetIterators) IntervalSet set; ASSERT_TRUE(set.begin() == set.end()); ASSERT_TRUE(set.cbegin() == set.cend()); - set.construct(); + set.build(); ASSERT_TRUE(set.begin() == set.end()); ASSERT_TRUE(set.cbegin() == set.cend()); } @@ -344,7 +344,7 @@ TEST(IntervalTree, IntervalMapBasic) ASSERT_TRUE(map.has(expected_interval.right)); } - map.construct(); + map.build(); ASSERT_TRUE(intervalMapToMap(map) == expected); @@ -401,7 +401,7 @@ TEST(IntervalTree, IntervalMapPoints) ASSERT_TRUE(map.has(expected_interval.right)); } - map.construct(); + map.build(); ASSERT_TRUE(intervalMapToMap(map) == expected); @@ -453,7 +453,7 @@ TEST(IntervalTree, IntervalMapIntersectingIntervals) ASSERT_TRUE(map.has(expected_interval.right)); } - map.construct(); + map.build(); ASSERT_TRUE(intervalMapToMap(map) == expected); @@ -476,7 +476,7 @@ TEST(IntervalTree, IntervalMapIterators) IntervalMap map; ASSERT_TRUE(map.begin() == map.end()); ASSERT_TRUE(map.cbegin() == map.cend()); - map.construct(); + map.build(); ASSERT_TRUE(map.begin() == map.end()); ASSERT_TRUE(map.cbegin() == map.cend()); } diff --git a/src/Dictionaries/RangeHashedDictionary.cpp b/src/Dictionaries/RangeHashedDictionary.cpp index 5b493c21357..f6cdd5679d8 100644 --- a/src/Dictionaries/RangeHashedDictionary.cpp +++ b/src/Dictionaries/RangeHashedDictionary.cpp @@ -307,7 +307,7 @@ void RangeHashedDictionary::loadData() updateData(); } - constructAttributeIntervalTrees(); + buildAttributeIntervalTrees(); if (require_nonempty && 0 == element_count) throw Exception(ErrorCodes::DICTIONARY_IS_EMPTY, @@ -546,7 +546,7 @@ void RangeHashedDictionary::blockToAttributes(const Block & } template -void RangeHashedDictionary::constructAttributeIntervalTrees() +void RangeHashedDictionary::buildAttributeIntervalTrees() { for (auto & attribute : attributes) { @@ -558,7 +558,7 @@ void RangeHashedDictionary::constructAttributeIntervalTrees auto & collection = std::get>(attribute.maps); for (auto & [_, ranges] : collection) - ranges.construct(); + ranges.build(); }; callOnDictionaryAttributeType(attribute.type, type_call); diff --git a/src/Dictionaries/RangeHashedDictionary.h b/src/Dictionaries/RangeHashedDictionary.h index 45d419996bc..f31d6415dc8 100644 --- a/src/Dictionaries/RangeHashedDictionary.h +++ b/src/Dictionaries/RangeHashedDictionary.h @@ -149,7 +149,7 @@ private: void blockToAttributes(const Block & block); - void constructAttributeIntervalTrees(); + void buildAttributeIntervalTrees(); template void setAttributeValueImpl(Attribute & attribute, KeyType key, const RangeInterval & interval, const Field & value); From 6536c3aaeecbc22deb4c7d383a679d4575dfe08b Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 18 Jan 2022 13:20:54 +0000 Subject: [PATCH 079/403] Added performance tests --- tests/performance/range_hashed_dictionary.xml | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 tests/performance/range_hashed_dictionary.xml diff --git a/tests/performance/range_hashed_dictionary.xml b/tests/performance/range_hashed_dictionary.xml new file mode 100644 index 00000000000..a070acfb4dd --- /dev/null +++ b/tests/performance/range_hashed_dictionary.xml @@ -0,0 +1,126 @@ + + + CREATE TABLE simple_key_range_hashed_dictionary_source_table + ( + id UInt64, + value UInt64, + start UInt64, + end UInt64 + ) ENGINE = Memory; + + + + CREATE TABLE complex_key_range_hashed_dictionary_source_table + ( + id UInt64, + id_key String, + value UInt64, + start UInt64, + end UInt64 + ) ENGINE = Memory; + + + + CREATE DICTIONARY simple_key_range_hashed_dictionary + ( + id UInt64, + value UInt64, + start UInt64, + end UInt64 + ) + PRIMARY KEY id + SOURCE(CLICKHOUSE(DB 'default' TABLE 'simple_key_range_hashed_dictionary_source_table')) + LAYOUT(RANGE_HASHED()) + RANGE(MIN start MAX end) + LIFETIME(MIN 0 MAX 1000); + + + + CREATE DICTIONARY complex_key_range_hashed_dictionary + ( + id UInt64, + id_key String, + value UInt64, + start UInt64, + end UInt64 + ) + PRIMARY KEY id, id_key + SOURCE(CLICKHOUSE(DB 'default' TABLE 'complex_key_range_hashed_dictionary_source_table')) + LAYOUT(COMPLEX_KEY_RANGE_HASHED()) + RANGE(MIN start MAX end) + LIFETIME(MIN 0 MAX 1000); + + + + INSERT INTO simple_key_range_hashed_dictionary_source_table + SELECT key, key, range_start * 2, range_start * 2 + 1 FROM + (SELECT number as key FROM numbers(10000)) as keys, + (SELECT number as range_start FROM numbers(10000)) as ranges; + + + + INSERT INTO complex_key_range_hashed_dictionary_source_table + SELECT key, toString(key), key, range_start * 2, range_start * 2 + 1 FROM + (SELECT number as key FROM numbers(10000)) as keys, + (SELECT number as range_start FROM numbers(10000)) as ranges; + + + + + elements_count + + 50000 + 75000 + + + + + + WITH rand64() % 10000 as key + SELECT dictGet('default.simple_key_range_hashed_dictionary', 'value', toUInt64(key), key) + FROM system.numbers + LIMIT {elements_count} + FORMAT Null; + + + + WITH rand64() % 10000 as key + SELECT dictHas('default.simple_key_range_hashed_dictionary', toUInt64(key), key) + FROM system.numbers + LIMIT {elements_count} + FORMAT Null; + + + + SELECT * FROM simple_key_range_hashed_dictionary + FORMAT Null; + + + + WITH (rand64() % toUInt64(10000), toString(rand64() % toUInt64(10000))) as key + SELECT dictGet('default.complex_key_range_hashed_dictionary', 'value', key, rand64() % toUInt64(10000)) + FROM system.numbers + LIMIT {elements_count} + FORMAT Null; + + + + WITH (rand64() % toUInt64(10000), toString(rand64() % toUInt64(10000))) as key + SELECT dictHas('default.complex_key_range_hashed_dictionary', key, rand64() % toUInt64(10000)) + FROM system.numbers + LIMIT {elements_count} + FORMAT Null; + + + + SELECT * FROM complex_key_range_hashed_dictionary + FORMAT Null; + + + DROP TABLE IF EXISTS simple_key_range_hashed_dictionary_source_table; + DROP TABLE IF EXISTS complex_key_range_hashed_dictionary_source_table; + + DROP DICTIONARY IF EXISTS simple_key_range_hashed_dictionary; + DROP DICTIONARY IF EXISTS complex_key_range_hashed_dictionary; + + From 562098779f91fef6639da5712529cf84406a70eb Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 18 Jan 2022 13:26:23 +0000 Subject: [PATCH 080/403] Fixed style check --- src/Common/IntervalTree.h | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/src/Common/IntervalTree.h b/src/Common/IntervalTree.h index 2668f11762a..13585a40bbb 100644 --- a/src/Common/IntervalTree.h +++ b/src/Common/IntervalTree.h @@ -221,14 +221,16 @@ public: if constexpr (is_empty_value) { - find(point, [&](auto &) { + find(point, [&](auto &) + { has_intervals = true; return false; }); } else { - find(point, [&](auto &, auto &) { + find(point, [&](auto &, auto &) + { has_intervals = true; return false; }); @@ -250,7 +252,11 @@ public: iterator end() { size_t end_index = findLastIteratorNodeIndex(); - size_t last_interval_index = end_index < nodes.size() ? nodes[end_index].sorted_intervals_range_size : 0; + size_t last_interval_index = 0; + + if (likely(end_index < nodes.size())) + nodes[end_index].sorted_intervals_range_size; + return Iterator(end_index, last_interval_index, this); } @@ -263,7 +269,10 @@ public: const_iterator end() const { size_t end_index = findLastIteratorNodeIndex(); - size_t last_interval_index = end_index < nodes.size() ? nodes[end_index].sorted_intervals_range_size : 0; + size_t last_interval_index = 0; + + if (likely(end_index < nodes.size())) + nodes[end_index].sorted_intervals_range_size; return Iterator(end_index, last_interval_index, this); } @@ -463,13 +472,15 @@ private: } } - std::sort(intervals_sorted_by_left_asc.begin(), intervals_sorted_by_left_asc.end(), [](auto & lhs, auto & rhs) { + std::sort(intervals_sorted_by_left_asc.begin(), intervals_sorted_by_left_asc.end(), [](auto & lhs, auto & rhs) + { auto & lhs_interval = getInterval(lhs); auto & rhs_interval = getInterval(rhs); return lhs_interval.left < rhs_interval.left; }); - std::sort(intervals_sorted_by_right_desc.begin(), intervals_sorted_by_right_desc.end(), [](auto & lhs, auto & rhs) { + std::sort(intervals_sorted_by_right_desc.begin(), intervals_sorted_by_right_desc.end(), [](auto & lhs, auto & rhs) + { auto & lhs_interval = getInterval(lhs); auto & rhs_interval = getInterval(rhs); return lhs_interval.right > rhs_interval.right; From 5e1e512bf27c92c47dbf3b2e8aeec658e9d38e8f Mon Sep 17 00:00:00 2001 From: liuneng1994 <1398775315@qq.com> Date: Tue, 18 Jan 2022 14:12:37 +0000 Subject: [PATCH 081/403] fix check style Signed-off-by: neng.liu --- src/Storages/HDFS/StorageHDFS.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index bf11623f3c5..372c9e04e08 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -72,8 +72,8 @@ namespace HDFSFileInfo ls; ls.file_info = hdfsListDirectory(fs.get(), prefix_without_globs.data(), &ls.length); - if (ls.file_info == nullptr && errno != ENOENT) { - // ignore file not found exception, keep throw other exception + if (ls.file_info == nullptr && errno != ENOENT) { // NOLINT + // ignore file not found exception, keep throw other exception, libhdfs3 doesn't have function to get exception type, so use errno. throw Exception( ErrorCodes::ACCESS_DENIED, "Cannot list directory {}: {}", prefix_without_globs, String(hdfsGetLastError())); } From 821c5dd9894595096a20d43d7a67fc753ba2d551 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 18 Jan 2022 14:46:19 +0000 Subject: [PATCH 082/403] Fix build --- src/Common/IntervalTree.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/IntervalTree.h b/src/Common/IntervalTree.h index 13585a40bbb..9d1acca91bd 100644 --- a/src/Common/IntervalTree.h +++ b/src/Common/IntervalTree.h @@ -255,7 +255,7 @@ public: size_t last_interval_index = 0; if (likely(end_index < nodes.size())) - nodes[end_index].sorted_intervals_range_size; + last_interval_index = nodes[end_index].sorted_intervals_range_size; return Iterator(end_index, last_interval_index, this); } @@ -272,7 +272,7 @@ public: size_t last_interval_index = 0; if (likely(end_index < nodes.size())) - nodes[end_index].sorted_intervals_range_size; + last_interval_index = nodes[end_index].sorted_intervals_range_size; return Iterator(end_index, last_interval_index, this); } From 4efadfad3cf0aa7d0263108fe6ff153af874bb05 Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 18 Jan 2022 22:26:13 +0300 Subject: [PATCH 083/403] Fix tests --- src/Parsers/ParserInsertQuery.cpp | 4 +++ src/Storages/HDFS/StorageHDFS.cpp | 30 ++++++++++++++----- src/Storages/HDFS/StorageHDFS.h | 2 +- src/Storages/StorageS3.cpp | 19 +++++++----- tests/integration/test_storage_hdfs/test.py | 10 +++---- .../configs/named_collections.xml | 5 ++++ tests/integration/test_storage_s3/test.py | 22 +++++++------- 7 files changed, 59 insertions(+), 33 deletions(-) diff --git a/src/Parsers/ParserInsertQuery.cpp b/src/Parsers/ParserInsertQuery.cpp index 568f486a5cf..631b4f7ed53 100644 --- a/src/Parsers/ParserInsertQuery.cpp +++ b/src/Parsers/ParserInsertQuery.cpp @@ -132,6 +132,7 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) Pos before_values = pos; String format_str; + bool values = false; /// VALUES or FORMAT or SELECT or WITH or WATCH. /// After FROM INFILE we expect FORMAT, SELECT, WITH or nothing. @@ -140,6 +141,7 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) /// If VALUES is defined in query, everything except setting will be parsed as data data = pos->begin; format_str = "Values"; + values = true; } else if (s_format.ignore(pos, expected)) { @@ -184,6 +186,8 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserSetQuery parser_settings(true); if (!parser_settings.parse(pos, settings_ast, expected)) return false; + if (values) + data = pos->begin; } if (select) diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index a5d01bb1428..72cb4941b8c 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -214,20 +214,34 @@ private: class HDFSSource::URISIterator::Impl { public: - Impl(const std::vector & uris_) : uris(uris_), index(0) + explicit Impl(const std::vector & uris_, ContextPtr context) { + auto path_and_uri = getPathFromUriAndUriWithoutPath(uris_[0]); + HDFSBuilderWrapper builder = createHDFSBuilder(path_and_uri.second + "/", context->getGlobalContext()->getConfigRef()); + HDFSFSPtr fs = createHDFSFS(builder.get()); + for (const auto & uri : uris_) + { + path_and_uri = getPathFromUriAndUriWithoutPath(uri); + if (!hdfsExists(fs.get(), path_and_uri.first.c_str())) + uris.push_back(uri); + } + uris_iter = uris.begin(); } String next() { - if (index == uris.size()) + std::lock_guard lock(mutex); + if (uris_iter == uris.end()) return ""; - return uris[index++]; + auto key = *uris_iter; + ++uris_iter; + return key; } private: - const std::vector & uris; - size_t index; + std::mutex mutex; + Strings uris; + Strings::iterator uris_iter; }; Block HDFSSource::getHeader(const StorageMetadataPtr & metadata_snapshot, bool need_path_column, bool need_file_column) @@ -263,8 +277,8 @@ String HDFSSource::DisclosedGlobIterator::next() return pimpl->next(); } -HDFSSource::URISIterator::URISIterator(const std::vector & uris_) - : pimpl(std::make_shared(uris_)) +HDFSSource::URISIterator::URISIterator(const std::vector & uris_, ContextPtr context) + : pimpl(std::make_shared(uris_, context)) { } @@ -501,7 +515,7 @@ Pipe StorageHDFS::read( } else { - auto uris_iterator = std::make_shared(uris); + auto uris_iterator = std::make_shared(uris, context_); iterator_wrapper = std::make_shared([uris_iterator]() { return uris_iterator->next(); diff --git a/src/Storages/HDFS/StorageHDFS.h b/src/Storages/HDFS/StorageHDFS.h index 53be640248c..8e455189bc6 100644 --- a/src/Storages/HDFS/StorageHDFS.h +++ b/src/Storages/HDFS/StorageHDFS.h @@ -99,7 +99,7 @@ public: class URISIterator { public: - URISIterator(const std::vector & uris_); + URISIterator(const std::vector & uris_, ContextPtr context); String next(); private: class Impl; diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 2fa5ee0fa64..e2294aaabc5 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -82,8 +82,6 @@ public: Impl(Aws::S3::S3Client & client_, const S3::URI & globbed_uri_) : client(client_), globbed_uri(globbed_uri_) { - std::lock_guard lock(mutex); - if (globbed_uri.bucket.find_first_of("*?{") != globbed_uri.bucket.npos) throw Exception("Expression can not have wildcards inside bucket name", ErrorCodes::UNEXPECTED_EXPRESSION); @@ -179,20 +177,24 @@ String StorageS3Source::DisclosedGlobIterator::next() class StorageS3Source::KeysIterator::Impl { public: - explicit Impl(const std::vector & keys_) : keys(keys_), index(0) + explicit Impl(const std::vector & keys_) : keys(keys_), keys_iter(keys.begin()) { } String next() { - if (index == keys.size()) + std::lock_guard lock(mutex); + if (keys_iter == keys.end()) return ""; - return keys[index++]; + auto key = *keys_iter; + ++keys_iter; + return key; } private: - const std::vector & keys; - size_t index; + std::mutex mutex; + Strings keys; + Strings::iterator keys_iter; }; StorageS3Source::KeysIterator::KeysIterator(const std::vector & keys_) : pimpl(std::make_shared(keys_)) @@ -849,9 +851,10 @@ ColumnsDescription StorageS3::getTableStructureFromDataImpl( const std::optional & format_settings, ContextPtr ctx) { + std::vector keys = {client_auth.uri.key}; auto read_buffer_creator = [&]() { - auto file_iterator = createFileIterator(client_auth, {client_auth.uri.key}, is_key_with_globs, distributed_processing, ctx); + auto file_iterator = createFileIterator(client_auth, keys, is_key_with_globs, distributed_processing, ctx); String current_key = (*file_iterator)(); if (current_key.empty()) throw Exception( diff --git a/tests/integration/test_storage_hdfs/test.py b/tests/integration/test_storage_hdfs/test.py index 1b569daef1b..7f4eccad823 100644 --- a/tests/integration/test_storage_hdfs/test.py +++ b/tests/integration/test_storage_hdfs/test.py @@ -366,12 +366,12 @@ def test_overwrite(started_cluster): hdfs_api = started_cluster.hdfs_api table_function = f"hdfs('hdfs://hdfs1:9000/data', 'Parquet', 'a Int32, b String')" - node1.query(f"create table test as {table_function}") - node1.query(f"insert into test select number, randomString(100) from numbers(5)") - node1.query_and_get_error(f"insert into test select number, randomString(100) FROM numbers(10)") - node1.query(f"insert into test select number, randomString(100) from numbers(10) settings hdfs_truncate_on_insert=1") + node1.query(f"create table test_overwrite as {table_function}") + node1.query(f"insert into test_overwrite select number, randomString(100) from numbers(5)") + node1.query_and_get_error(f"insert into test_overwrite select number, randomString(100) FROM numbers(10)") + node1.query(f"insert into test_overwrite select number, randomString(100) from numbers(10) settings hdfs_truncate_on_insert=1") - result = node1.query(f"select count() from test") + result = node1.query(f"select count() from test_overwrite") assert(int(result) == 10) diff --git a/tests/integration/test_storage_s3/configs/named_collections.xml b/tests/integration/test_storage_s3/configs/named_collections.xml index ef21ced4d0c..f22440d17c9 100644 --- a/tests/integration/test_storage_s3/configs/named_collections.xml +++ b/tests/integration/test_storage_s3/configs/named_collections.xml @@ -10,6 +10,11 @@ minio minio123 + + http://minio1:9001/root/test_parquet_gz + minio + minio123 + http://minio1:9001/root/test_orc minio diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 703329ce9cb..a804053d4fd 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -136,7 +136,7 @@ def test_put(started_cluster, maybe_auth, positive, compression): values_csv = "1,2,3\n3,2,1\n78,43,45\n" filename = "test.csv" put_query = f"""insert into table function s3('http://{started_cluster.minio_ip}:{started_cluster.minio_port}/{bucket}/{filename}', - {maybe_auth}'CSV', '{table_format}', {compression}) values {values}""" + {maybe_auth}'CSV', '{table_format}', {compression}) values settings s3_truncate_on_insert=1 {values}""" try: run_query(instance, put_query) @@ -298,7 +298,7 @@ def test_put_csv(started_cluster, maybe_auth, positive): instance = started_cluster.instances["dummy"] # type: ClickHouseInstance table_format = "column1 UInt32, column2 UInt32, column3 UInt32" filename = "test.csv" - put_query = "insert into table function s3('http://{}:{}/{}/{}', {}'CSV', '{}') format CSV".format( + put_query = "insert into table function s3('http://{}:{}/{}/{}', {}'CSV', '{}') format CSV settings s3_truncate_on_insert=1".format( started_cluster.minio_ip, MINIO_INTERNAL_PORT, bucket, filename, maybe_auth, table_format) csv_data = "8,9,16\n11,18,13\n22,14,2\n" @@ -322,7 +322,7 @@ def test_put_get_with_redirect(started_cluster): values = "(1, 1, 1), (1, 1, 1), (11, 11, 11)" values_csv = "1,1,1\n1,1,1\n11,11,11\n" filename = "test.csv" - query = "insert into table function s3('http://{}:{}/{}/{}', 'CSV', '{}') values {}".format( + query = "insert into table function s3('http://{}:{}/{}/{}', 'CSV', '{}') values settings s3_truncate_on_insert=1 {}".format( started_cluster.minio_redirect_host, started_cluster.minio_redirect_port, bucket, filename, table_format, values) run_query(instance, query) @@ -350,12 +350,12 @@ def test_put_with_zero_redirect(started_cluster): filename = "test.csv" # Should work without redirect - query = "insert into table function s3('http://{}:{}/{}/{}', 'CSV', '{}') values {}".format( + query = "insert into table function s3('http://{}:{}/{}/{}', 'CSV', '{}') values settings s3_truncate_on_insert=1 {}".format( started_cluster.minio_ip, MINIO_INTERNAL_PORT, bucket, filename, table_format, values) run_query(instance, query) # Should not work with redirect - query = "insert into table function s3('http://{}:{}/{}/{}', 'CSV', '{}') values {}".format( + query = "insert into table function s3('http://{}:{}/{}/{}', 'CSV', '{}') values settings s3_truncate_on_insert=1 {}".format( started_cluster.minio_redirect_host, started_cluster.minio_redirect_port, bucket, filename, table_format, values) exception_raised = False try: @@ -805,13 +805,13 @@ def test_seekable_formats(started_cluster): instance = started_cluster.instances["dummy"] # type: ClickHouseInstance table_function = f"s3(s3_parquet, structure='a Int32, b String', format='Parquet')" - instance.query(f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(5000000)") + instance.query(f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(5000000) settings s3_truncate_on_insert=1") result = instance.query(f"SELECT count() FROM {table_function}") assert(int(result) == 5000000) table_function = f"s3(s3_orc, structure='a Int32, b String', format='ORC')" - exec_query_with_retry(instance, f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(5000000)") + exec_query_with_retry(instance, f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(5000000) settings s3_truncate_on_insert=1") result = instance.query(f"SELECT count() FROM {table_function}") assert(int(result) == 5000000) @@ -827,14 +827,14 @@ def test_seekable_formats_url(started_cluster): instance = started_cluster.instances["dummy"] table_function = f"s3(s3_parquet, structure='a Int32, b String', format='Parquet')" - instance.query(f"insert into table function {table_function} select number, randomString(100) from numbers(5000000)") + instance.query(f"insert into table function {table_function} select number, randomString(100) from numbers(5000000) settings s3_truncate_on_insert=1") table_function = f"url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_parquet', 'Parquet', 'a Int32, b String')" result = instance.query(f"SELECT count() FROM {table_function}") assert(int(result) == 5000000) table_function = f"s3(s3_orc, structure='a Int32, b String', format='ORC')" - exec_query_with_retry(instance, f"insert into table function {table_function} select number, randomString(100) from numbers(5000000)") + exec_query_with_retry(instance, f"insert into table function {table_function} select number, randomString(100) from numbers(5000000) settings s3_truncate_on_insert=1") table_function = f"url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_orc', 'ORC', 'a Int32, b String')" result = instance.query(f"SELECT count() FROM {table_function}") @@ -939,7 +939,7 @@ def test_create_new_files_on_insert(started_cluster): table_function = f"s3(s3_parquet, structure='a Int32, b String', format='Parquet')" instance.query(f"create table test_multiple_inserts as {table_function}") instance.query(f"truncate table test_multiple_inserts") - instance.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(10)") + instance.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(10) settings s3_truncate_on_insert=1") instance.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(20) settings s3_create_new_file_on_insert=1") instance.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(30) settings s3_create_new_file_on_insert=1") @@ -951,7 +951,7 @@ def test_create_new_files_on_insert(started_cluster): table_function = f"s3(s3_parquet_gz, structure='a Int32, b String', format='Parquet')" instance.query(f"create table test_multiple_inserts as {table_function}") instance.query(f"truncate table test_multiple_inserts") - instance.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(10)") + instance.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(10) settings s3_truncate_on_insert=1") instance.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(20) settings s3_create_new_file_on_insert=1") instance.query(f"insert into test_multiple_inserts select number, randomString(100) from numbers(30) settings s3_create_new_file_on_insert=1") From c9e93b21c2b830c6716b665b90431578c2589727 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 23:19:46 +0300 Subject: [PATCH 084/403] Fix tiny race between count() and INSERT/merges/... in MergeTree Before it was possible to return stale counter from StorageMergeTree::totalRows() (that is used for optimize_trivial_count_query) since the accounting is done in two steps: - subtract old number of rows <-- here the number can be zero, even though there are rows --> - add new number of rows This was found by CI [1] in 01615_random_one_shard_insertion test: Here you can see that INSERT went to both tables:
2022.01.16 09:07:34.288252 [ 154369 ] {a1905be0-93da-460c-8c6f-9b5adace72a0} DistributedBlockOutputStream: It took 0.035197041 sec. to insert 100 blocks, 2841.1479249065287 rows per second. Insertion status: Wrote 54 blocks and 54 rows on shard 0 replica 0, localhost:9000 (average 0 ms per block, the slowest block 1 ms) Wrote 46 blocks and 46 rows on shard 1 replica 0, localhost:9000 (average 0 ms per block, the slowest block 1 ms)
But the test fails, since select from shard1.tbl returns 0, and the problem was concurrent merge:
2022.01.16 09:07:34.289470 [ 146495 ] {cd9d4cf2-7131-4179-b0b2-3aeec4045755} executeQuery: (from [::1]:42082) (comment: 01615_random_one_shard_insertion.sql) select count() != 0 from shard_0.tbl; 2022.01.16 09:07:34.289564 [ 375 ] {c7a885fa-4ef4-4dcf-a4de-1650d44fa0ab::all_1_54_9} MergeTask::MergeProjectionsStage: Merge sorted 54 rows, containing 1 columns (1 merged, 0 gathered) in 0.00171193 sec., 31543.345814373253 rows/sec., 246.43 KiB> 2022.01.16 09:07:34.289810 [ 375 ] {c7a885fa-4ef4-4dcf-a4de-1650d44fa0ab::all_1_54_9} shard_0.tbl (c7a885fa-4ef4-4dcf-a4de-1650d44fa0ab): Renaming temporary part tmp_merge_all_1_54_9 to all_1_54_9. 2022.01.16 09:07:34.289858 [ 146495 ] {cd9d4cf2-7131-4179-b0b2-3aeec4045755} ContextAccess (default): Access granted: SELECT(number) ON shard_0.tbl 2022.01.16 09:07:34.289897 [ 375 ] {c7a885fa-4ef4-4dcf-a4de-1650d44fa0ab::all_1_54_9} shard_0.tbl (c7a885fa-4ef4-4dcf-a4de-1650d44fa0ab) (MergerMutator): Merged 6 parts: from all_1_49_8 to all_54_54_0 2022.01.16 09:07:34.289920 [ 146495 ] {cd9d4cf2-7131-4179-b0b2-3aeec4045755} InterpreterSelectQuery: WithMergeableState -> Complete 2022.01.16 09:07:34.289987 [ 375 ] {} MemoryTracker: Peak memory usage Mutate/Merge: 3.12 MiB. 2022.01.16 09:07:34.290305 [ 154344 ] {cd9d4cf2-7131-4179-b0b2-3aeec4045755} MergingAggregatedTransform: Reading blocks of partially aggregated data. 2022.01.16 09:07:34.290332 [ 154344 ] {cd9d4cf2-7131-4179-b0b2-3aeec4045755} MergingAggregatedTransform: Read 1 blocks of partially aggregated data, total 1 rows. 2022.01.16 09:07:34.290343 [ 154344 ] {cd9d4cf2-7131-4179-b0b2-3aeec4045755} Aggregator: Merging partially aggregated single-level data. 2022.01.16 09:07:34.290358 [ 154344 ] {cd9d4cf2-7131-4179-b0b2-3aeec4045755} Aggregator: Merged partially aggregated single-level data. 2022.01.16 09:07:34.290366 [ 154344 ] {cd9d4cf2-7131-4179-b0b2-3aeec4045755} Aggregator: Converting aggregated data to blocks 2022.01.16 09:07:34.290391 [ 154344 ] {cd9d4cf2-7131-4179-b0b2-3aeec4045755} Aggregator: Converted aggregated data to blocks. 1 rows, 8.00 B in 1.0939e-05 sec. (91416.034 rows/sec., 714.19 KiB/sec.) 2022.01.16 09:07:34.290709 [ 146495 ] {cd9d4cf2-7131-4179-b0b2-3aeec4045755} executeQuery: Read 1 rows, 4.01 KiB in 0.001187722 sec., 841 rows/sec., 3.30 MiB/sec. 2022.01.16 09:07:34.290774 [ 146495 ] {cd9d4cf2-7131-4179-b0b2-3aeec4045755} MemoryTracker: Peak memory usage (for query): 0.00 B.
[1]: https://s3.amazonaws.com/clickhouse-test-reports/33675/7848ea7d609e4c720e8e4494eb6207c0751f5aea/stateless_tests__ubsan__actions_.html This also fixes a race between DROP TABLE check and INSERT/merges. v0: use Active parts instead. v2: fix total counters accounting instead. --- src/Storages/MergeTree/MergeTreeData.cpp | 31 ++++++++++++------------ src/Storages/MergeTree/MergeTreeData.h | 4 +-- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 54705a3c405..edbf561293b 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -2553,11 +2553,13 @@ bool MergeTreeData::renameTempPartAndReplace( ++reduce_parts; } - decreaseDataVolume(reduce_bytes, reduce_rows, reduce_parts); - modifyPartState(part_it, DataPartState::Active); addPartContributionToColumnAndSecondaryIndexSizes(part); - addPartContributionToDataVolume(part); + + ssize_t diff_bytes = part->getBytesOnDisk() - reduce_bytes; + ssize_t diff_rows = part->rows_count - reduce_rows; + ssize_t diff_parts = 1 - reduce_parts; + increaseDataVolume(diff_bytes, diff_rows, diff_parts); } auto part_in_memory = asInMemoryPart(part); @@ -3086,8 +3088,9 @@ void MergeTreeData::swapActivePart(MergeTreeData::DataPartPtr part_copy) auto part_it = data_parts_indexes.insert(part_copy).first; modifyPartState(part_it, DataPartState::Active); - removePartContributionToDataVolume(original_active_part); - addPartContributionToDataVolume(part_copy); + ssize_t diff_bytes = part_copy->getBytesOnDisk() - original_active_part->getBytesOnDisk(); + ssize_t diff_rows = part_copy->rows_count - original_active_part->rows_count; + increaseDataVolume(diff_bytes, diff_rows, /* parts= */ 0); auto disk = original_active_part->volume->getDisk(); String marker_path = fs::path(original_active_part->getFullRelativePath()) / IMergeTreeDataPart::DELETE_ON_DESTROY_MARKER_FILE_NAME; @@ -4288,8 +4291,11 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(MergeTreeData: data.addPartContributionToColumnAndSecondaryIndexSizes(part); } } - data.decreaseDataVolume(reduce_bytes, reduce_rows, reduce_parts); - data.increaseDataVolume(add_bytes, add_rows, add_parts); + + ssize_t diff_bytes = add_bytes - reduce_bytes; + ssize_t diff_rows = add_rows - reduce_rows; + ssize_t diff_parts = add_parts - reduce_parts; + data.increaseDataVolume(diff_bytes, diff_rows, diff_parts); } clear(); @@ -5674,23 +5680,16 @@ void MergeTreeData::addPartContributionToDataVolume(const DataPartPtr & part) void MergeTreeData::removePartContributionToDataVolume(const DataPartPtr & part) { - decreaseDataVolume(part->getBytesOnDisk(), part->rows_count, 1); + increaseDataVolume(-part->getBytesOnDisk(), -part->rows_count, -1); } -void MergeTreeData::increaseDataVolume(size_t bytes, size_t rows, size_t parts) +void MergeTreeData::increaseDataVolume(ssize_t bytes, ssize_t rows, ssize_t parts) { total_active_size_bytes.fetch_add(bytes, std::memory_order_acq_rel); total_active_size_rows.fetch_add(rows, std::memory_order_acq_rel); total_active_size_parts.fetch_add(parts, std::memory_order_acq_rel); } -void MergeTreeData::decreaseDataVolume(size_t bytes, size_t rows, size_t parts) -{ - total_active_size_bytes.fetch_sub(bytes, std::memory_order_acq_rel); - total_active_size_rows.fetch_sub(rows, std::memory_order_acq_rel); - total_active_size_parts.fetch_sub(parts, std::memory_order_acq_rel); -} - void MergeTreeData::setDataVolume(size_t bytes, size_t rows, size_t parts) { total_active_size_bytes.store(bytes, std::memory_order_release); diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 4c58a53f368..62335fa74ec 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -1159,9 +1159,7 @@ private: void addPartContributionToDataVolume(const DataPartPtr & part); void removePartContributionToDataVolume(const DataPartPtr & part); - void increaseDataVolume(size_t bytes, size_t rows, size_t parts); - void decreaseDataVolume(size_t bytes, size_t rows, size_t parts); - + void increaseDataVolume(ssize_t bytes, ssize_t rows, ssize_t parts); void setDataVolume(size_t bytes, size_t rows, size_t parts); std::atomic total_active_size_bytes = 0; From b0de356f4b989b4a1e44ed8764fc83af0c91458b Mon Sep 17 00:00:00 2001 From: Yohann Jardin Date: Wed, 19 Jan 2022 00:11:28 +0100 Subject: [PATCH 085/403] fix/doc: update output type of tupleHammingDistance --- .../sql-reference/functions/tuple-functions.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/tuple-functions.md b/docs/en/sql-reference/functions/tuple-functions.md index 0ddd628d9c2..8502fcdcf66 100644 --- a/docs/en/sql-reference/functions/tuple-functions.md +++ b/docs/en/sql-reference/functions/tuple-functions.md @@ -134,7 +134,23 @@ Tuples should have the same type of the elements. - The Hamming distance. -Type: [UInt8](../../sql-reference/data-types/int-uint.md). +Type: The result type is calculed the same way it is for [Arithmetic functions](../../sql-reference/functions/arithmetic-functions.md), based on the number of elements in the input tuples. + +``` sql +SELECT + toTypeName(tupleHammingDistance(tuple(0), tuple(0))) AS t1, + toTypeName(tupleHammingDistance((0, 0), (0, 0))) AS t2, + toTypeName(tupleHammingDistance((0, 0, 0), (0, 0, 0))) AS t3, + toTypeName(tupleHammingDistance((0, 0, 0, 0), (0, 0, 0, 0))) AS t4, + toTypeName(tupleHammingDistance((0, 0, 0, 0, 0), (0, 0, 0, 0, 0))) AS t5 +``` + +``` text +┌─t1────┬─t2─────┬─t3─────┬─t4─────┬─t5─────┐ +│ UInt8 │ UInt16 │ UInt32 │ UInt64 │ UInt64 │ +└───────┴────────┴────────┴────────┴────────┘ +``` + **Examples** From b008ee1c61361dc69c40bf8539b5aa1c5b274bb6 Mon Sep 17 00:00:00 2001 From: cnmade Date: Wed, 19 Jan 2022 09:18:07 +0800 Subject: [PATCH 086/403] Translate zh/sql-reference/statements/alter/ttl.md --- docs/zh/sql-reference/statements/alter/ttl.md | 86 ++++++++++++++++++- 1 file changed, 85 insertions(+), 1 deletion(-) mode change 120000 => 100644 docs/zh/sql-reference/statements/alter/ttl.md diff --git a/docs/zh/sql-reference/statements/alter/ttl.md b/docs/zh/sql-reference/statements/alter/ttl.md deleted file mode 120000 index 94a112e7a17..00000000000 --- a/docs/zh/sql-reference/statements/alter/ttl.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/alter/ttl.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/alter/ttl.md b/docs/zh/sql-reference/statements/alter/ttl.md new file mode 100644 index 00000000000..ca011a2a12f --- /dev/null +++ b/docs/zh/sql-reference/statements/alter/ttl.md @@ -0,0 +1,85 @@ +--- +toc_priority: 44 +toc_title: TTL +--- + +# 表的 TTL 操作 {#manipulations-with-table-ttl} + +## 修改 MODIFY TTL {#modify-ttl} + +你能修改 [表 TTL](../../../engines/table-engines/mergetree-family/mergetree.md#mergetree-table-ttl) ，命令语法如下所示： + +``` sql +ALTER TABLE table_name MODIFY TTL ttl_expression; +``` + +## 移除 REMOVE TTL {#remove-ttl} + +TTL 属性可以用下列命令从表中移除: + +```sql +ALTER TABLE table_name REMOVE TTL +``` + +**示例** + +创建一个表，带有 `TTL` 属性如下所示: + +```sql +CREATE TABLE table_with_ttl +( + event_time DateTime, + UserID UInt64, + Comment String +) +ENGINE MergeTree() +ORDER BY tuple() +TTL event_time + INTERVAL 3 MONTH; +SETTINGS min_bytes_for_wide_part = 0; + +INSERT INTO table_with_ttl VALUES (now(), 1, 'username1'); + +INSERT INTO table_with_ttl VALUES (now() - INTERVAL 4 MONTH, 2, 'username2'); +``` + +运行命令 `OPTIMIZE` 强制清理 `TTL`: + +```sql +OPTIMIZE TABLE table_with_ttl FINAL; +SELECT * FROM table_with_ttl FORMAT PrettyCompact; +``` +第二行记录被从表中删除掉了. + +```text +┌─────────event_time────┬──UserID─┬─────Comment──┐ +│ 2020-12-11 12:44:57 │ 1 │ username1 │ +└───────────────────────┴─────────┴──────────────┘ +``` + +现在用下面的命令，把表的 `TTL` 移除掉: + +```sql +ALTER TABLE table_with_ttl REMOVE TTL; +``` + +重新插入上面的数据，并尝试再次运行 `OPTIMIZE` 命令清理 `TTL` 属性 : + +```sql +INSERT INTO table_with_ttl VALUES (now() - INTERVAL 4 MONTH, 2, 'username2'); +OPTIMIZE TABLE table_with_ttl FINAL; +SELECT * FROM table_with_ttl FORMAT PrettyCompact; +``` + +可以看得到 `TTL` 这个属性已经没了，并且可以看得到第二行记录并没有被删除: + +```text +┌─────────event_time────┬──UserID─┬─────Comment──┐ +│ 2020-12-11 12:44:57 │ 1 │ username1 │ +│ 2020-08-11 12:44:57 │ 2 │ username2 │ +└───────────────────────┴─────────┴──────────────┘ +``` + +**更多参考** + +- 关于 [TTL 表达式](../../../sql-reference/statements/create/table.md#ttl-expression). +- 修改列 [with TTL](../../../sql-reference/statements/alter/column.md#alter_modify-column). From 94a24906641e05a8315daafaf2f7f7bbcd272a5f Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 19 Jan 2022 10:13:12 +0800 Subject: [PATCH 087/403] Fix style checking --- src/Functions/FunctionsBinaryRepr.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/FunctionsBinaryRepr.cpp b/src/Functions/FunctionsBinaryRepr.cpp index 7cc0fc324ab..b8733cfc644 100644 --- a/src/Functions/FunctionsBinaryRepr.cpp +++ b/src/Functions/FunctionsBinaryRepr.cpp @@ -519,7 +519,7 @@ public: char * begin = reinterpret_cast(&out_vec[pos]); char * end = begin; - // use executeOnUInt instead of using executeOneString + // use executeOnUInt instead of using executeOneString // because the latter one outputs the string in the memory order Impl::executeOneUInt(uuid[i].toUnderType().items[0], end, false, false); Impl::executeOneUInt(uuid[i].toUnderType().items[1], end, false, true); From 13cee6c1847902cc971465824737ab2b4b1f4b68 Mon Sep 17 00:00:00 2001 From: Vxider Date: Wed, 19 Jan 2022 02:24:27 +0000 Subject: [PATCH 088/403] update code style --- src/Storages/WindowView/StorageWindowView.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp index 9d0d2aeec87..37c913f58a9 100644 --- a/src/Storages/WindowView/StorageWindowView.cpp +++ b/src/Storages/WindowView/StorageWindowView.cpp @@ -488,7 +488,8 @@ std::pair StorageWindowView::getNewBlocks(UInt32 watermark) auto pipe = QueryPipelineBuilder::getPipe(std::move(builder)); pipes.emplace_back(std::move(pipe)); - auto creator = [&](const StorageID & blocks_id_global) { + auto creator = [&](const StorageID & blocks_id_global) + { auto parent_table_metadata = getParentStorage()->getInMemoryMetadataPtr(); auto required_columns = parent_table_metadata->getColumns(); required_columns.add(ColumnDescription("____timestamp", std::make_shared())); @@ -1245,7 +1246,8 @@ void StorageWindowView::writeIntoWindowView( Pipes pipes; pipes.emplace_back(std::move(pipe)); - auto creator = [&](const StorageID & blocks_id_global) { + auto creator = [&](const StorageID & blocks_id_global) + { auto parent_metadata = window_view.getParentStorage()->getInMemoryMetadataPtr(); auto required_columns = parent_metadata->getColumns(); required_columns.add(ColumnDescription("____timestamp", std::make_shared())); @@ -1261,7 +1263,8 @@ void StorageWindowView::writeIntoWindowView( QueryProcessingStage::WithMergeableState); builder = select_block.buildQueryPipeline(); - builder.addSimpleTransform([&](const Block & current_header) { + builder.addSimpleTransform([&](const Block & current_header) + { return std::make_shared( current_header, local_context->getSettingsRef().min_insert_block_size_rows, From 654e585fd6415ab176cc38426ab2d722bacd5617 Mon Sep 17 00:00:00 2001 From: Lemore <34434448+acelemore@users.noreply.github.com> Date: Wed, 19 Jan 2022 14:18:36 +0800 Subject: [PATCH 089/403] Update Chinese localization It's better than the machine-translate, i think :) --- docs/zh/operations/settings/settings-users.md | 59 +++++++++---------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/docs/zh/operations/settings/settings-users.md b/docs/zh/operations/settings/settings-users.md index ae75dddab58..d89b880328a 100644 --- a/docs/zh/operations/settings/settings-users.md +++ b/docs/zh/operations/settings/settings-users.md @@ -1,5 +1,5 @@ --- -machine_translated: true +machine_translated: false machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_priority: 63 toc_title: "\u7528\u6237\u8BBE\u7F6E" @@ -7,12 +7,12 @@ toc_title: "\u7528\u6237\u8BBE\u7F6E" # 用户设置 {#user-settings} -该 `users` 一节 `user.xml` 配置文件包含用户设置。 +`user.xml` 中的 `users` 配置段包含了用户配置 -!!! note "信息" +!!! note "提示" ClickHouse还支持 [SQL驱动的工作流](../access-rights.md#access-control) 用于管理用户。 我们建议使用它。 -的结构 `users` 科: +`users` 配置段的结构: ``` xml @@ -43,21 +43,21 @@ toc_title: "\u7528\u6237\u8BBE\u7F6E" ``` -### 用户名称/密码 {#user-namepassword} +### user_name/password {#user-namepassword} 密码可以以明文或SHA256（十六进制格式）指定。 -- 以明文形式分配密码 (**不推荐**），把它放在一个 `password` 元素。 +- 以明文形式分配密码 (**不推荐**），把它放在一个 `password` 配置段中。 例如, `qwerty`. 密码可以留空。 -- 要使用其SHA256散列分配密码，请将其放置在 `password_sha256_hex` 元素。 +- 要使用SHA256加密后的密码，请将其放置在 `password_sha256_hex` 配置段。 例如, `65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5`. - 如何从shell生成密码的示例: + 从shell生成加密密码的示例: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-' @@ -65,19 +65,19 @@ toc_title: "\u7528\u6237\u8BBE\u7F6E" -- 为了与MySQL客户端兼容，密码可以在双SHA1哈希中指定。 放进去 `password_double_sha1_hex` 元素。 +- 为了与MySQL客户端兼容，密码可以设置为双SHA1哈希加密, 请将其放置在 `password_double_sha1_hex` 配置段。 例如, `08b4a0f1de6ad37da17359e592c8d74788a83eb0`. - 如何从shell生成密码的示例: + 从shell生成密码的示例: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha1sum | tr -d '-' | xxd -r -p | sha1sum | tr -d '-' 结果的第一行是密码。 第二行是相应的双SHA1哈希。 -### 访问管理 {#access_management-user-setting} +### access_management {#access_management-user-setting} -此设置启用禁用使用SQL驱动 [访问控制和帐户管理](../access-rights.md#access-control) 对于用户。 +此设置可为用户启用或禁用 SQL-driven [访问控制和帐户管理](../access-rights.md#access-control) 。 可能的值: @@ -86,42 +86,42 @@ toc_title: "\u7528\u6237\u8BBE\u7F6E" 默认值：0。 -### 用户名称/网络 {#user-namenetworks} +### user_name/networks {#user-namenetworks} -用户可以从中连接到ClickHouse服务器的网络列表。 +用户访问来源列表 列表中的每个元素都可以具有以下形式之一: -- `` — IP address or network mask. +- `` — IP地址或网络掩码 例: `213.180.204.3`, `10.0.0.1/8`, `10.0.0.1/255.255.255.0`, `2a02:6b8::3`, `2a02:6b8::3/64`, `2a02:6b8::3/ffff:ffff:ffff:ffff::`. -- `` — Hostname. +- `` — 域名 示例: `example01.host.ru`. - 要检查访问，将执行DNS查询，并将所有返回的IP地址与对等地址进行比较。 + 为检查访问，将执行DNS查询，并将所有返回的IP地址与对端地址进行比较。 -- `` — Regular expression for hostnames. +- `` — 域名的正则表达式. 示例, `^example\d\d-\d\d-\d\.host\.ru$` - 要检查访问，a [DNS PTR查询](https://en.wikipedia.org/wiki/Reverse_DNS_lookup) 对对等体地址执行，然后应用指定的正则表达式。 然后，对PTR查询的结果执行另一个DNS查询，并将所有接收到的地址与对等地址进行比较。 我们强烈建议正则表达式以$结尾。 + 为检查访问，[DNS PTR查询](https://en.wikipedia.org/wiki/Reverse_DNS_lookup) 对对端地址执行，然后应用指定的正则表达式。 然后，以PTR查询的结果执行另一个DNS查询，并将所有接收到的地址与对端地址进行比较. 我们强烈建议正则表达式以$结尾. DNS请求的所有结果都将被缓存，直到服务器重新启动。 **例** -要从任何网络打开用户的访问权限，请指定: +要开启任意来源网络的访问, 请指定: ``` xml ::/0 ``` !!! warning "警告" - 从任何网络开放访问是不安全的，除非你有一个防火墙正确配置或服务器没有直接连接到互联网。 + 从任何网络开放访问是不安全的，除非你有一个正确配置的防火墙, 或者服务器没有直接连接到互联网。 -若要仅从本地主机打开访问权限，请指定: +若要限定本机访问, 请指定: ``` xml ::1 @@ -130,22 +130,21 @@ DNS请求的所有结果都将被缓存，直到服务器重新启动。 ### user_name/profile {#user-nameprofile} -您可以为用户分配设置配置文件。 设置配置文件在单独的部分配置 `users.xml` 文件 有关详细信息，请参阅 [设置配置文件](settings-profiles.md). +您可以为用户分配设置配置文件。 设置配置文件在`users.xml` 中有单独的配置段. 有关详细信息，请参阅 [设置配置文件](settings-profiles.md). -### 用户名称/配额 {#user-namequota} +### user_name/quota {#user-namequota} -配额允许您在一段时间内跟踪或限制资源使用情况。 配额在配置 `quotas` -一节 `users.xml` 配置文件。 +配额允许您在一段时间内跟踪或限制资源使用情况。 配额在`users.xml` 中的 `quotas` 配置段下. 您可以为用户分配配额。 有关配额配置的详细说明，请参阅 [配额](../quotas.md#quotas). -### 用户名/数据库 {#user-namedatabases} +### user_name/databases {#user-namedatabases} -在本节中，您可以限制ClickHouse返回的行 `SELECT` 由当前用户进行的查询，从而实现基本的行级安全性。 +在本配置段中，您可以限制ClickHouse中由当前用户进行的 `SELECT` 查询所返回的行，从而实现基本的行级安全性。 **示例** -以下配置强制该用户 `user1` 只能看到的行 `table1` 作为结果 `SELECT` 查询，其中的值 `id` 场是1000。 +以下配置使用户 `user1` 通过SELECT查询只能得到table1中id为1000的行 ``` xml @@ -159,6 +158,6 @@ DNS请求的所有结果都将被缓存，直到服务器重新启动。 ``` -该 `filter` 可以是导致任何表达式 [UInt8](../../sql-reference/data-types/int-uint.md)-键入值。 它通常包含比较和逻辑运算符。 从行 `database_name.table1` 其中，不会为此用户返回为0的筛选结果。 过滤是不兼容的 `PREWHERE` 操作和禁用 `WHERE→PREWHERE` 优化。 +该 `filter` 可以是[UInt8](../../sql-reference/data-types/int-uint.md)编码的任何表达式。 它通常包含比较和逻辑运算符, 当filter返回0时, database_name.table1 的该行结果将不会返回给用户.过滤不兼容 `PREWHERE` 操作并禁用 `WHERE→PREWHERE` 优化。 [原始文章](https://clickhouse.com/docs/en/operations/settings/settings_users/) From 5e58767de8a2f1f2bac23c5793d1b378ad02f324 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Tue, 11 Jan 2022 20:24:01 -0800 Subject: [PATCH 090/403] add check for h3 empty column arguments --- src/Functions/h3GetFaces.cpp | 9 +++++++++ src/Functions/h3IsPentagon.cpp | 11 ++++++++++- src/Functions/h3IsResClassIII.cpp | 10 +++++++++- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/Functions/h3GetFaces.cpp b/src/Functions/h3GetFaces.cpp index c0300e7212b..b85ad37c04f 100644 --- a/src/Functions/h3GetFaces.cpp +++ b/src/Functions/h3GetFaces.cpp @@ -19,6 +19,7 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int ILLEGAL_COLUMN; } namespace @@ -52,6 +53,14 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { const auto * column = checkAndGetColumn(arguments[0].column.get()); + if (!column) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64.", + arguments[0].type->getName(), + 1, + getName()); + const auto & data = column->getData(); auto result_column_data = ColumnUInt8::create(); diff --git a/src/Functions/h3IsPentagon.cpp b/src/Functions/h3IsPentagon.cpp index a6726fe1656..78c93be75a6 100644 --- a/src/Functions/h3IsPentagon.cpp +++ b/src/Functions/h3IsPentagon.cpp @@ -17,6 +17,7 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int ILLEGAL_COLUMN; } namespace @@ -50,13 +51,21 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { const auto * column = checkAndGetColumn(arguments[0].column.get()); + if (!column) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[0].type->getName(), + 1, + getName()); + const auto & data = column->getData(); auto dst = ColumnVector::create(); auto & dst_data = dst->getData(); dst_data.resize(input_rows_count); - for (size_t row = 0 ; row < input_rows_count ; ++row) + for (size_t row = 0; row < input_rows_count; ++row) { UInt8 res = isPentagon(data[row]); dst_data[row] = res; diff --git a/src/Functions/h3IsResClassIII.cpp b/src/Functions/h3IsResClassIII.cpp index c6b79d404a4..923985b3c8f 100644 --- a/src/Functions/h3IsResClassIII.cpp +++ b/src/Functions/h3IsResClassIII.cpp @@ -17,6 +17,7 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int ILLEGAL_COLUMN; } namespace @@ -50,13 +51,20 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { const auto * column = checkAndGetColumn(arguments[0].column.get()); + if (!column) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[0].type->getName(), + 1, + getName()); const auto & data = column->getData(); auto dst = ColumnVector::create(); auto & dst_data = dst->getData(); dst_data.resize(input_rows_count); - for (size_t row = 0 ; row < input_rows_count ; ++row) + for (size_t row = 0; row < input_rows_count; ++row) { UInt8 res = isResClassIII(data[row]); dst_data[row] = res; From 28a7366130fde5e4e6f61b949ea40a3ac5afb59f Mon Sep 17 00:00:00 2001 From: bharatnc Date: Tue, 11 Jan 2022 20:52:05 -0800 Subject: [PATCH 091/403] fix spacing --- src/Functions/h3IsResClassIII.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Functions/h3IsResClassIII.cpp b/src/Functions/h3IsResClassIII.cpp index 923985b3c8f..a6bc02ae5b2 100644 --- a/src/Functions/h3IsResClassIII.cpp +++ b/src/Functions/h3IsResClassIII.cpp @@ -58,6 +58,7 @@ public: arguments[0].type->getName(), 1, getName()); + const auto & data = column->getData(); auto dst = ColumnVector::create(); From a2a5beaedc9fa0f28f1c597f48a3de777305f7ee Mon Sep 17 00:00:00 2001 From: bharatnc Date: Tue, 11 Jan 2022 22:54:28 -0800 Subject: [PATCH 092/403] fix style check --- src/Functions/h3IsResClassIII.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/h3IsResClassIII.cpp b/src/Functions/h3IsResClassIII.cpp index a6bc02ae5b2..3fd9e1e62b0 100644 --- a/src/Functions/h3IsResClassIII.cpp +++ b/src/Functions/h3IsResClassIII.cpp @@ -58,7 +58,7 @@ public: arguments[0].type->getName(), 1, getName()); - + const auto & data = column->getData(); auto dst = ColumnVector::create(); From 4d3bb1584d9cd7d63e1995b307013c0cb8f1db21 Mon Sep 17 00:00:00 2001 From: liuneng1994 <1398775315@qq.com> Date: Wed, 19 Jan 2022 02:50:07 +0000 Subject: [PATCH 093/403] fix Dereference of null pointer error Signed-off-by: neng.liu --- src/Storages/HDFS/StorageHDFS.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index 372c9e04e08..c3240672391 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -53,6 +53,7 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ACCESS_DENIED; extern const int CANNOT_EXTRACT_TABLE_STRUCTURE; + extern const int LOGICAL_ERROR; } namespace { @@ -72,7 +73,8 @@ namespace HDFSFileInfo ls; ls.file_info = hdfsListDirectory(fs.get(), prefix_without_globs.data(), &ls.length); - if (ls.file_info == nullptr && errno != ENOENT) { // NOLINT + if (ls.file_info == nullptr && errno != ENOENT) // NOLINT + { // ignore file not found exception, keep throw other exception, libhdfs3 doesn't have function to get exception type, so use errno. throw Exception( ErrorCodes::ACCESS_DENIED, "Cannot list directory {}: {}", prefix_without_globs, String(hdfsGetLastError())); @@ -80,6 +82,8 @@ namespace Strings result; for (int i = 0; i < ls.length; ++i) { + if (ls.file_info == nullptr) + throw Exception(ErrorCodes::LOGICAL_ERROR, "file_info shouldn't be null"); const String full_path = String(ls.file_info[i].mName); const size_t last_slash = full_path.rfind('/'); const String file_name = full_path.substr(last_slash); From 53fde56cb7e77166d8a434ec192200ee83ebc4a6 Mon Sep 17 00:00:00 2001 From: feng lv Date: Wed, 19 Jan 2022 06:02:24 +0000 Subject: [PATCH 094/403] rewrite bitHammingDistance with FunctionBinaryArithmetic fix fix typo --- src/Functions/FunctionBinaryArithmetic.h | 3 + src/Functions/IsOperation.h | 3 + src/Functions/bitHammingDistance.cpp | 151 ++--------------------- 3 files changed, 18 insertions(+), 139 deletions(-) diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index a1c3320f88a..4aaaf37e6cf 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -139,6 +139,9 @@ public: Case::allow_decimal && IsDataTypeDecimal && IsFloatingPoint, DataTypeFloat64>, + Case::bit_hamming_distance && IsIntegral && IsIntegral, + DataTypeUInt8>, + /// Decimal Real is not supported (traditional DBs convert Decimal Real to Real) Case && !IsIntegralOrExtendedOrDecimal, InvalidType>, Case && !IsIntegralOrExtendedOrDecimal, InvalidType>, diff --git a/src/Functions/IsOperation.h b/src/Functions/IsOperation.h index 5af8ae77727..de7701db59a 100644 --- a/src/Functions/IsOperation.h +++ b/src/Functions/IsOperation.h @@ -19,6 +19,7 @@ template struct EqualsOp; template struct NotEqualsOp; template struct LessOrEqualsOp; template struct GreaterOrEqualsOp; +template struct BitHammingDistanceImpl; template struct SignImpl; @@ -55,6 +56,8 @@ struct IsOperation static constexpr bool least = IsSameOperation::value; static constexpr bool greatest = IsSameOperation::value; + static constexpr bool bit_hamming_distance = IsSameOperation::value; + static constexpr bool division = div_floating || div_int || div_int_or_zero; static constexpr bool allow_decimal = plus || minus || multiply || division || least || greatest; diff --git a/src/Functions/bitHammingDistance.cpp b/src/Functions/bitHammingDistance.cpp index bbb60a764fe..afc281e7e76 100644 --- a/src/Functions/bitHammingDistance.cpp +++ b/src/Functions/bitHammingDistance.cpp @@ -1,159 +1,32 @@ -#include -#include +#include #include -#include -#include -#include namespace DB { -namespace ErrorCodes -{ - extern const int ILLEGAL_COLUMN; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; -} - - template struct BitHammingDistanceImpl { using ResultType = UInt8; + static const constexpr bool allow_fixed_string = false; + static const constexpr bool allow_string_integer = false; - static void NO_INLINE vectorVector(const PaddedPODArray & a, const PaddedPODArray & b, PaddedPODArray & c) + template + static inline NO_SANITIZE_UNDEFINED Result apply(A a, B b) { - size_t size = a.size(); - for (size_t i = 0; i < size; ++i) - c[i] = apply(a[i], b[i]); - } - - static void NO_INLINE vectorConstant(const PaddedPODArray & a, B b, PaddedPODArray & c) - { - size_t size = a.size(); - for (size_t i = 0; i < size; ++i) - c[i] = apply(a[i], b); - } - - static void NO_INLINE constantVector(A a, const PaddedPODArray & b, PaddedPODArray & c) - { - size_t size = b.size(); - for (size_t i = 0; i < size; ++i) - c[i] = apply(a, b[i]); - } - -private: - static inline UInt8 apply(UInt64 a, UInt64 b) - { - UInt64 res = a ^ b; + UInt64 res = static_cast(a) ^ static_cast(b); return __builtin_popcountll(res); } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = false; /// special type handling, some other time +#endif }; -template -bool castType(const IDataType * type, F && f) +struct NameBitHammingDistance { - return castTypeToEither< - DataTypeInt8, - DataTypeInt16, - DataTypeInt32, - DataTypeInt64, - DataTypeUInt8, - DataTypeUInt16, - DataTypeUInt32, - DataTypeUInt64>(type, std::forward(f)); -} - -template -static bool castBothTypes(const IDataType * left, const IDataType * right, F && f) -{ - return castType(left, [&](const auto & left_) { return castType(right, [&](const auto & right_) { return f(left_, right_); }); }); -} - -// bitHammingDistance function: (Integer, Integer) -> UInt8 -class FunctionBitHammingDistance : public IFunction -{ -public: static constexpr auto name = "bitHammingDistance"; - using ResultType = UInt8; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } - - String getName() const override { return name; } - - size_t getNumberOfArguments() const override { return 2; } - - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - if (!isInteger(arguments[0])) - throw Exception( - "Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - if (!isInteger(arguments[1])) - throw Exception( - "Illegal type " + arguments[1]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - return std::make_shared(); - } - - bool useDefaultImplementationForConstants() const override { return true; } - - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override - { - const auto * left_generic = arguments[0].type.get(); - const auto * right_generic = arguments[1].type.get(); - ColumnPtr result_column; - bool valid = castBothTypes(left_generic, right_generic, [&](const auto & left, const auto & right) - { - using LeftDataType = std::decay_t; - using RightDataType = std::decay_t; - using T0 = typename LeftDataType::FieldType; - using T1 = typename RightDataType::FieldType; - using ColVecT0 = ColumnVector; - using ColVecT1 = ColumnVector; - using ColVecResult = ColumnVector; - - using OpImpl = BitHammingDistanceImpl; - - const auto * const col_left_raw = arguments[0].column.get(); - const auto * const col_right_raw = arguments[1].column.get(); - - typename ColVecResult::MutablePtr col_res = nullptr; - col_res = ColVecResult::create(); - - auto & vec_res = col_res->getData(); - vec_res.resize(input_rows_count); - - if (auto col_left_const = checkAndGetColumnConst(col_left_raw)) - { - if (auto col_right = checkAndGetColumn(col_right_raw)) - { - // constant integer - non-constant integer - OpImpl::constantVector(col_left_const->template getValue(), col_right->getData(), vec_res); - } - else - return false; - } - else if (auto col_left = checkAndGetColumn(col_left_raw)) - { - if (auto col_right = checkAndGetColumn(col_right_raw)) - // non-constant integer - non-constant integer - OpImpl::vectorVector(col_left->getData(), col_right->getData(), vec_res); - else if (auto col_right_const = checkAndGetColumnConst(col_right_raw)) - // non-constant integer - constant integer - OpImpl::vectorConstant(col_left->getData(), col_right_const->template getValue(), vec_res); - else - return false; - } - else - return false; - - result_column = std::move(col_res); - return true; - }); - if (!valid) - throw Exception(getName() + "'s arguments do not match the expected data types", ErrorCodes::ILLEGAL_COLUMN); - - return result_column; - } }; +using FunctionBitHammingDistance = BinaryArithmeticOverloadResolver; void registerFunctionBitHammingDistance(FunctionFactory & factory) { From d0c72f6df4f30855f99b0337ab19f60c3adf3cb4 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Wed, 19 Jan 2022 17:15:36 +0800 Subject: [PATCH 095/403] Update test case Signed-off-by: Frank Chen --- .../0_stateless/02128_hex_bin_on_uuid.reference | 2 +- tests/queries/0_stateless/02128_hex_bin_on_uuid.sql | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/queries/0_stateless/02128_hex_bin_on_uuid.reference b/tests/queries/0_stateless/02128_hex_bin_on_uuid.reference index d0b85f7c688..32d25b95178 100644 --- a/tests/queries/0_stateless/02128_hex_bin_on_uuid.reference +++ b/tests/queries/0_stateless/02128_hex_bin_on_uuid.reference @@ -1,5 +1,5 @@ 32 1 0000000080e746f800009d773a2fd319 -1 128 +00000000000000000000000000000000100000001110011101000110111110000000000000000000100111010111011100111010001011111101001100011001 diff --git a/tests/queries/0_stateless/02128_hex_bin_on_uuid.sql b/tests/queries/0_stateless/02128_hex_bin_on_uuid.sql index 6f6bc64542a..30c0c4b7629 100644 --- a/tests/queries/0_stateless/02128_hex_bin_on_uuid.sql +++ b/tests/queries/0_stateless/02128_hex_bin_on_uuid.sql @@ -2,17 +2,15 @@ select length(hex(generateUUIDv4())); with generateUUIDv4() as uuid, - hex(reverse(reinterpretAsString(uuid))) as str1, - hex(uuid) as str2 + replace(toString(uuid), '-', '') as str1, + lower(hex(uuid)) as str2 select str1 = str2; -- hex on UUID always generate 32 characters even there're leading zeros select lower(hex(toUUID('00000000-80e7-46f8-0000-9d773a2fd319'))); -with generateUUIDv4() as uuid, - bin(reverse(reinterpretAsString(uuid))) as bin_str1, - bin(uuid) as bin_str2 -select bin_str1 = bin_str2; +-- length should be 128 +select length(bin(generateUUIDv4())); -- bin on UUID always generate 128 characters even there're leading zeros -select length(bin(toUUID('00000000-80e7-46f8-0000-9d773a2fd319'))); \ No newline at end of file +select bin(toUUID('00000000-80e7-46f8-0000-9d773a2fd319')); \ No newline at end of file From a166963d919b83ea1a0a04b1b0c7e239a6b69271 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Wed, 19 Jan 2022 13:14:03 +0300 Subject: [PATCH 096/403] Update 02168_avro_bug.sql --- tests/queries/0_stateless/02168_avro_bug.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02168_avro_bug.sql b/tests/queries/0_stateless/02168_avro_bug.sql index e50c78fb0b8..ac98119845f 100644 --- a/tests/queries/0_stateless/02168_avro_bug.sql +++ b/tests/queries/0_stateless/02168_avro_bug.sql @@ -1,4 +1,4 @@ --- Tags: no-fasttest +-- Tags: no-fasttest, no-parallel insert into table function file('data.avro', 'Parquet', 'x UInt64') select * from numbers(10); insert into table function file('data.avro', 'Parquet', 'x UInt64') select * from numbers(10); -- { serverError CANNOT_APPEND_TO_FILE } insert into table function file('data.avro', 'Parquet', 'x UInt64') select * from numbers(10); -- { serverError CANNOT_APPEND_TO_FILE } From d3accef87d7db639f9628f12541f4a3abb21e295 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 12 Jan 2022 18:11:51 +0100 Subject: [PATCH 097/403] Rename release action to docs to reflect the content --- .github/workflows/{release.yml => docs_release.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{release.yml => docs_release.yml} (100%) diff --git a/.github/workflows/release.yml b/.github/workflows/docs_release.yml similarity index 100% rename from .github/workflows/release.yml rename to .github/workflows/docs_release.yml From 7fd629004b38238d44e765b341eedf87beb85851 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 12 Jan 2022 18:11:58 +0100 Subject: [PATCH 098/403] Improve push_to_artifactory - Add tgz archives - Add --all argument for all packages download - Add -n to skip pushing to artifactory - Add help for artifactory-url - Make 3.8 compatible - Add cleanup `refs/tags/` from --release argument --- tests/ci/push_to_artifactory.py | 66 ++++++++++++++++++++++++++++----- 1 file changed, 56 insertions(+), 10 deletions(-) diff --git a/tests/ci/push_to_artifactory.py b/tests/ci/push_to_artifactory.py index f7ee495b7df..add76780c4e 100755 --- a/tests/ci/push_to_artifactory.py +++ b/tests/ci/push_to_artifactory.py @@ -9,6 +9,19 @@ from artifactory import ArtifactorySaaSPath from build_download_helper import dowload_build_with_progress +# Py 3.8 removeprefix and removesuffix +def removeprefix(string: str, prefix: str): + if string.startswith(prefix): + return string[len(prefix) :] + return string + + +def removesuffix(string: str, suffix: str): + if string.endswith(suffix): + return string[: -len(suffix)] + return string + + # Necessary ENV variables def getenv(name, default=None): env = os.getenv(name, default) @@ -44,10 +57,12 @@ class Packages: for name, arch in self.packages ) + self.tgz = tuple("{}-{}.tgz".format(name, version) for name, _ in self.packages) + def arch(self, deb_pkg: str) -> str: if deb_pkg not in self.deb: raise ValueError("{} not in {}".format(deb_pkg, self.deb)) - return deb_pkg.removesuffix(".deb").split("_")[-1] + return removesuffix(deb_pkg, ".deb").split("_")[-1] @staticmethod def path(package): @@ -97,17 +112,23 @@ class S3: for package in self.packages.rpm: self.download_package(package) + def download_tgz(self): + for package in self.packages.tgz: + self.download_package(package) + class Release: def __init__(self, name: str) -> str: r = re.compile(r"^v\d{2}[.]\d+[.]\d+[.]\d+-(testing|prestable|stable|lts)$") + # Automatically remove refs/tags/ if full refname passed here + name = removeprefix(name, "refs/tags/") if not r.match(name): raise argparse.ArgumentTypeError( - "release name does not match " + f"release name {name} does not match " "v12.1.2.15-(testing|prestable|stable|lts) pattern" ) self._name = name - self._version = self._name.removeprefix("v") + self._version = removeprefix(self._name, "v") self._version = self.version.split("-")[0] self._version_parts = tuple(self.version.split(".")) self._type = self._name.split("-")[-1] @@ -193,7 +214,8 @@ def parse_args() -> argparse.Namespace: "--release", required=True, type=Release, - help="release name, e.g. v12.13.14.15-prestable", + help="release name, e.g. v12.13.14.15-prestable; 'refs/tags/' " + "prefix is striped automatically", ) parser.add_argument( "--pull-request", @@ -216,6 +238,9 @@ def parse_args() -> argparse.Namespace: help="check name, a part of bucket path, " "will be converted to lower case with spaces->underscore", ) + parser.add_argument( + "--all", action="store_true", help="implies all deb, rpm and tgz" + ) parser.add_argument( "--deb", action="store_true", help="if Debian packages should be processed" ) @@ -223,12 +248,27 @@ def parse_args() -> argparse.Namespace: "--rpm", action="store_true", help="if RPM packages should be processed" ) parser.add_argument( - "--artifactory-url", default="https://clickhousedb.jfrog.io/artifactory" + "--tgz", + action="store_true", + help="if tgz archives should be processed. They aren't pushed to artifactory", + ) + parser.add_argument( + "--artifactory-url", + default="https://clickhousedb.jfrog.io/artifactory", + help="SaaS Artifactory url", + ) + parser.add_argument( + "-n", + "--no-artifactory", + action="store_true", + help="do not push packages to artifactory", ) args = parser.parse_args() - if not args.deb and not args.rpm: - parser.error("at least one of --deb and --rpm should be specified") + if args.all: + args.deb = args.rpm = args.tgz = True + if not (args.deb or args.rpm or args.tgz): + parser.error("at least one of --deb, --rpm or --tgz should be specified") args.check_name = args.check_name.lower().replace(" ", "_") if args.pull_request == 0: args.pull_request = ".".join(args.release.version_parts[:2]) @@ -245,13 +285,19 @@ def main(): args.check_name, args.release.version, ) - art_client = Artifactory(args.artifactory_url, args.release.type) + if not args.no_artifactory: + art_client = Artifactory(args.artifactory_url, args.release.type) + if args.deb: s3.download_deb() - art_client.deploy_deb(s3.packages) + if not args.no_artifactory: + art_client.deploy_deb(s3.packages) if args.rpm: s3.download_rpm() - art_client.deploy_rpm(s3.packages) + if not args.no_artifactory: + art_client.deploy_rpm(s3.packages) + if args.tgz: + s3.download_tgz() if __name__ == "__main__": From e01662927823123a81c2378222353d2da20e5c6b Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 12 Jan 2022 18:12:01 +0100 Subject: [PATCH 099/403] Create release workflow --- .github/workflows/release.yml | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000000..77bc285196c --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,39 @@ +name: ReleaseWorkflow +# - Gets artifacts from S3 +# - Sends it to JFROG Artifactory +# - Adds them to the release assets + +on: # yamllint disable-line rule:truthy + release: + types: + - published + +jobs: + ReleasePublish: + runs-on: [self-hosted, style-checker] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + JFROG_API_KEY=${{ secrets.JFROG_KEY_API_PACKAGES }} + TEMP_PATH=${{runner.temp}}/release_packages + REPO_COPY=${{runner.temp}}/release_packages/ClickHouse + EOF + - name: Check out repository code + uses: actions/checkout@v2 + - name: Download packages and push to Artifactory + env: + run: | + rm -rf "$TEMP_PATH" && mkdir -p "$REPO_COPY" + cp -r "$GITHUB_WORKSPACE" "$REPO_COPY" + cd "$REPO_COPY" + python3 ./tests/ci/push_to_artifactory.py --release "${{ github.ref }}" \ + --commit '${{ github.sha }}' --all + - name: Upload packages to release assets + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: ${{runner.temp}}/release_packages/* + overwrite: true + tag: ${{ github.ref }} + file_glob: true From 13368bf7087116c03f66f40cadf47f1ea7d4b1f5 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 17 Jan 2022 10:41:24 +0100 Subject: [PATCH 100/403] Fix some linter issues --- tests/ci/push_to_artifactory.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/ci/push_to_artifactory.py b/tests/ci/push_to_artifactory.py index add76780c4e..fd3f81b5d58 100755 --- a/tests/ci/push_to_artifactory.py +++ b/tests/ci/push_to_artifactory.py @@ -4,15 +4,16 @@ import argparse import logging import os import re +from typing import Tuple -from artifactory import ArtifactorySaaSPath +from artifactory import ArtifactorySaaSPath # type: ignore from build_download_helper import dowload_build_with_progress # Py 3.8 removeprefix and removesuffix def removeprefix(string: str, prefix: str): if string.startswith(prefix): - return string[len(prefix) :] + return string[len(prefix) :] # noqa: ignore E203, false positive return string @@ -118,7 +119,7 @@ class S3: class Release: - def __init__(self, name: str) -> str: + def __init__(self, name: str): r = re.compile(r"^v\d{2}[.]\d+[.]\d+[.]\d+-(testing|prestable|stable|lts)$") # Automatically remove refs/tags/ if full refname passed here name = removeprefix(name, "refs/tags/") @@ -138,7 +139,7 @@ class Release: return self._version @property - def version_parts(self) -> str: + def version_parts(self) -> Tuple[str, ...]: return self._version_parts @property @@ -162,7 +163,8 @@ class Artifactory: comp = "main" arch = packages.arch(package) logging.info( - "Deploy %s(distribution=%s;component=%s;architecture=%s) to artifactory", + "Deploy %s(distribution=%s;component=%s;architecture=%s) " + "to artifactory", path, dist, comp, From 3142ae998b8257f2ddd65890ff200f909dc5a898 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 17 Jan 2022 19:16:06 +0100 Subject: [PATCH 101/403] Add option to skip redownload, push tgz --- tests/ci/push_to_artifactory.py | 61 ++++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 12 deletions(-) diff --git a/tests/ci/push_to_artifactory.py b/tests/ci/push_to_artifactory.py index fd3f81b5d58..2e41a6700c6 100755 --- a/tests/ci/push_to_artifactory.py +++ b/tests/ci/push_to_artifactory.py @@ -92,6 +92,7 @@ class S3: commit: str, check_name: str, version: str, + force_download: bool, ): self._common = dict( bucket_name=bucket_name, @@ -99,9 +100,12 @@ class S3: commit=commit, check_name=check_name, ) + self.force_download = force_download self.packages = Packages(version) def download_package(self, package): + if not self.force_download and os.path.exists(Packages.path(package)): + return url = self.template.format_map({**self._common, "package": package}) dowload_build_with_progress(url, Packages.path(package)) @@ -148,11 +152,14 @@ class Release: class Artifactory: - def __init__(self, url: str, release: str, deb_repo="deb", rpm_repo="rpm"): + def __init__( + self, url: str, release: str, deb_repo="deb", rpm_repo="rpm", tgz_repo="tgz" + ): self._url = url self._release = release self._deb_url = "/".join((self._url, deb_repo, "pool", self._release)) + "/" self._rpm_url = "/".join((self._url, rpm_repo, self._release)) + "/" + self._tgz_url = "/".join((self._url, tgz_repo, self._release)) + "/" # check the credentials ENVs for early exit self.__path_helper("_deb", "") @@ -170,13 +177,19 @@ class Artifactory: comp, arch, ) - self.deb(package).deploy_deb(path, dist, comp, arch) + self.deb_path(package).deploy_deb(path, dist, comp, arch) def deploy_rpm(self, packages: Packages): for package in packages.rpm: path = packages.path(package) logging.info("Deploy %s to artifactory", path) - self.rpm(package).deploy_file(path) + self.rpm_path(package).deploy_file(path) + + def deploy_tgz(self, packages: Packages): + for package in packages.tgz: + path = packages.path(package) + logging.info("Deploy %s to artifactory", path) + self.tgz_path(package).deploy_file(path) def __path_helper(self, name, package) -> ArtifactorySaaSPath: url = "/".join((getattr(self, name + "_url"), package)) @@ -189,12 +202,15 @@ class Artifactory: raise KeyError("Neither JFROG_API_KEY nor JFROG_TOKEN env are defined") return path - def deb(self, package) -> ArtifactorySaaSPath: + def deb_path(self, package) -> ArtifactorySaaSPath: return self.__path_helper("_deb", package) - def rpm(self, package) -> ArtifactorySaaSPath: + def rpm_path(self, package) -> ArtifactorySaaSPath: return self.__path_helper("_rpm", package) + def tgz_path(self, package) -> ArtifactorySaaSPath: + return self.__path_helper("_tgz", package) + def commit(name): r = re.compile(r"^([0-9]|[a-f]){40}$") @@ -265,6 +281,11 @@ def parse_args() -> argparse.Namespace: action="store_true", help="do not push packages to artifactory", ) + parser.add_argument( + "--no-force-download", + action="store_true", + help="do not download packages again if they exist already", + ) args = parser.parse_args() if args.all: @@ -277,6 +298,24 @@ def parse_args() -> argparse.Namespace: return args +def process_deb(s3: S3, art_client: Artifactory): + s3.download_deb() + if art_client is not None: + art_client.deploy_deb(s3.packages) + + +def process_rpm(s3: S3, art_client: Artifactory): + s3.download_rpm() + if art_client is not None: + art_client.deploy_rpm(s3.packages) + + +def process_tgz(s3: S3, art_client: Artifactory): + s3.download_tgz() + if art_client is not None: + art_client.deploy_tgz(s3.packages) + + def main(): logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") args = parse_args() @@ -286,20 +325,18 @@ def main(): args.commit, args.check_name, args.release.version, + not args.no_force_download, ) + art_client = None if not args.no_artifactory: art_client = Artifactory(args.artifactory_url, args.release.type) if args.deb: - s3.download_deb() - if not args.no_artifactory: - art_client.deploy_deb(s3.packages) + process_deb(s3, art_client) if args.rpm: - s3.download_rpm() - if not args.no_artifactory: - art_client.deploy_rpm(s3.packages) + process_rpm(s3, art_client) if args.tgz: - s3.download_tgz() + process_tgz(s3, art_client) if __name__ == "__main__": From d9a1284f2de8d3b3045b2d4eb6ecef5055f16759 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 17 Jan 2022 19:47:32 +0100 Subject: [PATCH 102/403] Improve typing and arguments --- tests/ci/push_to_artifactory.py | 64 ++++++++++++++++----------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/tests/ci/push_to_artifactory.py b/tests/ci/push_to_artifactory.py index 2e41a6700c6..d2241d10b6c 100755 --- a/tests/ci/push_to_artifactory.py +++ b/tests/ci/push_to_artifactory.py @@ -24,7 +24,7 @@ def removesuffix(string: str, suffix: str): # Necessary ENV variables -def getenv(name, default=None): +def getenv(name: str, default: str = None): env = os.getenv(name, default) if env is not None: return env @@ -66,8 +66,8 @@ class Packages: return removesuffix(deb_pkg, ".deb").split("_")[-1] @staticmethod - def path(package): - return os.path.join(TEMP_PATH, package) + def path(package_file: str) -> str: + return os.path.join(TEMP_PATH, package_file) class S3: @@ -103,23 +103,23 @@ class S3: self.force_download = force_download self.packages = Packages(version) - def download_package(self, package): - if not self.force_download and os.path.exists(Packages.path(package)): + def download_package(self, package_file: str): + if not self.force_download and os.path.exists(Packages.path(package_file)): return - url = self.template.format_map({**self._common, "package": package}) - dowload_build_with_progress(url, Packages.path(package)) + url = self.template.format_map({**self._common, "package": package_file}) + dowload_build_with_progress(url, Packages.path(package_file)) def download_deb(self): - for package in self.packages.deb: - self.download_package(package) + for package_file in self.packages.deb: + self.download_package(package_file) def download_rpm(self): - for package in self.packages.rpm: - self.download_package(package) + for package_file in self.packages.rpm: + self.download_package(package_file) def download_tgz(self): - for package in self.packages.tgz: - self.download_package(package) + for package_file in self.packages.tgz: + self.download_package(package_file) class Release: @@ -164,11 +164,11 @@ class Artifactory: self.__path_helper("_deb", "") def deploy_deb(self, packages: Packages): - for package in packages.deb: - path = packages.path(package) + for package_file in packages.deb: + path = packages.path(package_file) dist = self._release comp = "main" - arch = packages.arch(package) + arch = packages.arch(package_file) logging.info( "Deploy %s(distribution=%s;component=%s;architecture=%s) " "to artifactory", @@ -177,22 +177,22 @@ class Artifactory: comp, arch, ) - self.deb_path(package).deploy_deb(path, dist, comp, arch) + self.deb_path(package_file).deploy_deb(path, dist, comp, arch) def deploy_rpm(self, packages: Packages): - for package in packages.rpm: - path = packages.path(package) + for package_file in packages.rpm: + path = packages.path(package_file) logging.info("Deploy %s to artifactory", path) - self.rpm_path(package).deploy_file(path) + self.rpm_path(package_file).deploy_file(path) def deploy_tgz(self, packages: Packages): - for package in packages.tgz: - path = packages.path(package) + for package_file in packages.tgz: + path = packages.path(package_file) logging.info("Deploy %s to artifactory", path) - self.tgz_path(package).deploy_file(path) + self.tgz_path(package_file).deploy_file(path) - def __path_helper(self, name, package) -> ArtifactorySaaSPath: - url = "/".join((getattr(self, name + "_url"), package)) + def __path_helper(self, name: str, package_file: str) -> ArtifactorySaaSPath: + url = "/".join((getattr(self, name + "_url"), package_file)) path = None if JFROG_API_KEY: path = ArtifactorySaaSPath(url, apikey=JFROG_API_KEY) @@ -202,17 +202,17 @@ class Artifactory: raise KeyError("Neither JFROG_API_KEY nor JFROG_TOKEN env are defined") return path - def deb_path(self, package) -> ArtifactorySaaSPath: - return self.__path_helper("_deb", package) + def deb_path(self, package_file: str) -> ArtifactorySaaSPath: + return self.__path_helper("_deb", package_file) - def rpm_path(self, package) -> ArtifactorySaaSPath: - return self.__path_helper("_rpm", package) + def rpm_path(self, package_file: str) -> ArtifactorySaaSPath: + return self.__path_helper("_rpm", package_file) - def tgz_path(self, package) -> ArtifactorySaaSPath: - return self.__path_helper("_tgz", package) + def tgz_path(self, package_file: str) -> ArtifactorySaaSPath: + return self.__path_helper("_tgz", package_file) -def commit(name): +def commit(name: str): r = re.compile(r"^([0-9]|[a-f]){40}$") if not r.match(name): raise argparse.ArgumentTypeError( From 1f72bde52fd51a14fb2602ffc762b0eed6cc551b Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 19 Jan 2022 10:41:58 +0000 Subject: [PATCH 103/403] Fix performance tests --- src/Common/IntervalTree.h | 26 +++++++++---------- tests/performance/range_hashed_dictionary.xml | 20 +++++++------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/Common/IntervalTree.h b/src/Common/IntervalTree.h index 9d1acca91bd..fd2fec528a4 100644 --- a/src/Common/IntervalTree.h +++ b/src/Common/IntervalTree.h @@ -85,7 +85,7 @@ struct IntervalTreeVoidValue * 1. Intervals that lie that are less than median point. Interval right is less than median point. * 2. Intervals that lie that are greater than median point. Interval right is less than median point. * 3. Intervals that intersect node sorted by left ascending. - * 4. Intervals that intersect node sorted by right ascending. + * 4. Intervals that intersect node sorted by right descending. * * If intervals in 1 group are not empty. Continue build left child recursively with intervals from 1 group. * If intervals in 2 group are not empty. Continue build right child recursively with intervals from 2 group. @@ -107,7 +107,7 @@ struct IntervalTreeVoidValue * Additional details: * 1. To improve cache locality tree is stored implicitly in array, after build method is called * other intervals cannot be added to the tree. - * 2. Additionaly to improve cache locality in tree node we store sorted intervals for all nodes in separate + * 2. Additionally to improve cache locality in tree node we store sorted intervals for all nodes in separate * array. In node we store only start of its sorted intervals, and also size of intersecting intervals. * If we need to retrieve intervals sorted by left ascending they will be stored in indexes * [sorted_intervals_start_index, sorted_intervals_start_index + intersecting_intervals_size). @@ -127,7 +127,7 @@ public: template , bool> = true> void emplace(Interval interval) { - assert(!tree_is_build); + assert(!tree_is_built); sorted_intervals.emplace_back(interval); increaseIntervalsSize(); } @@ -135,7 +135,7 @@ public: template , bool> = true, typename... Args> void emplace(Interval interval, Args &&... args) { - assert(!tree_is_build); + assert(!tree_is_built); sorted_intervals.emplace_back( std::piecewise_construct, std::forward_as_tuple(interval), std::forward_as_tuple(std::forward(args)...)); increaseIntervalsSize(); @@ -144,7 +144,7 @@ public: template , bool> = true> void insert(Interval interval) { - assert(!tree_is_build); + assert(!tree_is_built); sorted_intervals.emplace_back(interval); increaseIntervalsSize(); } @@ -152,27 +152,27 @@ public: template , bool> = true> void insert(Interval interval, const Value & value) { - assert(!tree_is_build); - sorted_intervals.emplace_back(std::piecewise_construct, interval, value); + assert(!tree_is_built); + sorted_intervals.emplace_back(interval, value); increaseIntervalsSize(); } template , bool> = true> void insert(Interval interval, Value && value) { - assert(!tree_is_build); - sorted_intervals.emplace_back(std::piecewise_construct, interval, std::move(value)); + assert(!tree_is_built); + sorted_intervals.emplace_back(interval, std::move(value)); increaseIntervalsSize(); } /// Build tree, after that intervals cannot be inserted, and only search or iteration can be performed. void build() { - assert(!tree_is_build); + assert(!tree_is_built); nodes.clear(); nodes.reserve(sorted_intervals.size()); buildTree(); - tree_is_build = true; + tree_is_built = true; } /** Find all intervals intersecting point. @@ -205,7 +205,7 @@ public: template void find(IntervalStorageType point, IntervalCallback && callback) const { - if (unlikely(!tree_is_build)) + if (unlikely(!tree_is_built)) { findIntervalsNonConstructedImpl(point, callback); return; @@ -629,7 +629,7 @@ private: std::vector nodes; std::vector sorted_intervals; size_t intervals_size = 0; - bool tree_is_build = false; + bool tree_is_built = false; static inline const Interval & getInterval(const IntervalWithValue & interval_with_value) { diff --git a/tests/performance/range_hashed_dictionary.xml b/tests/performance/range_hashed_dictionary.xml index a070acfb4dd..bdf949cd1ff 100644 --- a/tests/performance/range_hashed_dictionary.xml +++ b/tests/performance/range_hashed_dictionary.xml @@ -55,28 +55,28 @@ INSERT INTO simple_key_range_hashed_dictionary_source_table SELECT key, key, range_start * 2, range_start * 2 + 1 FROM (SELECT number as key FROM numbers(10000)) as keys, - (SELECT number as range_start FROM numbers(10000)) as ranges; + (SELECT number as range_start FROM numbers(1000)) as ranges; INSERT INTO complex_key_range_hashed_dictionary_source_table SELECT key, toString(key), key, range_start * 2, range_start * 2 + 1 FROM (SELECT number as key FROM numbers(10000)) as keys, - (SELECT number as range_start FROM numbers(10000)) as ranges; + (SELECT number as range_start FROM numbers(1000)) as ranges; elements_count - 50000 - 75000 + 500000 + 750000 - WITH rand64() % 10000 as key + WITH rand64() % 5000 as key SELECT dictGet('default.simple_key_range_hashed_dictionary', 'value', toUInt64(key), key) FROM system.numbers LIMIT {elements_count} @@ -84,7 +84,7 @@ - WITH rand64() % 10000 as key + WITH rand64() % 5000 as key SELECT dictHas('default.simple_key_range_hashed_dictionary', toUInt64(key), key) FROM system.numbers LIMIT {elements_count} @@ -97,16 +97,16 @@ - WITH (rand64() % toUInt64(10000), toString(rand64() % toUInt64(10000))) as key - SELECT dictGet('default.complex_key_range_hashed_dictionary', 'value', key, rand64() % toUInt64(10000)) + WITH (rand64() % toUInt64(5000) as key, toString(key) as key_id) as complex_key + SELECT dictGet('default.complex_key_range_hashed_dictionary', 'value', complex_key, key) FROM system.numbers LIMIT {elements_count} FORMAT Null; - WITH (rand64() % toUInt64(10000), toString(rand64() % toUInt64(10000))) as key - SELECT dictHas('default.complex_key_range_hashed_dictionary', key, rand64() % toUInt64(10000)) + WITH (rand64() % toUInt64(5000) as key, toString(key) as key_id) as complex_key + SELECT dictHas('default.complex_key_range_hashed_dictionary', complex_key, key) FROM system.numbers LIMIT {elements_count} FORMAT Null; From c4cd8c6da49570dd2734e9e8b979361a1657bc23 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 19 Jan 2022 14:22:32 +0300 Subject: [PATCH 104/403] Fix parsing query INSERT INTO ... VALUES SETTINGS ... (...), ... --- src/Parsers/ParserInsertQuery.cpp | 4 ++++ .../0_stateless/02180_insert_into_values_settings.reference | 0 .../queries/0_stateless/02180_insert_into_values_settings.sql | 4 ++++ 3 files changed, 8 insertions(+) create mode 100644 tests/queries/0_stateless/02180_insert_into_values_settings.reference create mode 100644 tests/queries/0_stateless/02180_insert_into_values_settings.sql diff --git a/src/Parsers/ParserInsertQuery.cpp b/src/Parsers/ParserInsertQuery.cpp index 568f486a5cf..44db07278c2 100644 --- a/src/Parsers/ParserInsertQuery.cpp +++ b/src/Parsers/ParserInsertQuery.cpp @@ -184,6 +184,10 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserSetQuery parser_settings(true); if (!parser_settings.parse(pos, settings_ast, expected)) return false; + /// In case of INSERT INTO ... VALUES SETTINGS ... (...), (...), ... + /// we should move data pointer after all settings. + if (data != nullptr) + data = pos->begin; } if (select) diff --git a/tests/queries/0_stateless/02180_insert_into_values_settings.reference b/tests/queries/0_stateless/02180_insert_into_values_settings.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02180_insert_into_values_settings.sql b/tests/queries/0_stateless/02180_insert_into_values_settings.sql new file mode 100644 index 00000000000..0a1468070c1 --- /dev/null +++ b/tests/queries/0_stateless/02180_insert_into_values_settings.sql @@ -0,0 +1,4 @@ +drop table if exists t; +create table t (x Bool) engine=Memory(); +insert into t values settings bool_true_representation='да' ('да'); +drop table t; From 9ea6b8c2d3c15143e5773d2cdf39e2785675b96f Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 19 Jan 2022 14:46:29 +0300 Subject: [PATCH 105/403] Some interm state --- src/Coordination/FourLetterCommand.cpp | 2 ++ src/Coordination/KeeperStateMachine.cpp | 14 +++++++++ src/Coordination/KeeperStateMachine.h | 4 ++- src/Coordination/KeeperStorage.cpp | 40 ++++++++++++++---------- src/Coordination/KeeperStorage.h | 9 +++++- src/Coordination/SnapshotableHashTable.h | 28 +++++++---------- src/Coordination/pathUtils.cpp | 8 ++--- src/Coordination/pathUtils.h | 4 +-- utils/keeper-bench/Generator.cpp | 2 +- 9 files changed, 69 insertions(+), 42 deletions(-) diff --git a/src/Coordination/FourLetterCommand.cpp b/src/Coordination/FourLetterCommand.cpp index 3d0ebe86bf3..4c76d052f9b 100644 --- a/src/Coordination/FourLetterCommand.cpp +++ b/src/Coordination/FourLetterCommand.cpp @@ -228,6 +228,8 @@ String MonitorCommand::run() print(ret, "watch_count", state_machine.getTotalWatchesCount()); print(ret, "ephemerals_count", state_machine.getTotalEphemeralNodesCount()); print(ret, "approximate_data_size", state_machine.getApproximateDataSize()); + print(ret, "key_arena_size", state_machine.getKeyArenaSize()); + print(ret, "latest_snapshot_size", state_machine.getLatestSnapshotBufSize()); #if defined(__linux__) || defined(__APPLE__) print(ret, "open_file_descriptor_count", getCurrentProcessFDCount()); diff --git a/src/Coordination/KeeperStateMachine.cpp b/src/Coordination/KeeperStateMachine.cpp index 98d522eaa9b..1c60b4d29ca 100644 --- a/src/Coordination/KeeperStateMachine.cpp +++ b/src/Coordination/KeeperStateMachine.cpp @@ -404,6 +404,20 @@ uint64_t KeeperStateMachine::getApproximateDataSize() const return storage->getApproximateDataSize(); } +uint64_t KeeperStateMachine::getKeyArenaSize() const +{ + std::lock_guard lock(storage_and_responses_lock); + return storage->getArenaDataSize(); +} + +uint64_t KeeperStateMachine::getLatestSnapshotBufSize() const +{ + std::lock_guard lock(snapshots_lock); + if (latest_snapshot_buf) + return latest_snapshot_buf->size(); + return 0; +} + ClusterConfigPtr KeeperStateMachine::getClusterConfig() const { std::lock_guard lock(cluster_config_lock); diff --git a/src/Coordination/KeeperStateMachine.h b/src/Coordination/KeeperStateMachine.h index 2803f4b9027..291b58e2498 100644 --- a/src/Coordination/KeeperStateMachine.h +++ b/src/Coordination/KeeperStateMachine.h @@ -97,6 +97,8 @@ public: uint64_t getSessionWithEphemeralNodesCount() const; uint64_t getTotalEphemeralNodesCount() const; uint64_t getApproximateDataSize() const; + uint64_t getKeyArenaSize() const; + uint64_t getLatestSnapshotBufSize() const; private: @@ -120,7 +122,7 @@ private: SnapshotsQueue & snapshots_queue; /// Mutex for snapshots - std::mutex snapshots_lock; + mutable std::mutex snapshots_lock; /// Lock for storage and responses_queue. It's important to process requests /// and push them to the responses queue while holding this lock. Otherwise diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index 08a91ed715a..bf891a91f7a 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -143,12 +143,12 @@ static KeeperStorage::ResponsesForSessions processWatchesImpl(const String & pat Strings paths_to_check_for_list_watches; if (event_type == Coordination::Event::CREATED) { - paths_to_check_for_list_watches.push_back(parent_path); /// Trigger list watches for parent + paths_to_check_for_list_watches.push_back(parent_path.toString()); /// Trigger list watches for parent } else if (event_type == Coordination::Event::DELETED) { paths_to_check_for_list_watches.push_back(path); /// Trigger both list watches for this path - paths_to_check_for_list_watches.push_back(parent_path); /// And for parent path + paths_to_check_for_list_watches.push_back(parent_path.toString()); /// And for parent path } /// CHANGED event never trigger list wathes @@ -285,8 +285,7 @@ struct KeeperStorageCreateRequestProcessor final : public KeeperStorageRequestPr response.error = Coordination::Error::ZNODEEXISTS; return { response_ptr, undo }; } - auto child_path = getBaseName(path_created); - if (child_path.empty()) + if (getBaseName(path_created).size == 0) { response.error = Coordination::Error::ZBADARGUMENTS; return { response_ptr, undo }; @@ -318,15 +317,17 @@ struct KeeperStorageCreateRequestProcessor final : public KeeperStorageRequestPr created_node.data = request.data; created_node.is_sequental = request.is_sequential; + auto [map_key, _] = container.insert(path_created, std::move(created_node)); + auto child_path = getBaseName(map_key); + int32_t parent_cversion = request.parent_cversion; int64_t prev_parent_zxid; int32_t prev_parent_cversion; container.updateValue(parent_path, [child_path, zxid, &prev_parent_zxid, parent_cversion, &prev_parent_cversion] (KeeperStorage::Node & parent) { - parent.children.insert(child_path); - parent.size_bytes += child_path.size(); + parent.size_bytes += child_path.size; prev_parent_cversion = parent.stat.cversion; prev_parent_zxid = parent.stat.pzxid; @@ -344,14 +345,12 @@ struct KeeperStorageCreateRequestProcessor final : public KeeperStorageRequestPr }); response.path_created = path_created; - container.insert(path_created, std::move(created_node)); if (request.is_ephemeral) ephemerals[session_id].emplace(path_created); undo = [&storage, prev_parent_zxid, prev_parent_cversion, session_id, path_created, is_ephemeral = request.is_ephemeral, parent_path, child_path, acl_id] { - storage.container.erase(path_created); storage.acl_map.removeUsage(acl_id); if (is_ephemeral) @@ -364,8 +363,10 @@ struct KeeperStorageCreateRequestProcessor final : public KeeperStorageRequestPr undo_parent.stat.cversion = prev_parent_cversion; undo_parent.stat.pzxid = prev_parent_zxid; undo_parent.children.erase(child_path); - undo_parent.size_bytes -= child_path.size(); + undo_parent.size_bytes -= child_path.size; }); + + storage.container.erase(path_created); }; response.error = Coordination::Error::ZOK; @@ -498,27 +499,27 @@ struct KeeperStorageRemoveRequestProcessor final : public KeeperStorageRequestPr --parent.stat.numChildren; ++parent.stat.cversion; parent.children.erase(child_basename); - parent.size_bytes -= child_basename.size(); + parent.size_bytes -= child_basename.size; }); response.error = Coordination::Error::ZOK; container.erase(request.path); - undo = [prev_node, &storage, path = request.path, child_basename] + undo = [prev_node, &storage, path = request.path] { if (prev_node.stat.ephemeralOwner != 0) storage.ephemerals[prev_node.stat.ephemeralOwner].emplace(path); storage.acl_map.addUsage(prev_node.acl_id); - storage.container.insert(path, prev_node); - storage.container.updateValue(parentPath(path), [&child_basename] (KeeperStorage::Node & parent) + auto [map_key, _] = storage.container.insert(path, prev_node); + storage.container.updateValue(parentPath(path), [child_name = getBaseName(map_key)] (KeeperStorage::Node & parent) { ++parent.stat.numChildren; --parent.stat.cversion; - parent.children.insert(child_basename); - parent.size_bytes += child_basename.size(); + parent.children.insert(child_name); + parent.size_bytes += child_name.size; }); }; } @@ -671,7 +672,12 @@ struct KeeperStorageListRequestProcessor final : public KeeperStorageRequestProc if (path_prefix.empty()) throw DB::Exception("Logical error: path cannot be empty", ErrorCodes::LOGICAL_ERROR); - response.names.insert(response.names.end(), it->value.children.begin(), it->value.children.end()); + response.names.reserve(it->value.children.size()); + + for (const auto child : it->value.children) + { + response.names.push_back(child.toString()); + } response.stat = it->value.stat; response.error = Coordination::Error::ZOK; @@ -1087,7 +1093,7 @@ KeeperStorage::ResponsesForSessions KeeperStorage::processRequest(const Coordina ++parent.stat.cversion; auto base_name = getBaseName(ephemeral_path); parent.children.erase(base_name); - parent.size_bytes -= base_name.size(); + parent.size_bytes -= base_name.size; }); auto responses = processWatchesImpl(ephemeral_path, watches, list_watches, Coordination::Event::DELETED); diff --git a/src/Coordination/KeeperStorage.h b/src/Coordination/KeeperStorage.h index 11d191b7f50..55d4e4ea7e7 100644 --- a/src/Coordination/KeeperStorage.h +++ b/src/Coordination/KeeperStorage.h @@ -17,7 +17,7 @@ namespace DB struct KeeperStorageRequestProcessor; using KeeperStorageRequestProcessorPtr = std::shared_ptr; using ResponseCallback = std::function; -using ChildrenSet = std::unordered_set; +using ChildrenSet = std::unordered_set; using SessionAndTimeout = std::unordered_map; struct KeeperStorageSnapshot; @@ -28,6 +28,7 @@ struct KeeperStorageSnapshot; class KeeperStorage { public: + struct Node { String data; @@ -203,6 +204,12 @@ public: return container.getApproximateDataSize(); } + uint64_t getArenaDataSize() const + { + return container.keyArenaSize(); + } + + uint64_t getTotalWatchesCount() const; uint64_t getWatchedPathsCount() const diff --git a/src/Coordination/SnapshotableHashTable.h b/src/Coordination/SnapshotableHashTable.h index e4c7ed7a872..c3c8f672d70 100644 --- a/src/Coordination/SnapshotableHashTable.h +++ b/src/Coordination/SnapshotableHashTable.h @@ -10,7 +10,6 @@ namespace DB { - template struct ListNode { @@ -21,7 +20,6 @@ struct ListNode bool free_key{false}; }; - template class SnapshotableHashTable { @@ -128,11 +126,9 @@ public: using iterator = typename List::iterator; using const_iterator = typename List::const_iterator; - using reverse_iterator = typename List::reverse_iterator; - using const_reverse_iterator = typename List::const_reverse_iterator; using ValueUpdater = std::function; - bool insert(const std::string & key, const V & value) + std::pair insert(const std::string & key, const V & value) { size_t hash_value = map.hash(key); auto it = map.find(key, hash_value); @@ -147,10 +143,10 @@ public: it->getMapped() = itr; updateDataSize(INSERT, key.size(), value.sizeInBytes(), 0); - return true; + return std::make_pair(it->getKey(), true); } - return false; + return std::make_pair(it->getKey(), false); } void insertOrReplace(const std::string & key, const V & value) @@ -216,7 +212,7 @@ public: return map.find(key) != map.end(); } - const_iterator updateValue(const std::string & key, ValueUpdater updater) + const_iterator updateValue(StringRef key, ValueUpdater updater) { size_t hash_value = map.hash(key); auto it = map.find(key, hash_value); @@ -243,11 +239,11 @@ public: ret = list_itr; } - updateDataSize(UPDATE_VALUE, key.size(), ret->value.sizeInBytes(), old_value_size); + updateDataSize(UPDATE_VALUE, key.size, ret->value.sizeInBytes(), old_value_size); return ret; } - const_iterator find(const std::string & key) const + const_iterator find(StringRef key) const { auto map_it = map.find(key); if (map_it != map.end()) @@ -256,7 +252,7 @@ public: } - const V & getValue(const std::string & key) const + const V & getValue(StringRef key) const { auto it = map.find(key); assert(it); @@ -318,15 +314,15 @@ public: return approximate_data_size; } + uint64_t keyArenaSize() const + { + return arena.size(); + } + iterator begin() { return list.begin(); } const_iterator begin() const { return list.cbegin(); } iterator end() { return list.end(); } const_iterator end() const { return list.cend(); } - - reverse_iterator rbegin() { return list.rbegin(); } - const_reverse_iterator rbegin() const { return list.crbegin(); } - reverse_iterator rend() { return list.rend(); } - const_reverse_iterator rend() const { return list.crend(); } }; diff --git a/src/Coordination/pathUtils.cpp b/src/Coordination/pathUtils.cpp index dbef6d6d2af..1e1da339d2e 100644 --- a/src/Coordination/pathUtils.cpp +++ b/src/Coordination/pathUtils.cpp @@ -21,18 +21,18 @@ static size_t findLastSlash(StringRef path) return std::string::npos; } -std::string parentPath(StringRef path) +StringRef parentPath(StringRef path) { auto rslash_pos = findLastSlash(path); if (rslash_pos > 0) - return std::string{path.data, rslash_pos}; + return StringRef{path.data, rslash_pos}; return "/"; } -std::string getBaseName(StringRef path) +StringRef getBaseName(StringRef path) { size_t basename_start = findLastSlash(path); - return std::string{path.data + basename_start + 1, path.size - basename_start - 1}; + return StringRef{path.data + basename_start + 1, path.size - basename_start - 1}; } } diff --git a/src/Coordination/pathUtils.h b/src/Coordination/pathUtils.h index c20a786a15c..69ed2d8b177 100644 --- a/src/Coordination/pathUtils.h +++ b/src/Coordination/pathUtils.h @@ -6,8 +6,8 @@ namespace DB { -std::string parentPath(StringRef path); +StringRef parentPath(StringRef path); -std::string getBaseName(StringRef path); +StringRef getBaseName(StringRef path); } diff --git a/utils/keeper-bench/Generator.cpp b/utils/keeper-bench/Generator.cpp index 0b35144b0ed..f7228c3f579 100644 --- a/utils/keeper-bench/Generator.cpp +++ b/utils/keeper-bench/Generator.cpp @@ -242,7 +242,7 @@ std::unique_ptr getGenerator(const std::string & name) } else if (name == "create_small_data") { - return std::make_unique("/create_generator", 5, 32); + return std::make_unique("/create_generator", 50, 32); } else if (name == "create_medium_data") { From 34765765ad349bf67bf4a7062b95ac60f87b4cbe Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 19 Jan 2022 11:53:39 +0000 Subject: [PATCH 106/403] Use tb64senc for base64Decode on aarch64 --- src/Functions/FunctionBase64Conversion.h | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/Functions/FunctionBase64Conversion.h b/src/Functions/FunctionBase64Conversion.h index 6b1234f6228..e257199b7c8 100644 --- a/src/Functions/FunctionBase64Conversion.h +++ b/src/Functions/FunctionBase64Conversion.h @@ -124,13 +124,26 @@ public: if constexpr (std::is_same_v) { + /* + * Some bug in sse arm64 implementation? + * `base64Encode(repeat('a', 46))` returns wrong padding character + */ +#if (__aarch64__) + outlen = tb64senc(reinterpret_cast(source), srclen, reinterpret_cast(dst_pos)); +#else outlen = _tb64e(reinterpret_cast(source), srclen, reinterpret_cast(dst_pos)); +#endif } else if constexpr (std::is_same_v) { if (srclen > 0) { - outlen = _tb64d(reinterpret_cast(source), srclen, reinterpret_cast(dst_pos)); +#if (__aarch64__) + outlen = tb64sdec(reinterpret_cast(source), srclen, reinterpret_cast(dst_pos)); +#else + outlen = _tb64d(reinterpret_cast(source), srclen, reinterpret_cast(dst_pos)); +#endif + if (!outlen) throw Exception("Failed to " + getName() + " input '" + String(reinterpret_cast(source), srclen) + "'", ErrorCodes::INCORRECT_DATA); } From e5613fce3c7f7d968ab906c3fc1b4309f1be3c62 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 19 Jan 2022 12:21:51 +0000 Subject: [PATCH 107/403] Support USE_* build flags in tests' tags --- tests/clickhouse-test | 9 +++++++++ tests/queries/0_stateless/01030_storage_hdfs_syntax.sql | 2 +- .../0_stateless/02004_max_hyperscan_regex_length.sql | 3 +-- tests/queries/0_stateless/02113_hdfs_assert.sh | 2 +- tests/queries/0_stateless/02114_hdfs_bad_url.sh | 2 +- tests/queries/1_stateful/00095_hyperscan_profiler.sql | 2 +- 6 files changed, 14 insertions(+), 6 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index cb8d5914362..4566cabf1e7 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -467,6 +467,10 @@ class TestCase: for build_flag in args.build_flags: if 'no-' + build_flag in tags: return FailureReason.BUILD + for tag in tags: + tag = tag.replace('-', '_') + if tag.startswith('use_') and tag not in args.build_flags: + return FailureReason.BUILD return None @@ -1069,6 +1073,11 @@ def collect_build_flags(args): if value == 0: result.append(BuildFlags.POLYMORPHIC_PARTS) + use_flags = clickhouse_execute(args, "SELECT name FROM system.build_options WHERE name like 'USE_%' AND value in ('ON', '1');") + for use_flag in use_flags.strip().splitlines(): + use_flag = use_flag.decode().lower() + result.append(use_flag) + return result diff --git a/tests/queries/0_stateless/01030_storage_hdfs_syntax.sql b/tests/queries/0_stateless/01030_storage_hdfs_syntax.sql index 1fb6770d49b..b679a0ccf9c 100644 --- a/tests/queries/0_stateless/01030_storage_hdfs_syntax.sql +++ b/tests/queries/0_stateless/01030_storage_hdfs_syntax.sql @@ -1,4 +1,4 @@ --- Tags: no-fasttest +-- Tags: no-fasttest, use-hdfs drop table if exists test_table_hdfs_syntax ; diff --git a/tests/queries/0_stateless/02004_max_hyperscan_regex_length.sql b/tests/queries/0_stateless/02004_max_hyperscan_regex_length.sql index 9d62093a702..168ccec37f0 100644 --- a/tests/queries/0_stateless/02004_max_hyperscan_regex_length.sql +++ b/tests/queries/0_stateless/02004_max_hyperscan_regex_length.sql @@ -1,5 +1,4 @@ --- Tags: no-debug, no-fasttest --- Tag no-fasttest: Hyperscan +-- Tags: no-debug, no-fasttest, use-hyperscan set max_hyperscan_regexp_length = 1; set max_hyperscan_regexp_total_length = 1; diff --git a/tests/queries/0_stateless/02113_hdfs_assert.sh b/tests/queries/0_stateless/02113_hdfs_assert.sh index 3adf1360c47..e4f97b1fbfd 100755 --- a/tests/queries/0_stateless/02113_hdfs_assert.sh +++ b/tests/queries/0_stateless/02113_hdfs_assert.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-fasttest +# Tags: no-fasttest, use-hdfs CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/02114_hdfs_bad_url.sh b/tests/queries/0_stateless/02114_hdfs_bad_url.sh index a05baf19e6f..be48a987f45 100755 --- a/tests/queries/0_stateless/02114_hdfs_bad_url.sh +++ b/tests/queries/0_stateless/02114_hdfs_bad_url.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-fasttest +# Tags: no-fasttest, use-hdfs CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/1_stateful/00095_hyperscan_profiler.sql b/tests/queries/1_stateful/00095_hyperscan_profiler.sql index c704907178c..66a5d98e389 100644 --- a/tests/queries/1_stateful/00095_hyperscan_profiler.sql +++ b/tests/queries/1_stateful/00095_hyperscan_profiler.sql @@ -1,4 +1,4 @@ --- Tags: no-debug +-- Tags: no-debug, use-hyperscan -- Check that server does not get segfault due to bad stack unwinding from Hyperscan From 62441f0a0f002ee6c2d272f754ccd1c03313a6eb Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Wed, 19 Jan 2022 20:27:11 +0800 Subject: [PATCH 108/403] Fix mutation when table contains projections (#33679) --- src/Interpreters/MutationsInterpreter.cpp | 7 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 14 +-- src/Storages/MergeTree/MergeTask.cpp | 2 +- src/Storages/MergeTree/MutateTask.cpp | 5 +- .../__init__.py | 0 .../configs/config.xml | 12 +++ .../configs/users.xml | 11 +++ .../test_mutations_with_projection/test.py | 88 +++++++++++++++++++ .../01710_projection_mutation.reference | 0 .../0_stateless/01710_projection_mutation.sql | 7 ++ 10 files changed, 131 insertions(+), 15 deletions(-) create mode 100644 tests/integration/test_mutations_with_projection/__init__.py create mode 100644 tests/integration/test_mutations_with_projection/configs/config.xml create mode 100644 tests/integration/test_mutations_with_projection/configs/users.xml create mode 100644 tests/integration/test_mutations_with_projection/test.py create mode 100644 tests/queries/0_stateless/01710_projection_mutation.reference create mode 100644 tests/queries/0_stateless/01710_projection_mutation.sql diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index ac8dcce35d0..1c7b970e731 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -226,7 +226,8 @@ bool isStorageTouchedByMutations( /// Interpreter must be alive, when we use result of execute() method. /// For some reason it may copy context and and give it into ExpressionTransform /// after that we will use context from destroyed stack frame in our stream. - InterpreterSelectQuery interpreter(select_query, context_copy, storage, metadata_snapshot, SelectQueryOptions().ignoreLimits()); + InterpreterSelectQuery interpreter( + select_query, context_copy, storage, metadata_snapshot, SelectQueryOptions().ignoreLimits().ignoreProjections()); auto io = interpreter.execute(); PullingPipelineExecutor executor(io.pipeline); @@ -291,7 +292,7 @@ MutationsInterpreter::MutationsInterpreter( , commands(std::move(commands_)) , context(Context::createCopy(context_)) , can_execute(can_execute_) - , select_limits(SelectQueryOptions().analyze(!can_execute).ignoreLimits()) + , select_limits(SelectQueryOptions().analyze(!can_execute).ignoreLimits().ignoreProjections()) { mutation_ast = prepare(!can_execute); } @@ -732,7 +733,7 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) const ASTPtr select_query = prepareInterpreterSelectQuery(stages_copy, /* dry_run = */ true); InterpreterSelectQuery interpreter{ select_query, context, storage, metadata_snapshot, - SelectQueryOptions().analyze(/* dry_run = */ false).ignoreLimits()}; + SelectQueryOptions().analyze(/* dry_run = */ false).ignoreLimits().ignoreProjections()}; auto first_stage_header = interpreter.getSampleBlock(); QueryPlan plan; diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 826589361e9..e4ca3e00c0f 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1310,7 +1310,7 @@ void IMergeTreeDataPart::remove() const void IMergeTreeDataPart::projectionRemove(const String & parent_to, bool keep_shared_data) const { - String to = parent_to + "/" + relative_path; + String to = fs::path(parent_to) / relative_path; auto disk = volume->getDisk(); if (checksums.empty()) { @@ -1320,7 +1320,7 @@ void IMergeTreeDataPart::projectionRemove(const String & parent_to, bool keep_sh "Cannot quickly remove directory {} by removing files; fallback to recursive removal. Reason: checksums.txt is missing", fullPath(disk, to)); /// If the part is not completely written, we cannot use fast path by listing files. - disk->removeSharedRecursive(to + "/", keep_shared_data); + disk->removeSharedRecursive(fs::path(to) / "", keep_shared_data); } else { @@ -1333,15 +1333,15 @@ void IMergeTreeDataPart::projectionRemove(const String & parent_to, bool keep_sh # pragma GCC diagnostic ignored "-Wunused-variable" #endif for (const auto & [file, _] : checksums.files) - disk->removeSharedFile(to + "/" + file, keep_shared_data); + disk->removeSharedFile(fs::path(to) / file, keep_shared_data); #if !defined(__clang__) # pragma GCC diagnostic pop #endif for (const auto & file : {"checksums.txt", "columns.txt"}) - disk->removeSharedFile(to + "/" + file, keep_shared_data); - disk->removeSharedFileIfExists(to + "/" + DEFAULT_COMPRESSION_CODEC_FILE_NAME, keep_shared_data); - disk->removeSharedFileIfExists(to + "/" + DELETE_ON_DESTROY_MARKER_FILE_NAME, keep_shared_data); + disk->removeSharedFile(fs::path(to) / file, keep_shared_data); + disk->removeSharedFileIfExists(fs::path(to) / DEFAULT_COMPRESSION_CODEC_FILE_NAME, keep_shared_data); + disk->removeSharedFileIfExists(fs::path(to) / DELETE_ON_DESTROY_MARKER_FILE_NAME, keep_shared_data); disk->removeSharedRecursive(to, keep_shared_data); } @@ -1351,7 +1351,7 @@ void IMergeTreeDataPart::projectionRemove(const String & parent_to, bool keep_sh LOG_ERROR(storage.log, "Cannot quickly remove directory {} by removing files; fallback to recursive removal. Reason: {}", fullPath(disk, to), getCurrentExceptionMessage(false)); - disk->removeSharedRecursive(to + "/", keep_shared_data); + disk->removeSharedRecursive(fs::path(to) / "", keep_shared_data); } } } diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index b4ecfbebdcb..3b6f3860ee1 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -120,7 +120,7 @@ bool MergeTask::ExecuteAndFinalizeHorizontalPart::prepare() ctx->disk = global_ctx->space_reservation->getDisk(); String local_part_path = global_ctx->data->relative_data_path; - String local_tmp_part_basename = local_tmp_prefix + global_ctx->future_part->name + (global_ctx->parent_part ? ".proj" : ""); + String local_tmp_part_basename = local_tmp_prefix + global_ctx->future_part->name + local_tmp_suffix; String local_new_part_tmp_path = local_part_path + local_tmp_part_basename + "/"; if (ctx->disk->exists(local_new_part_tmp_path)) diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 86a692c8a48..c7ce80756ea 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -650,7 +650,6 @@ public: ".tmp_proj"); next_level_parts.push_back(executeHere(tmp_part_merge_task)); - next_level_parts.back()->is_temp = true; } @@ -1081,9 +1080,7 @@ private: ctx->disk->createDirectories(destination); for (auto p_it = ctx->disk->iterateDirectory(it->path()); p_it->isValid(); p_it->next()) { - String p_destination = destination + "/"; - String p_file_name = p_it->name(); - p_destination += p_it->name(); + String p_destination = fs::path(destination) / p_it->name(); ctx->disk->createHardLink(p_it->path(), p_destination); } } diff --git a/tests/integration/test_mutations_with_projection/__init__.py b/tests/integration/test_mutations_with_projection/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_mutations_with_projection/configs/config.xml b/tests/integration/test_mutations_with_projection/configs/config.xml new file mode 100644 index 00000000000..9183a737238 --- /dev/null +++ b/tests/integration/test_mutations_with_projection/configs/config.xml @@ -0,0 +1,12 @@ + + + + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/clickhouse-server.err.log + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + + /var/lib/clickhouse/ + users.xml + diff --git a/tests/integration/test_mutations_with_projection/configs/users.xml b/tests/integration/test_mutations_with_projection/configs/users.xml new file mode 100644 index 00000000000..37e6c66b5a5 --- /dev/null +++ b/tests/integration/test_mutations_with_projection/configs/users.xml @@ -0,0 +1,11 @@ + + + + + + 1000 + 1000 + 1000 + + + diff --git a/tests/integration/test_mutations_with_projection/test.py b/tests/integration/test_mutations_with_projection/test.py new file mode 100644 index 00000000000..da34764067f --- /dev/null +++ b/tests/integration/test_mutations_with_projection/test.py @@ -0,0 +1,88 @@ +import time + +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +instance_test_mutations = cluster.add_instance( + "test_mutations_with_projection", + main_configs=["configs/config.xml"], + user_configs=["configs/users.xml"], +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + instance_test_mutations.query( + """ + CREATE TABLE video_log + ( + `datetime` DateTime, -- 20,000 records per second + `user_id` UInt64, -- Cardinality == 100,000,000 + `device_id` UInt64, -- Cardinality == 200,000,000 + `video_id` UInt64, -- Cardinality == 100,00000 + `domain` LowCardinality(String), -- Cardinality == 100 + `bytes` UInt64, -- Ranging from 128 to 1152 + `duration` UInt64, -- Ranging from 100 to 400 + PROJECTION p_norm (SELECT datetime, device_id, bytes, duration ORDER BY device_id), + PROJECTION p_agg (SELECT toStartOfHour(datetime) AS hour, domain, sum(bytes), avg(duration) GROUP BY hour, domain) + ) + ENGINE = MergeTree + PARTITION BY toDate(datetime) -- Daily partitioning + ORDER BY (user_id, device_id, video_id) -- Can only favor one column here + SETTINGS index_granularity = 1000; + """ + ) + + instance_test_mutations.query( + """CREATE TABLE rng (`user_id_raw` UInt64, `device_id_raw` UInt64, `video_id_raw` UInt64, `domain_raw` UInt64, `bytes_raw` UInt64, `duration_raw` UInt64) ENGINE = GenerateRandom(1024);""" + ) + + instance_test_mutations.query( + """INSERT INTO video_log SELECT toUnixTimestamp(toDateTime(today())) + (rowNumberInAllBlocks() / 20000), user_id_raw % 100000000 AS user_id, device_id_raw % 200000000 AS device_id, video_id_raw % 100000000 AS video_id, domain_raw % 100, (bytes_raw % 1024) + 128, (duration_raw % 300) + 100 FROM rng LIMIT 500000;""" + ) + + instance_test_mutations.query("""OPTIMIZE TABLE video_log FINAL;""") + + yield cluster + finally: + cluster.shutdown() + + +def test_mutations_with_multi_level_merge_of_projections(started_cluster): + try: + instance_test_mutations.query( + """ALTER TABLE video_log UPDATE bytes = bytes + 10086 WHERE 1;""" + ) + + def count_and_changed(): + return instance_test_mutations.query( + "SELECT count(), countIf(bytes > 10000) FROM video_log SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT CSV" + ).splitlines() + + all_done = False + for wait_times_for_mutation in range( + 100 + ): # wait for replication 80 seconds max + time.sleep(0.8) + + if count_and_changed() == ["500000,500000"]: + all_done = True + break + + print( + instance_test_mutations.query( + "SELECT mutation_id, command, parts_to_do, is_done, latest_failed_part, latest_fail_reason, parts_to_do_names FROM system.mutations WHERE table = 'video_log' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT TSVWithNames" + ) + ) + + assert (count_and_changed(), all_done) == (["500000,500000"], True) + assert instance_test_mutations.query( + f"SELECT DISTINCT arraySort(projections) FROM system.parts WHERE table = 'video_log' SETTINGS force_index_by_date = 0, force_primary_key = 0 FORMAT TSVRaw" + ).splitlines() == ["['p_agg','p_norm']"] + + finally: + instance_test_mutations.query(f"""DROP TABLE video_log""") diff --git a/tests/queries/0_stateless/01710_projection_mutation.reference b/tests/queries/0_stateless/01710_projection_mutation.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01710_projection_mutation.sql b/tests/queries/0_stateless/01710_projection_mutation.sql new file mode 100644 index 00000000000..ab3fbd117d0 --- /dev/null +++ b/tests/queries/0_stateless/01710_projection_mutation.sql @@ -0,0 +1,7 @@ +DROP TABLE IF EXISTS t; + +CREATE TABLE t (`key` UInt32, `created_at` Date, `value` UInt32, PROJECTION xxx (SELECT key, created_at, sum(value) GROUP BY key, created_at)) ENGINE = MergeTree PARTITION BY toYYYYMM(created_at) ORDER BY key; + +INSERT INTO t SELECT 1 AS key, today() + (number % 30), number FROM numbers(1000); + +ALTER TABLE t UPDATE value = 0 WHERE (value > 0) AND (created_at >= '2021-12-21') SETTINGS allow_experimental_projection_optimization = 1; From 4dd8b65a1c0169d377e675b5cbb288726ce71294 Mon Sep 17 00:00:00 2001 From: Bharat Nallan Date: Wed, 19 Jan 2022 05:04:23 -0800 Subject: [PATCH 109/403] add h3ToCenterChild function (#33313) --- docs/en/sql-reference/functions/geo/h3.md | 37 +++++ src/Functions/h3ToCenterChild.cpp | 115 +++++++++++++++ src/Functions/registerFunctionsGeo.cpp | 2 + .../02155_h3_to_center_child.reference | 120 ++++++++++++++++ .../0_stateless/02155_h3_to_center_child.sql | 135 ++++++++++++++++++ 5 files changed, 409 insertions(+) create mode 100644 src/Functions/h3ToCenterChild.cpp create mode 100644 tests/queries/0_stateless/02155_h3_to_center_child.reference create mode 100644 tests/queries/0_stateless/02155_h3_to_center_child.sql diff --git a/docs/en/sql-reference/functions/geo/h3.md b/docs/en/sql-reference/functions/geo/h3.md index fcd78c4452f..19246ee94fe 100644 --- a/docs/en/sql-reference/functions/geo/h3.md +++ b/docs/en/sql-reference/functions/geo/h3.md @@ -812,4 +812,41 @@ Result: └─────────────────────┘ ``` +## h3ToCenterChild {#h3tocenterchild} + +Returns the center child (finer) [H3](#h3index) index contained by given [H3](#h3index) at the given resolution. + +**Syntax** + +``` sql +h3ToCenterChild(index, resolution) +``` + +**Parameter** + +- `index` — Hexagon index number. Type: [UInt64](../../../sql-reference/data-types/int-uint.md). +- `resolution` — Index resolution. Range: `[0, 15]`. Type: [UInt8](../../../sql-reference/data-types/int-uint.md). + +**Returned values** + +- [H3](#h3index) index of the center child contained by given [H3](#h3index) at the given resolution. + +Type: [UInt64](../../../sql-reference/data-types/int-uint.md). + +**Example** + +Query: + +``` sql +SELECT h3ToCenterChild(577023702256844799,1) AS centerToChild; +``` + +Result: + +``` text +┌──────centerToChild─┐ +│ 581496515558637567 │ +└────────────────────┘ +``` + [Original article](https://clickhouse.com/docs/en/sql-reference/functions/geo/h3) diff --git a/src/Functions/h3ToCenterChild.cpp b/src/Functions/h3ToCenterChild.cpp new file mode 100644 index 00000000000..d9bc31b8f19 --- /dev/null +++ b/src/Functions/h3ToCenterChild.cpp @@ -0,0 +1,115 @@ +#include "config_functions.h" + +#if USE_H3 + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ARGUMENT_OUT_OF_BOUND; + extern const int ILLEGAL_COLUMN; +} + +namespace +{ + class FunctionH3ToCenterChild : public IFunction + { + public: + static constexpr auto name = "h3ToCenterChild"; + + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + std::string getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 2; } + bool useDefaultImplementationForConstants() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + const auto * arg = arguments[0].get(); + if (!WhichDataType(arg).isUInt64()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arg->getName(), 1, getName()); + + arg = arguments[1].get(); + if (!WhichDataType(arg).isUInt8()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt8", + arg->getName(), 2, getName()); + + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto * col_hindex = checkAndGetColumn(arguments[0].column.get()); + if (!col_hindex) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64.", + arguments[0].type->getName(), + 1, + getName()); + const auto & data_hindex = col_hindex->getData(); + + const auto * col_resolution = checkAndGetColumn(arguments[1].column.get()); + if (!col_resolution) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt8.", + arguments[0].type->getName(), + 1, + getName()); + const auto & data_resolution = col_resolution->getData(); + + auto dst = ColumnVector::create(); + auto & dst_data = dst->getData(); + dst_data.resize(input_rows_count); + + for (size_t row = 0; row < input_rows_count; ++row) + { + if (data_resolution[row] > MAX_H3_RES) + throw Exception( + ErrorCodes::ARGUMENT_OUT_OF_BOUND, + "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is {}", + toString(data_resolution[row]), + getName(), + toString(MAX_H3_RES)); + + UInt64 res = cellToCenterChild(data_hindex[row], data_resolution[row]); + + dst_data[row] = res; + } + return dst; + } +}; + +} + +void registerFunctionH3ToCenterChild(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + +#endif diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index e3e1a44663f..33b15d91e60 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -36,6 +36,7 @@ void registerFunctionH3KRing(FunctionFactory &); void registerFunctionH3GetBaseCell(FunctionFactory &); void registerFunctionH3ToParent(FunctionFactory &); void registerFunctionH3ToChildren(FunctionFactory &); +void registerFunctionH3ToCenterChild(FunctionFactory &); void registerFunctionH3IndexesAreNeighbors(FunctionFactory &); void registerFunctionStringToH3(FunctionFactory &); void registerFunctionH3ToString(FunctionFactory &); @@ -96,6 +97,7 @@ void registerFunctionsGeo(FunctionFactory & factory) registerFunctionH3GetBaseCell(factory); registerFunctionH3ToParent(factory); registerFunctionH3ToChildren(factory); + registerFunctionH3ToCenterChild(factory); registerFunctionH3IndexesAreNeighbors(factory); registerFunctionStringToH3(factory); registerFunctionH3ToString(factory); diff --git a/tests/queries/0_stateless/02155_h3_to_center_child.reference b/tests/queries/0_stateless/02155_h3_to_center_child.reference new file mode 100644 index 00000000000..332120f290a --- /dev/null +++ b/tests/queries/0_stateless/02155_h3_to_center_child.reference @@ -0,0 +1,120 @@ +581496515558637567 +585996266895310847 +590499385486344191 +595002924984172543 +599506517095350271 +604010115783196671 +608513715293126655 +613017314905817087 +617520914531352575 +622024514158493695 +626528113785835519 +631031713413202431 +635535313040572479 +640038912667942919 +644542512295313408 +586018257127866367 +590521375718899711 +595024915216728063 +599528507327905791 +604032106015752191 +608535705525682175 +613039305138372607 +617542904763908095 +622046504391049215 +626550104018391039 +631053703645757951 +635557303273127999 +640060902900498439 +644564502527868928 +590524674253783039 +595028213751611391 +599531805862789119 +604035404550635519 +608539004060565503 +613042603673255935 +617546203298791423 +622049802925932543 +626553402553274367 +631057002180641279 +635560601808011327 +640064201435381767 +644567801062752256 +595028557348995071 +599532149460172799 +604035748148019199 +608539347657949183 +613042947270639615 +617546546896175103 +622050146523316223 +626553746150658047 +631057345778024959 +635560945405395007 +640064545032765447 +644568144660135936 +599532200999780351 +604035799687626751 +608539399197556735 +613042998810247167 +617546598435782655 +622050198062923775 +626553797690265599 +631057397317632511 +635560996945002559 +640064596572372999 +644568196199743488 +604035805056335871 +608539404566265855 +613043004178956287 +617546603804491775 +622050203431632895 +626553803058974719 +631057402686341631 +635561002313711679 +640064601941082119 +644568201568452608 +608539405371572223 +613043004984262655 +617546604609798143 +622050204236939263 +626553803864281087 +631057403491647999 +635561003119018047 +640064602746388487 +644568202373758976 +612640339485786111 +617143939111321599 +621647538738462719 +626151138365804543 +630654737993171455 +635158337620541503 +639661937247911943 +644165536875282432 +617143939115515903 +621647538742657023 +626151138369998847 +630654737997365759 +635158337624735807 +639661937252106247 +644165536879476736 +621647538742657023 +626151138369998847 +630654737997365759 +635158337624735807 +639661937252106247 +644165536879476736 +626151138369998847 +630654737997365759 +635158337624735807 +639661937252106247 +644165536879476736 +630654737997365759 +635158337624735807 +639661937252106247 +644165536879476736 +635158337624735807 +639661937252106247 +644165536879476736 +639661937252106247 +644165536879476736 +644165536879476736 diff --git a/tests/queries/0_stateless/02155_h3_to_center_child.sql b/tests/queries/0_stateless/02155_h3_to_center_child.sql new file mode 100644 index 00000000000..28abe99c4b8 --- /dev/null +++ b/tests/queries/0_stateless/02155_h3_to_center_child.sql @@ -0,0 +1,135 @@ +-- Tags: no-fasttest + +DROP TABLE IF EXISTS h3_indexes; + +--Note: id column just exists to keep the test results sorted. +-- Order is not guaranteed with h3_index or res columns as we test the same h3_index at various resolutions. +CREATE TABLE h3_indexes (id UInt8, h3_index UInt64, res UInt8) ENGINE = Memory; + +-- Test cases taken from fixture: https://github.com/uber/h3/blob/master/src/apps/testapps/testCellToCenterChild.c + +INSERT INTO h3_indexes VALUES (1,577023702256844799,1); +INSERT INTO h3_indexes VALUES (2,577023702256844799,2); +INSERT INTO h3_indexes VALUES (3,577023702256844799,3); +INSERT INTO h3_indexes VALUES (4,577023702256844799,4); +INSERT INTO h3_indexes VALUES (5,577023702256844799,5); +INSERT INTO h3_indexes VALUES (6,577023702256844799,6); +INSERT INTO h3_indexes VALUES (7,577023702256844799,7); +INSERT INTO h3_indexes VALUES (8,577023702256844799,8); +INSERT INTO h3_indexes VALUES (9,577023702256844799,9); +INSERT INTO h3_indexes VALUES (10,577023702256844799,10); +INSERT INTO h3_indexes VALUES (11,577023702256844799,11); +INSERT INTO h3_indexes VALUES (12,577023702256844799,12); +INSERT INTO h3_indexes VALUES (13,577023702256844799,13); +INSERT INTO h3_indexes VALUES (14,577023702256844799,14); +INSERT INTO h3_indexes VALUES (15,577023702256844799,15); +INSERT INTO h3_indexes VALUES (16,581518505791193087,2); +INSERT INTO h3_indexes VALUES (17,581518505791193087,3); +INSERT INTO h3_indexes VALUES (18,581518505791193087,4); +INSERT INTO h3_indexes VALUES (19,581518505791193087,5); +INSERT INTO h3_indexes VALUES (20,581518505791193087,6); +INSERT INTO h3_indexes VALUES (21,581518505791193087,7); +INSERT INTO h3_indexes VALUES (22,581518505791193087,8); +INSERT INTO h3_indexes VALUES (23,581518505791193087,9); +INSERT INTO h3_indexes VALUES (24,581518505791193087,10); +INSERT INTO h3_indexes VALUES (25,581518505791193087,11); +INSERT INTO h3_indexes VALUES (26,581518505791193087,12); +INSERT INTO h3_indexes VALUES (27,581518505791193087,13); +INSERT INTO h3_indexes VALUES (28,581518505791193087,14); +INSERT INTO h3_indexes VALUES (29,581518505791193087,15); +INSERT INTO h3_indexes VALUES (30,586021555662749695,3); +INSERT INTO h3_indexes VALUES (31,586021555662749695,4); +INSERT INTO h3_indexes VALUES (32,586021555662749695,5); +INSERT INTO h3_indexes VALUES (33,586021555662749695,6); +INSERT INTO h3_indexes VALUES (34,586021555662749695,7); +INSERT INTO h3_indexes VALUES (35,586021555662749695,8); +INSERT INTO h3_indexes VALUES (36,586021555662749695,9); +INSERT INTO h3_indexes VALUES (37,586021555662749695,10); +INSERT INTO h3_indexes VALUES (38,586021555662749695,11); +INSERT INTO h3_indexes VALUES (39,586021555662749695,12); +INSERT INTO h3_indexes VALUES (40,586021555662749695,13); +INSERT INTO h3_indexes VALUES (41,586021555662749695,14); +INSERT INTO h3_indexes VALUES (42,586021555662749695,15); +INSERT INTO h3_indexes VALUES (43,590525017851166719,4); +INSERT INTO h3_indexes VALUES (44,590525017851166719,5); +INSERT INTO h3_indexes VALUES (45,590525017851166719,6); +INSERT INTO h3_indexes VALUES (46,590525017851166719,7); +INSERT INTO h3_indexes VALUES (47,590525017851166719,8); +INSERT INTO h3_indexes VALUES (48,590525017851166719,9); +INSERT INTO h3_indexes VALUES (49,590525017851166719,10); +INSERT INTO h3_indexes VALUES (50,590525017851166719,11); +INSERT INTO h3_indexes VALUES (51,590525017851166719,12); +INSERT INTO h3_indexes VALUES (52,590525017851166719,13); +INSERT INTO h3_indexes VALUES (53,590525017851166719,14); +INSERT INTO h3_indexes VALUES (54,590525017851166719,15); +INSERT INTO h3_indexes VALUES (55,595028608888602623,5); +INSERT INTO h3_indexes VALUES (56,595028608888602623,6); +INSERT INTO h3_indexes VALUES (57,595028608888602623,7); +INSERT INTO h3_indexes VALUES (58,595028608888602623,8); +INSERT INTO h3_indexes VALUES (59,595028608888602623,9); +INSERT INTO h3_indexes VALUES (60,595028608888602623,10); +INSERT INTO h3_indexes VALUES (61,595028608888602623,11); +INSERT INTO h3_indexes VALUES (62,595028608888602623,12); +INSERT INTO h3_indexes VALUES (63,595028608888602623,13); +INSERT INTO h3_indexes VALUES (64,595028608888602623,14); +INSERT INTO h3_indexes VALUES (65,595028608888602623,15); +INSERT INTO h3_indexes VALUES (66,599532206368489471,6); +INSERT INTO h3_indexes VALUES (67,599532206368489471,7); +INSERT INTO h3_indexes VALUES (68,599532206368489471,8); +INSERT INTO h3_indexes VALUES (69,599532206368489471,9); +INSERT INTO h3_indexes VALUES (70,599532206368489471,10); +INSERT INTO h3_indexes VALUES (71,599532206368489471,11); +INSERT INTO h3_indexes VALUES (72,599532206368489471,12); +INSERT INTO h3_indexes VALUES (73,599532206368489471,13); +INSERT INTO h3_indexes VALUES (74,599532206368489471,14); +INSERT INTO h3_indexes VALUES (75,599532206368489471,15); +INSERT INTO h3_indexes VALUES (76,604035805861642239,7); +INSERT INTO h3_indexes VALUES (77,604035805861642239,8); +INSERT INTO h3_indexes VALUES (78,604035805861642239,9); +INSERT INTO h3_indexes VALUES (79,604035805861642239,10); +INSERT INTO h3_indexes VALUES (80,604035805861642239,11); +INSERT INTO h3_indexes VALUES (81,604035805861642239,12); +INSERT INTO h3_indexes VALUES (82,604035805861642239,13); +INSERT INTO h3_indexes VALUES (83,604035805861642239,14); +INSERT INTO h3_indexes VALUES (84,604035805861642239,15); +INSERT INTO h3_indexes VALUES (85,608136739873095679,8); +INSERT INTO h3_indexes VALUES (86,608136739873095679,9); +INSERT INTO h3_indexes VALUES (87,608136739873095679,10); +INSERT INTO h3_indexes VALUES (88,608136739873095679,11); +INSERT INTO h3_indexes VALUES (89,608136739873095679,12); +INSERT INTO h3_indexes VALUES (90,608136739873095679,13); +INSERT INTO h3_indexes VALUES (91,608136739873095679,14); +INSERT INTO h3_indexes VALUES (92,608136739873095679,15); +INSERT INTO h3_indexes VALUES (93,612640339489980415,9); +INSERT INTO h3_indexes VALUES (94,612640339489980415,10); +INSERT INTO h3_indexes VALUES (95,612640339489980415,11); +INSERT INTO h3_indexes VALUES (96,612640339489980415,12); +INSERT INTO h3_indexes VALUES (97,612640339489980415,13); +INSERT INTO h3_indexes VALUES (98,612640339489980415,14); +INSERT INTO h3_indexes VALUES (99,612640339489980415,15); +INSERT INTO h3_indexes VALUES (100,617143939115515903,10); +INSERT INTO h3_indexes VALUES (101,617143939115515903,11); +INSERT INTO h3_indexes VALUES (102,617143939115515903,12); +INSERT INTO h3_indexes VALUES (103,617143939115515903,13); +INSERT INTO h3_indexes VALUES (104,617143939115515903,14); +INSERT INTO h3_indexes VALUES (105,617143939115515903,15); +INSERT INTO h3_indexes VALUES (106,621647538742657023,11); +INSERT INTO h3_indexes VALUES (107,621647538742657023,12); +INSERT INTO h3_indexes VALUES (108,621647538742657023,13); +INSERT INTO h3_indexes VALUES (109,621647538742657023,14); +INSERT INTO h3_indexes VALUES (110,621647538742657023,15); +INSERT INTO h3_indexes VALUES (111,626151138369998847,12); +INSERT INTO h3_indexes VALUES (112,626151138369998847,13); +INSERT INTO h3_indexes VALUES (113,626151138369998847,14); +INSERT INTO h3_indexes VALUES (114,626151138369998847,15); +INSERT INTO h3_indexes VALUES (115,630654737997365759,13); +INSERT INTO h3_indexes VALUES (116,630654737997365759,14); +INSERT INTO h3_indexes VALUES (117,630654737997365759,15); +INSERT INTO h3_indexes VALUES (118,635158337624735807,14); +INSERT INTO h3_indexes VALUES (119,635158337624735807,15); +INSERT INTO h3_indexes VALUES (120,639661937252106247,15); + + +SELECT h3ToCenterChild(h3_index,res) FROM h3_indexes ORDER BY id; + +DROP TABLE h3_indexes; From 044cc3a00e179a5ba181360af15ea976735b3fa8 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 19 Jan 2022 16:38:11 +0300 Subject: [PATCH 110/403] Control snapshot size better --- src/Coordination/KeeperSnapshotManager.cpp | 4 ++-- src/Coordination/KeeperStorage.h | 4 ++-- src/Coordination/SnapshotableHashTable.h | 26 ++++++++++++++++------ 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/Coordination/KeeperSnapshotManager.cpp b/src/Coordination/KeeperSnapshotManager.cpp index fc90ce860d0..7cdea53d7be 100644 --- a/src/Coordination/KeeperSnapshotManager.cpp +++ b/src/Coordination/KeeperSnapshotManager.cpp @@ -335,8 +335,8 @@ KeeperStorageSnapshot::KeeperStorageSnapshot(KeeperStorage * storage_, uint64_t , session_id(storage->session_id_counter) , cluster_config(cluster_config_) { - storage->enableSnapshotMode(); snapshot_container_size = storage->container.snapshotSize(); + storage->enableSnapshotMode(snapshot_container_size); begin = storage->getSnapshotIteratorBegin(); session_and_timeout = storage->getActiveSessions(); acl_map = storage->acl_map.getMapping(); @@ -349,8 +349,8 @@ KeeperStorageSnapshot::KeeperStorageSnapshot(KeeperStorage * storage_, const Sna , session_id(storage->session_id_counter) , cluster_config(cluster_config_) { - storage->enableSnapshotMode(); snapshot_container_size = storage->container.snapshotSize(); + storage->enableSnapshotMode(snapshot_container_size); begin = storage->getSnapshotIteratorBegin(); session_and_timeout = storage->getActiveSessions(); acl_map = storage->acl_map.getMapping(); diff --git a/src/Coordination/KeeperStorage.h b/src/Coordination/KeeperStorage.h index 55d4e4ea7e7..fc0f9884c2c 100644 --- a/src/Coordination/KeeperStorage.h +++ b/src/Coordination/KeeperStorage.h @@ -159,9 +159,9 @@ public: /// Set of methods for creating snapshots /// Turn on snapshot mode, so data inside Container is not deleted, but replaced with new version. - void enableSnapshotMode() + void enableSnapshotMode(size_t up_to_size) { - container.enableSnapshotMode(); + container.enableSnapshotMode(up_to_size); } /// Turn off snapshot mode. diff --git a/src/Coordination/SnapshotableHashTable.h b/src/Coordination/SnapshotableHashTable.h index c3c8f672d70..5637e80c0d3 100644 --- a/src/Coordination/SnapshotableHashTable.h +++ b/src/Coordination/SnapshotableHashTable.h @@ -33,6 +33,7 @@ private: List list; IndexMap map; bool snapshot_mode{false}; + size_t snapshot_up_to_size = 0; ArenaWithFreeLists arena; uint64_t approximate_data_size{0}; @@ -225,13 +226,22 @@ public: if (snapshot_mode) { - auto elem_copy = *(list_itr); - list_itr->active_in_map = false; + size_t distance = std::distance(list.begin(), list_itr); - updater(elem_copy.value); - auto itr = list.insert(list.end(), elem_copy); - it->getMapped() = itr; - ret = itr; + if (distance < snapshot_up_to_size) + { + auto elem_copy = *(list_itr); + list_itr->active_in_map = false; + updater(elem_copy.value); + auto itr = list.insert(list.end(), elem_copy); + it->getMapped() = itr; + ret = itr; + } + else + { + updater(list_itr->value); + ret = list_itr; + } } else { @@ -289,14 +299,16 @@ public: updateDataSize(CLEAR, 0, 0, 0); } - void enableSnapshotMode() + void enableSnapshotMode(size_t up_to_size) { snapshot_mode = true; + snapshot_up_to_size = up_to_size; } void disableSnapshotMode() { snapshot_mode = false; + snapshot_up_to_size = 0; } size_t size() const From 35b6b11a5dbc7a78fc5acb7de0d204911267a9c8 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 19 Jan 2022 16:47:12 +0300 Subject: [PATCH 111/403] Fix hash --- src/Coordination/KeeperStorage.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Coordination/KeeperStorage.h b/src/Coordination/KeeperStorage.h index fc0f9884c2c..20e026362d4 100644 --- a/src/Coordination/KeeperStorage.h +++ b/src/Coordination/KeeperStorage.h @@ -17,7 +17,7 @@ namespace DB struct KeeperStorageRequestProcessor; using KeeperStorageRequestProcessorPtr = std::shared_ptr; using ResponseCallback = std::function; -using ChildrenSet = std::unordered_set; +using ChildrenSet = std::unordered_set; using SessionAndTimeout = std::unordered_map; struct KeeperStorageSnapshot; From 91d2bcc4d0608c90cb4c80fd511281348c6d5b5e Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Wed, 19 Jan 2022 11:25:35 -0400 Subject: [PATCH 112/403] Update index.md --- docs/en/sql-reference/window-functions/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/window-functions/index.md b/docs/en/sql-reference/window-functions/index.md index e62808a46bd..0a55eafc7ab 100644 --- a/docs/en/sql-reference/window-functions/index.md +++ b/docs/en/sql-reference/window-functions/index.md @@ -3,7 +3,7 @@ toc_priority: 62 toc_title: Window Functions --- -# [experimental] Window Functions +# Window Functions ClickHouse supports the standard grammar for defining windows and window functions. The following features are currently supported: From 9cb136f5b0d1360b4dcd5f971e6e72af5e383c82 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 19 Jan 2022 17:00:41 +0100 Subject: [PATCH 113/403] Use workflow names in approve lambda --- tests/ci/workflow_approve_rerun_lambda/app.py | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/tests/ci/workflow_approve_rerun_lambda/app.py b/tests/ci/workflow_approve_rerun_lambda/app.py index 988feef2f6d..be5a3d2c5cc 100644 --- a/tests/ci/workflow_approve_rerun_lambda/app.py +++ b/tests/ci/workflow_approve_rerun_lambda/app.py @@ -6,8 +6,8 @@ import fnmatch from collections import namedtuple import jwt -import requests -import boto3 +import requests # type: ignore +import boto3 # type: ignore API_URL = "https://api.github.com/repos/ClickHouse/ClickHouse" @@ -52,18 +52,19 @@ TRUSTED_ORG_IDS = { 54801242, # clickhouse } -# See {API_URL}/actions/workflows +# See https://api.github.com/repos/ClickHouse/ClickHouse/actions/workflows +# Use ID to not inject a malicious workflow TRUSTED_WORKFLOW_IDS = { 14586616, # Cancel workflows, always trusted } NEED_RERUN_WORKFLOWS = { - 14738810, # DocsRelease - 15834118, # Docs - 15522500, # MasterCI - 15516108, # ReleaseCI - 15797242, # BackportPR - 16441423, # PullRequestCI + "BackportPR", + "Docs", + "DocsRelease", + "MasterCI", + "PullRequestCI", + "ReleaseCI", } # Individual trusted contirbutors who are not in any trusted organization. @@ -392,10 +393,10 @@ def main(event): "completed and failed, let's check for rerun", ) - if workflow_description.workflow_id not in NEED_RERUN_WORKFLOWS: + if workflow_description.name not in NEED_RERUN_WORKFLOWS: print( "Workflow", - workflow_description.workflow_id, + workflow_description.name, "not in list of rerunable workflows", ) return @@ -437,7 +438,8 @@ def main(event): print(f"Totally have {len(changed_files)} changed files in PR:", changed_files) if check_suspicious_changed_files(changed_files): print( - f"Pull Request {pull_request['number']} has suspicious changes, label it for manuall approve" + f"Pull Request {pull_request['number']} has suspicious changes, " + "label it for manuall approve" ) label_manual_approve(pull_request, token) else: From 6092ed48bd603dd3887f8c658e91408a7b5419c3 Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 19 Jan 2022 19:04:57 +0300 Subject: [PATCH 114/403] Fix bug in client that led to 'Connection reset by peer' in server --- src/Client/ClientBase.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index eb00ee349ee..8db1f63aab0 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -1194,7 +1194,7 @@ bool ClientBase::receiveEndOfQuery() case Protocol::Server::Progress: onProgress(packet.progress); - return true; + break; default: throw NetException( From fd6a728953a92a86d92cf593895ac0f7376ff820 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Wed, 19 Jan 2022 16:08:56 +0000 Subject: [PATCH 115/403] Dictionaries read keys array copy fix --- src/Dictionaries/CacheDictionary.cpp | 3 ++- src/Dictionaries/DictionaryHelpers.h | 9 +++++++++ src/Dictionaries/FlatDictionary.cpp | 3 ++- src/Dictionaries/HashedArrayDictionary.cpp | 7 ++++++- src/Dictionaries/HashedDictionary.cpp | 7 ++++++- src/Dictionaries/RangeHashedDictionary.cpp | 23 +++++++++++++++------- 6 files changed, 41 insertions(+), 11 deletions(-) diff --git a/src/Dictionaries/CacheDictionary.cpp b/src/Dictionaries/CacheDictionary.cpp index c21ea763ac3..4b242ee7fd9 100644 --- a/src/Dictionaries/CacheDictionary.cpp +++ b/src/Dictionaries/CacheDictionary.cpp @@ -494,7 +494,8 @@ Pipe CacheDictionary::read(const Names & column_names, size if constexpr (dictionary_key_type == DictionaryKeyType::Simple) { auto keys = cache_storage_ptr->getCachedSimpleKeys(); - key_columns = {ColumnWithTypeAndName(getColumnFromPODArray(keys), std::make_shared(), dict_struct.id->name)}; + auto keys_column = getColumnFromPODArray(std::move(keys)); + key_columns = {ColumnWithTypeAndName(std::move(keys_column), std::make_shared(), dict_struct.id->name)}; } else { diff --git a/src/Dictionaries/DictionaryHelpers.h b/src/Dictionaries/DictionaryHelpers.h index 1e6a4a5fb44..5c2b6b27afd 100644 --- a/src/Dictionaries/DictionaryHelpers.h +++ b/src/Dictionaries/DictionaryHelpers.h @@ -682,6 +682,15 @@ static ColumnPtr getColumnFromPODArray(const PaddedPODArray & array) return column_vector; } +template +static ColumnPtr getColumnFromPODArray(PaddedPODArray && array) +{ + auto column_vector = ColumnVector::create(); + column_vector->getData() = std::move(array); + + return column_vector; +} + template static ColumnPtr getColumnFromPODArray(const PaddedPODArray & array, size_t start, size_t length) { diff --git a/src/Dictionaries/FlatDictionary.cpp b/src/Dictionaries/FlatDictionary.cpp index 5d26ad3ebc2..9bf6bf97c8d 100644 --- a/src/Dictionaries/FlatDictionary.cpp +++ b/src/Dictionaries/FlatDictionary.cpp @@ -547,7 +547,8 @@ Pipe FlatDictionary::read(const Names & column_names, size_t max_block_size, siz if (loaded_keys[key_index]) keys.push_back(key_index); - ColumnsWithTypeAndName key_columns = {ColumnWithTypeAndName(getColumnFromPODArray(keys), std::make_shared(), dict_struct.id->name)}; + auto keys_column = getColumnFromPODArray(std::move(keys)); + ColumnsWithTypeAndName key_columns = {ColumnWithTypeAndName(std::move(keys_column), std::make_shared(), dict_struct.id->name)}; std::shared_ptr dictionary = shared_from_this(); auto coordinator = DictionarySourceCoordinator::create(dictionary, column_names, std::move(key_columns), max_block_size); diff --git a/src/Dictionaries/HashedArrayDictionary.cpp b/src/Dictionaries/HashedArrayDictionary.cpp index 148aaafb160..55a3adc32ae 100644 --- a/src/Dictionaries/HashedArrayDictionary.cpp +++ b/src/Dictionaries/HashedArrayDictionary.cpp @@ -753,9 +753,14 @@ Pipe HashedArrayDictionary::read(const Names & column_names ColumnsWithTypeAndName key_columns; if constexpr (dictionary_key_type == DictionaryKeyType::Simple) - key_columns = {ColumnWithTypeAndName(getColumnFromPODArray(keys), std::make_shared(), dict_struct.id->name)}; + { + auto keys_column = getColumnFromPODArray(std::move(keys)); + key_columns = {ColumnWithTypeAndName(std::move(keys_column), std::make_shared(), dict_struct.id->name)}; + } else + { key_columns = deserializeColumnsWithTypeAndNameFromKeys(dict_struct, keys, 0, keys.size()); + } std::shared_ptr dictionary = shared_from_this(); auto coordinator = DictionarySourceCoordinator::create(dictionary, column_names, std::move(key_columns), max_block_size); diff --git a/src/Dictionaries/HashedDictionary.cpp b/src/Dictionaries/HashedDictionary.cpp index 7025c771e8f..8417455087e 100644 --- a/src/Dictionaries/HashedDictionary.cpp +++ b/src/Dictionaries/HashedDictionary.cpp @@ -661,9 +661,14 @@ Pipe HashedDictionary::read(const Names & column_na ColumnsWithTypeAndName key_columns; if constexpr (dictionary_key_type == DictionaryKeyType::Simple) - key_columns = {ColumnWithTypeAndName(getColumnFromPODArray(keys), std::make_shared(), dict_struct.id->name)}; + { + auto keys_column = getColumnFromPODArray(std::move(keys)); + key_columns = {ColumnWithTypeAndName(std::move(keys_column), std::make_shared(), dict_struct.id->name)}; + } else + { key_columns = deserializeColumnsWithTypeAndNameFromKeys(dict_struct, keys, 0, keys.size()); + } std::shared_ptr dictionary = shared_from_this(); auto coordinator = DictionarySourceCoordinator::create(dictionary, column_names, std::move(key_columns), max_block_size); diff --git a/src/Dictionaries/RangeHashedDictionary.cpp b/src/Dictionaries/RangeHashedDictionary.cpp index 2d98583d4a3..17ed871eefa 100644 --- a/src/Dictionaries/RangeHashedDictionary.cpp +++ b/src/Dictionaries/RangeHashedDictionary.cpp @@ -715,19 +715,28 @@ Pipe RangeHashedDictionary::read(const Names & column_names using RangeType = typename LeftDataType::FieldType; PaddedPODArray keys; - PaddedPODArray start_dates; - PaddedPODArray end_dates; - getKeysAndDates(keys, start_dates, end_dates); + PaddedPODArray range_start; + PaddedPODArray range_end; + getKeysAndDates(keys, range_start, range_end); - range_min_column = ColumnWithTypeAndName{getColumnFromPODArray(start_dates), dict_struct.range_min->type, dict_struct.range_min->name}; - range_max_column = ColumnWithTypeAndName{getColumnFromPODArray(end_dates), dict_struct.range_max->type, dict_struct.range_max->name}; + auto date_column = getColumnFromPODArray(makeDateKeys(range_start, range_end)); + + auto range_start_column = getColumnFromPODArray(std::move(range_start)); + range_min_column = ColumnWithTypeAndName{std::move(range_start_column), dict_struct.range_min->type, dict_struct.range_min->name}; + + auto range_end_column = getColumnFromPODArray(std::move(range_end)); + range_max_column = ColumnWithTypeAndName{std::move(range_end_column), dict_struct.range_max->type, dict_struct.range_max->name}; if constexpr (dictionary_key_type == DictionaryKeyType::Simple) - key_columns = {ColumnWithTypeAndName(getColumnFromPODArray(keys), std::make_shared(), dict_struct.id->name)}; + { + auto keys_column = getColumnFromPODArray(std::move(keys)); + key_columns = {ColumnWithTypeAndName(std::move(keys_column), std::make_shared(), dict_struct.id->name)}; + } else + { key_columns = deserializeColumnsWithTypeAndNameFromKeys(dict_struct, keys, 0, keys.size()); + } - auto date_column = getColumnFromPODArray(makeDateKeys(start_dates, end_dates)); key_columns.emplace_back(ColumnWithTypeAndName{std::move(date_column), std::make_shared(), ""}); return true; From d8062e910a568b3fed4cf0db890508e7bfe81dcc Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Wed, 19 Jan 2022 20:16:51 +0300 Subject: [PATCH 116/403] Remove debug logging from TableFunctionFile --- src/TableFunctions/TableFunctionFile.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/TableFunctions/TableFunctionFile.cpp b/src/TableFunctions/TableFunctionFile.cpp index 71aba5494e8..6e288f9fa1e 100644 --- a/src/TableFunctions/TableFunctionFile.cpp +++ b/src/TableFunctions/TableFunctionFile.cpp @@ -16,7 +16,6 @@ StoragePtr TableFunctionFile::getStorage(const String & source, ContextPtr global_context, const std::string & table_name, const std::string & compression_method_) const { - LOG_DEBUG(&Poco::Logger::get("TableFunctionFile"), "getStorage"); // For `file` table function, we are going to use format settings from the // query context. StorageFile::CommonArguments args{ From e3de3889e3cf0c6c0373446b2a2a69d69a16e36e Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 19 Jan 2022 20:53:39 +0300 Subject: [PATCH 117/403] Revert "Revert "Ignore parse failure of opentelemetry header (#32116)" (#33594)" (#33595) --- src/Interpreters/OpenTelemetrySpanLog.cpp | 7 ++++--- src/Server/HTTPHandler.cpp | 8 ++------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/src/Interpreters/OpenTelemetrySpanLog.cpp b/src/Interpreters/OpenTelemetrySpanLog.cpp index 4c415800a20..10de6ba0e7b 100644 --- a/src/Interpreters/OpenTelemetrySpanLog.cpp +++ b/src/Interpreters/OpenTelemetrySpanLog.cpp @@ -202,7 +202,6 @@ bool OpenTelemetryTraceContext::parseTraceparentHeader(const std::string & trace ++data; UInt128 trace_id_128 = readHex(data); - trace_id = trace_id_128; data += 32; if (*data != '-') @@ -212,7 +211,7 @@ bool OpenTelemetryTraceContext::parseTraceparentHeader(const std::string & trace } ++data; - span_id = readHex(data); + UInt64 span_id_64 = readHex(data); data += 16; if (*data != '-') @@ -222,7 +221,9 @@ bool OpenTelemetryTraceContext::parseTraceparentHeader(const std::string & trace } ++data; - trace_flags = readHex(data); + this->trace_flags = readHex(data); + this->trace_id = trace_id_128; + this->span_id = span_id_64; return true; } diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 727781ea4a3..5253e66be92 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -96,7 +96,6 @@ namespace ErrorCodes extern const int REQUIRED_PASSWORD; extern const int AUTHENTICATION_FAILED; - extern const int BAD_REQUEST_PARAMETER; extern const int INVALID_SESSION_TIMEOUT; extern const int HTTP_LENGTH_REQUIRED; } @@ -491,12 +490,9 @@ void HTTPHandler::processQuery( { std::string opentelemetry_traceparent = request.get("traceparent"); std::string error; - if (!client_info.client_trace_context.parseTraceparentHeader( - opentelemetry_traceparent, error)) + if (!client_info.client_trace_context.parseTraceparentHeader(opentelemetry_traceparent, error)) { - throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER, - "Failed to parse OpenTelemetry traceparent header '{}': {}", - opentelemetry_traceparent, error); + LOG_DEBUG(log, "Failed to parse OpenTelemetry traceparent header '{}': {}", opentelemetry_traceparent, error); } client_info.client_trace_context.tracestate = request.get("tracestate", ""); } From 4810d13afffd3a8b2bd569736da16eb93ee03658 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 20 Jan 2022 02:59:09 +0300 Subject: [PATCH 118/403] use connection pool for redis dictionary --- src/Dictionaries/RedisDictionarySource.cpp | 197 ++++++++++-------- src/Dictionaries/RedisDictionarySource.h | 63 ++++-- src/Dictionaries/RedisSource.cpp | 23 +- src/Dictionaries/RedisSource.h | 13 +- .../test_dictionaries_redis/test_long.py | 51 +++++ 5 files changed, 219 insertions(+), 128 deletions(-) create mode 100644 tests/integration/test_dictionaries_redis/test_long.py diff --git a/src/Dictionaries/RedisDictionarySource.cpp b/src/Dictionaries/RedisDictionarySource.cpp index 24a14d8cc80..b17d8a1c7f7 100644 --- a/src/Dictionaries/RedisDictionarySource.cpp +++ b/src/Dictionaries/RedisDictionarySource.cpp @@ -3,26 +3,6 @@ #include "DictionaryStructure.h" #include "registerDictionaries.h" -namespace DB -{ - -void registerDictionarySourceRedis(DictionarySourceFactory & factory) -{ - auto create_table_source = [=](const DictionaryStructure & dict_struct, - const Poco::Util::AbstractConfiguration & config, - const String & config_prefix, - Block & sample_block, - ContextPtr /* global_context */, - const std::string & /* default_database */, - bool /* created_from_ddl */) -> DictionarySourcePtr { - return std::make_unique(dict_struct, config, config_prefix + ".redis", sample_block); - }; - factory.registerSource("redis", create_table_source); -} - -} - - #include #include #include @@ -33,7 +13,6 @@ void registerDictionarySourceRedis(DictionarySourceFactory & factory) #include "RedisSource.h" - namespace DB { namespace ErrorCodes @@ -42,27 +21,46 @@ namespace DB extern const int INVALID_CONFIG_PARAMETER; extern const int INTERNAL_REDIS_ERROR; extern const int LOGICAL_ERROR; + extern const int TIMEOUT_EXCEEDED; } + void registerDictionarySourceRedis(DictionarySourceFactory & factory) + { + auto create_table_source = [=](const DictionaryStructure & dict_struct, + const Poco::Util::AbstractConfiguration & config, + const String & config_prefix, + Block & sample_block, + ContextPtr /* global_context */, + const std::string & /* default_database */, + bool /* created_from_ddl */) -> DictionarySourcePtr { - static const size_t max_block_size = 8192; + return std::make_unique(dict_struct, config, config_prefix + ".redis", sample_block); + }; + + factory.registerSource("redis", create_table_source); + } + + static constexpr size_t REDIS_MAX_BLOCK_SIZE = DEFAULT_BLOCK_SIZE; + static constexpr size_t REDIS_LOCK_ACQUIRE_TIMEOUT_MS = 5000; RedisDictionarySource::RedisDictionarySource( - const DictionaryStructure & dict_struct_, - const String & host_, - UInt16 port_, - UInt8 db_index_, - const String & password_, - RedisStorageType storage_type_, - const Block & sample_block_) - : dict_struct{dict_struct_} - , host{host_} - , port{port_} - , db_index{db_index_} - , password{password_} - , storage_type{storage_type_} - , sample_block{sample_block_} - , client{std::make_shared(host, port)} + const DictionaryStructure & dict_struct_, + const String & host_, + UInt16 port_, + UInt8 db_index_, + const String & password_, + RedisStorageType storage_type_, + size_t pool_size_, + const Block & sample_block_) + : dict_struct{dict_struct_} + , host{host_} + , port{port_} + , db_index{db_index_} + , password{password_} + , storage_type{storage_type_} + , pool_size(pool_size_) + , pool(std::make_shared(pool_size_)) + , sample_block{sample_block_} { if (dict_struct.attributes.size() != 1) throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, @@ -87,61 +85,39 @@ namespace DB key.name, key.type->getName()); } - - if (!password.empty()) - { - RedisCommand command("AUTH"); - command << password; - String reply = client->execute(command); - if (reply != "OK") - throw Exception(ErrorCodes::INTERNAL_REDIS_ERROR, - "Authentication failed with reason {}", - reply); - } - - if (db_index != 0) - { - RedisCommand command("SELECT"); - command << std::to_string(db_index); - String reply = client->execute(command); - if (reply != "OK") - throw Exception(ErrorCodes::INTERNAL_REDIS_ERROR, - "Selecting database with index {} failed with reason {}", - DB::toString(db_index), - reply); - } } RedisDictionarySource::RedisDictionarySource( - const DictionaryStructure & dict_struct_, - const Poco::Util::AbstractConfiguration & config_, - const String & config_prefix_, - Block & sample_block_) - : RedisDictionarySource( + const DictionaryStructure & dict_struct_, + const Poco::Util::AbstractConfiguration & config_, + const String & config_prefix_, + Block & sample_block_) + : RedisDictionarySource( dict_struct_, config_.getString(config_prefix_ + ".host"), config_.getUInt(config_prefix_ + ".port"), config_.getUInt(config_prefix_ + ".db_index", 0), - config_.getString(config_prefix_ + ".password",""), + config_.getString(config_prefix_ + ".password", ""), parseStorageType(config_.getString(config_prefix_ + ".storage_type", "")), + config_.getUInt(config_prefix_ + ".pool_size", 16), sample_block_) { } - RedisDictionarySource::RedisDictionarySource(const RedisDictionarySource & other) - : RedisDictionarySource{other.dict_struct, - other.host, - other.port, - other.db_index, - other.password, - other.storage_type, - other.sample_block} + : RedisDictionarySource{ + other.dict_struct, + other.host, + other.port, + other.db_index, + other.password, + other.storage_type, + other.pool_size, + other.sample_block} { } - RedisDictionarySource::~RedisDictionarySource() = default; static String storageTypeToKeyType(RedisStorageType type) @@ -161,21 +137,20 @@ namespace DB Pipe RedisDictionarySource::loadAll() { - if (!client->isConnected()) - client->connect(host, port); + auto conn = getConnection(); RedisCommand command_for_keys("KEYS"); command_for_keys << "*"; /// Get only keys for specified storage type. - auto all_keys = client->execute(command_for_keys); + auto all_keys = conn->client->execute(command_for_keys); if (all_keys.isNull()) - return Pipe(std::make_shared(client, RedisArray{}, storage_type, sample_block, max_block_size)); + return Pipe(std::make_shared(std::move(conn), RedisArray{}, storage_type, sample_block, REDIS_MAX_BLOCK_SIZE)); RedisArray keys; auto key_type = storageTypeToKeyType(storage_type); for (const auto & key : all_keys) - if (key_type == client->execute(RedisCommand("TYPE").addRedisType(key))) + if (key_type == conn->client->execute(RedisCommand("TYPE").addRedisType(key))) keys.addRedisType(std::move(key)); if (storage_type == RedisStorageType::HASH_MAP) @@ -186,7 +161,7 @@ namespace DB RedisCommand command_for_secondary_keys("HKEYS"); command_for_secondary_keys.addRedisType(key); - auto secondary_keys = client->execute(command_for_secondary_keys); + auto secondary_keys = conn->client->execute(command_for_secondary_keys); RedisArray primary_with_secondary; primary_with_secondary.addRedisType(key); @@ -194,7 +169,7 @@ namespace DB { primary_with_secondary.addRedisType(secondary_key); /// Do not store more than max_block_size values for one request. - if (primary_with_secondary.size() == max_block_size + 1) + if (primary_with_secondary.size() == REDIS_MAX_BLOCK_SIZE + 1) { hkeys.add(primary_with_secondary); primary_with_secondary.clear(); @@ -209,14 +184,12 @@ namespace DB keys = std::move(hkeys); } - return Pipe(std::make_shared(client, std::move(keys), storage_type, sample_block, max_block_size)); + return Pipe(std::make_shared(std::move(conn), std::move(keys), storage_type, sample_block, REDIS_MAX_BLOCK_SIZE)); } - Pipe RedisDictionarySource::loadIds(const std::vector & ids) { - if (!client->isConnected()) - client->connect(host, port); + auto conn = getConnection(); if (storage_type == RedisStorageType::HASH_MAP) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Cannot use loadIds with 'hash_map' storage type"); @@ -229,13 +202,12 @@ namespace DB for (UInt64 id : ids) keys << DB::toString(id); - return Pipe(std::make_shared(client, std::move(keys), storage_type, sample_block, max_block_size)); + return Pipe(std::make_shared(std::move(conn), std::move(keys), storage_type, sample_block, REDIS_MAX_BLOCK_SIZE)); } Pipe RedisDictionarySource::loadKeys(const Columns & key_columns, const std::vector & requested_rows) { - if (!client->isConnected()) - client->connect(host, port); + auto conn = getConnection(); if (key_columns.size() != dict_struct.key->size()) throw Exception(ErrorCodes::LOGICAL_ERROR, "The size of key_columns does not equal to the size of dictionary key"); @@ -250,7 +222,7 @@ namespace DB if (isInteger(type)) key << DB::toString(key_columns[i]->get64(row)); else if (isString(type)) - key << get((*key_columns[i])[row]); + key << get((*key_columns[i])[row]); else throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected type of key in Redis dictionary"); } @@ -258,15 +230,56 @@ namespace DB keys.add(key); } - return Pipe(std::make_shared(client, std::move(keys), storage_type, sample_block, max_block_size)); + return Pipe(std::make_shared(std::move(conn), std::move(keys), storage_type, sample_block, REDIS_MAX_BLOCK_SIZE)); } - String RedisDictionarySource::toString() const { return "Redis: " + host + ':' + DB::toString(port); } + auto RedisDictionarySource::getConnection() const -> ConnectionPtr + { + ClientPtr client; + bool ok = pool->tryBorrowObject(client, + [] { return std::make_unique(); }, + REDIS_LOCK_ACQUIRE_TIMEOUT_MS); + + if (!ok) + throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, + "Could not get connection from pool, timeout exceeded {} seconds", + REDIS_LOCK_ACQUIRE_TIMEOUT_MS); + + if (!client->isConnected()) + { + client->connect(host, port); + + if (!password.empty()) + { + RedisCommand command("AUTH"); + command << password; + String reply = client->execute(command); + if (reply != "OK") + throw Exception(ErrorCodes::INTERNAL_REDIS_ERROR, + "Authentication failed with reason {}", reply); + } + + if (db_index != 0) + { + RedisCommand command("SELECT"); + command << std::to_string(db_index); + String reply = client->execute(command); + if (reply != "OK") + throw Exception(ErrorCodes::INTERNAL_REDIS_ERROR, + "Selecting database with index {} failed with reason {}", + DB::toString(db_index), + reply); + } + } + + return std::make_unique(pool, std::move(client)); + } + RedisStorageType RedisDictionarySource::parseStorageType(const String & storage_type_str) { if (storage_type_str == "hash_map") diff --git a/src/Dictionaries/RedisDictionarySource.h b/src/Dictionaries/RedisDictionarySource.h index eff97dede0c..2b5f3fcf0f5 100644 --- a/src/Dictionaries/RedisDictionarySource.h +++ b/src/Dictionaries/RedisDictionarySource.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include "DictionaryStructure.h" #include "IDictionarySource.h" @@ -20,13 +21,13 @@ namespace Poco } } - namespace DB { -namespace ErrorCodes -{ - extern const int NOT_IMPLEMENTED; -} + namespace ErrorCodes + { + extern const int NOT_IMPLEMENTED; + } + enum class RedisStorageType { SIMPLE, @@ -37,23 +38,46 @@ namespace ErrorCodes class RedisDictionarySource final : public IDictionarySource { RedisDictionarySource( - const DictionaryStructure & dict_struct, - const std::string & host, - UInt16 port, - UInt8 db_index, - const std::string & password, - RedisStorageType storage_type, - const Block & sample_block); + const DictionaryStructure & dict_struct, + const std::string & host, + UInt16 port, + UInt8 db_index, + const std::string & password, + RedisStorageType storage_type, + size_t pool_size, + const Block & sample_block); public: using RedisArray = Poco::Redis::Array; using RedisCommand = Poco::Redis::Command; + using ClientPtr = std::unique_ptr; + using Pool = BorrowedObjectPool; + using PoolPtr = std::shared_ptr; + + struct Connection + { + Connection(PoolPtr pool_, ClientPtr client_) + : pool(std::move(pool_)), client(std::move(client_)) + { + } + + ~Connection() + { + pool->returnObject(std::move(client)); + } + + PoolPtr pool; + ClientPtr client; + }; + + using ConnectionPtr = std::unique_ptr; + RedisDictionarySource( - const DictionaryStructure & dict_struct, - const Poco::Util::AbstractConfiguration & config, - const std::string & config_prefix, - Block & sample_block); + const DictionaryStructure & dict_struct, + const Poco::Util::AbstractConfiguration & config, + const std::string & config_prefix, + Block & sample_block); RedisDictionarySource(const RedisDictionarySource & other); @@ -81,6 +105,7 @@ namespace ErrorCodes std::string toString() const override; private: + ConnectionPtr getConnection() const; static RedisStorageType parseStorageType(const std::string& storage_type); const DictionaryStructure dict_struct; @@ -89,9 +114,9 @@ namespace ErrorCodes const UInt8 db_index; const std::string password; const RedisStorageType storage_type; + const size_t pool_size; + + PoolPtr pool; Block sample_block; - - std::shared_ptr client; }; - } diff --git a/src/Dictionaries/RedisSource.cpp b/src/Dictionaries/RedisSource.cpp index ad5cf8a0977..6089b836d98 100644 --- a/src/Dictionaries/RedisSource.cpp +++ b/src/Dictionaries/RedisSource.cpp @@ -30,20 +30,22 @@ namespace DB RedisSource::RedisSource( - const std::shared_ptr & client_, - const RedisArray & keys_, - const RedisStorageType & storage_type_, - const DB::Block & sample_block, - const size_t max_block_size_) - : SourceWithProgress(sample_block) - , client(client_), keys(keys_), storage_type(storage_type_), max_block_size{max_block_size_} + ConnectionPtr connection_, + const RedisArray & keys_, + const RedisStorageType & storage_type_, + const DB::Block & sample_block, + size_t max_block_size_) + : SourceWithProgress(sample_block) + , connection(std::move(connection_)) + , keys(keys_) + , storage_type(storage_type_) + , max_block_size{max_block_size_} { description.init(sample_block); } RedisSource::~RedisSource() = default; - namespace { using ValueType = ExternalResultDescription::ValueType; @@ -121,7 +123,6 @@ namespace DB } } - Chunk RedisSource::generate() { if (keys.isNull() || description.sample_block.rows() == 0 || cursor >= keys.size()) @@ -168,7 +169,7 @@ namespace DB for (const auto & elem : keys_array) command_for_values.addRedisType(elem); - auto values = client->execute(command_for_values); + auto values = connection->client->execute(command_for_values); if (keys_array.size() != values.size() + 1) // 'HMGET' primary_key secondary_keys throw Exception(ErrorCodes::NUMBER_OF_COLUMNS_DOESNT_MATCH, @@ -199,7 +200,7 @@ namespace DB for (size_t i = 0; i < need_values; ++i) command_for_values.add(keys.get(cursor + i)); - auto values = client->execute(command_for_values); + auto values = connection->client->execute(command_for_values); if (values.size() != need_values) throw Exception(ErrorCodes::INTERNAL_REDIS_ERROR, "Inconsistent sizes of keys and values in Redis request"); diff --git a/src/Dictionaries/RedisSource.h b/src/Dictionaries/RedisSource.h index db2e643eb4e..24507998f58 100644 --- a/src/Dictionaries/RedisSource.h +++ b/src/Dictionaries/RedisSource.h @@ -24,13 +24,14 @@ namespace DB public: using RedisArray = Poco::Redis::Array; using RedisBulkString = Poco::Redis::BulkString; + using ConnectionPtr = RedisDictionarySource::ConnectionPtr; RedisSource( - const std::shared_ptr & client_, - const Poco::Redis::Array & keys_, - const RedisStorageType & storage_type_, - const Block & sample_block, - const size_t max_block_size); + ConnectionPtr connection_, + const Poco::Redis::Array & keys_, + const RedisStorageType & storage_type_, + const Block & sample_block, + size_t max_block_size); ~RedisSource() override; @@ -39,7 +40,7 @@ namespace DB private: Chunk generate() override; - std::shared_ptr client; + ConnectionPtr connection; Poco::Redis::Array keys; RedisStorageType storage_type; const size_t max_block_size; diff --git a/tests/integration/test_dictionaries_redis/test_long.py b/tests/integration/test_dictionaries_redis/test_long.py new file mode 100644 index 00000000000..3f29403df62 --- /dev/null +++ b/tests/integration/test_dictionaries_redis/test_long.py @@ -0,0 +1,51 @@ +import pytest +from helpers.cluster import ClickHouseCluster +import redis + +cluster = ClickHouseCluster(__file__) + +node = cluster.add_instance('node', with_redis=True) + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + + N = 1000 + client = redis.Redis(host='localhost', port=cluster.redis_port, password='clickhouse', db=0) + client.flushdb() + for i in range(N): + client.hset('2020-10-10', i, i) + + node.query(""" + CREATE DICTIONARY redis_dict + ( + date String, + id UInt64, + value UInt64 + ) + PRIMARY KEY date, id + SOURCE(REDIS(HOST '{}' PORT 6379 STORAGE_TYPE 'hash_map' DB_INDEX 0 PASSWORD 'clickhouse')) + LAYOUT(COMPLEX_KEY_DIRECT()) + """.format(cluster.redis_host) + ) + + node.query(""" + CREATE TABLE redis_dictionary_test + ( + date Date, + id UInt64 + ) + ENGINE = MergeTree ORDER BY id""" + ) + + node.query("INSERT INTO default.redis_dictionary_test SELECT '2020-10-10', number FROM numbers(1000000)") + + yield cluster + + finally: + cluster.shutdown() + +def test_redis_dict_long(start_cluster): + assert node.query("SELECT count(), uniqExact(date), uniqExact(id) FROM redis_dict") == "1000\t1\t1000\n" + assert node.query("SELECT count(DISTINCT dictGet('redis_dict', 'value', tuple(date, id % 1000))) FROM redis_dictionary_test") == "1000\n" From c49a49b41ff9c2eeda615677ad8fd0d64d4ae261 Mon Sep 17 00:00:00 2001 From: cnmade Date: Thu, 20 Jan 2022 09:50:25 +0800 Subject: [PATCH 119/403] Translate zh/sql-reference/statements/alter/update.md --- .../sql-reference/statements/alter/update.md | 30 ++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) mode change 120000 => 100644 docs/zh/sql-reference/statements/alter/update.md diff --git a/docs/zh/sql-reference/statements/alter/update.md b/docs/zh/sql-reference/statements/alter/update.md deleted file mode 120000 index fa9be21c070..00000000000 --- a/docs/zh/sql-reference/statements/alter/update.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/alter/update.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/alter/update.md b/docs/zh/sql-reference/statements/alter/update.md new file mode 100644 index 00000000000..08eccdf1aa2 --- /dev/null +++ b/docs/zh/sql-reference/statements/alter/update.md @@ -0,0 +1,29 @@ +--- +toc_priority: 40 +toc_title: UPDATE +--- + +# ALTER TABLE … UPDATE 语句 {#alter-table-update-statements} + +``` sql +ALTER TABLE [db.]table UPDATE column1 = expr1 [, ...] WHERE filter_expr +``` + +操作与指定过滤表达式相匹配的数据。作为一个[变更 mutation](../../../sql-reference/statements/alter/index.md#mutations)来实现. + +!!! note "Note" + `ALTER TABLE` 的前缀使这个语法与其他大多数支持SQL的系统不同。它的目的是表明，与OLTP数据库中的类似查询不同，这是一个繁重的操作，不是为频繁使用而设计。 + +`filter_expr`必须是`UInt8`类型。这个查询将指定列的值更新为行中相应表达式的值，对于这些行，`filter_expr`取值为非零。使用`CAST`操作符将数值映射到列的类型上。不支持更新用于计算主键或分区键的列。 + +一个查询可以包含几个由逗号分隔的命令。 + +查询处理的同步性由 [mutations_sync](../../../operations/settings/settings.md#mutations_sync) 设置定义。 默认情况下，它是异步操作。 + + +**更多详情请参阅** + +- [变更 Mutations](../../../sql-reference/statements/alter/index.md#mutations) +- [ALTER查询的同步性问题](../../../sql-reference/statements/alter/index.md#synchronicity-of-alter-queries) +- [mutations_sync](../../../operations/settings/settings.md#mutations_sync) setting + From 6a0ba9eeb247e3db5e6ad2fe101d601ca93d5745 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 20 Jan 2022 06:37:23 +0300 Subject: [PATCH 120/403] Remove confusion. People often put changelog entry into the detailed description. And they never fill "documentation draft". --- .github/PULL_REQUEST_TEMPLATE.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 82a16d0589f..6540b60476f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -13,13 +13,4 @@ Changelog entry (a user-readable short description of the changes that goes to C ... -Detailed description / Documentation draft: -... - - -> By adding documentation, you'll allow users to try your new feature immediately, not when someone else will have time to document it later. Documentation is necessary for all features that affect user experience in any way. You can add brief documentation draft above, or add documentation right into your patch as Markdown files in [docs](https://github.com/ClickHouse/ClickHouse/tree/master/docs) folder. - -> If you are doing this for the first time, it's recommended to read the lightweight [Contributing to ClickHouse Documentation](https://github.com/ClickHouse/ClickHouse/tree/master/docs/README.md) guide first. - - > Information about CI checks: https://clickhouse.tech/docs/en/development/continuous-integration/ From 5dee76cda7cd7cb02877fe4cc86f824c512f01ae Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 16 Jan 2022 12:20:20 +0300 Subject: [PATCH 121/403] Remove unbundled zstd support --- CMakeLists.txt | 1 - cmake/find/rocksdb.cmake | 5 +---- cmake/find/zstd.cmake | 27 ------------------------- contrib/CMakeLists.txt | 5 +---- contrib/arrow-cmake/CMakeLists.txt | 9 +-------- contrib/librdkafka-cmake/CMakeLists.txt | 7 +++---- contrib/rocksdb-cmake/CMakeLists.txt | 2 +- contrib/zstd-cmake/CMakeLists.txt | 8 ++++---- src/CMakeLists.txt | 10 ++------- 9 files changed, 13 insertions(+), 61 deletions(-) delete mode 100644 cmake/find/zstd.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 4c9a972a4e4..c3d819d5813 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -495,7 +495,6 @@ include (cmake/find/ldap.cmake) # after ssl include (cmake/find/icu.cmake) include (cmake/find/xz.cmake) include (cmake/find/zlib.cmake) -include (cmake/find/zstd.cmake) include (cmake/find/ltdl.cmake) # for odbc # openssl, zlib before poco include (cmake/find/sparsehash.cmake) diff --git a/cmake/find/rocksdb.cmake b/cmake/find/rocksdb.cmake index 10592d1d037..c7d7c526297 100644 --- a/cmake/find/rocksdb.cmake +++ b/cmake/find/rocksdb.cmake @@ -36,16 +36,14 @@ if (NOT USE_INTERNAL_ROCKSDB_LIBRARY) endif() find_package(BZip2) - find_library(ZSTD_LIBRARY zstd) find_library(LZ4_LIBRARY lz4) find_library(GFLAGS_LIBRARY gflags) - if(SNAPPY_LIBRARY AND ZLIB_LIBRARY AND LZ4_LIBRARY AND BZIP2_FOUND AND ZSTD_LIBRARY AND GFLAGS_LIBRARY) + if(SNAPPY_LIBRARY AND ZLIB_LIBRARY AND LZ4_LIBRARY AND BZIP2_FOUND AND GFLAGS_LIBRARY) list (APPEND ROCKSDB_LIBRARY ${SNAPPY_LIBRARY}) list (APPEND ROCKSDB_LIBRARY ${ZLIB_LIBRARY}) list (APPEND ROCKSDB_LIBRARY ${LZ4_LIBRARY}) list (APPEND ROCKSDB_LIBRARY ${BZIP2_LIBRARY}) - list (APPEND ROCKSDB_LIBRARY ${ZSTD_LIBRARY}) list (APPEND ROCKSDB_LIBRARY ${GFLAGS_LIBRARY}) else() message (${RECONFIGURE_MESSAGE_LEVEL} @@ -53,7 +51,6 @@ if (NOT USE_INTERNAL_ROCKSDB_LIBRARY) " zlib=${ZLIB_LIBRARY} ;" " lz4=${LZ4_LIBRARY} ;" " bz2=${BZIP2_LIBRARY} ;" - " zstd=${ZSTD_LIBRARY} ;" " gflags=${GFLAGS_LIBRARY} ;") endif() endif () diff --git a/cmake/find/zstd.cmake b/cmake/find/zstd.cmake deleted file mode 100644 index 2b8dd53fbc3..00000000000 --- a/cmake/find/zstd.cmake +++ /dev/null @@ -1,27 +0,0 @@ -option (USE_INTERNAL_ZSTD_LIBRARY "Set to FALSE to use system zstd library instead of bundled" ON) - -if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/zstd/lib/zstd.h") - if(USE_INTERNAL_ZSTD_LIBRARY) - message(WARNING "submodule contrib/zstd is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal zstd library") - set(USE_INTERNAL_ZSTD_LIBRARY 0) - endif() - set(MISSING_INTERNAL_ZSTD_LIBRARY 1) -endif() - -if (NOT USE_INTERNAL_ZSTD_LIBRARY) - find_library (ZSTD_LIBRARY zstd) - find_path (ZSTD_INCLUDE_DIR NAMES zstd.h PATHS ${ZSTD_INCLUDE_PATHS}) - if (NOT ZSTD_LIBRARY OR NOT ZSTD_INCLUDE_DIR) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system zstd library") - endif () -endif () - -if (ZSTD_LIBRARY AND ZSTD_INCLUDE_DIR) -elseif (NOT MISSING_INTERNAL_ZSTD_LIBRARY) - set (USE_INTERNAL_ZSTD_LIBRARY 1) - set (ZSTD_LIBRARY zstd) - set (ZSTD_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/zstd/lib) -endif () - -message (STATUS "Using zstd: ${ZSTD_INCLUDE_DIR} : ${ZSTD_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 17954159b3a..bf2325453b8 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -56,13 +56,10 @@ endif() add_subdirectory (poco-cmake) add_subdirectory (croaring-cmake) +add_subdirectory (zstd-cmake) # TODO: refactor the contrib libraries below this comment. -if (USE_INTERNAL_ZSTD_LIBRARY) - add_subdirectory (zstd-cmake) -endif () - if (USE_INTERNAL_RE2_LIBRARY) add_subdirectory (re2-cmake) endif () diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index a28a83a87c5..6a481f412c8 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -311,10 +311,6 @@ if (ZLIB_INCLUDE_DIR AND ZLIB_LIBRARIES) set(ARROW_WITH_ZLIB 1) endif () -if (ZSTD_INCLUDE_DIR AND ZSTD_LIBRARY) - set(ARROW_WITH_ZSTD 1) -endif () - add_definitions(-DARROW_WITH_LZ4) SET(ARROW_SRCS "${LIBRARY_DIR}/util/compression_lz4.cc" ${ARROW_SRCS}) @@ -355,10 +351,7 @@ endif () if (ARROW_WITH_ZLIB) target_link_libraries(${ARROW_LIBRARY} PRIVATE ${ZLIB_LIBRARIES}) endif () -if (ARROW_WITH_ZSTD) - target_link_libraries(${ARROW_LIBRARY} PRIVATE ${ZSTD_LIBRARY}) - target_include_directories(${ARROW_LIBRARY} SYSTEM BEFORE PRIVATE ${ZLIB_INCLUDE_DIR}) -endif () +target_link_libraries(${ARROW_LIBRARY} PRIVATE ch_contrib::zstd) target_include_directories(${ARROW_LIBRARY} SYSTEM PRIVATE ${ORC_INCLUDE_DIR}) target_include_directories(${ARROW_LIBRARY} SYSTEM PRIVATE ${ORC_SOURCE_SRC_DIR}) diff --git a/contrib/librdkafka-cmake/CMakeLists.txt b/contrib/librdkafka-cmake/CMakeLists.txt index 97b6a7e1ec5..1a2821991a2 100644 --- a/contrib/librdkafka-cmake/CMakeLists.txt +++ b/contrib/librdkafka-cmake/CMakeLists.txt @@ -114,11 +114,10 @@ endif() add_library(rdkafka ${SRCS}) target_compile_options(rdkafka PRIVATE -fno-sanitize=undefined) # target_include_directories(rdkafka SYSTEM PUBLIC include) -target_include_directories(rdkafka SYSTEM PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") # for "librdkafka/rdkafka.h" -target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR}) # Because weird logic with "include_next" is used. +target_include_directories(rdkafka SYSTEM PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") # for "librdkafka/rdkafka.h" +target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR}) # Because weird logic with "include_next" is used. target_include_directories(rdkafka SYSTEM PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/auxdir") # for "../config.h" -target_include_directories(rdkafka SYSTEM PRIVATE "${ZSTD_INCLUDE_DIR}/common") # Because wrong path to "zstd_errors.h" is used. -target_link_libraries(rdkafka PRIVATE lz4 ${ZLIB_LIBRARIES} ${ZSTD_LIBRARY}) +target_link_libraries(rdkafka PRIVATE lz4 ${ZLIB_LIBRARIES} ch_contrib::zstd) if(OPENSSL_SSL_LIBRARY AND OPENSSL_CRYPTO_LIBRARY) target_link_libraries(rdkafka PRIVATE ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY}) endif() diff --git a/contrib/rocksdb-cmake/CMakeLists.txt b/contrib/rocksdb-cmake/CMakeLists.txt index db0b3942b79..749147136f4 100644 --- a/contrib/rocksdb-cmake/CMakeLists.txt +++ b/contrib/rocksdb-cmake/CMakeLists.txt @@ -66,7 +66,7 @@ else() include_directories("${ZSTD_INCLUDE_DIR}/dictBuilder") include_directories("${ZSTD_INCLUDE_DIR}/deprecated") - list(APPEND THIRDPARTY_LIBS zstd) + list(APPEND THIRDPARTY_LIBS ch_contrib::zstd) endif() endif() diff --git a/contrib/zstd-cmake/CMakeLists.txt b/contrib/zstd-cmake/CMakeLists.txt index 226ee1a8067..4949c3f30d5 100644 --- a/contrib/zstd-cmake/CMakeLists.txt +++ b/contrib/zstd-cmake/CMakeLists.txt @@ -148,7 +148,7 @@ IF (ZSTD_LEGACY_SUPPORT) "${LIBRARY_LEGACY_DIR}/zstd_v07.h") ENDIF (ZSTD_LEGACY_SUPPORT) -ADD_LIBRARY(zstd ${Sources} ${Headers}) - -target_include_directories (zstd PUBLIC ${LIBRARY_DIR}) -target_compile_options(zstd PRIVATE -fno-sanitize=undefined) +add_library(_zstd ${Sources} ${Headers}) +add_library(ch_contrib::zstd ALIAS _zstd) +target_include_directories(_zstd BEFORE PUBLIC ${LIBRARY_DIR}) +target_compile_options(_zstd PRIVATE -fno-sanitize=undefined) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8d14bd327b1..e05bb14dbda 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -393,14 +393,8 @@ dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${PDQSORT_INCLUDE_DIR}) target_include_directories(clickhouse_common_io SYSTEM BEFORE PUBLIC ${MINISELECT_INCLUDE_DIR}) dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${MINISELECT_INCLUDE_DIR}) -if (ZSTD_LIBRARY) - dbms_target_link_libraries(PRIVATE ${ZSTD_LIBRARY}) - target_link_libraries (clickhouse_common_io PUBLIC ${ZSTD_LIBRARY}) - target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${ZSTD_INCLUDE_DIR}) - if (NOT USE_INTERNAL_ZSTD_LIBRARY AND ZSTD_INCLUDE_DIR) - dbms_target_include_directories(SYSTEM BEFORE PRIVATE ${ZSTD_INCLUDE_DIR}) - endif () -endif() +dbms_target_link_libraries(PRIVATE ch_contrib::zstd) +target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::zstd) if (XZ_LIBRARY) target_link_libraries (clickhouse_common_io PUBLIC ${XZ_LIBRARY}) From ca8525a733f6e916a4bf7a07fe0da9bedd8d175b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 16 Jan 2022 13:34:54 +0300 Subject: [PATCH 122/403] Remove unbundled zlib support --- CMakeLists.txt | 1 - cmake/find/llvm.cmake | 2 +- cmake/find/rocksdb.cmake | 7 +--- cmake/find/zlib.cmake | 42 ------------------- contrib/CMakeLists.txt | 10 +---- contrib/arrow-cmake/CMakeLists.txt | 14 ++----- contrib/boost-cmake/CMakeLists.txt | 2 +- contrib/cassandra-cmake/CMakeLists.txt | 4 +- contrib/grpc-cmake/CMakeLists.txt | 7 +--- contrib/librdkafka-cmake/CMakeLists.txt | 2 +- contrib/libxml2-cmake/CMakeLists.txt | 3 +- .../mariadb-connector-c-cmake/CMakeLists.txt | 6 +-- contrib/poco-cmake/Foundation/CMakeLists.txt | 2 +- contrib/protobuf-cmake/CMakeLists.txt | 3 +- contrib/rocksdb-cmake/CMakeLists.txt | 2 +- contrib/zlib-ng-cmake/CMakeLists.txt | 11 ++--- src/CMakeLists.txt | 2 +- src/Common/mysqlxx/CMakeLists.txt | 2 +- src/Functions/CMakeLists.txt | 2 +- src/IO/examples/CMakeLists.txt | 2 +- 20 files changed, 27 insertions(+), 99 deletions(-) delete mode 100644 cmake/find/zlib.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index c3d819d5813..7bd57a2927b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -494,7 +494,6 @@ include (cmake/find/ssl.cmake) include (cmake/find/ldap.cmake) # after ssl include (cmake/find/icu.cmake) include (cmake/find/xz.cmake) -include (cmake/find/zlib.cmake) include (cmake/find/ltdl.cmake) # for odbc # openssl, zlib before poco include (cmake/find/sparsehash.cmake) diff --git a/cmake/find/llvm.cmake b/cmake/find/llvm.cmake index ece5d5434a0..81cafb5eec4 100644 --- a/cmake/find/llvm.cmake +++ b/cmake/find/llvm.cmake @@ -74,6 +74,6 @@ LLVMDemangle # else() # set (result "LLVM") # endif () -# list (APPEND result ${CMAKE_DL_LIBS} ${ZLIB_LIBRARIES}) +# list (APPEND result ${CMAKE_DL_LIBS} ch_contrib::zlib) # set (${REQUIRED_LLVM_LIBRARIES} ${result} PARENT_SCOPE) #endfunction() diff --git a/cmake/find/rocksdb.cmake b/cmake/find/rocksdb.cmake index c7d7c526297..d5f5d453abd 100644 --- a/cmake/find/rocksdb.cmake +++ b/cmake/find/rocksdb.cmake @@ -31,24 +31,19 @@ if (NOT USE_INTERNAL_ROCKSDB_LIBRARY) if (NOT SNAPPY_LIBRARY) include(cmake/find/snappy.cmake) endif() - if (NOT ZLIB_LIBRARY) - include(cmake/find/zlib.cmake) - endif() find_package(BZip2) find_library(LZ4_LIBRARY lz4) find_library(GFLAGS_LIBRARY gflags) - if(SNAPPY_LIBRARY AND ZLIB_LIBRARY AND LZ4_LIBRARY AND BZIP2_FOUND AND GFLAGS_LIBRARY) + if(SNAPPY_LIBRARY AND LZ4_LIBRARY AND BZIP2_FOUND AND GFLAGS_LIBRARY) list (APPEND ROCKSDB_LIBRARY ${SNAPPY_LIBRARY}) - list (APPEND ROCKSDB_LIBRARY ${ZLIB_LIBRARY}) list (APPEND ROCKSDB_LIBRARY ${LZ4_LIBRARY}) list (APPEND ROCKSDB_LIBRARY ${BZIP2_LIBRARY}) list (APPEND ROCKSDB_LIBRARY ${GFLAGS_LIBRARY}) else() message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system rocksdb: snappy=${SNAPPY_LIBRARY} ;" - " zlib=${ZLIB_LIBRARY} ;" " lz4=${LZ4_LIBRARY} ;" " bz2=${BZIP2_LIBRARY} ;" " gflags=${GFLAGS_LIBRARY} ;") diff --git a/cmake/find/zlib.cmake b/cmake/find/zlib.cmake deleted file mode 100644 index c2ee8217afa..00000000000 --- a/cmake/find/zlib.cmake +++ /dev/null @@ -1,42 +0,0 @@ -option (USE_INTERNAL_ZLIB_LIBRARY "Set to FALSE to use system zlib library instead of bundled" ON) - -if (NOT MSVC) - set (INTERNAL_ZLIB_NAME "zlib-ng" CACHE INTERNAL "") -else () - set (INTERNAL_ZLIB_NAME "zlib" CACHE INTERNAL "") - if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}") - message (WARNING "Will use standard zlib, please clone manually:\n git clone https://github.com/madler/zlib.git ${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal zlib library") - endif () -endif () - -if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}/zlib.h") - if(USE_INTERNAL_ZLIB_LIBRARY) - message(WARNING "submodule contrib/${INTERNAL_ZLIB_NAME} is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal zlib library") - endif() - set(USE_INTERNAL_ZLIB_LIBRARY 0) - set(MISSING_INTERNAL_ZLIB_LIBRARY 1) -endif() - -if (NOT USE_INTERNAL_ZLIB_LIBRARY) - find_package (ZLIB) - if (NOT ZLIB_FOUND) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system zlib library") - else() - set (ZLIB_NAME "libz") - endif() -endif () - -if (NOT ZLIB_FOUND AND NOT MISSING_INTERNAL_ZLIB_LIBRARY) - set (USE_INTERNAL_ZLIB_LIBRARY 1) - set (ZLIB_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}" "${ClickHouse_BINARY_DIR}/contrib/${INTERNAL_ZLIB_NAME}" CACHE INTERNAL "") # generated zconf.h - set (ZLIB_INCLUDE_DIRS ${ZLIB_INCLUDE_DIR}) # for poco - set (ZLIB_INCLUDE_DIRECTORIES ${ZLIB_INCLUDE_DIR}) # for protobuf - set (ZLIB_FOUND 1) # for poco - set (ZLIB_LIBRARIES zlib CACHE INTERNAL "") - set (ZLIB_LIBRARY_NAME ${ZLIB_LIBRARIES}) # for cassandra - set (ZLIB_NAME "${INTERNAL_ZLIB_NAME}") -endif () - -message (STATUS "Using ${ZLIB_NAME}: ${ZLIB_INCLUDE_DIR} : ${ZLIB_LIBRARIES}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index bf2325453b8..1a6d0c875bf 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -57,6 +57,7 @@ endif() add_subdirectory (poco-cmake) add_subdirectory (croaring-cmake) add_subdirectory (zstd-cmake) +add_subdirectory (zlib-ng-cmake) # TODO: refactor the contrib libraries below this comment. @@ -76,14 +77,6 @@ if (USE_INTERNAL_FARMHASH_LIBRARY) add_subdirectory (libfarmhash) endif () -if (USE_INTERNAL_ZLIB_LIBRARY) - if (INTERNAL_ZLIB_NAME STREQUAL "zlib-ng") - add_subdirectory (zlib-ng-cmake) - else () - add_subdirectory (${INTERNAL_ZLIB_NAME}) - endif () -endif () - if (USE_INTERNAL_H3_LIBRARY) add_subdirectory(h3-cmake) endif () @@ -105,7 +98,6 @@ endif () if (USE_INTERNAL_RDKAFKA_LIBRARY) add_subdirectory (librdkafka-cmake) - target_include_directories(rdkafka BEFORE PRIVATE ${ZLIB_INCLUDE_DIR}) if(OPENSSL_INCLUDE_DIR) target_include_directories(rdkafka BEFORE PRIVATE ${OPENSSL_INCLUDE_DIR}) endif() diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 6a481f412c8..5a61bca2152 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -307,10 +307,6 @@ if (SNAPPY_INCLUDE_DIR AND SNAPPY_LIBRARY) set(ARROW_WITH_SNAPPY 1) endif () -if (ZLIB_INCLUDE_DIR AND ZLIB_LIBRARIES) - set(ARROW_WITH_ZLIB 1) -endif () - add_definitions(-DARROW_WITH_LZ4) SET(ARROW_SRCS "${LIBRARY_DIR}/util/compression_lz4.cc" ${ARROW_SRCS}) @@ -319,10 +315,8 @@ if (ARROW_WITH_SNAPPY) SET(ARROW_SRCS "${LIBRARY_DIR}/util/compression_snappy.cc" ${ARROW_SRCS}) endif () -if (ARROW_WITH_ZLIB) - add_definitions(-DARROW_WITH_ZLIB) - SET(ARROW_SRCS "${LIBRARY_DIR}/util/compression_zlib.cc" ${ARROW_SRCS}) -endif () +add_definitions(-DARROW_WITH_ZLIB) +SET(ARROW_SRCS "${LIBRARY_DIR}/util/compression_zlib.cc" ${ARROW_SRCS}) if (ARROW_WITH_ZSTD) add_definitions(-DARROW_WITH_ZSTD) @@ -348,9 +342,7 @@ target_link_libraries(${ARROW_LIBRARY} PRIVATE lz4) if (ARROW_WITH_SNAPPY) target_link_libraries(${ARROW_LIBRARY} PRIVATE ${SNAPPY_LIBRARY}) endif () -if (ARROW_WITH_ZLIB) - target_link_libraries(${ARROW_LIBRARY} PRIVATE ${ZLIB_LIBRARIES}) -endif () +target_link_libraries(${ARROW_LIBRARY} PRIVATE ch_contrib::zlib) target_link_libraries(${ARROW_LIBRARY} PRIVATE ch_contrib::zstd) target_include_directories(${ARROW_LIBRARY} SYSTEM PRIVATE ${ORC_INCLUDE_DIR}) diff --git a/contrib/boost-cmake/CMakeLists.txt b/contrib/boost-cmake/CMakeLists.txt index 4a21b8a0e2d..b0cf461a52a 100644 --- a/contrib/boost-cmake/CMakeLists.txt +++ b/contrib/boost-cmake/CMakeLists.txt @@ -101,7 +101,7 @@ if (NOT EXTERNAL_BOOST_FOUND) add_library (_boost_iostreams ${SRCS_IOSTREAMS}) add_library (boost::iostreams ALIAS _boost_iostreams) target_include_directories (_boost_iostreams PRIVATE ${LIBRARY_DIR}) - target_link_libraries (_boost_iostreams PRIVATE ${ZLIB_LIBRARIES}) + target_link_libraries (_boost_iostreams PRIVATE ch_contrib::zlib) # program_options diff --git a/contrib/cassandra-cmake/CMakeLists.txt b/contrib/cassandra-cmake/CMakeLists.txt index a8f2bec5e2b..15565ec0a8d 100644 --- a/contrib/cassandra-cmake/CMakeLists.txt +++ b/contrib/cassandra-cmake/CMakeLists.txt @@ -50,7 +50,7 @@ add_library(minizip OBJECT ${CASS_SRC_DIR}/third_party/minizip/zip.c ${CASS_SRC_DIR}/third_party/minizip/unzip.c) -target_link_libraries(minizip zlib) +target_link_libraries(minizip ch_contrib::zlib) target_compile_definitions(minizip PRIVATE "-Dz_crc_t=unsigned long") list(APPEND INCLUDE_DIRS @@ -115,7 +115,7 @@ add_library(cassandra $ $) -target_link_libraries(cassandra zlib) +target_link_libraries(cassandra ch_contrib::zlib) add_library(cassandra_static ALIAS cassandra) target_include_directories(cassandra PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${INCLUDE_DIRS}) target_compile_definitions(cassandra PRIVATE CASS_BUILDING) diff --git a/contrib/grpc-cmake/CMakeLists.txt b/contrib/grpc-cmake/CMakeLists.txt index b93968f62f9..ed8160ab2a1 100644 --- a/contrib/grpc-cmake/CMakeLists.txt +++ b/contrib/grpc-cmake/CMakeLists.txt @@ -10,12 +10,9 @@ set(_gRPC_RE2_INCLUDE_DIR "${RE2_INCLUDE_DIR}") set(_gRPC_RE2_LIBRARIES "${RE2_LIBRARY}") # Use zlib from ClickHouse contrib, not from gRPC third_party. -if(NOT ZLIB_INCLUDE_DIRS) - message(FATAL_ERROR " grpc: The location of the \"zlib\" library is unknown") -endif() set(gRPC_ZLIB_PROVIDER "clickhouse" CACHE STRING "" FORCE) -set(_gRPC_ZLIB_INCLUDE_DIR "${ZLIB_INCLUDE_DIRS}") -set(_gRPC_ZLIB_LIBRARIES "${ZLIB_LIBRARIES}") +set(_gRPC_ZLIB_INCLUDE_DIR "") +set(_gRPC_ZLIB_LIBRARIES ch_contrib::zlib) # Use protobuf from ClickHouse contrib, not from gRPC third_party. if(NOT Protobuf_INCLUDE_DIR OR NOT Protobuf_LIBRARY) diff --git a/contrib/librdkafka-cmake/CMakeLists.txt b/contrib/librdkafka-cmake/CMakeLists.txt index 1a2821991a2..9fb69a49e87 100644 --- a/contrib/librdkafka-cmake/CMakeLists.txt +++ b/contrib/librdkafka-cmake/CMakeLists.txt @@ -117,7 +117,7 @@ target_compile_options(rdkafka PRIVATE -fno-sanitize=undefined) target_include_directories(rdkafka SYSTEM PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") # for "librdkafka/rdkafka.h" target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR}) # Because weird logic with "include_next" is used. target_include_directories(rdkafka SYSTEM PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/auxdir") # for "../config.h" -target_link_libraries(rdkafka PRIVATE lz4 ${ZLIB_LIBRARIES} ch_contrib::zstd) +target_link_libraries(rdkafka PRIVATE lz4 ch_contrib::zlib ch_contrib::zstd) if(OPENSSL_SSL_LIBRARY AND OPENSSL_CRYPTO_LIBRARY) target_link_libraries(rdkafka PRIVATE ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY}) endif() diff --git a/contrib/libxml2-cmake/CMakeLists.txt b/contrib/libxml2-cmake/CMakeLists.txt index 8fda0399ea3..48bf87e71f6 100644 --- a/contrib/libxml2-cmake/CMakeLists.txt +++ b/contrib/libxml2-cmake/CMakeLists.txt @@ -52,11 +52,10 @@ set(SRCS ) add_library(libxml2 ${SRCS}) -target_link_libraries(libxml2 PRIVATE ${ZLIB_LIBRARIES}) +target_link_libraries(libxml2 PRIVATE ch_contrib::zlib) if(M_LIBRARY) target_link_libraries(libxml2 PRIVATE ${M_LIBRARY}) endif() target_include_directories(libxml2 PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/include") target_include_directories(libxml2 PUBLIC "${LIBXML2_SOURCE_DIR}/include") -target_include_directories(libxml2 SYSTEM BEFORE PRIVATE ${ZLIB_INCLUDE_DIR}) diff --git a/contrib/mariadb-connector-c-cmake/CMakeLists.txt b/contrib/mariadb-connector-c-cmake/CMakeLists.txt index 7c3f25cdf87..b4c163ac4d8 100644 --- a/contrib/mariadb-connector-c-cmake/CMakeLists.txt +++ b/contrib/mariadb-connector-c-cmake/CMakeLists.txt @@ -80,7 +80,7 @@ set(SIZEOF_SIZE_T 8) set(SOCKET_SIZE_TYPE socklen_t) -set(SYSTEM_LIBS ${SYSTEM_LIBS} zlib) +set(SYSTEM_LIBS ${SYSTEM_LIBS} ch_contrib::zlib) if(CMAKE_HAVE_PTHREAD_H) set(CMAKE_REQUIRED_INCLUDES pthread.h) @@ -222,10 +222,6 @@ if(ICONV_INCLUDE_DIR) endif() add_definitions(-DLIBICONV_PLUG) -if(ZLIB_FOUND AND WITH_EXTERNAL_ZLIB) - include_directories(${ZLIB_INCLUDE_DIR}) -endif() - if(WITH_DYNCOL) set(LIBMARIADB_SOURCES ${LIBMARIADB_SOURCES} ${CC_SOURCE_DIR}/libmariadb/mariadb_dyncol.c) endif() diff --git a/contrib/poco-cmake/Foundation/CMakeLists.txt b/contrib/poco-cmake/Foundation/CMakeLists.txt index 0c13d109344..e094ec329a7 100644 --- a/contrib/poco-cmake/Foundation/CMakeLists.txt +++ b/contrib/poco-cmake/Foundation/CMakeLists.txt @@ -223,7 +223,7 @@ if (USE_INTERNAL_POCO_LIBRARY) POCO_OS_FAMILY_UNIX ) target_include_directories (_poco_foundation SYSTEM PUBLIC "${LIBRARY_DIR}/Foundation/include") - target_link_libraries (_poco_foundation PRIVATE Poco::Foundation::PCRE ${ZLIB_LIBRARIES} lz4) + target_link_libraries (_poco_foundation PRIVATE Poco::Foundation::PCRE ch_contrib::zlib lz4) else () add_library (Poco::Foundation UNKNOWN IMPORTED GLOBAL) diff --git a/contrib/protobuf-cmake/CMakeLists.txt b/contrib/protobuf-cmake/CMakeLists.txt index 92eec444e44..55fde879d07 100644 --- a/contrib/protobuf-cmake/CMakeLists.txt +++ b/contrib/protobuf-cmake/CMakeLists.txt @@ -8,7 +8,6 @@ add_definitions(-DHAVE_PTHREAD) add_definitions(-DHAVE_ZLIB) include_directories( - ${ZLIB_INCLUDE_DIRECTORIES} ${protobuf_binary_dir} ${protobuf_source_dir}/src) @@ -114,7 +113,7 @@ if (ENABLE_FUZZING) target_compile_options(libprotobuf PRIVATE "-fsanitize-recover=all") endif() target_link_libraries(libprotobuf pthread) -target_link_libraries(libprotobuf ${ZLIB_LIBRARIES}) +target_link_libraries(libprotobuf ch_contrib::zlib) if(${CMAKE_SYSTEM_NAME} STREQUAL "Android") target_link_libraries(libprotobuf log) endif() diff --git a/contrib/rocksdb-cmake/CMakeLists.txt b/contrib/rocksdb-cmake/CMakeLists.txt index 749147136f4..e38572a9c19 100644 --- a/contrib/rocksdb-cmake/CMakeLists.txt +++ b/contrib/rocksdb-cmake/CMakeLists.txt @@ -51,7 +51,7 @@ else() if(WITH_ZLIB) add_definitions(-DZLIB) - list(APPEND THIRDPARTY_LIBS zlib) + list(APPEND THIRDPARTY_LIBS ch_contrib::zlib) endif() if(WITH_LZ4) diff --git a/contrib/zlib-ng-cmake/CMakeLists.txt b/contrib/zlib-ng-cmake/CMakeLists.txt index bf5bc0d7f1c..0b7564c6420 100644 --- a/contrib/zlib-ng-cmake/CMakeLists.txt +++ b/contrib/zlib-ng-cmake/CMakeLists.txt @@ -130,8 +130,9 @@ set(ZLIB_SRCS set(ZLIB_ALL_SRCS ${ZLIB_SRCS} ${ZLIB_ARCH_SRCS}) -add_library(zlib ${ZLIB_ALL_SRCS}) -add_library(zlibstatic ALIAS zlib) +add_library(_zlib ${ZLIB_ALL_SRCS}) +add_library(zlibstatic ALIAS _zlib) +add_library(ch_contrib::zlib ALIAS _zlib) # https://github.com/zlib-ng/zlib-ng/pull/733 # This is disabed by default @@ -153,9 +154,9 @@ configure_file(${SOURCE_DIR}/zlib.pc.cmakein ${ZLIB_PC} @ONLY) configure_file(${CMAKE_CURRENT_BINARY_DIR}/zconf.h.cmakein ${CMAKE_CURRENT_BINARY_DIR}/zconf.h @ONLY) # We should use same defines when including zlib.h as used when zlib compiled -target_compile_definitions (zlib PUBLIC ZLIB_COMPAT WITH_GZFILEOP) +target_compile_definitions (_zlib PUBLIC ZLIB_COMPAT WITH_GZFILEOP) if (ARCH_AMD64 OR ARCH_AARCH64) - target_compile_definitions (zlib PUBLIC X86_64 UNALIGNED_OK) + target_compile_definitions (_zlib PUBLIC X86_64 UNALIGNED_OK) endif () -target_include_directories(zlib SYSTEM PUBLIC ${SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) +target_include_directories(_zlib SYSTEM BEFORE PUBLIC ${SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e05bb14dbda..a91b7ddcdd8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -329,7 +329,7 @@ target_link_libraries(clickhouse_common_io boost::program_options boost::system ${CITYHASH_LIBRARIES} - ${ZLIB_LIBRARIES} + ch_contrib::zlib pcg_random Poco::Foundation ) diff --git a/src/Common/mysqlxx/CMakeLists.txt b/src/Common/mysqlxx/CMakeLists.txt index 76005651e61..8821834eef8 100644 --- a/src/Common/mysqlxx/CMakeLists.txt +++ b/src/Common/mysqlxx/CMakeLists.txt @@ -16,7 +16,7 @@ target_include_directories (mysqlxx PUBLIC .) target_link_libraries (mysqlxx clickhouse_common_io ${MYSQLCLIENT_LIBRARIES} - ${ZLIB_LIBRARIES} + ch_contrib::zlib ) if (ENABLE_TESTS) diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 120eac7bb5f..9ab839f9246 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -25,7 +25,7 @@ target_link_libraries(clickhouse_functions murmurhash PRIVATE - ${ZLIB_LIBRARIES} + ch_contrib::zlib boost::filesystem divide_impl ) diff --git a/src/IO/examples/CMakeLists.txt b/src/IO/examples/CMakeLists.txt index 0d335d66d27..d79aaa48d83 100644 --- a/src/IO/examples/CMakeLists.txt +++ b/src/IO/examples/CMakeLists.txt @@ -65,7 +65,7 @@ add_executable (parse_date_time_best_effort parse_date_time_best_effort.cpp) target_link_libraries (parse_date_time_best_effort PRIVATE clickhouse_common_io) add_executable (zlib_ng_bug zlib_ng_bug.cpp) -target_link_libraries (zlib_ng_bug PRIVATE ${ZLIB_LIBRARIES}) +target_link_libraries (zlib_ng_bug PRIVATE ch_contrib::zlib) add_executable (dragonbox_test dragonbox_test.cpp) target_link_libraries (dragonbox_test PRIVATE dragonbox_to_chars) From 6fadf4c66fd467d2f35da4370dbe1dd0c49de416 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 16 Jan 2022 13:44:23 +0300 Subject: [PATCH 123/403] Remove unbundled rocksdb support --- CMakeLists.txt | 1 - cmake/find/rocksdb.cmake | 63 ---------------------------- contrib/CMakeLists.txt | 4 +- contrib/rocksdb-cmake/CMakeLists.txt | 17 +++++--- src/CMakeLists.txt | 8 ++-- src/configure_config.cmake | 3 ++ 6 files changed, 20 insertions(+), 76 deletions(-) delete mode 100644 cmake/find/rocksdb.cmake create mode 100644 src/configure_config.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 7bd57a2927b..3ba9b7beedd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -528,7 +528,6 @@ include (cmake/find/fastops.cmake) include (cmake/find/odbc.cmake) include (cmake/find/nanodbc.cmake) include (cmake/find/sqlite.cmake) -include (cmake/find/rocksdb.cmake) include (cmake/find/libpqxx.cmake) include (cmake/find/nuraft.cmake) include (cmake/find/yaml-cpp.cmake) diff --git a/cmake/find/rocksdb.cmake b/cmake/find/rocksdb.cmake deleted file mode 100644 index d5f5d453abd..00000000000 --- a/cmake/find/rocksdb.cmake +++ /dev/null @@ -1,63 +0,0 @@ -if (OS_DARWIN AND ARCH_AARCH64) - set (ENABLE_ROCKSDB OFF CACHE INTERNAL "") -endif() - -option(ENABLE_ROCKSDB "Enable ROCKSDB" ${ENABLE_LIBRARIES}) - -if (NOT ENABLE_ROCKSDB) - if (USE_INTERNAL_ROCKSDB_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal rocksdb library with ENABLE_ROCKSDB=OFF") - endif() - return() -endif() - -option(USE_INTERNAL_ROCKSDB_LIBRARY "Set to FALSE to use system ROCKSDB library instead of bundled" ON) - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/rocksdb/include") - if (USE_INTERNAL_ROCKSDB_LIBRARY) - message (WARNING "submodule contrib is missing. to fix try run: \n git submodule update --init") - message(${RECONFIGURE_MESSAGE_LEVEL} "cannot find internal rocksdb") - endif() - set (MISSING_INTERNAL_ROCKSDB 1) -endif () - -if (NOT USE_INTERNAL_ROCKSDB_LIBRARY) - find_library (ROCKSDB_LIBRARY rocksdb) - find_path (ROCKSDB_INCLUDE_DIR NAMES rocksdb/db.h PATHS ${ROCKSDB_INCLUDE_PATHS}) - if (NOT ROCKSDB_LIBRARY OR NOT ROCKSDB_INCLUDE_DIR) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system rocksdb library") - endif() - - if (NOT SNAPPY_LIBRARY) - include(cmake/find/snappy.cmake) - endif() - - find_package(BZip2) - find_library(LZ4_LIBRARY lz4) - find_library(GFLAGS_LIBRARY gflags) - - if(SNAPPY_LIBRARY AND LZ4_LIBRARY AND BZIP2_FOUND AND GFLAGS_LIBRARY) - list (APPEND ROCKSDB_LIBRARY ${SNAPPY_LIBRARY}) - list (APPEND ROCKSDB_LIBRARY ${LZ4_LIBRARY}) - list (APPEND ROCKSDB_LIBRARY ${BZIP2_LIBRARY}) - list (APPEND ROCKSDB_LIBRARY ${GFLAGS_LIBRARY}) - else() - message (${RECONFIGURE_MESSAGE_LEVEL} - "Can't find system rocksdb: snappy=${SNAPPY_LIBRARY} ;" - " lz4=${LZ4_LIBRARY} ;" - " bz2=${BZIP2_LIBRARY} ;" - " gflags=${GFLAGS_LIBRARY} ;") - endif() -endif () - -if(ROCKSDB_LIBRARY AND ROCKSDB_INCLUDE_DIR) - set(USE_ROCKSDB 1) -elseif (NOT MISSING_INTERNAL_ROCKSDB) - set (USE_INTERNAL_ROCKSDB_LIBRARY 1) - - set (ROCKSDB_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/rocksdb/include") - set (ROCKSDB_LIBRARY "rocksdb") - set (USE_ROCKSDB 1) -endif () - -message (STATUS "Using ROCKSDB=${USE_ROCKSDB}: ${ROCKSDB_INCLUDE_DIR} : ${ROCKSDB_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 1a6d0c875bf..64813ddbb2e 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -58,6 +58,7 @@ add_subdirectory (poco-cmake) add_subdirectory (croaring-cmake) add_subdirectory (zstd-cmake) add_subdirectory (zlib-ng-cmake) +add_subdirectory (rocksdb-cmake) # TODO: refactor the contrib libraries below this comment. @@ -258,9 +259,6 @@ if (USE_KRB5) endif() endif() -if (USE_INTERNAL_ROCKSDB_LIBRARY) - add_subdirectory(rocksdb-cmake) -endif() if (USE_LIBPQXX) add_subdirectory (libpq-cmake) diff --git a/contrib/rocksdb-cmake/CMakeLists.txt b/contrib/rocksdb-cmake/CMakeLists.txt index e38572a9c19..a6cd981058c 100644 --- a/contrib/rocksdb-cmake/CMakeLists.txt +++ b/contrib/rocksdb-cmake/CMakeLists.txt @@ -1,3 +1,10 @@ +option (ENABLE_ROCKSDB "Enable rocksdb library" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_ROCKSDB) + message (STATUS "Not using rocksdb") + return() +endif() + ## this file is extracted from `contrib/rocksdb/CMakeLists.txt` set(ROCKSDB_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/rocksdb") list(APPEND CMAKE_MODULE_PATH "${ROCKSDB_SOURCE_DIR}/cmake/modules/") @@ -538,8 +545,8 @@ if(WITH_FOLLY_DISTRIBUTED_MUTEX) "${ROCKSDB_SOURCE_DIR}/third-party/folly/folly/synchronization/WaitOptions.cpp") endif() -set(ROCKSDB_STATIC_LIB rocksdb) - -add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES}) -target_link_libraries(${ROCKSDB_STATIC_LIB} PRIVATE - ${THIRDPARTY_LIBS} ${SYSTEM_LIBS}) +add_library(_rocksdb STATIC ${SOURCES}) +add_library(ch_contrib::rocksdb ALIAS _rocksdb) +target_link_libraries(_rocksdb PRIVATE ${THIRDPARTY_LIBS} ${SYSTEM_LIBS}) +# SYSTEM is required to overcome some issues +target_include_directories(_rocksdb SYSTEM BEFORE INTERFACE "${ROCKSDB_SOURCE_DIR}/include") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a91b7ddcdd8..921b9a61c6b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -23,6 +23,7 @@ set (CONFIG_COMMON "${CMAKE_CURRENT_BINARY_DIR}/Common/config.h") include (../cmake/version.cmake) message (STATUS "Will build ${VERSION_FULL} revision ${VERSION_REVISION} ${VERSION_OFFICIAL}") +include (configure_config.cmake) configure_file (Common/config.h.in ${CONFIG_COMMON}) configure_file (Common/config_version.h.in ${CONFIG_VERSION}) configure_file (Core/config_core.h.in "${CMAKE_CURRENT_BINARY_DIR}/Core/include/config_core.h") @@ -100,7 +101,7 @@ if (USE_LIBPQXX) add_headers_and_sources(dbms Storages/PostgreSQL) endif() -if (USE_ROCKSDB) +if (TARGET ch_contrib::rocksdb) add_headers_and_sources(dbms Storages/RocksDB) endif() @@ -510,9 +511,8 @@ if (USE_ORC) dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${ORC_INCLUDE_DIR} "${CMAKE_BINARY_DIR}/contrib/orc/c++/include") endif () -if (USE_ROCKSDB) - dbms_target_link_libraries(PUBLIC ${ROCKSDB_LIBRARY}) - dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${ROCKSDB_INCLUDE_DIR}) +if (TARGET ch_contrib::rocksdb) + dbms_target_link_libraries(PUBLIC ch_contrib::rocksdb) endif() if (USE_LIBPQXX) diff --git a/src/configure_config.cmake b/src/configure_config.cmake new file mode 100644 index 00000000000..9ddd6bee5db --- /dev/null +++ b/src/configure_config.cmake @@ -0,0 +1,3 @@ +if (TARGET ch_contrib::rocksdb) + set(USE_ROCKSDB 1) +endif() From a752e12df9fe4e278c588be9b25500bb01df2267 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 16 Jan 2022 13:55:08 +0300 Subject: [PATCH 124/403] Remove unbundled bzip2 support --- CMakeLists.txt | 1 - cmake/find/bzip2.cmake | 19 ------------------- cmake/find/parquet.cmake | 4 ++-- contrib/CMakeLists.txt | 5 +---- contrib/bzip2-cmake/CMakeLists.txt | 16 +++++++++++++--- src/CMakeLists.txt | 5 ++--- src/configure_config.cmake | 3 +++ 7 files changed, 21 insertions(+), 32 deletions(-) delete mode 100644 cmake/find/bzip2.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 3ba9b7beedd..29bb40365c0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -533,7 +533,6 @@ include (cmake/find/nuraft.cmake) include (cmake/find/yaml-cpp.cmake) include (cmake/find/s2geometry.cmake) include (cmake/find/nlp.cmake) -include (cmake/find/bzip2.cmake) include (cmake/find/filelog.cmake) if(NOT USE_INTERNAL_PARQUET_LIBRARY) diff --git a/cmake/find/bzip2.cmake b/cmake/find/bzip2.cmake deleted file mode 100644 index 5e6a6fb5841..00000000000 --- a/cmake/find/bzip2.cmake +++ /dev/null @@ -1,19 +0,0 @@ -option(ENABLE_BZIP2 "Enable bzip2 compression support" ${ENABLE_LIBRARIES}) - -if (NOT ENABLE_BZIP2) - message (STATUS "bzip2 compression disabled") - return() -endif() - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/bzip2/bzlib.h") - message (WARNING "submodule contrib/bzip2 is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal bzip2 library") - set (USE_NLP 0) - return() -endif () - -set (USE_BZIP2 1) -set (BZIP2_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/bzip2") -set (BZIP2_LIBRARY bzip2) - -message (STATUS "Using bzip2=${USE_BZIP2}: ${BZIP2_INCLUDE_DIR} : ${BZIP2_LIBRARY}") diff --git a/cmake/find/parquet.cmake b/cmake/find/parquet.cmake index 48c2bb7babb..08db7be7f83 100644 --- a/cmake/find/parquet.cmake +++ b/cmake/find/parquet.cmake @@ -86,7 +86,7 @@ if(NOT USE_INTERNAL_PARQUET_LIBRARY) add_library(imported_arrow_deps STATIC ${ARROW_OTHER_OBJS}) set(ARROW_LIBRARY ${ARROW_STATIC_LIB} - imported_arrow_deps ${THRIFT_LIBRARY} ${UTF8_PROC_LIBRARY} ${BZIP2_LIBRARIES} ${SNAPPY_LIBRARY}) + imported_arrow_deps ${THRIFT_LIBRARY} ${UTF8_PROC_LIBRARY} bzip2 ${SNAPPY_LIBRARY}) else() message(WARNING "Using external static Arrow does not always work. " "Could not find arrow_bundled_dependencies.a. If compilation fails, " @@ -100,7 +100,7 @@ if(NOT USE_INTERNAL_PARQUET_LIBRARY) set(PARQUET_LIBRARY ${PARQUET_SHARED_LIB}) endif() - if(ARROW_INCLUDE_DIR AND ARROW_LIBRARY AND PARQUET_INCLUDE_DIR AND PARQUET_LIBRARY AND THRIFT_LIBRARY AND UTF8_PROC_LIBRARY AND BZIP2_FOUND) + if(ARROW_INCLUDE_DIR AND ARROW_LIBRARY AND PARQUET_INCLUDE_DIR AND PARQUET_LIBRARY AND THRIFT_LIBRARY AND UTF8_PROC_LIBRARY) set(USE_PARQUET 1) set(EXTERNAL_PARQUET_FOUND 1) else() diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 64813ddbb2e..88f7800fe39 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -59,6 +59,7 @@ add_subdirectory (croaring-cmake) add_subdirectory (zstd-cmake) add_subdirectory (zlib-ng-cmake) add_subdirectory (rocksdb-cmake) +add_subdirectory (bzip2-cmake) # TODO: refactor the contrib libraries below this comment. @@ -277,10 +278,6 @@ if (USE_NLP) add_subdirectory(lemmagen-c-cmake) endif() -if (USE_BZIP2) - add_subdirectory(bzip2-cmake) -endif() - if (USE_SQLITE) add_subdirectory(sqlite-cmake) endif() diff --git a/contrib/bzip2-cmake/CMakeLists.txt b/contrib/bzip2-cmake/CMakeLists.txt index a9d2efa43c1..2e01a624000 100644 --- a/contrib/bzip2-cmake/CMakeLists.txt +++ b/contrib/bzip2-cmake/CMakeLists.txt @@ -1,3 +1,9 @@ +option(ENABLE_BZIP2 "Enable bzip2 compression support" ${ENABLE_LIBRARIES}) +if (NOT ENABLE_BZIP2) + message (STATUS "bzip2 compression disabled") + return() +endif() + set(BZIP2_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/bzip2") set(BZIP2_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/bzip2") @@ -18,6 +24,10 @@ configure_file ( "${BZIP2_BINARY_DIR}/bz_version.h" ) -add_library(bzip2 ${SRCS}) - -target_include_directories(bzip2 PUBLIC "${BZIP2_SOURCE_DIR}" "${BZIP2_BINARY_DIR}") +add_library(_bzip2 ${SRCS}) +add_library(ch_contrib::bzip2 ALIAS _bzip2) +# To avoid -Wreserved-id-macro we use SYSTEM: +# +# clickhouse/contrib/bzip2/bzlib.h:23:9: error: macro name is a reserved identifier [-Werror,-Wreserved-id-macro] +# #define _BZLIB_H +target_include_directories(_bzip2 SYSTEM BEFORE PUBLIC "${BZIP2_SOURCE_DIR}" "${BZIP2_BINARY_DIR}") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 921b9a61c6b..6c871a2a599 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -534,9 +534,8 @@ if (USE_NLP) dbms_target_link_libraries (PUBLIC lemmagen) endif() -if (USE_BZIP2) - target_link_libraries (clickhouse_common_io PRIVATE ${BZIP2_LIBRARY}) - target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${BZIP2_INCLUDE_DIR}) +if (TARGET ch_contrib::bzip2) + target_link_libraries (clickhouse_common_io PRIVATE ch_contrib::bzip2) endif() if(USE_SIMDJSON) diff --git a/src/configure_config.cmake b/src/configure_config.cmake index 9ddd6bee5db..2355c3fc802 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -1,3 +1,6 @@ if (TARGET ch_contrib::rocksdb) set(USE_ROCKSDB 1) endif() +if (TARGET ch_contrib::bzip2) + set(USE_BZIP2 1) +endif() From b4ad324a880f6677c49d13e7bab4aac12c469d84 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 16 Jan 2022 21:02:32 +0300 Subject: [PATCH 125/403] Remove unbundled parquet/arrow support --- CMakeLists.txt | 6 - cmake/Modules/FindArrow.cmake | 433 ----------------------------- cmake/Modules/FindParquet.cmake | 132 --------- cmake/find/orc.cmake | 11 +- cmake/find/parquet.cmake | 168 ----------- contrib/CMakeLists.txt | 25 +- contrib/arrow-cmake/CMakeLists.txt | 110 ++++++-- src/CMakeLists.txt | 10 +- src/Formats/CMakeLists.txt | 4 + 9 files changed, 95 insertions(+), 804 deletions(-) delete mode 100644 cmake/Modules/FindArrow.cmake delete mode 100644 cmake/Modules/FindParquet.cmake delete mode 100644 cmake/find/parquet.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 29bb40365c0..4cbb921b190 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -520,7 +520,6 @@ include (cmake/find/curl.cmake) include (cmake/find/s3.cmake) include (cmake/find/blob_storage.cmake) include (cmake/find/base64.cmake) -include (cmake/find/parquet.cmake) # uses protobuf and thrift include (cmake/find/simdjson.cmake) include (cmake/find/fast_float.cmake) include (cmake/find/rapidjson.cmake) @@ -534,12 +533,7 @@ include (cmake/find/yaml-cpp.cmake) include (cmake/find/s2geometry.cmake) include (cmake/find/nlp.cmake) include (cmake/find/filelog.cmake) - -if(NOT USE_INTERNAL_PARQUET_LIBRARY) - set (ENABLE_ORC OFF CACHE INTERNAL "") -endif() include (cmake/find/orc.cmake) - include (cmake/find/avro.cmake) include (cmake/find/msgpack.cmake) include (cmake/find/cassandra.cmake) diff --git a/cmake/Modules/FindArrow.cmake b/cmake/Modules/FindArrow.cmake deleted file mode 100644 index 5bd111de1e3..00000000000 --- a/cmake/Modules/FindArrow.cmake +++ /dev/null @@ -1,433 +0,0 @@ -# https://github.com/apache/arrow/blob/master/cpp/cmake_modules/FindArrow.cmake - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# - Find Arrow (arrow/api.h, libarrow.a, libarrow.so) -# This module defines -# ARROW_FOUND, whether Arrow has been found -# ARROW_FULL_SO_VERSION, full shared object version of found Arrow "100.0.0" -# ARROW_IMPORT_LIB, path to libarrow's import library (Windows only) -# ARROW_INCLUDE_DIR, directory containing headers -# ARROW_LIBS, deprecated. Use ARROW_LIB_DIR instead -# ARROW_LIB_DIR, directory containing Arrow libraries -# ARROW_SHARED_IMP_LIB, deprecated. Use ARROW_IMPORT_LIB instead -# ARROW_SHARED_LIB, path to libarrow's shared library -# ARROW_SO_VERSION, shared object version of found Arrow such as "100" -# ARROW_STATIC_LIB, path to libarrow.a -# ARROW_VERSION, version of found Arrow -# ARROW_VERSION_MAJOR, major version of found Arrow -# ARROW_VERSION_MINOR, minor version of found Arrow -# ARROW_VERSION_PATCH, patch version of found Arrow - -if(DEFINED ARROW_FOUND) - return() -endif() - -include(FindPkgConfig) -include(FindPackageHandleStandardArgs) - -set(ARROW_SEARCH_LIB_PATH_SUFFIXES) -if(CMAKE_LIBRARY_ARCHITECTURE) - list(APPEND ARROW_SEARCH_LIB_PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}") -endif() -list(APPEND ARROW_SEARCH_LIB_PATH_SUFFIXES - "lib64" - "lib32" - "lib" - "bin") -set(ARROW_CONFIG_SUFFIXES - "_RELEASE" - "_RELWITHDEBINFO" - "_MINSIZEREL" - "_DEBUG" - "") -if(CMAKE_BUILD_TYPE) - string(TOUPPER ${CMAKE_BUILD_TYPE} ARROW_CONFIG_SUFFIX_PREFERRED) - set(ARROW_CONFIG_SUFFIX_PREFERRED "_${ARROW_CONFIG_SUFFIX_PREFERRED}") - list(INSERT ARROW_CONFIG_SUFFIXES 0 "${ARROW_CONFIG_SUFFIX_PREFERRED}") -endif() - -if(NOT DEFINED ARROW_MSVC_STATIC_LIB_SUFFIX) - if(MSVC) - set(ARROW_MSVC_STATIC_LIB_SUFFIX "_static") - else() - set(ARROW_MSVC_STATIC_LIB_SUFFIX "") - endif() -endif() - -# Internal function. -# -# Set shared library name for ${base_name} to ${output_variable}. -# -# Example: -# arrow_build_shared_library_name(ARROW_SHARED_LIBRARY_NAME arrow) -# # -> ARROW_SHARED_LIBRARY_NAME=libarrow.so on Linux -# # -> ARROW_SHARED_LIBRARY_NAME=libarrow.dylib on macOS -# # -> ARROW_SHARED_LIBRARY_NAME=arrow.dll with MSVC on Windows -# # -> ARROW_SHARED_LIBRARY_NAME=libarrow.dll with MinGW on Windows -function(arrow_build_shared_library_name output_variable base_name) - set(${output_variable} - "${CMAKE_SHARED_LIBRARY_PREFIX}${base_name}${CMAKE_SHARED_LIBRARY_SUFFIX}" - PARENT_SCOPE) -endfunction() - -# Internal function. -# -# Set import library name for ${base_name} to ${output_variable}. -# This is useful only for MSVC build. Import library is used only -# with MSVC build. -# -# Example: -# arrow_build_import_library_name(ARROW_IMPORT_LIBRARY_NAME arrow) -# # -> ARROW_IMPORT_LIBRARY_NAME=arrow on Linux (meaningless) -# # -> ARROW_IMPORT_LIBRARY_NAME=arrow on macOS (meaningless) -# # -> ARROW_IMPORT_LIBRARY_NAME=arrow.lib with MSVC on Windows -# # -> ARROW_IMPORT_LIBRARY_NAME=libarrow.dll.a with MinGW on Windows -function(arrow_build_import_library_name output_variable base_name) - set(${output_variable} - "${CMAKE_IMPORT_LIBRARY_PREFIX}${base_name}${CMAKE_IMPORT_LIBRARY_SUFFIX}" - PARENT_SCOPE) -endfunction() - -# Internal function. -# -# Set static library name for ${base_name} to ${output_variable}. -# -# Example: -# arrow_build_static_library_name(ARROW_STATIC_LIBRARY_NAME arrow) -# # -> ARROW_STATIC_LIBRARY_NAME=libarrow.a on Linux -# # -> ARROW_STATIC_LIBRARY_NAME=libarrow.a on macOS -# # -> ARROW_STATIC_LIBRARY_NAME=arrow.lib with MSVC on Windows -# # -> ARROW_STATIC_LIBRARY_NAME=libarrow.dll.a with MinGW on Windows -function(arrow_build_static_library_name output_variable base_name) - set( - ${output_variable} - "${CMAKE_STATIC_LIBRARY_PREFIX}${base_name}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}" - PARENT_SCOPE) -endfunction() - -# Internal function. -# -# Set macro value for ${macro_name} in ${header_content} to ${output_variable}. -# -# Example: -# arrow_extract_macro_value(version_major -# "ARROW_VERSION_MAJOR" -# "#define ARROW_VERSION_MAJOR 1.0.0") -# # -> version_major=1.0.0 -function(arrow_extract_macro_value output_variable macro_name header_content) - string(REGEX MATCH "#define +${macro_name} +[^\r\n]+" macro_definition - "${header_content}") - string(REGEX - REPLACE "^#define +${macro_name} +(.+)$" "\\1" macro_value "${macro_definition}") - set(${output_variable} "${macro_value}" PARENT_SCOPE) -endfunction() - -# Internal macro only for arrow_find_package. -# -# Find package in HOME. -macro(arrow_find_package_home) - find_path(${prefix}_include_dir "${header_path}" - PATHS "${home}" - PATH_SUFFIXES "include" - NO_DEFAULT_PATH) - set(include_dir "${${prefix}_include_dir}") - set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE) - - if(MSVC) - set(CMAKE_SHARED_LIBRARY_SUFFIXES_ORIGINAL ${CMAKE_FIND_LIBRARY_SUFFIXES}) - # .dll isn't found by find_library with MSVC because .dll isn't included in - # CMAKE_FIND_LIBRARY_SUFFIXES. - list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES "${CMAKE_SHARED_LIBRARY_SUFFIX}") - endif() - find_library(${prefix}_shared_lib - NAMES "${shared_lib_name}" - PATHS "${home}" - PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES} - NO_DEFAULT_PATH) - if(MSVC) - set(CMAKE_SHARED_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_ORIGINAL}) - endif() - set(shared_lib "${${prefix}_shared_lib}") - set(${prefix}_SHARED_LIB "${shared_lib}" PARENT_SCOPE) - if(shared_lib) - add_library(${target_shared} SHARED IMPORTED) - set_target_properties(${target_shared} PROPERTIES IMPORTED_LOCATION "${shared_lib}") - if(include_dir) - set_target_properties(${target_shared} - PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}") - endif() - find_library(${prefix}_import_lib - NAMES "${import_lib_name}" - PATHS "${home}" - PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES} - NO_DEFAULT_PATH) - set(import_lib "${${prefix}_import_lib}") - set(${prefix}_IMPORT_LIB "${import_lib}" PARENT_SCOPE) - if(import_lib) - set_target_properties(${target_shared} PROPERTIES IMPORTED_IMPLIB "${import_lib}") - endif() - endif() - - find_library(${prefix}_static_lib - NAMES "${static_lib_name}" - PATHS "${home}" - PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES} - NO_DEFAULT_PATH) - set(static_lib "${${prefix}_static_lib}") - set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE) - if(static_lib) - add_library(${target_static} STATIC IMPORTED) - set_target_properties(${target_static} PROPERTIES IMPORTED_LOCATION "${static_lib}") - if(include_dir) - set_target_properties(${target_static} - PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}") - endif() - endif() -endmacro() - -# Internal macro only for arrow_find_package. -# -# Find package by CMake package configuration. -macro(arrow_find_package_cmake_package_configuration) - find_package(${cmake_package_name} CONFIG) - if(${cmake_package_name}_FOUND) - set(${prefix}_USE_CMAKE_PACKAGE_CONFIG TRUE PARENT_SCOPE) - if(TARGET ${target_shared}) - foreach(suffix ${ARROW_CONFIG_SUFFIXES}) - get_target_property(shared_lib ${target_shared} IMPORTED_LOCATION${suffix}) - if(shared_lib) - # Remove shared library version: - # libarrow.so.100.0.0 -> libarrow.so - # Because ARROW_HOME and pkg-config approaches don't add - # shared library version. - string(REGEX - REPLACE "(${CMAKE_SHARED_LIBRARY_SUFFIX})[.0-9]+$" "\\1" shared_lib - "${shared_lib}") - set(${prefix}_SHARED_LIB "${shared_lib}" PARENT_SCOPE) - break() - endif() - endforeach() - endif() - if(TARGET ${target_static}) - foreach(suffix ${ARROW_CONFIG_SUFFIXES}) - get_target_property(static_lib ${target_static} IMPORTED_LOCATION${suffix}) - if(static_lib) - set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE) - break() - endif() - endforeach() - endif() - endif() -endmacro() - -# Internal macro only for arrow_find_package. -# -# Find package by pkg-config. -macro(arrow_find_package_pkg_config) - pkg_check_modules(${prefix}_PC ${pkg_config_name}) - if(${prefix}_PC_FOUND) - set(${prefix}_USE_PKG_CONFIG TRUE PARENT_SCOPE) - - set(include_dir "${${prefix}_PC_INCLUDEDIR}") - set(lib_dir "${${prefix}_PC_LIBDIR}") - set(shared_lib_paths "${${prefix}_PC_LINK_LIBRARIES}") - # Use the first shared library path as the IMPORTED_LOCATION - # for ${target_shared}. This assumes that the first shared library - # path is the shared library path for this module. - list(GET shared_lib_paths 0 first_shared_lib_path) - # Use the rest shared library paths as the INTERFACE_LINK_LIBRARIES - # for ${target_shared}. This assumes that the rest shared library - # paths are dependency library paths for this module. - list(LENGTH shared_lib_paths n_shared_lib_paths) - if(n_shared_lib_paths LESS_EQUAL 1) - set(rest_shared_lib_paths) - else() - list(SUBLIST - shared_lib_paths - 1 - -1 - rest_shared_lib_paths) - endif() - - set(${prefix}_VERSION "${${prefix}_PC_VERSION}" PARENT_SCOPE) - set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE) - set(${prefix}_SHARED_LIB "${first_shared_lib_path}" PARENT_SCOPE) - - add_library(${target_shared} SHARED IMPORTED) - set_target_properties(${target_shared} - PROPERTIES INTERFACE_INCLUDE_DIRECTORIES - "${include_dir}" - INTERFACE_LINK_LIBRARIES - "${rest_shared_lib_paths}" - IMPORTED_LOCATION - "${first_shared_lib_path}") - get_target_property(shared_lib ${target_shared} IMPORTED_LOCATION) - - find_library(${prefix}_static_lib - NAMES "${static_lib_name}" - PATHS "${lib_dir}" - NO_DEFAULT_PATH) - set(static_lib "${${prefix}_static_lib}") - set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE) - if(static_lib) - add_library(${target_static} STATIC IMPORTED) - set_target_properties(${target_static} - PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}" - IMPORTED_LOCATION "${static_lib}") - endif() - endif() -endmacro() - -function(arrow_find_package - prefix - home - base_name - header_path - cmake_package_name - pkg_config_name) - arrow_build_shared_library_name(shared_lib_name ${base_name}) - arrow_build_import_library_name(import_lib_name ${base_name}) - arrow_build_static_library_name(static_lib_name ${base_name}) - - set(target_shared ${base_name}_shared) - set(target_static ${base_name}_static) - - if(home) - arrow_find_package_home() - set(${prefix}_FIND_APPROACH "HOME: ${home}" PARENT_SCOPE) - else() - arrow_find_package_cmake_package_configuration() - if(${cmake_package_name}_FOUND) - set(${prefix}_FIND_APPROACH - "CMake package configuration: ${cmake_package_name}" - PARENT_SCOPE) - else() - arrow_find_package_pkg_config() - set(${prefix}_FIND_APPROACH "pkg-config: ${pkg_config_name}" PARENT_SCOPE) - endif() - endif() - - if(NOT include_dir) - if(TARGET ${target_shared}) - get_target_property(include_dir ${target_shared} INTERFACE_INCLUDE_DIRECTORIES) - elseif(TARGET ${target_static}) - get_target_property(include_dir ${target_static} INTERFACE_INCLUDE_DIRECTORIES) - endif() - endif() - if(include_dir) - set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE) - endif() - - if(shared_lib) - get_filename_component(lib_dir "${shared_lib}" DIRECTORY) - elseif(static_lib) - get_filename_component(lib_dir "${static_lib}" DIRECTORY) - else() - set(lib_dir NOTFOUND) - endif() - set(${prefix}_LIB_DIR "${lib_dir}" PARENT_SCOPE) - # For backward compatibility - set(${prefix}_LIBS "${lib_dir}" PARENT_SCOPE) -endfunction() - -if(NOT "$ENV{ARROW_HOME}" STREQUAL "") - file(TO_CMAKE_PATH "$ENV{ARROW_HOME}" ARROW_HOME) -endif() -arrow_find_package(ARROW - "${ARROW_HOME}" - arrow - arrow/api.h - Arrow - arrow) - -if(ARROW_HOME) - if(ARROW_INCLUDE_DIR) - file(READ "${ARROW_INCLUDE_DIR}/arrow/util/config.h" ARROW_CONFIG_H_CONTENT) - arrow_extract_macro_value(ARROW_VERSION_MAJOR "ARROW_VERSION_MAJOR" - "${ARROW_CONFIG_H_CONTENT}") - arrow_extract_macro_value(ARROW_VERSION_MINOR "ARROW_VERSION_MINOR" - "${ARROW_CONFIG_H_CONTENT}") - arrow_extract_macro_value(ARROW_VERSION_PATCH "ARROW_VERSION_PATCH" - "${ARROW_CONFIG_H_CONTENT}") - if("${ARROW_VERSION_MAJOR}" STREQUAL "" - OR "${ARROW_VERSION_MINOR}" STREQUAL "" - OR "${ARROW_VERSION_PATCH}" STREQUAL "") - set(ARROW_VERSION "0.0.0") - else() - set(ARROW_VERSION - "${ARROW_VERSION_MAJOR}.${ARROW_VERSION_MINOR}.${ARROW_VERSION_PATCH}") - endif() - - arrow_extract_macro_value(ARROW_SO_VERSION_QUOTED "ARROW_SO_VERSION" - "${ARROW_CONFIG_H_CONTENT}") - string(REGEX REPLACE "^\"(.+)\"$" "\\1" ARROW_SO_VERSION "${ARROW_SO_VERSION_QUOTED}") - arrow_extract_macro_value(ARROW_FULL_SO_VERSION_QUOTED "ARROW_FULL_SO_VERSION" - "${ARROW_CONFIG_H_CONTENT}") - string(REGEX - REPLACE "^\"(.+)\"$" "\\1" ARROW_FULL_SO_VERSION - "${ARROW_FULL_SO_VERSION_QUOTED}") - endif() -else() - if(ARROW_USE_CMAKE_PACKAGE_CONFIG) - find_package(Arrow CONFIG) - elseif(ARROW_USE_PKG_CONFIG) - pkg_get_variable(ARROW_SO_VERSION arrow so_version) - pkg_get_variable(ARROW_FULL_SO_VERSION arrow full_so_version) - endif() -endif() - -set(ARROW_ABI_VERSION ${ARROW_SO_VERSION}) - -mark_as_advanced(ARROW_ABI_VERSION - ARROW_CONFIG_SUFFIXES - ARROW_FULL_SO_VERSION - ARROW_IMPORT_LIB - ARROW_INCLUDE_DIR - ARROW_LIBS - ARROW_LIB_DIR - ARROW_SEARCH_LIB_PATH_SUFFIXES - ARROW_SHARED_IMP_LIB - ARROW_SHARED_LIB - ARROW_SO_VERSION - ARROW_STATIC_LIB - ARROW_VERSION - ARROW_VERSION_MAJOR - ARROW_VERSION_MINOR - ARROW_VERSION_PATCH) - -find_package_handle_standard_args(Arrow REQUIRED_VARS - # The first required variable is shown - # in the found message. So this list is - # not sorted alphabetically. - ARROW_INCLUDE_DIR - ARROW_LIB_DIR - ARROW_FULL_SO_VERSION - ARROW_SO_VERSION - VERSION_VAR - ARROW_VERSION) -set(ARROW_FOUND ${Arrow_FOUND}) - -if(Arrow_FOUND AND NOT Arrow_FIND_QUIETLY) - message(STATUS "Arrow version: ${ARROW_VERSION} (${ARROW_FIND_APPROACH})") - message(STATUS "Arrow SO and ABI version: ${ARROW_SO_VERSION}") - message(STATUS "Arrow full SO version: ${ARROW_FULL_SO_VERSION}") - message(STATUS "Found the Arrow core shared library: ${ARROW_SHARED_LIB}") - message(STATUS "Found the Arrow core import library: ${ARROW_IMPORT_LIB}") - message(STATUS "Found the Arrow core static library: ${ARROW_STATIC_LIB}") -endif() diff --git a/cmake/Modules/FindParquet.cmake b/cmake/Modules/FindParquet.cmake deleted file mode 100644 index 654020c0b87..00000000000 --- a/cmake/Modules/FindParquet.cmake +++ /dev/null @@ -1,132 +0,0 @@ -# https://github.com/apache/arrow/blob/master/cpp/cmake_modules/FindParquet.cmake - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# - Find Parquet (parquet/api/reader.h, libparquet.a, libparquet.so) -# -# This module requires Arrow from which it uses -# arrow_find_package() -# -# This module defines -# PARQUET_FOUND, whether Parquet has been found -# PARQUET_IMPORT_LIB, path to libparquet's import library (Windows only) -# PARQUET_INCLUDE_DIR, directory containing headers -# PARQUET_LIBS, deprecated. Use PARQUET_LIB_DIR instead -# PARQUET_LIB_DIR, directory containing Parquet libraries -# PARQUET_SHARED_IMP_LIB, deprecated. Use PARQUET_IMPORT_LIB instead -# PARQUET_SHARED_LIB, path to libparquet's shared library -# PARQUET_SO_VERSION, shared object version of found Parquet such as "100" -# PARQUET_STATIC_LIB, path to libparquet.a - -if(DEFINED PARQUET_FOUND) - return() -endif() - -set(find_package_arguments) -if(${CMAKE_FIND_PACKAGE_NAME}_FIND_VERSION) - list(APPEND find_package_arguments "${${CMAKE_FIND_PACKAGE_NAME}_FIND_VERSION}") -endif() -if(${CMAKE_FIND_PACKAGE_NAME}_FIND_REQUIRED) - list(APPEND find_package_arguments REQUIRED) -endif() -if(${CMAKE_FIND_PACKAGE_NAME}_FIND_QUIETLY) - list(APPEND find_package_arguments QUIET) -endif() -find_package(Arrow ${find_package_arguments}) - -if(NOT "$ENV{PARQUET_HOME}" STREQUAL "") - file(TO_CMAKE_PATH "$ENV{PARQUET_HOME}" PARQUET_HOME) -endif() - -if((NOT PARQUET_HOME) AND ARROW_HOME) - set(PARQUET_HOME ${ARROW_HOME}) -endif() - -if(ARROW_FOUND) - arrow_find_package(PARQUET - "${PARQUET_HOME}" - parquet - parquet/api/reader.h - Parquet - parquet) - if(PARQUET_HOME) - if(PARQUET_INCLUDE_DIR) - file(READ "${PARQUET_INCLUDE_DIR}/parquet/parquet_version.h" - PARQUET_VERSION_H_CONTENT) - arrow_extract_macro_value(PARQUET_VERSION_MAJOR "PARQUET_VERSION_MAJOR" - "${PARQUET_VERSION_H_CONTENT}") - arrow_extract_macro_value(PARQUET_VERSION_MINOR "PARQUET_VERSION_MINOR" - "${PARQUET_VERSION_H_CONTENT}") - arrow_extract_macro_value(PARQUET_VERSION_PATCH "PARQUET_VERSION_PATCH" - "${PARQUET_VERSION_H_CONTENT}") - if("${PARQUET_VERSION_MAJOR}" STREQUAL "" - OR "${PARQUET_VERSION_MINOR}" STREQUAL "" - OR "${PARQUET_VERSION_PATCH}" STREQUAL "") - set(PARQUET_VERSION "0.0.0") - else() - set(PARQUET_VERSION - "${PARQUET_VERSION_MAJOR}.${PARQUET_VERSION_MINOR}.${PARQUET_VERSION_PATCH}") - endif() - - arrow_extract_macro_value(PARQUET_SO_VERSION_QUOTED "PARQUET_SO_VERSION" - "${PARQUET_VERSION_H_CONTENT}") - string(REGEX - REPLACE "^\"(.+)\"$" "\\1" PARQUET_SO_VERSION "${PARQUET_SO_VERSION_QUOTED}") - arrow_extract_macro_value(PARQUET_FULL_SO_VERSION_QUOTED "PARQUET_FULL_SO_VERSION" - "${PARQUET_VERSION_H_CONTENT}") - string(REGEX - REPLACE "^\"(.+)\"$" "\\1" PARQUET_FULL_SO_VERSION - "${PARQUET_FULL_SO_VERSION_QUOTED}") - endif() - else() - if(PARQUET_USE_CMAKE_PACKAGE_CONFIG) - find_package(Parquet CONFIG) - elseif(PARQUET_USE_PKG_CONFIG) - pkg_get_variable(PARQUET_SO_VERSION parquet so_version) - pkg_get_variable(PARQUET_FULL_SO_VERSION parquet full_so_version) - endif() - endif() - set(PARQUET_ABI_VERSION "${PARQUET_SO_VERSION}") -endif() - -mark_as_advanced(PARQUET_ABI_VERSION - PARQUET_IMPORT_LIB - PARQUET_INCLUDE_DIR - PARQUET_LIBS - PARQUET_LIB_DIR - PARQUET_SHARED_IMP_LIB - PARQUET_SHARED_LIB - PARQUET_SO_VERSION - PARQUET_STATIC_LIB - PARQUET_VERSION) - -find_package_handle_standard_args(Parquet - REQUIRED_VARS - PARQUET_INCLUDE_DIR - PARQUET_LIB_DIR - PARQUET_SO_VERSION - VERSION_VAR - PARQUET_VERSION) -set(PARQUET_FOUND ${Parquet_FOUND}) - -if(Parquet_FOUND AND NOT Parquet_FIND_QUIETLY) - message(STATUS "Parquet version: ${PARQUET_VERSION} (${PARQUET_FIND_APPROACH})") - message(STATUS "Found the Parquet shared library: ${PARQUET_SHARED_LIB}") - message(STATUS "Found the Parquet import library: ${PARQUET_IMPORT_LIB}") - message(STATUS "Found the Parquet static library: ${PARQUET_STATIC_LIB}") -endif() diff --git a/cmake/find/orc.cmake b/cmake/find/orc.cmake index a5c3f57468a..c96c3ea5d5a 100644 --- a/cmake/find/orc.cmake +++ b/cmake/find/orc.cmake @@ -7,12 +7,7 @@ if(NOT ENABLE_ORC) return() endif() -if (USE_INTERNAL_PARQUET_LIBRARY) - option(USE_INTERNAL_ORC_LIBRARY "Set to FALSE to use system ORC instead of bundled (experimental set to OFF on your own risk)" - ON) -elseif(USE_INTERNAL_ORC_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Currently internal ORC can be build only with bundled Parquet") -endif() +option(USE_INTERNAL_ORC_LIBRARY "Set to FALSE to use system ORC instead of bundled (experimental set to OFF on your own risk)" ON) include(cmake/find/snappy.cmake) @@ -42,7 +37,7 @@ endif () if (ORC_LIBRARY AND ORC_INCLUDE_DIR) set(USE_ORC 1) -elseif(NOT MISSING_INTERNAL_ORC_LIBRARY AND ARROW_LIBRARY AND SNAPPY_LIBRARY) # (LIBGSASL_LIBRARY AND LIBXML2_LIBRARY) +elseif(NOT MISSING_INTERNAL_ORC_LIBRARY AND SNAPPY_LIBRARY) # (LIBGSASL_LIBRARY AND LIBXML2_LIBRARY) set(ORC_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include") set(ORC_LIBRARY orc) set(USE_ORC 1) @@ -50,7 +45,7 @@ elseif(NOT MISSING_INTERNAL_ORC_LIBRARY AND ARROW_LIBRARY AND SNAPPY_LIBRARY) # else() message (${RECONFIGURE_MESSAGE_LEVEL} "Can't enable ORC support - missing dependencies. Missing internal orc=${MISSING_INTERNAL_ORC_LIBRARY}. " - "arrow=${ARROW_LIBRARY} snappy=${SNAPPY_LIBRARY}") + "snappy=${SNAPPY_LIBRARY}") set(USE_INTERNAL_ORC_LIBRARY 0) endif() diff --git a/cmake/find/parquet.cmake b/cmake/find/parquet.cmake deleted file mode 100644 index 08db7be7f83..00000000000 --- a/cmake/find/parquet.cmake +++ /dev/null @@ -1,168 +0,0 @@ -if (Protobuf_PROTOC_EXECUTABLE) - option (ENABLE_PARQUET "Enable parquet" ${ENABLE_LIBRARIES}) -elseif(ENABLE_PARQUET OR USE_INTERNAL_PARQUET_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use parquet without protoc executable") -endif() - -if (NOT ENABLE_PARQUET) - if(USE_INTERNAL_PARQUET_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal parquet with ENABLE_PARQUET=OFF") - endif() - message(STATUS "Building without Parquet support") - return() -endif() - -if (NOT OS_FREEBSD) # Freebsd: ../contrib/arrow/cpp/src/arrow/util/bit-util.h:27:10: fatal error: endian.h: No such file or directory - option(USE_INTERNAL_PARQUET_LIBRARY "Set to FALSE to use system parquet library instead of bundled" ON) -elseif(USE_INTERNAL_PARQUET_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Using internal parquet is not supported on freebsd") -endif() - -if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/CMakeLists.txt") - if(USE_INTERNAL_PARQUET_LIBRARY) - message(WARNING "submodule contrib/arrow (required for Parquet) is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal parquet library") - set(USE_INTERNAL_PARQUET_LIBRARY 0) - endif() - set(MISSING_INTERNAL_PARQUET_LIBRARY 1) -endif() - -if (NOT SNAPPY_LIBRARY) - include(cmake/find/snappy.cmake) -endif() - -if(NOT USE_INTERNAL_PARQUET_LIBRARY) - find_package(Arrow) - find_package(Parquet) - find_library(UTF8_PROC_LIBRARY utf8proc) - find_package(BZip2) - - if(USE_STATIC_LIBRARIES) - find_library(ARROW_DEPS_LIBRARY arrow_bundled_dependencies) - - if (ARROW_DEPS_LIBRARY) - set(ARROW_IMPORT_OBJ_DIR "${CMAKE_CURRENT_BINARY_DIR}/contrib/arrow-cmake/imported-objects") - set(ARROW_OTHER_OBJS - "${ARROW_IMPORT_OBJ_DIR}/jemalloc.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/arena.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/background_thread.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/base.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/bin.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/bitmap.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/ckh.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/ctl.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/div.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/extent.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/extent_dss.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/extent_mmap.pic.o" - # skip hash - "${ARROW_IMPORT_OBJ_DIR}/hook.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/large.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/log.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/malloc_io.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/mutex.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/mutex_pool.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/nstime.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/pages.pic.o" - # skip prng - "${ARROW_IMPORT_OBJ_DIR}/prof.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/rtree.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/stats.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/sc.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/sz.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/tcache.pic.o" - # skip ticker - "${ARROW_IMPORT_OBJ_DIR}/tsd.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/test_hooks.pic.o" - "${ARROW_IMPORT_OBJ_DIR}/witness.pic.o" - ) - add_custom_command(OUTPUT ${ARROW_OTHER_OBJS} - COMMAND - mkdir -p "${ARROW_IMPORT_OBJ_DIR}" && - cd "${ARROW_IMPORT_OBJ_DIR}" && - "${CMAKE_AR}" x "${ARROW_DEPS_LIBRARY}" - ) - set_source_files_properties(jemalloc.pic.o PROPERTIES EXTERNAL_OBJECT true GENERATED true) - add_library(imported_arrow_deps STATIC ${ARROW_OTHER_OBJS}) - - set(ARROW_LIBRARY ${ARROW_STATIC_LIB} - imported_arrow_deps ${THRIFT_LIBRARY} ${UTF8_PROC_LIBRARY} bzip2 ${SNAPPY_LIBRARY}) - else() - message(WARNING "Using external static Arrow does not always work. " - "Could not find arrow_bundled_dependencies.a. If compilation fails, " - "Try: -D\"USE_INTERNAL_PARQUET_LIBRARY\"=ON or -D\"ENABLE_PARQUET\"=OFF or " - "-D\"USE_STATIC_LIBRARIES\"=OFF") - set(ARROW_LIBRARY ${ARROW_STATIC_LIB}) - endif() - set(PARQUET_LIBRARY ${PARQUET_STATIC_LIB}) - else() - set(ARROW_LIBRARY ${ARROW_SHARED_LIB}) - set(PARQUET_LIBRARY ${PARQUET_SHARED_LIB}) - endif() - - if(ARROW_INCLUDE_DIR AND ARROW_LIBRARY AND PARQUET_INCLUDE_DIR AND PARQUET_LIBRARY AND THRIFT_LIBRARY AND UTF8_PROC_LIBRARY) - set(USE_PARQUET 1) - set(EXTERNAL_PARQUET_FOUND 1) - else() - message (${RECONFIGURE_MESSAGE_LEVEL} - "Can't find system parquet: arrow=${ARROW_INCLUDE_DIR}:${ARROW_LIBRARY} ;" - " parquet=${PARQUET_INCLUDE_DIR}:${PARQUET_LIBRARY} ;" - " thrift=${THRIFT_LIBRARY} ;") - set(EXTERNAL_PARQUET_FOUND 0) - endif() -endif() - -if(NOT EXTERNAL_PARQUET_FOUND AND NOT MISSING_INTERNAL_PARQUET_LIBRARY AND NOT OS_FREEBSD) - if(SNAPPY_LIBRARY) - set(CAN_USE_INTERNAL_PARQUET_LIBRARY 1) - else() - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal parquet library without snappy") - endif() - - include(CheckCXXSourceCompiles) - if(NOT USE_INTERNAL_DOUBLE_CONVERSION_LIBRARY) - set(CMAKE_REQUIRED_LIBRARIES ${DOUBLE_CONVERSION_LIBRARIES}) - set(CMAKE_REQUIRED_INCLUDES ${DOUBLE_CONVERSION_INCLUDE_DIR}) - check_cxx_source_compiles(" - #include - int main() { static const int flags_ = double_conversion::StringToDoubleConverter::ALLOW_CASE_INSENSIBILITY; return 0;} - " HAVE_DOUBLE_CONVERSION_ALLOW_CASE_INSENSIBILITY) - - if(NOT HAVE_DOUBLE_CONVERSION_ALLOW_CASE_INSENSIBILITY) # HAVE_STD_RANDOM_SHUFFLE - message (${RECONFIGURE_MESSAGE_LEVEL} "Disabling internal parquet library because arrow is broken (can't use old double_conversion)") - set(CAN_USE_INTERNAL_PARQUET_LIBRARY 0) - endif() - endif() - - if(NOT CAN_USE_INTERNAL_PARQUET_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal parquet") - set(USE_INTERNAL_PARQUET_LIBRARY 0) - else() - set(USE_INTERNAL_PARQUET_LIBRARY 1) - - if(MAKE_STATIC_LIBRARIES) - set(FLATBUFFERS_LIBRARY flatbuffers) - set(ARROW_LIBRARY arrow_static) - set(PARQUET_LIBRARY parquet_static) - else() - set(FLATBUFFERS_LIBRARY flatbuffers_shared) - set(ARROW_LIBRARY arrow_shared) - set(PARQUET_LIBRARY parquet_shared) - endif() - - set(USE_PARQUET 1) - set(USE_ORC 1) - set(USE_ARROW 1) - endif() -elseif(OS_FREEBSD) - message (${RECONFIGURE_MESSAGE_LEVEL} "Using internal parquet library on FreeBSD is not supported") -endif() - -if(USE_PARQUET) - message(STATUS "Using Parquet: arrow=${ARROW_LIBRARY}:${ARROW_INCLUDE_DIR} ;" - " parquet=${PARQUET_LIBRARY}:${PARQUET_INCLUDE_DIR} ;" - " thrift=${THRIFT_LIBRARY} ;" - " flatbuffers=${FLATBUFFERS_LIBRARY}") -else() - message(STATUS "Building without Parquet support") -endif() diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 88f7800fe39..45965c738c5 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -60,6 +60,8 @@ add_subdirectory (zstd-cmake) add_subdirectory (zlib-ng-cmake) add_subdirectory (rocksdb-cmake) add_subdirectory (bzip2-cmake) +# parquet/arrow +add_subdirectory (arrow-cmake) # TODO: refactor the contrib libraries below this comment. @@ -121,29 +123,6 @@ if(USE_INTERNAL_SNAPPY_LIBRARY) set (SNAPPY_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/snappy") endif() -if (USE_INTERNAL_PARQUET_LIBRARY) - # We dont use arrow's cmakefiles because they uses too many depends and download some libs in compile time - # But you can update auto-generated parquet files manually: - # cd {BUILD_DIR}/contrib/arrow/cpp/src/parquet && mkdir -p build && cd build - # cmake .. -DARROW_COMPUTE=ON -DARROW_PARQUET=ON -DARROW_SIMD_LEVEL=NONE -DARROW_VERBOSE_THIRDPARTY_BUILD=ON - # -DARROW_BUILD_SHARED=1 -DARROW_BUILD_UTILITIES=OFF -DARROW_BUILD_INTEGRATION=OFF - # -DBoost_FOUND=1 -DARROW_TEST_LINKAGE="shared" - # make -j8 - # copy {BUILD_DIR}/contrib/arrow/cpp/src/parquet/*.cpp,*.h -> {BUILD_DIR}/contrib/arrow-cmake/cpp/src/parquet/ - - # Also useful parquet reader: - # cd {BUILD_DIR}/contrib/arrow/cpp && mkdir -p build && cd build - # cmake .. -DARROW_PARQUET=1 -DARROW_WITH_SNAPPY=1 -DPARQUET_BUILD_EXECUTABLES=1 - # make -j8 - # {BUILD_DIR}/contrib/arrow/cpp/build/release/parquet-reader some_file.parquet - - add_subdirectory(arrow-cmake) - - # The library is large - avoid bloat. - target_compile_options (${ARROW_LIBRARY} PRIVATE -g0) - target_compile_options (${PARQUET_LIBRARY} PRIVATE -g0) -endif() - if (USE_INTERNAL_AVRO_LIBRARY) add_subdirectory(avro-cmake) endif() diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 5a61bca2152..1a0e8c988e3 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -1,3 +1,44 @@ +# We dont use arrow's cmakefiles because they uses too many depends and download some libs in compile time +# But you can update auto-generated parquet files manually: +# cd {BUILD_DIR}/contrib/arrow/cpp/src/parquet && mkdir -p build && cd build +# cmake .. -DARROW_COMPUTE=ON -DARROW_PARQUET=ON -DARROW_SIMD_LEVEL=NONE -DARROW_VERBOSE_THIRDPARTY_BUILD=ON +# -DARROW_BUILD_SHARED=1 -DARROW_BUILD_UTILITIES=OFF -DARROW_BUILD_INTEGRATION=OFF +# -DBoost_FOUND=1 -DARROW_TEST_LINKAGE="shared" +# make -j8 +# copy {BUILD_DIR}/contrib/arrow/cpp/src/parquet/*.cpp,*.h -> {BUILD_DIR}/contrib/arrow-cmake/cpp/src/parquet/ + +# Also useful parquet reader: +# cd {BUILD_DIR}/contrib/arrow/cpp && mkdir -p build && cd build +# cmake .. -DARROW_PARQUET=1 -DARROW_WITH_SNAPPY=1 -DPARQUET_BUILD_EXECUTABLES=1 +# make -j8 +# {BUILD_DIR}/contrib/arrow/cpp/build/release/parquet-reader some_file.parquet + +set (ENABLE_PARQUET_DEFAULT ${ENABLE_LIBRARIES}) +if (OS_FREEBSD) + set (ENABLE_PARQUET_DEFAULT OFF) +endif() +option (ENABLE_PARQUET "Enable parquet" ${ENABLE_PARQUET_DEFAULT}) + +if (NOT ENABLE_PARQUET) + message(STATUS "Building without Parquet support") + return() +endif() + +if (NOT Protobuf_PROTOC_EXECUTABLE) + message (FATAL_ERROR "Can't use parquet without protoc executable") +endif() + +# Freebsd: ../contrib/arrow/cpp/src/arrow/util/bit-util.h:27:10: fatal error: endian.h: No such file or directory +if (OS_FREEBSD) + message (FATAL_ERROR "Using internal parquet library on FreeBSD is not supported") +endif() + +if(MAKE_STATIC_LIBRARIES) + set(FLATBUFFERS_LIBRARY flatbuffers) +else() + set(FLATBUFFERS_LIBRARY flatbuffers_shared) +endif() + set (CMAKE_CXX_STANDARD 17) set(ARROW_VERSION "6.0.1") @@ -324,36 +365,37 @@ if (ARROW_WITH_ZSTD) endif () -add_library(${ARROW_LIBRARY} ${ARROW_SRCS}) +add_library(_arrow ${ARROW_SRCS}) # Arrow dependencies -add_dependencies(${ARROW_LIBRARY} ${FLATBUFFERS_LIBRARY}) +add_dependencies(_arrow ${FLATBUFFERS_LIBRARY}) -target_link_libraries(${ARROW_LIBRARY} PRIVATE ${FLATBUFFERS_LIBRARY} boost::filesystem) +target_link_libraries(_arrow PRIVATE ${FLATBUFFERS_LIBRARY} boost::filesystem) if (USE_INTERNAL_PROTOBUF_LIBRARY) - add_dependencies(${ARROW_LIBRARY} protoc) + add_dependencies(_arrow protoc) endif () -target_include_directories(${ARROW_LIBRARY} SYSTEM PUBLIC ${ARROW_SRC_DIR}) -target_include_directories(${ARROW_LIBRARY} SYSTEM PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/cpp/src") -target_link_libraries(${ARROW_LIBRARY} PRIVATE ${DOUBLE_CONVERSION_LIBRARIES} ${Protobuf_LIBRARY}) -target_link_libraries(${ARROW_LIBRARY} PRIVATE lz4) +target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ARROW_SRC_DIR}) +target_include_directories(_arrow SYSTEM BEFORE PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/cpp/src") +target_link_libraries(_arrow PRIVATE ${DOUBLE_CONVERSION_LIBRARIES} ${Protobuf_LIBRARY}) +target_link_libraries(_arrow PRIVATE lz4) if (ARROW_WITH_SNAPPY) - target_link_libraries(${ARROW_LIBRARY} PRIVATE ${SNAPPY_LIBRARY}) + target_link_libraries(_arrow PRIVATE ${SNAPPY_LIBRARY}) endif () -target_link_libraries(${ARROW_LIBRARY} PRIVATE ch_contrib::zlib) -target_link_libraries(${ARROW_LIBRARY} PRIVATE ch_contrib::zstd) +target_link_libraries(_arrow PRIVATE ch_contrib::zlib) +target_link_libraries(_arrow PRIVATE ch_contrib::zstd) -target_include_directories(${ARROW_LIBRARY} SYSTEM PRIVATE ${ORC_INCLUDE_DIR}) -target_include_directories(${ARROW_LIBRARY} SYSTEM PRIVATE ${ORC_SOURCE_SRC_DIR}) -target_include_directories(${ARROW_LIBRARY} SYSTEM PRIVATE ${ORC_SOURCE_WRAP_DIR}) -target_include_directories(${ARROW_LIBRARY} SYSTEM PRIVATE ${GOOGLE_PROTOBUF_DIR}) -target_include_directories(${ARROW_LIBRARY} SYSTEM PRIVATE ${ORC_BUILD_SRC_DIR}) -target_include_directories(${ARROW_LIBRARY} SYSTEM PRIVATE ${ORC_BUILD_INCLUDE_DIR}) -target_include_directories(${ARROW_LIBRARY} SYSTEM PRIVATE ${ORC_ADDITION_SOURCE_DIR}) -target_include_directories(${ARROW_LIBRARY} SYSTEM PRIVATE ${FLATBUFFERS_INCLUDE_DIR}) -target_include_directories(${ARROW_LIBRARY} SYSTEM PRIVATE ${HDFS_INCLUDE_DIR}) +target_include_directories(_arrow SYSTEM PRIVATE ${ORC_INCLUDE_DIR}) +target_include_directories(_arrow SYSTEM PRIVATE ${ORC_BUILD_INCLUDE_DIR}) +target_include_directories(_arrow SYSTEM PRIVATE ${ORC_SOURCE_SRC_DIR}) +target_include_directories(_arrow SYSTEM PRIVATE ${ORC_SOURCE_WRAP_DIR}) +target_include_directories(_arrow SYSTEM PRIVATE ${GOOGLE_PROTOBUF_DIR}) +target_include_directories(_arrow SYSTEM PRIVATE ${ORC_BUILD_SRC_DIR}) +target_include_directories(_arrow SYSTEM PRIVATE ${ORC_ADDITION_SOURCE_DIR}) +target_include_directories(_arrow SYSTEM PRIVATE ${ARROW_SRC_DIR}) +target_include_directories(_arrow SYSTEM PRIVATE ${FLATBUFFERS_INCLUDE_DIR}) +target_include_directories(_arrow SYSTEM PRIVATE ${HDFS_INCLUDE_DIR}) # === parquet @@ -396,13 +438,25 @@ set(PARQUET_SRCS "${GEN_LIBRARY_DIR}/parquet_types.cpp" ) #list(TRANSFORM PARQUET_SRCS PREPEND "${LIBRARY_DIR}/") # cmake 3.12 -add_library(${PARQUET_LIBRARY} ${PARQUET_SRCS}) -target_include_directories(${PARQUET_LIBRARY} SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src" "${CMAKE_CURRENT_SOURCE_DIR}/cpp/src" PRIVATE ${OPENSSL_INCLUDE_DIR}) -target_link_libraries(${PARQUET_LIBRARY} PUBLIC ${ARROW_LIBRARY} PRIVATE ${THRIFT_LIBRARY} boost::headers_only boost::regex ${OPENSSL_LIBRARIES}) +add_library(_parquet ${PARQUET_SRCS}) +add_library(ch_contrib::parquet ALIAS _parquet) +target_include_directories(_parquet SYSTEM BEFORE + PUBLIC + "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src" + "${CMAKE_CURRENT_SOURCE_DIR}/cpp/src" + PRIVATE + ${OPENSSL_INCLUDE_DIR}) +target_link_libraries(_parquet + PUBLIC _arrow + PRIVATE + ${THRIFT_LIBRARY} + boost::headers_only + boost::regex + ${OPENSSL_LIBRARIES}) if (SANITIZE STREQUAL "undefined") - target_compile_options(${PARQUET_LIBRARY} PRIVATE -fno-sanitize=undefined) - target_compile_options(${ARROW_LIBRARY} PRIVATE -fno-sanitize=undefined) + target_compile_options(_parquet PRIVATE -fno-sanitize=undefined) + target_compile_options(_arrow PRIVATE -fno-sanitize=undefined) endif () # === tools @@ -411,5 +465,9 @@ set(TOOLS_DIR "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/tools/parquet") set(PARQUET_TOOLS parquet_dump_schema parquet_reader parquet_scan) foreach (TOOL ${PARQUET_TOOLS}) add_executable(${TOOL} "${TOOLS_DIR}/${TOOL}.cc") - target_link_libraries(${TOOL} PRIVATE ${PARQUET_LIBRARY}) + target_link_libraries(${TOOL} PRIVATE _parquet) endforeach () + +# The library is large - avoid bloat. +target_compile_options (_arrow PRIVATE -g0) +target_compile_options (_parquet PRIVATE -g0) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6c871a2a599..e3fb771af08 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -411,14 +411,8 @@ if (USE_CAPNP) dbms_target_link_libraries (PRIVATE ${CAPNP_LIBRARIES}) endif () -if (USE_PARQUET) - dbms_target_link_libraries(PRIVATE ${PARQUET_LIBRARY}) - if (NOT USE_INTERNAL_PARQUET_LIBRARY) - dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${PARQUET_INCLUDE_DIR} ${ARROW_INCLUDE_DIR}) - if (USE_STATIC_LIBRARIES) - dbms_target_link_libraries(PRIVATE ${ARROW_LIBRARY}) - endif() - endif () +if (TARGET ch_contrib::parquet) + dbms_target_link_libraries (PRIVATE ch_contrib::parquet) endif () if (USE_AVRO) diff --git a/src/Formats/CMakeLists.txt b/src/Formats/CMakeLists.txt index 12def0fb1d0..237bf206d5a 100644 --- a/src/Formats/CMakeLists.txt +++ b/src/Formats/CMakeLists.txt @@ -1 +1,5 @@ +if (TARGET ch_contrib::parquet) + set(USE_PARQUET 1) + set(USE_ARROW 1) +endif() configure_file(config_formats.h.in ${ConfigIncludePath}/config_formats.h) From d1b2bd5fbe0756e064454fe942658e73d110d1a6 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 16 Jan 2022 22:34:00 +0300 Subject: [PATCH 126/403] Remove unbundled avro support --- CMakeLists.txt | 1 - cmake/find/avro.cmake | 35 ------------------------------- contrib/CMakeLists.txt | 5 +---- contrib/avro-cmake/CMakeLists.txt | 29 +++++++++++++++++-------- src/CMakeLists.txt | 5 ++--- src/Formats/CMakeLists.txt | 3 +++ 6 files changed, 26 insertions(+), 52 deletions(-) delete mode 100644 cmake/find/avro.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 4cbb921b190..69860900a4d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -534,7 +534,6 @@ include (cmake/find/s2geometry.cmake) include (cmake/find/nlp.cmake) include (cmake/find/filelog.cmake) include (cmake/find/orc.cmake) -include (cmake/find/avro.cmake) include (cmake/find/msgpack.cmake) include (cmake/find/cassandra.cmake) include (cmake/find/sentry.cmake) diff --git a/cmake/find/avro.cmake b/cmake/find/avro.cmake deleted file mode 100644 index a70fb92c122..00000000000 --- a/cmake/find/avro.cmake +++ /dev/null @@ -1,35 +0,0 @@ -# Needed when using Apache Avro serialization format -option (ENABLE_AVRO "Enable Avro" ${ENABLE_LIBRARIES}) - -if (NOT ENABLE_AVRO) - if (USE_INTERNAL_AVRO_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal avro library with ENABLE_AVRO=OFF") - endif() - return() -endif() - -option (USE_INTERNAL_AVRO_LIBRARY "Set to FALSE to use system avro library instead of bundled" ON) - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/avro/lang") - if (USE_INTERNAL_AVRO_LIBRARY) - message(WARNING "submodule contrib/avro is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot find internal avro") - set(USE_INTERNAL_AVRO_LIBRARY 0) - endif() - set(MISSING_INTERNAL_AVRO_LIBRARY 1) -endif() - -if (NOT USE_INTERNAL_AVRO_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Using system avro library is not supported yet") -elseif(NOT MISSING_INTERNAL_AVRO_LIBRARY) - include(cmake/find/snappy.cmake) - set(AVROCPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/avro/lang/c++/include") - set(AVROCPP_LIBRARY avrocpp) - set(USE_INTERNAL_AVRO_LIBRARY 1) -endif () - -if (AVROCPP_LIBRARY AND AVROCPP_INCLUDE_DIR) - set(USE_AVRO 1) -endif() - -message (STATUS "Using avro=${USE_AVRO}: ${AVROCPP_INCLUDE_DIR} : ${AVROCPP_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 45965c738c5..c8c4c5e17f1 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -62,6 +62,7 @@ add_subdirectory (rocksdb-cmake) add_subdirectory (bzip2-cmake) # parquet/arrow add_subdirectory (arrow-cmake) +add_subdirectory (avro-cmake) # requires: snappy # TODO: refactor the contrib libraries below this comment. @@ -123,10 +124,6 @@ if(USE_INTERNAL_SNAPPY_LIBRARY) set (SNAPPY_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/snappy") endif() -if (USE_INTERNAL_AVRO_LIBRARY) - add_subdirectory(avro-cmake) -endif() - if(USE_INTERNAL_GTEST_LIBRARY) add_subdirectory(googletest-cmake) elseif(GTEST_SRC_DIR) diff --git a/contrib/avro-cmake/CMakeLists.txt b/contrib/avro-cmake/CMakeLists.txt index b56afd1598c..9c73bd12b1f 100644 --- a/contrib/avro-cmake/CMakeLists.txt +++ b/contrib/avro-cmake/CMakeLists.txt @@ -1,3 +1,11 @@ +# Needed when using Apache Avro serialization format +option (ENABLE_AVRO "Enable Avro" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_AVRO) + message(STATUS "Not using avro") + return() +endif() + set(AVROCPP_ROOT_DIR "${CMAKE_SOURCE_DIR}/contrib/avro/lang/c++") set(AVROCPP_INCLUDE_DIR "${AVROCPP_ROOT_DIR}/api") set(AVROCPP_SOURCE_DIR "${AVROCPP_ROOT_DIR}/impl") @@ -40,17 +48,19 @@ set (AVROCPP_SOURCE_FILES "${AVROCPP_SOURCE_DIR}/Validator.cc" ) -add_library (avrocpp ${AVROCPP_SOURCE_FILES}) -set_target_properties (avrocpp PROPERTIES VERSION ${AVRO_VERSION_MAJOR}.${AVRO_VERSION_MINOR}) +add_library (_avrocpp ${AVROCPP_SOURCE_FILES}) +add_library (ch_contrib::avrocpp ALIAS _avrocpp) +set_target_properties (_avrocpp PROPERTIES VERSION ${AVRO_VERSION_MAJOR}.${AVRO_VERSION_MINOR}) -target_include_directories(avrocpp SYSTEM PUBLIC ${AVROCPP_INCLUDE_DIR}) +target_include_directories(_avrocpp SYSTEM PUBLIC ${AVROCPP_INCLUDE_DIR}) -target_link_libraries (avrocpp PRIVATE boost::headers_only boost::iostreams) +target_link_libraries (_avrocpp PRIVATE boost::headers_only boost::iostreams) +# include(cmake/find/snappy.cmake) if (SNAPPY_INCLUDE_DIR AND SNAPPY_LIBRARY) - target_compile_definitions (avrocpp PUBLIC SNAPPY_CODEC_AVAILABLE) - target_include_directories (avrocpp PRIVATE ${SNAPPY_INCLUDE_DIR}) - target_link_libraries (avrocpp PRIVATE ${SNAPPY_LIBRARY}) + target_compile_definitions (_avrocpp PUBLIC SNAPPY_CODEC_AVAILABLE) + target_include_directories (_avrocpp PRIVATE ${SNAPPY_INCLUDE_DIR}) + target_link_libraries (_avrocpp PRIVATE ${SNAPPY_LIBRARY}) endif () if (COMPILER_GCC) @@ -59,11 +69,12 @@ elseif (COMPILER_CLANG) set (SUPPRESS_WARNINGS -Wno-non-virtual-dtor) endif () -target_compile_options(avrocpp PRIVATE ${SUPPRESS_WARNINGS}) +target_compile_options(_avrocpp PRIVATE ${SUPPRESS_WARNINGS}) # create a symlink to include headers with ADD_CUSTOM_TARGET(avro_symlink_headers ALL COMMAND ${CMAKE_COMMAND} -E make_directory "${AVROCPP_ROOT_DIR}/include" COMMAND ${CMAKE_COMMAND} -E create_symlink "${AVROCPP_ROOT_DIR}/api" "${AVROCPP_ROOT_DIR}/include/avro" ) -add_dependencies(avrocpp avro_symlink_headers) +add_dependencies(_avrocpp avro_symlink_headers) +target_include_directories(_avrocpp SYSTEM BEFORE PUBLIC "${AVROCPP_ROOT_DIR}/include") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e3fb771af08..c509b4e5b57 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -415,9 +415,8 @@ if (TARGET ch_contrib::parquet) dbms_target_link_libraries (PRIVATE ch_contrib::parquet) endif () -if (USE_AVRO) - dbms_target_link_libraries(PRIVATE ${AVROCPP_LIBRARY}) - dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${AVROCPP_INCLUDE_DIR}) +if (TARGET ch_contrib::avrocpp) + dbms_target_link_libraries(PRIVATE ch_contrib::avrocpp) endif () if (OPENSSL_CRYPTO_LIBRARY) diff --git a/src/Formats/CMakeLists.txt b/src/Formats/CMakeLists.txt index 237bf206d5a..53f838863c1 100644 --- a/src/Formats/CMakeLists.txt +++ b/src/Formats/CMakeLists.txt @@ -1,3 +1,6 @@ +if (TARGET ch_contrib::avrocpp) + set(USE_AVRO 1) +endif() if (TARGET ch_contrib::parquet) set(USE_PARQUET 1) set(USE_ARROW 1) From ab8cdb198f0fd47b8402022830e8410b2d71f2dc Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 16 Jan 2022 22:37:27 +0300 Subject: [PATCH 127/403] Remove unbundled orc support --- CMakeLists.txt | 1 - cmake/find/orc.cmake | 52 ------------------------------ contrib/CMakeLists.txt | 2 +- contrib/arrow-cmake/CMakeLists.txt | 4 +-- src/CMakeLists.txt | 5 --- src/Formats/CMakeLists.txt | 1 + 6 files changed, 4 insertions(+), 61 deletions(-) delete mode 100644 cmake/find/orc.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 69860900a4d..5f4de8d26a4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -533,7 +533,6 @@ include (cmake/find/yaml-cpp.cmake) include (cmake/find/s2geometry.cmake) include (cmake/find/nlp.cmake) include (cmake/find/filelog.cmake) -include (cmake/find/orc.cmake) include (cmake/find/msgpack.cmake) include (cmake/find/cassandra.cmake) include (cmake/find/sentry.cmake) diff --git a/cmake/find/orc.cmake b/cmake/find/orc.cmake deleted file mode 100644 index c96c3ea5d5a..00000000000 --- a/cmake/find/orc.cmake +++ /dev/null @@ -1,52 +0,0 @@ -option (ENABLE_ORC "Enable ORC" ${ENABLE_LIBRARIES}) - -if(NOT ENABLE_ORC) - if(USE_INTERNAL_ORC_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal ORC library with ENABLE_ORC=OFF") - endif() - return() -endif() - -option(USE_INTERNAL_ORC_LIBRARY "Set to FALSE to use system ORC instead of bundled (experimental set to OFF on your own risk)" ON) - -include(cmake/find/snappy.cmake) - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include/orc/OrcFile.hh") - if(USE_INTERNAL_ORC_LIBRARY) - message(WARNING "submodule contrib/orc is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal ORC") - set(USE_INTERNAL_ORC_LIBRARY 0) - endif() - set(MISSING_INTERNAL_ORC_LIBRARY 1) -endif () - -if (NOT USE_INTERNAL_ORC_LIBRARY) - find_package(orc) - if (NOT ORC_LIBRARY OR NOT ORC_INCLUDE_DIR) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system ORC") - endif () -endif () - -#if (USE_INTERNAL_ORC_LIBRARY) -#find_path(CYRUS_SASL_INCLUDE_DIR sasl/sasl.h) -#find_library(CYRUS_SASL_SHARED_LIB sasl2) -#if (NOT CYRUS_SASL_INCLUDE_DIR OR NOT CYRUS_SASL_SHARED_LIB) -# set(USE_ORC 0) -#endif() -#endif() - -if (ORC_LIBRARY AND ORC_INCLUDE_DIR) - set(USE_ORC 1) -elseif(NOT MISSING_INTERNAL_ORC_LIBRARY AND SNAPPY_LIBRARY) # (LIBGSASL_LIBRARY AND LIBXML2_LIBRARY) - set(ORC_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include") - set(ORC_LIBRARY orc) - set(USE_ORC 1) - set(USE_INTERNAL_ORC_LIBRARY 1) -else() - message (${RECONFIGURE_MESSAGE_LEVEL} - "Can't enable ORC support - missing dependencies. Missing internal orc=${MISSING_INTERNAL_ORC_LIBRARY}. " - "snappy=${SNAPPY_LIBRARY}") - set(USE_INTERNAL_ORC_LIBRARY 0) -endif() - -message (STATUS "Using internal=${USE_INTERNAL_ORC_LIBRARY} orc=${USE_ORC}: ${ORC_INCLUDE_DIR} : ${ORC_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index c8c4c5e17f1..67bd4e054c7 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -60,7 +60,7 @@ add_subdirectory (zstd-cmake) add_subdirectory (zlib-ng-cmake) add_subdirectory (rocksdb-cmake) add_subdirectory (bzip2-cmake) -# parquet/arrow +# parquet/arrow/orc add_subdirectory (arrow-cmake) add_subdirectory (avro-cmake) # requires: snappy diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 1a0e8c988e3..8919820c2d9 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -386,8 +386,8 @@ endif () target_link_libraries(_arrow PRIVATE ch_contrib::zlib) target_link_libraries(_arrow PRIVATE ch_contrib::zstd) -target_include_directories(_arrow SYSTEM PRIVATE ${ORC_INCLUDE_DIR}) -target_include_directories(_arrow SYSTEM PRIVATE ${ORC_BUILD_INCLUDE_DIR}) +target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ORC_INCLUDE_DIR}) +target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ORC_BUILD_INCLUDE_DIR}) target_include_directories(_arrow SYSTEM PRIVATE ${ORC_SOURCE_SRC_DIR}) target_include_directories(_arrow SYSTEM PRIVATE ${ORC_SOURCE_WRAP_DIR}) target_include_directories(_arrow SYSTEM PRIVATE ${GOOGLE_PROTOBUF_DIR}) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c509b4e5b57..abb4647aa9b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -499,11 +499,6 @@ endif() target_link_libraries (clickhouse_common_io PUBLIC ${FAST_FLOAT_LIBRARY}) target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${FAST_FLOAT_INCLUDE_DIR}) -if (USE_ORC) - dbms_target_link_libraries(PUBLIC ${ORC_LIBRARIES}) - dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${ORC_INCLUDE_DIR} "${CMAKE_BINARY_DIR}/contrib/orc/c++/include") -endif () - if (TARGET ch_contrib::rocksdb) dbms_target_link_libraries(PUBLIC ch_contrib::rocksdb) endif() diff --git a/src/Formats/CMakeLists.txt b/src/Formats/CMakeLists.txt index 53f838863c1..dce3f0a1491 100644 --- a/src/Formats/CMakeLists.txt +++ b/src/Formats/CMakeLists.txt @@ -4,5 +4,6 @@ endif() if (TARGET ch_contrib::parquet) set(USE_PARQUET 1) set(USE_ARROW 1) + set(USE_ORC 1) endif() configure_file(config_formats.h.in ${ConfigIncludePath}/config_formats.h) From 1145e32af6cea32231c6e4677e95a436844552ec Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 16 Jan 2022 23:00:25 +0300 Subject: [PATCH 128/403] Remove unbundled snappy support --- cmake/find/snappy.cmake | 21 --------------------- contrib/CMakeLists.txt | 13 +++---------- contrib/arrow-cmake/CMakeLists.txt | 14 +++----------- contrib/avro-cmake/CMakeLists.txt | 9 +++------ contrib/rocksdb-cmake/CMakeLists.txt | 6 +++++- contrib/snappy-cmake/CMakeLists.txt | 9 +++++---- src/CMakeLists.txt | 5 ++--- src/Formats/CMakeLists.txt | 3 +++ src/configure_config.cmake | 3 +++ 9 files changed, 27 insertions(+), 56 deletions(-) delete mode 100644 cmake/find/snappy.cmake diff --git a/cmake/find/snappy.cmake b/cmake/find/snappy.cmake deleted file mode 100644 index 245b3a9a2ff..00000000000 --- a/cmake/find/snappy.cmake +++ /dev/null @@ -1,21 +0,0 @@ -option(USE_SNAPPY "Enable snappy library" ON) - -if(NOT USE_SNAPPY) - if (USE_INTERNAL_SNAPPY_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal snappy library with USE_SNAPPY=OFF") - endif() - return() -endif() - -option (USE_INTERNAL_SNAPPY_LIBRARY "Set to FALSE to use system snappy library instead of bundled" ON) - -if(NOT USE_INTERNAL_SNAPPY_LIBRARY) - find_library(SNAPPY_LIBRARY snappy) - if (NOT SNAPPY_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system snappy library") - endif() -else () - set(SNAPPY_LIBRARY snappy) -endif() - -message (STATUS "Using snappy: ${SNAPPY_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 67bd4e054c7..e97c7275b6f 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -58,10 +58,11 @@ add_subdirectory (poco-cmake) add_subdirectory (croaring-cmake) add_subdirectory (zstd-cmake) add_subdirectory (zlib-ng-cmake) -add_subdirectory (rocksdb-cmake) add_subdirectory (bzip2-cmake) +add_subdirectory (snappy-cmake) +add_subdirectory (rocksdb-cmake) # parquet/arrow/orc -add_subdirectory (arrow-cmake) +add_subdirectory (arrow-cmake) # requires: snappy add_subdirectory (avro-cmake) # requires: snappy # TODO: refactor the contrib libraries below this comment. @@ -116,14 +117,6 @@ if (ENABLE_ICU AND USE_INTERNAL_ICU_LIBRARY) add_subdirectory (icu-cmake) endif () -if(USE_INTERNAL_SNAPPY_LIBRARY) - set(SNAPPY_BUILD_TESTS 0 CACHE INTERNAL "") - - add_subdirectory(snappy-cmake) - - set (SNAPPY_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/snappy") -endif() - if(USE_INTERNAL_GTEST_LIBRARY) add_subdirectory(googletest-cmake) elseif(GTEST_SRC_DIR) diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 8919820c2d9..6b73add6bd2 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -344,17 +344,11 @@ set(ARROW_SRCS ${ORC_SRCS} ) -if (SNAPPY_INCLUDE_DIR AND SNAPPY_LIBRARY) - set(ARROW_WITH_SNAPPY 1) -endif () - add_definitions(-DARROW_WITH_LZ4) SET(ARROW_SRCS "${LIBRARY_DIR}/util/compression_lz4.cc" ${ARROW_SRCS}) -if (ARROW_WITH_SNAPPY) - add_definitions(-DARROW_WITH_SNAPPY) - SET(ARROW_SRCS "${LIBRARY_DIR}/util/compression_snappy.cc" ${ARROW_SRCS}) -endif () +add_definitions(-DARROW_WITH_SNAPPY) +SET(ARROW_SRCS "${LIBRARY_DIR}/util/compression_snappy.cc" ${ARROW_SRCS}) add_definitions(-DARROW_WITH_ZLIB) SET(ARROW_SRCS "${LIBRARY_DIR}/util/compression_zlib.cc" ${ARROW_SRCS}) @@ -380,9 +374,7 @@ target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ARROW_SRC_DIR}) target_include_directories(_arrow SYSTEM BEFORE PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/cpp/src") target_link_libraries(_arrow PRIVATE ${DOUBLE_CONVERSION_LIBRARIES} ${Protobuf_LIBRARY}) target_link_libraries(_arrow PRIVATE lz4) -if (ARROW_WITH_SNAPPY) - target_link_libraries(_arrow PRIVATE ${SNAPPY_LIBRARY}) -endif () +target_link_libraries(_arrow PRIVATE ch_contrib::snappy) target_link_libraries(_arrow PRIVATE ch_contrib::zlib) target_link_libraries(_arrow PRIVATE ch_contrib::zstd) diff --git a/contrib/avro-cmake/CMakeLists.txt b/contrib/avro-cmake/CMakeLists.txt index 9c73bd12b1f..d91ce40dd54 100644 --- a/contrib/avro-cmake/CMakeLists.txt +++ b/contrib/avro-cmake/CMakeLists.txt @@ -56,12 +56,9 @@ target_include_directories(_avrocpp SYSTEM PUBLIC ${AVROCPP_INCLUDE_DIR}) target_link_libraries (_avrocpp PRIVATE boost::headers_only boost::iostreams) -# include(cmake/find/snappy.cmake) -if (SNAPPY_INCLUDE_DIR AND SNAPPY_LIBRARY) - target_compile_definitions (_avrocpp PUBLIC SNAPPY_CODEC_AVAILABLE) - target_include_directories (_avrocpp PRIVATE ${SNAPPY_INCLUDE_DIR}) - target_link_libraries (_avrocpp PRIVATE ${SNAPPY_LIBRARY}) -endif () +target_compile_definitions (_avrocpp PUBLIC SNAPPY_CODEC_AVAILABLE) +target_include_directories (_avrocpp PRIVATE ${SNAPPY_INCLUDE_DIR}) +target_link_libraries (_avrocpp PRIVATE ch_contrib::snappy) if (COMPILER_GCC) set (SUPPRESS_WARNINGS -Wno-non-virtual-dtor) diff --git a/contrib/rocksdb-cmake/CMakeLists.txt b/contrib/rocksdb-cmake/CMakeLists.txt index a6cd981058c..05964d18033 100644 --- a/contrib/rocksdb-cmake/CMakeLists.txt +++ b/contrib/rocksdb-cmake/CMakeLists.txt @@ -13,6 +13,10 @@ set(PORTABLE ON) ## always disable jemalloc for rocksdb by default ## because it introduces non-standard jemalloc APIs option(WITH_JEMALLOC "build with JeMalloc" OFF) +set(USE_SNAPPY OFF) +if (TARGET ch_contrib::snappy) + set(USE_SNAPPY ON) +endif() option(WITH_SNAPPY "build with SNAPPY" ${USE_SNAPPY}) ## lz4, zlib, zstd is enabled in ClickHouse by default option(WITH_LZ4 "build with lz4" ON) @@ -53,7 +57,7 @@ else() if(WITH_SNAPPY) add_definitions(-DSNAPPY) - list(APPEND THIRDPARTY_LIBS snappy) + list(APPEND THIRDPARTY_LIBS ch_contrib::snappy) endif() if(WITH_ZLIB) diff --git a/contrib/snappy-cmake/CMakeLists.txt b/contrib/snappy-cmake/CMakeLists.txt index 289f8908436..0997ea207e0 100644 --- a/contrib/snappy-cmake/CMakeLists.txt +++ b/contrib/snappy-cmake/CMakeLists.txt @@ -30,8 +30,9 @@ configure_file( "${SOURCE_DIR}/snappy-stubs-public.h.in" "${CMAKE_CURRENT_BINARY_DIR}/snappy-stubs-public.h") -add_library(snappy "") -target_sources(snappy +add_library(_snappy "") +add_library(ch_contrib::snappy ALIAS _snappy) +target_sources(_snappy PRIVATE "${SOURCE_DIR}/snappy-internal.h" "${SOURCE_DIR}/snappy-stubs-internal.h" @@ -40,5 +41,5 @@ target_sources(snappy "${SOURCE_DIR}/snappy-stubs-internal.cc" "${SOURCE_DIR}/snappy.cc") -target_include_directories(snappy SYSTEM PUBLIC ${SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) -target_compile_definitions(snappy PRIVATE -DHAVE_CONFIG_H) +target_include_directories(_snappy SYSTEM BEFORE PUBLIC ${SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) +target_compile_definitions(_snappy PRIVATE -DHAVE_CONFIG_H) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index abb4647aa9b..9af3ad29858 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -471,9 +471,8 @@ if (USE_BROTLI) target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${BROTLI_INCLUDE_DIR}) endif() -if (USE_SNAPPY) - target_link_libraries (clickhouse_common_io PUBLIC ${SNAPPY_LIBRARY}) - target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${SNAPPY_INCLUDE_DIR}) +if (TARGET ch_contrib::snappy) + target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::snappy) endif() if (USE_AMQPCPP) diff --git a/src/Formats/CMakeLists.txt b/src/Formats/CMakeLists.txt index dce3f0a1491..9734ee1bd9c 100644 --- a/src/Formats/CMakeLists.txt +++ b/src/Formats/CMakeLists.txt @@ -6,4 +6,7 @@ if (TARGET ch_contrib::parquet) set(USE_ARROW 1) set(USE_ORC 1) endif() +if (TARGET ch_contrib::snappy) + set(USE_SNAPPY 1) +endif() configure_file(config_formats.h.in ${ConfigIncludePath}/config_formats.h) diff --git a/src/configure_config.cmake b/src/configure_config.cmake index 2355c3fc802..57162e6e595 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -4,3 +4,6 @@ endif() if (TARGET ch_contrib::bzip2) set(USE_BZIP2 1) endif() +if (TARGET ch_contrib::snappy) + set(USE_SNAPPY 1) +endif() From 4bd49138b41be9b4894149277ee8657c4a80a885 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 08:06:53 +0300 Subject: [PATCH 129/403] Remove unbundled hive support --- CMakeLists.txt | 1 - cmake/find/hive-metastore.cmake | 26 --------------------- contrib/CMakeLists.txt | 5 +--- contrib/hive-metastore-cmake/CMakeLists.txt | 14 ++++++++--- src/CMakeLists.txt | 7 +++--- src/configure_config.cmake | 3 +++ 6 files changed, 18 insertions(+), 38 deletions(-) delete mode 100644 cmake/find/hive-metastore.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f4de8d26a4..e175458f963 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -538,7 +538,6 @@ include (cmake/find/cassandra.cmake) include (cmake/find/sentry.cmake) include (cmake/find/datasketches.cmake) include (cmake/find/libprotobuf-mutator.cmake) -include (cmake/find/hive-metastore.cmake) set (USE_INTERNAL_CITYHASH_LIBRARY ON CACHE INTERNAL "") find_contrib_lib(cityhash) diff --git a/cmake/find/hive-metastore.cmake b/cmake/find/hive-metastore.cmake deleted file mode 100644 index bc283cf8bd2..00000000000 --- a/cmake/find/hive-metastore.cmake +++ /dev/null @@ -1,26 +0,0 @@ -option(ENABLE_HIVE "Enable Hive" ${ENABLE_LIBRARIES}) - -if (NOT ENABLE_HIVE) - message("Hive disabled") - return() -endif() - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/hive-metastore") - message(WARNING "submodule contrib/hive-metastore is missing. to fix try run: \n git submodule update --init") - set(USE_HIVE 0) -elseif (NOT USE_THRIFT) - message(WARNING "Thrift is not found, which is needed by Hive") - set(USE_HIVE 0) -elseif (NOT USE_HDFS) - message(WARNING "HDFS is not found, which is needed by Hive") - set(USE_HIVE 0) -elseif (NOT USE_ORC OR NOT USE_ARROW OR NOT USE_PARQUET) - message(WARNING "ORC/Arrow/Parquet is not found, which are needed by Hive") - set(USE_HIVE 0) -else() - set(USE_HIVE 1) - set(HIVE_METASTORE_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/hive-metastore) - set(HIVE_METASTORE_LIBRARY hivemetastore) -endif() - -message (STATUS "Using_Hive=${USE_HIVE}: ${HIVE_METASTORE_INCLUDE_DIR} : ${HIVE_METASTORE_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index e97c7275b6f..22881290dec 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -64,6 +64,7 @@ add_subdirectory (rocksdb-cmake) # parquet/arrow/orc add_subdirectory (arrow-cmake) # requires: snappy add_subdirectory (avro-cmake) # requires: snappy +add_subdirectory (hive-metastore-cmake) # requires: thrift/avro/arrow # TODO: refactor the contrib libraries below this comment. @@ -255,10 +256,6 @@ if (USE_S2_GEOMETRY) add_subdirectory(s2geometry-cmake) endif() -if (USE_HIVE) - add_subdirectory (hive-metastore-cmake) -endif() - # Put all targets defined here and in subdirectories under "contrib/" folders in GUI-based IDEs. # Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear # in "contrib/..." as originally planned, so we workaround this by fixing FOLDER properties of all targets manually, diff --git a/contrib/hive-metastore-cmake/CMakeLists.txt b/contrib/hive-metastore-cmake/CMakeLists.txt index c92405fa4e8..016c7618ddb 100644 --- a/contrib/hive-metastore-cmake/CMakeLists.txt +++ b/contrib/hive-metastore-cmake/CMakeLists.txt @@ -1,9 +1,17 @@ +option(ENABLE_HIVE "Enable Hive" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_HIVE) + message("Hive disabled") + return() +endif() + set (SRCS ${ClickHouse_SOURCE_DIR}/contrib/hive-metastore/hive_metastore_constants.cpp ${ClickHouse_SOURCE_DIR}/contrib/hive-metastore/hive_metastore_types.cpp ${ClickHouse_SOURCE_DIR}/contrib/hive-metastore/ThriftHiveMetastore.cpp ) -add_library(${HIVE_METASTORE_LIBRARY} ${SRCS}) -target_link_libraries(${HIVE_METASTORE_LIBRARY} PUBLIC ${THRIFT_LIBRARY}) -target_include_directories(${HIVE_METASTORE_LIBRARY} SYSTEM PUBLIC ${HIVE_METASTORE_INCLUDE_DIR}) +add_library(_hivemetastore ${SRCS}) +add_library(ch_contrib::hivemetastore ALIAS _hivemetastore) +target_link_libraries(_hivemetastore PUBLIC ${THRIFT_LIBRARY}) +target_include_directories(_hivemetastore SYSTEM BEFORE PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/hive-metastore) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9af3ad29858..4a10f90f942 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -120,7 +120,7 @@ if (USE_HDFS) endif() add_headers_and_sources(dbms Storages/Cache) -if (USE_HIVE) +if (TARGET ch_contrib::hivemetastore AND TARGET ch_contrib::hdfs) add_headers_and_sources(dbms Storages/Hive) endif() @@ -444,9 +444,8 @@ if (USE_HDFS) dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${HDFS3_INCLUDE_DIR}) endif() -if (USE_HIVE) - dbms_target_link_libraries(PRIVATE hivemetastore) - dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/hive-metastore) +if (TARGET ch_contrib::hivemetastore) + dbms_target_link_libraries(PRIVATE ch_contrib::hivemetastore) endif() diff --git a/src/configure_config.cmake b/src/configure_config.cmake index 57162e6e595..1bac4e07fb1 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -7,3 +7,6 @@ endif() if (TARGET ch_contrib::snappy) set(USE_SNAPPY 1) endif() +if (TARGET ch_contrib::hivemetastore) + set(USE_HIVE 1) +endif() From 313716be9435da17acfa2e0569a020b8764cf63e Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 08:10:57 +0300 Subject: [PATCH 130/403] Remove unbundled thrift support --- CMakeLists.txt | 1 - cmake/find/thrift.cmake | 34 --------------------- contrib/CMakeLists.txt | 7 ++--- contrib/arrow-cmake/CMakeLists.txt | 2 +- contrib/hive-metastore-cmake/CMakeLists.txt | 2 +- contrib/thrift-cmake/CMakeLists.txt | 14 +++++++-- src/Common/examples/CMakeLists.txt | 4 +-- 7 files changed, 17 insertions(+), 47 deletions(-) delete mode 100644 cmake/find/thrift.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index e175458f963..3c424b07584 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -510,7 +510,6 @@ include (cmake/find/h3.cmake) include (cmake/find/libxml2.cmake) include (cmake/find/brotli.cmake) include (cmake/find/protobuf.cmake) -include (cmake/find/thrift.cmake) include (cmake/find/grpc.cmake) include (cmake/find/pdqsort.cmake) include (cmake/find/miniselect.cmake) diff --git a/cmake/find/thrift.cmake b/cmake/find/thrift.cmake deleted file mode 100644 index 08eeb60915e..00000000000 --- a/cmake/find/thrift.cmake +++ /dev/null @@ -1,34 +0,0 @@ -option(ENABLE_THRIFT "Enable Thrift" ${ENABLE_LIBRARIES}) - -if (NOT ENABLE_THRIFT) - message (STATUS "thrift disabled") - set(USE_INTERNAL_THRIFT_LIBRARY 0) - return() -endif() - -option(USE_INTERNAL_THRIFT_LIBRARY "Set to FALSE to use system thrift library instead of bundled" ON) -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/thrift") - if (USE_INTERNAL_THRIFT_LIBRARY) - message (WARNING "submodule contrib/thrift is missing. to fix try run: \n git submodule update --init --recursive") - set(USE_INTERNAL_THRIFT_LIBRARY 0) - endif () -endif() - -if (USE_INTERNAL_THRIFT_LIBRARY) - if (MAKE_STATIC_LIBRARIES) - set(THRIFT_LIBRARY thrift_static) - else() - set(THRIFT_LIBRARY thrift) - endif() - set (THRIFT_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp/src") - set(USE_THRIFT 1) -else() - find_library(THRIFT_LIBRARY thrift) - if (NOT THRIFT_LIBRARY) - set(USE_THRIFT 0) - else() - set(USE_THRIFT 1) - endif() -endif () - -message (STATUS "Using thrift=${USE_THRIFT}: ${THRIFT_INCLUDE_DIR} : ${THRIFT_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 22881290dec..a7c06f31717 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -61,8 +61,9 @@ add_subdirectory (zlib-ng-cmake) add_subdirectory (bzip2-cmake) add_subdirectory (snappy-cmake) add_subdirectory (rocksdb-cmake) +add_subdirectory (thrift-cmake) # parquet/arrow/orc -add_subdirectory (arrow-cmake) # requires: snappy +add_subdirectory (arrow-cmake) # requires: snappy, thrift add_subdirectory (avro-cmake) # requires: snappy add_subdirectory (hive-metastore-cmake) # requires: thrift/avro/arrow @@ -166,10 +167,6 @@ if (USE_INTERNAL_PROTOBUF_LIBRARY) add_subdirectory(protobuf-cmake) endif () -if (USE_INTERNAL_THRIFT_LIBRARY) - add_subdirectory(thrift-cmake) -endif () - if (USE_INTERNAL_HDFS3_LIBRARY) add_subdirectory(libhdfs3-cmake) endif () diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 6b73add6bd2..09a0158771e 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -441,7 +441,7 @@ target_include_directories(_parquet SYSTEM BEFORE target_link_libraries(_parquet PUBLIC _arrow PRIVATE - ${THRIFT_LIBRARY} + ch_contrib::thrift boost::headers_only boost::regex ${OPENSSL_LIBRARIES}) diff --git a/contrib/hive-metastore-cmake/CMakeLists.txt b/contrib/hive-metastore-cmake/CMakeLists.txt index 016c7618ddb..4a1d90b7cef 100644 --- a/contrib/hive-metastore-cmake/CMakeLists.txt +++ b/contrib/hive-metastore-cmake/CMakeLists.txt @@ -13,5 +13,5 @@ set (SRCS add_library(_hivemetastore ${SRCS}) add_library(ch_contrib::hivemetastore ALIAS _hivemetastore) -target_link_libraries(_hivemetastore PUBLIC ${THRIFT_LIBRARY}) +target_link_libraries(_hivemetastore PUBLIC ch_contrib::thrift) target_include_directories(_hivemetastore SYSTEM BEFORE PUBLIC ${ClickHouse_SOURCE_DIR}/contrib/hive-metastore) diff --git a/contrib/thrift-cmake/CMakeLists.txt b/contrib/thrift-cmake/CMakeLists.txt index 088dd0a969b..2a62a6fe7ab 100644 --- a/contrib/thrift-cmake/CMakeLists.txt +++ b/contrib/thrift-cmake/CMakeLists.txt @@ -1,3 +1,10 @@ +option(ENABLE_THRIFT "Enable Thrift" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_THRIFT) + message (STATUS "thrift disabled") + return() +endif() + set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp") set(thriftcpp_SOURCES "${LIBRARY_DIR}/src/thrift/TApplicationException.cpp" @@ -82,6 +89,7 @@ configure_file("${CMAKE_CURRENT_SOURCE_DIR}/build/cmake/config.h.in" "${CMAKE_CU include_directories("${CMAKE_CURRENT_BINARY_DIR}") -add_library(${THRIFT_LIBRARY} ${thriftcpp_SOURCES} ${thriftcpp_threads_SOURCES}) -target_include_directories(${THRIFT_LIBRARY} SYSTEM PUBLIC "${THRIFT_INCLUDE_DIR}" ${CMAKE_CURRENT_BINARY_DIR}) -target_link_libraries (${THRIFT_LIBRARY} PUBLIC boost::headers_only) +add_library(_thrift ${thriftcpp_SOURCES} ${thriftcpp_threads_SOURCES}) +add_library(ch_contrib::thrift ALIAS _thrift) +target_include_directories(_thrift SYSTEM PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp/src" ${CMAKE_CURRENT_BINARY_DIR}) +target_link_libraries (_thrift PUBLIC boost::headers_only) diff --git a/src/Common/examples/CMakeLists.txt b/src/Common/examples/CMakeLists.txt index 7b21591f83e..86b88cda24a 100644 --- a/src/Common/examples/CMakeLists.txt +++ b/src/Common/examples/CMakeLists.txt @@ -81,8 +81,8 @@ target_link_libraries (shell_command_inout PRIVATE clickhouse_common_io) add_executable (executable_udf executable_udf.cpp) target_link_libraries (executable_udf PRIVATE dbms) -add_executable(hive_metastore_client hive_metastore_client.cpp) -target_link_libraries (hive_metastore_client PUBLIC hivemetastore ${THRIFT_LIBRARY}) +add_executable (hive_metastore_client hive_metastore_client.cpp) +target_link_libraries (hive_metastore_client PUBLIC hivemetastore ch_contrib::thrift) add_executable (interval_tree interval_tree.cpp) target_link_libraries (interval_tree PRIVATE dbms) From 61b781971cb9e97879ac4584e5d82a9302908bdf Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 08:21:56 +0300 Subject: [PATCH 131/403] Remove unbundled librdkafka/cppkafka support --- CMakeLists.txt | 1 - cmake/find/rdkafka.cmake | 68 ------------------------- contrib/CMakeLists.txt | 13 +---- contrib/cppkafka-cmake/CMakeLists.txt | 16 ++++-- contrib/librdkafka-cmake/CMakeLists.txt | 52 ++++++++++--------- src/CMakeLists.txt | 9 ++-- src/configure_config.cmake | 3 ++ 7 files changed, 48 insertions(+), 114 deletions(-) delete mode 100644 cmake/find/rdkafka.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c424b07584..88365c70d5f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -501,7 +501,6 @@ include (cmake/find/re2.cmake) include (cmake/find/krb5.cmake) include (cmake/find/libgsasl.cmake) include (cmake/find/cyrus-sasl.cmake) -include (cmake/find/rdkafka.cmake) include (cmake/find/libuv.cmake) # for amqpcpp and cassandra include (cmake/find/amqpcpp.cmake) include (cmake/find/capnp.cmake) diff --git a/cmake/find/rdkafka.cmake b/cmake/find/rdkafka.cmake deleted file mode 100644 index cad267bacff..00000000000 --- a/cmake/find/rdkafka.cmake +++ /dev/null @@ -1,68 +0,0 @@ -option (ENABLE_RDKAFKA "Enable kafka" ${ENABLE_LIBRARIES}) - -if (NOT ENABLE_RDKAFKA) - if (USE_INTERNAL_RDKAFKA_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal librdkafka with ENABLE_RDKAFKA=OFF") - endif() - return() -endif() - -option (USE_INTERNAL_RDKAFKA_LIBRARY "Set to FALSE to use system librdkafka instead of the bundled" ON) - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cppkafka/src") - if(USE_INTERNAL_RDKAFKA_LIBRARY) - message (WARNING "submodule contrib/cppkafka is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal cppkafka") - set (USE_INTERNAL_RDKAFKA_LIBRARY 0) - endif() - set (MISSING_INTERNAL_CPPKAFKA_LIBRARY 1) -endif () - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/src") - if(USE_INTERNAL_RDKAFKA_LIBRARY OR MISSING_INTERNAL_CPPKAFKA_LIBRARY) - message (WARNING "submodule contrib/librdkafka is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal rdkafka") - set (USE_INTERNAL_RDKAFKA_LIBRARY 0) - endif() - set (MISSING_INTERNAL_RDKAFKA_LIBRARY 1) -endif () - -if (NOT USE_INTERNAL_RDKAFKA_LIBRARY) - find_library (RDKAFKA_LIB rdkafka) - find_path (RDKAFKA_INCLUDE_DIR NAMES librdkafka/rdkafka.h PATHS ${RDKAFKA_INCLUDE_PATHS}) - if (NOT RDKAFKA_LIB OR NOT RDKAFKA_INCLUDE_DIR) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system librdkafka") - endif() - - if (USE_STATIC_LIBRARIES AND NOT OS_FREEBSD) - find_library (SASL2_LIBRARY sasl2) - if (NOT SASL2_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system sasl2 library needed for static librdkafka") - endif() - endif () - set (CPPKAFKA_LIBRARY cppkafka) -endif () - -if (RDKAFKA_LIB AND RDKAFKA_INCLUDE_DIR) - set (USE_RDKAFKA 1) - add_library (rdkafka_imp UNKNOWN IMPORTED) - set_target_properties (rdkafka_imp PROPERTIES IMPORTED_LOCATION ${RDKAFKA_LIB}) - set_target_properties (rdkafka_imp PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${RDKAFKA_INCLUDE_DIR}) - - set (RDKAFKA_LIBRARY rdkafka_imp ${OPENSSL_LIBRARIES}) - set (CPPKAFKA_LIBRARY cppkafka) - if (SASL2_LIBRARY) - list (APPEND RDKAFKA_LIBRARY ${SASL2_LIBRARY}) - endif () - if (LZ4_LIBRARY) - list (APPEND RDKAFKA_LIBRARY ${LZ4_LIBRARY}) - endif () -elseif (NOT MISSING_INTERNAL_RDKAFKA_LIBRARY AND NOT MISSING_INTERNAL_CPPKAFKA_LIBRARY) - set (USE_INTERNAL_RDKAFKA_LIBRARY 1) - set (RDKAFKA_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/src") - set (RDKAFKA_LIBRARY rdkafka) - set (CPPKAFKA_LIBRARY cppkafka) - set (USE_RDKAFKA 1) -endif () - -message (STATUS "Using librdkafka=${USE_RDKAFKA}: ${RDKAFKA_INCLUDE_DIR} : ${RDKAFKA_LIBRARY} ${CPPKAFKA_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index a7c06f31717..8b8f74bb698 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -66,6 +66,8 @@ add_subdirectory (thrift-cmake) add_subdirectory (arrow-cmake) # requires: snappy, thrift add_subdirectory (avro-cmake) # requires: snappy add_subdirectory (hive-metastore-cmake) # requires: thrift/avro/arrow +add_subdirectory (librdkafka-cmake) +add_subdirectory (cppkafka-cmake) # TODO: refactor the contrib libraries below this comment. @@ -104,17 +106,6 @@ if (USE_INTERNAL_MYSQL_LIBRARY) add_subdirectory (mariadb-connector-c-cmake) endif () -if (USE_INTERNAL_RDKAFKA_LIBRARY) - add_subdirectory (librdkafka-cmake) - if(OPENSSL_INCLUDE_DIR) - target_include_directories(rdkafka BEFORE PRIVATE ${OPENSSL_INCLUDE_DIR}) - endif() -endif () - -if (USE_RDKAFKA) - add_subdirectory (cppkafka-cmake) -endif() - if (ENABLE_ICU AND USE_INTERNAL_ICU_LIBRARY) add_subdirectory (icu-cmake) endif () diff --git a/contrib/cppkafka-cmake/CMakeLists.txt b/contrib/cppkafka-cmake/CMakeLists.txt index 0bc33ada529..87bf2356a80 100644 --- a/contrib/cppkafka-cmake/CMakeLists.txt +++ b/contrib/cppkafka-cmake/CMakeLists.txt @@ -1,3 +1,8 @@ +if (NOT ENABLE_KAFKA) + message(STATUS "Not using librdkafka (skip cppkafka)") + return() +endif() + set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/cppkafka") set(SRCS @@ -22,12 +27,13 @@ set(SRCS "${LIBRARY_DIR}/src/topic.cpp" ) -add_library(cppkafka ${SRCS}) +add_library(_cppkafka ${SRCS}) +add_library(ch_contrib::cppkafka ALIAS _cppkafka) -target_link_libraries(cppkafka +target_link_libraries(_cppkafka PRIVATE - ${RDKAFKA_LIBRARY} + ch_contrib::rdkafka boost::headers_only ) -target_include_directories(cppkafka PRIVATE "${LIBRARY_DIR}/include/cppkafka") -target_include_directories(cppkafka SYSTEM BEFORE PUBLIC "${LIBRARY_DIR}/include") +target_include_directories(_cppkafka PRIVATE "${LIBRARY_DIR}/include/cppkafka") +target_include_directories(_cppkafka SYSTEM BEFORE PUBLIC "${LIBRARY_DIR}/include") diff --git a/contrib/librdkafka-cmake/CMakeLists.txt b/contrib/librdkafka-cmake/CMakeLists.txt index 9fb69a49e87..0505d06d2fc 100644 --- a/contrib/librdkafka-cmake/CMakeLists.txt +++ b/contrib/librdkafka-cmake/CMakeLists.txt @@ -1,3 +1,10 @@ +option (ENABLE_KAFKA "Enable kafka" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_KAFKA) + message(STATUS "Not using librdkafka") + return() +endif() + set(RDKAFKA_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/src") set(SRCS @@ -85,19 +92,13 @@ if(${ENABLE_CYRUS_SASL}) set(WITH_SASL_CYRUS 1) endif() -if(OPENSSL_FOUND) - message (STATUS "librdkafka with SSL support") - set(WITH_SSL 1) - - if(${ENABLE_CYRUS_SASL}) - set(WITH_SASL_SCRAM 1) - set(WITH_SASL_OAUTHBEARER 1) - endif() -endif() - -if(WITH_SSL) - list(APPEND SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka_ssl.c") +message (STATUS "librdkafka with SSL support") +set(WITH_SSL 1) +if(${ENABLE_CYRUS_SASL}) + set(WITH_SASL_SCRAM 1) + set(WITH_SASL_OAUTHBEARER 1) endif() +list(APPEND SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka_ssl.c") if(WITH_SASL_CYRUS) list(APPEND SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_cyrus.c") # needed to support Kerberos, requires cyrus-sasl @@ -111,18 +112,23 @@ if(WITH_SASL_OAUTHBEARER) list(APPEND SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_oauthbearer.c") endif() -add_library(rdkafka ${SRCS}) -target_compile_options(rdkafka PRIVATE -fno-sanitize=undefined) -# target_include_directories(rdkafka SYSTEM PUBLIC include) -target_include_directories(rdkafka SYSTEM PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") # for "librdkafka/rdkafka.h" -target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR}) # Because weird logic with "include_next" is used. -target_include_directories(rdkafka SYSTEM PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/auxdir") # for "../config.h" -target_link_libraries(rdkafka PRIVATE lz4 ch_contrib::zlib ch_contrib::zstd) -if(OPENSSL_SSL_LIBRARY AND OPENSSL_CRYPTO_LIBRARY) - target_link_libraries(rdkafka PRIVATE ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY}) -endif() +add_library(_rdkafka ${SRCS}) +add_library(ch_contrib::rdkafka ALIAS _rdkafka) + +target_compile_options(_rdkafka PRIVATE -fno-sanitize=undefined) +# target_include_directories(_rdkafka SYSTEM PUBLIC include) +target_include_directories(_rdkafka SYSTEM PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") # for "librdkafka/rdkafka.h" +target_include_directories(_rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR}) # Because weird logic with "include_next" is used. +target_include_directories(_rdkafka SYSTEM PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/auxdir") # for "../config.h" +target_link_libraries(_rdkafka + PRIVATE + lz4 + ch_contrib::zlib + ch_contrib::zstd + OpenSSL::Crypto OpenSSL::SSL +) if(${ENABLE_CYRUS_SASL}) - target_link_libraries(rdkafka PRIVATE ${CYRUS_SASL_LIBRARY}) + target_link_libraries(_rdkafka PRIVATE ${CYRUS_SASL_LIBRARY}) endif() file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/auxdir") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4a10f90f942..97bcedf9ff6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -87,7 +87,7 @@ if (USE_SQLITE) add_headers_and_sources(dbms Databases/SQLite) endif() -if(USE_RDKAFKA) +if (TARGET ch_contrib::rdkafka) add_headers_and_sources(dbms Storages/Kafka) endif() @@ -339,11 +339,8 @@ target_link_libraries(clickhouse_common_io # That way we we can redirect malloc/free functions avoiding circular dependencies dbms_target_link_libraries(PUBLIC roaring) -if (USE_RDKAFKA) - dbms_target_link_libraries(PRIVATE ${CPPKAFKA_LIBRARY} ${RDKAFKA_LIBRARY}) - if(NOT USE_INTERNAL_RDKAFKA_LIBRARY) - dbms_target_include_directories(SYSTEM BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR}) - endif() +if (TARGET ch_contrib::rdkafka) + dbms_target_link_libraries(PRIVATE ch_contrib::rdkafka ch_contrib::cppkafka) endif() if (USE_CYRUS_SASL) diff --git a/src/configure_config.cmake b/src/configure_config.cmake index 1bac4e07fb1..2a084d328a2 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -10,3 +10,6 @@ endif() if (TARGET ch_contrib::hivemetastore) set(USE_HIVE 1) endif() +if (TARGET ch_contrib::rdkafka) + set(USE_RDKAFKA 1) +endif() From 8485abd52b116b7944225410aed74aee9741a5c4 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 08:28:05 +0300 Subject: [PATCH 132/403] Remove unbundled lz4 support --- contrib/arrow-cmake/CMakeLists.txt | 2 +- contrib/librdkafka-cmake/CMakeLists.txt | 2 +- contrib/lz4-cmake/CMakeLists.txt | 47 ++++++------------- contrib/poco-cmake/Foundation/CMakeLists.txt | 6 ++- contrib/rocksdb-cmake/CMakeLists.txt | 2 +- src/CMakeLists.txt | 4 +- src/Compression/CMakeLists.txt | 2 +- src/Compression/fuzzers/CMakeLists.txt | 2 +- src/Functions/CMakeLists.txt | 2 +- ...StorageSystemBuildOptions.generated.cpp.in | 2 +- utils/compressor/CMakeLists.txt | 2 +- 11 files changed, 29 insertions(+), 44 deletions(-) diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 09a0158771e..8adae333965 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -373,7 +373,7 @@ endif () target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ARROW_SRC_DIR}) target_include_directories(_arrow SYSTEM BEFORE PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/cpp/src") target_link_libraries(_arrow PRIVATE ${DOUBLE_CONVERSION_LIBRARIES} ${Protobuf_LIBRARY}) -target_link_libraries(_arrow PRIVATE lz4) +target_link_libraries(_arrow PRIVATE ch_contrib::lz4) target_link_libraries(_arrow PRIVATE ch_contrib::snappy) target_link_libraries(_arrow PRIVATE ch_contrib::zlib) target_link_libraries(_arrow PRIVATE ch_contrib::zstd) diff --git a/contrib/librdkafka-cmake/CMakeLists.txt b/contrib/librdkafka-cmake/CMakeLists.txt index 0505d06d2fc..b8135a927e0 100644 --- a/contrib/librdkafka-cmake/CMakeLists.txt +++ b/contrib/librdkafka-cmake/CMakeLists.txt @@ -122,7 +122,7 @@ target_include_directories(_rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR}) target_include_directories(_rdkafka SYSTEM PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/auxdir") # for "../config.h" target_link_libraries(_rdkafka PRIVATE - lz4 + ch_contrib::lz4 ch_contrib::zlib ch_contrib::zstd OpenSSL::Crypto OpenSSL::SSL diff --git a/contrib/lz4-cmake/CMakeLists.txt b/contrib/lz4-cmake/CMakeLists.txt index 2c412d6e36a..68662b12c85 100644 --- a/contrib/lz4-cmake/CMakeLists.txt +++ b/contrib/lz4-cmake/CMakeLists.txt @@ -1,37 +1,18 @@ -option (USE_INTERNAL_LZ4_LIBRARY "Use internal lz4 library" ON) +# lz4 is the main compression method, cannot be disabled. -if (NOT USE_INTERNAL_LZ4_LIBRARY) - find_library (LIBRARY_LZ4 lz4) - find_path (INCLUDE_LZ4 lz4.h) +set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/lz4") +set (SRCS + "${LIBRARY_DIR}/lib/lz4.c" + "${LIBRARY_DIR}/lib/lz4hc.c" + "${LIBRARY_DIR}/lib/lz4frame.c" + "${LIBRARY_DIR}/lib/xxhash.c" +) - if (LIBRARY_LZ4 AND INCLUDE_LZ4) - set(EXTERNAL_LZ4_LIBRARY_FOUND 1) - add_library (lz4 INTERFACE) - set_property (TARGET lz4 PROPERTY INTERFACE_LINK_LIBRARIES ${LIBRARY_LZ4}) - set_property (TARGET lz4 PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_LZ4}) - set_property (TARGET lz4 APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS USE_XXHASH=0) - else() - set(EXTERNAL_LZ4_LIBRARY_FOUND 0) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system lz4") - endif() -endif() +add_library (_lz4 ${SRCS}) +add_library (ch_contrib::lz4 ALIAS _lz4) -if (NOT EXTERNAL_LZ4_LIBRARY_FOUND) - set (USE_INTERNAL_LZ4_LIBRARY 1) - set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/lz4") - - set (SRCS - "${LIBRARY_DIR}/lib/lz4.c" - "${LIBRARY_DIR}/lib/lz4hc.c" - "${LIBRARY_DIR}/lib/lz4frame.c" - "${LIBRARY_DIR}/lib/xxhash.c" - ) - - add_library (lz4 ${SRCS}) - - target_compile_definitions (lz4 PUBLIC LZ4_DISABLE_DEPRECATE_WARNINGS=1 USE_XXHASH=1) - if (SANITIZE STREQUAL "undefined") - target_compile_options (lz4 PRIVATE -fno-sanitize=undefined) - endif () - target_include_directories(lz4 PUBLIC "${LIBRARY_DIR}/lib") +target_compile_definitions (_lz4 PUBLIC LZ4_DISABLE_DEPRECATE_WARNINGS=1 USE_XXHASH=1) +if (SANITIZE STREQUAL "undefined") + target_compile_options (_lz4 PRIVATE -fno-sanitize=undefined) endif () +target_include_directories(_lz4 PUBLIC "${LIBRARY_DIR}/lib") diff --git a/contrib/poco-cmake/Foundation/CMakeLists.txt b/contrib/poco-cmake/Foundation/CMakeLists.txt index e094ec329a7..8ed4dfa6c66 100644 --- a/contrib/poco-cmake/Foundation/CMakeLists.txt +++ b/contrib/poco-cmake/Foundation/CMakeLists.txt @@ -223,7 +223,11 @@ if (USE_INTERNAL_POCO_LIBRARY) POCO_OS_FAMILY_UNIX ) target_include_directories (_poco_foundation SYSTEM PUBLIC "${LIBRARY_DIR}/Foundation/include") - target_link_libraries (_poco_foundation PRIVATE Poco::Foundation::PCRE ch_contrib::zlib lz4) + target_link_libraries (_poco_foundation + PRIVATE + Poco::Foundation::PCRE + ch_contrib::zlib + ch_contrib::lz4) else () add_library (Poco::Foundation UNKNOWN IMPORTED GLOBAL) diff --git a/contrib/rocksdb-cmake/CMakeLists.txt b/contrib/rocksdb-cmake/CMakeLists.txt index 05964d18033..f91e7199727 100644 --- a/contrib/rocksdb-cmake/CMakeLists.txt +++ b/contrib/rocksdb-cmake/CMakeLists.txt @@ -67,7 +67,7 @@ else() if(WITH_LZ4) add_definitions(-DLZ4) - list(APPEND THIRDPARTY_LIBS lz4) + list(APPEND THIRDPARTY_LIBS ch_contrib::lz4) endif() if(WITH_ZSTD) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 97bcedf9ff6..991ed315346 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -368,7 +368,7 @@ dbms_target_link_libraries ( clickhouse_common_zookeeper clickhouse_dictionaries_embedded clickhouse_parsers - lz4 + ch_contrib::lz4 Poco::JSON Poco::MongoDB string_utils @@ -507,7 +507,7 @@ if (USE_DATASKETCHES) target_include_directories (clickhouse_aggregate_functions SYSTEM BEFORE PRIVATE ${DATASKETCHES_INCLUDE_DIR}) endif () -target_link_libraries (clickhouse_common_io PRIVATE lz4) +target_link_libraries (clickhouse_common_io PRIVATE ch_contrib::lz4) dbms_target_link_libraries(PRIVATE _boost_context) diff --git a/src/Compression/CMakeLists.txt b/src/Compression/CMakeLists.txt index 34369d8dbc8..efa3f2b1c09 100644 --- a/src/Compression/CMakeLists.txt +++ b/src/Compression/CMakeLists.txt @@ -6,7 +6,7 @@ if (ENABLE_FUZZING) list(REMOVE_ITEM ${fuzz_compression_sources} CompressionFactoryAdditions.cpp) add_library(fuzz_compression ${fuzz_compression_headers} ${fuzz_compression_sources}) - target_link_libraries(fuzz_compression PUBLIC clickhouse_parsers clickhouse_common_io common lz4) + target_link_libraries(fuzz_compression PUBLIC clickhouse_parsers clickhouse_common_io common ch_contrib::lz4) endif() if (ENABLE_EXAMPLES) diff --git a/src/Compression/fuzzers/CMakeLists.txt b/src/Compression/fuzzers/CMakeLists.txt index 189aea66a92..db1573f1354 100644 --- a/src/Compression/fuzzers/CMakeLists.txt +++ b/src/Compression/fuzzers/CMakeLists.txt @@ -8,7 +8,7 @@ add_executable (compressed_buffer_fuzzer compressed_buffer_fuzzer.cpp) target_link_libraries (compressed_buffer_fuzzer PRIVATE dbms ${LIB_FUZZING_ENGINE}) add_executable (lz4_decompress_fuzzer lz4_decompress_fuzzer.cpp) -target_link_libraries (lz4_decompress_fuzzer PUBLIC dbms lz4 ${LIB_FUZZING_ENGINE}) +target_link_libraries (lz4_decompress_fuzzer PUBLIC dbms ch_contrib::lz4 ${LIB_FUZZING_ENGINE}) add_executable (delta_decompress_fuzzer delta_decompress_fuzzer.cpp) target_link_libraries (delta_decompress_fuzzer PRIVATE dbms ${LIB_FUZZING_ENGINE}) diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 9ab839f9246..7cb4371bb2f 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -79,7 +79,7 @@ if(USE_BASE64) target_include_directories(clickhouse_functions SYSTEM PRIVATE ${BASE64_INCLUDE_DIR}) endif() -target_link_libraries(clickhouse_functions PRIVATE lz4) +target_link_libraries(clickhouse_functions PRIVATE ch_contrib::lz4) if (USE_H3) target_link_libraries(clickhouse_functions PRIVATE ${H3_LIBRARY}) diff --git a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in index 9435bdcc65b..01a79e7c48d 100644 --- a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in +++ b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in @@ -35,7 +35,7 @@ const char * auto_config_build[] "USE_RDKAFKA", "@USE_RDKAFKA@", "USE_CAPNP", "@USE_CAPNP@", "USE_BASE64", "@USE_BASE64@", - "USE_XXHASH", "@USE_INTERNAL_LZ4_LIBRARY@", + "USE_XXHASH", "USE_HDFS", "@USE_HDFS@", "USE_SNAPPY", "@USE_SNAPPY@", "USE_PARQUET", "@USE_PARQUET@", diff --git a/utils/compressor/CMakeLists.txt b/utils/compressor/CMakeLists.txt index 43cde973846..7553ad42446 100644 --- a/utils/compressor/CMakeLists.txt +++ b/utils/compressor/CMakeLists.txt @@ -1,2 +1,2 @@ add_executable (decompress_perf decompress_perf.cpp) -target_link_libraries(decompress_perf PRIVATE dbms lz4) +target_link_libraries(decompress_perf PRIVATE dbms ch_contrib::lz4) From cf30669cc9d18a29ff095b2971ee7f7985a47dd3 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 08:45:10 +0300 Subject: [PATCH 133/403] Remove unbundled openssl/boringssl (crypo/ssl) support --- CMakeLists.txt | 1 - cmake/find/amqpcpp.cmake | 2 +- cmake/find/grpc.cmake | 8 +- cmake/find/ldap.cmake | 6 +- cmake/find/mysqlclient.cmake | 2 +- cmake/find/s2geometry.cmake | 8 +- cmake/find/ssl.cmake | 133 ------------------ contrib/CMakeLists.txt | 8 +- contrib/amqpcpp-cmake/CMakeLists.txt | 2 +- contrib/arrow-cmake/CMakeLists.txt | 6 +- contrib/aws-s3-cmake/CMakeLists.txt | 4 +- contrib/azure-cmake/CMakeLists.txt | 4 +- contrib/boringssl-cmake/CMakeLists.txt | 13 ++ contrib/grpc-cmake/CMakeLists.txt | 4 +- contrib/krb5-cmake/CMakeLists.txt | 5 +- contrib/libhdfs3-cmake/CMakeLists.txt | 6 +- .../mariadb-connector-c-cmake/CMakeLists.txt | 3 +- contrib/nuraft-cmake/CMakeLists.txt | 6 +- contrib/openldap-cmake/CMakeLists.txt | 9 +- contrib/s2geometry-cmake/CMakeLists.txt | 4 +- src/CMakeLists.txt | 6 +- src/Common/examples/CMakeLists.txt | 4 +- src/Core/examples/CMakeLists.txt | 3 - src/Functions/CMakeLists.txt | 4 +- src/configure_config.cmake | 3 + 25 files changed, 58 insertions(+), 196 deletions(-) delete mode 100644 cmake/find/ssl.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 88365c70d5f..15ab3351a4f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -490,7 +490,6 @@ include (GNUInstallDirs) include (cmake/contrib_finder.cmake) find_contrib_lib(double-conversion) # Must be before parquet -include (cmake/find/ssl.cmake) include (cmake/find/ldap.cmake) # after ssl include (cmake/find/icu.cmake) include (cmake/find/xz.cmake) diff --git a/cmake/find/amqpcpp.cmake b/cmake/find/amqpcpp.cmake index e033bea439f..43f88fae6c9 100644 --- a/cmake/find/amqpcpp.cmake +++ b/cmake/find/amqpcpp.cmake @@ -17,7 +17,7 @@ if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/src") endif () set (USE_AMQPCPP 1) -set (AMQPCPP_LIBRARY amqp-cpp ${OPENSSL_LIBRARIES}) +set (AMQPCPP_LIBRARY amqp-cpp OpenSSL::Crypto OpenSSL::SSL) set (AMQPCPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/include") list (APPEND AMQPCPP_INCLUDE_DIR diff --git a/cmake/find/grpc.cmake b/cmake/find/grpc.cmake index 92a85b0df04..a627f0fc0b6 100644 --- a/cmake/find/grpc.cmake +++ b/cmake/find/grpc.cmake @@ -33,11 +33,11 @@ if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/grpc/CMakeLists.txt") set(MISSING_INTERNAL_GRPC_LIBRARY 1) endif() -if(USE_SSL) +#if(USE_SSL) set(gRPC_USE_UNSECURE_LIBRARIES FALSE) -else() - set(gRPC_USE_UNSECURE_LIBRARIES TRUE) -endif() +#else() +# set(gRPC_USE_UNSECURE_LIBRARIES TRUE) +#endif() if(NOT USE_INTERNAL_GRPC_LIBRARY) find_package(gRPC) diff --git a/cmake/find/ldap.cmake b/cmake/find/ldap.cmake index d0d1e54bfec..a5ed2786746 100644 --- a/cmake/find/ldap.cmake +++ b/cmake/find/ldap.cmake @@ -70,9 +70,9 @@ if (NOT OPENLDAP_FOUND AND NOT MISSING_INTERNAL_LDAP_LIBRARY) if (NOT _ldap_supported_platform) message (WARNING "LDAP support using the bundled library is not implemented for ${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_PROCESSOR} platform.") message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot enable LDAP support") - elseif (NOT USE_SSL) - message (WARNING "LDAP support using the bundled library is not possible if SSL is not used.") - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot enable LDAP support") + #elseif (NOT TARGET OpenSSL::SSL) + # message (WARNING "LDAP support using the bundled library is not possible if SSL is not used.") + # message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot enable LDAP support") else () set (USE_INTERNAL_LDAP_LIBRARY 1) set (OPENLDAP_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/openldap") diff --git a/cmake/find/mysqlclient.cmake b/cmake/find/mysqlclient.cmake index 746775410cb..f191d06b17e 100644 --- a/cmake/find/mysqlclient.cmake +++ b/cmake/find/mysqlclient.cmake @@ -1,4 +1,4 @@ -if(OS_LINUX AND OPENSSL_FOUND) +if(OS_LINUX AND TARGET OpenSSL::SSL) option(ENABLE_MYSQL "Enable MySQL" ${ENABLE_LIBRARIES}) else () option(ENABLE_MYSQL "Enable MySQL" FALSE) diff --git a/cmake/find/s2geometry.cmake b/cmake/find/s2geometry.cmake index 348805b342e..ccbf72e0c41 100644 --- a/cmake/find/s2geometry.cmake +++ b/cmake/find/s2geometry.cmake @@ -7,13 +7,13 @@ if (ENABLE_S2_GEOMETRY) set (ENABLE_S2_GEOMETRY 0) set (USE_S2_GEOMETRY 0) else() - if (OPENSSL_FOUND) + #if (TARGET OpenSSL::SSL) set (S2_GEOMETRY_LIBRARY s2) set (S2_GEOMETRY_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/s2geometry/src/s2) set (USE_S2_GEOMETRY 1) - else() - message (WARNING "S2 uses OpenSSL, but the latter is absent.") - endif() + #else() + # message (WARNING "S2 uses OpenSSL, but the latter is absent.") + #endif() endif() if (NOT USE_S2_GEOMETRY) diff --git a/cmake/find/ssl.cmake b/cmake/find/ssl.cmake deleted file mode 100644 index 1ac6a54ed20..00000000000 --- a/cmake/find/ssl.cmake +++ /dev/null @@ -1,133 +0,0 @@ -# Needed when securely connecting to an external server, e.g. -# clickhouse-client --host ... --secure -option(ENABLE_SSL "Enable ssl" ${ENABLE_LIBRARIES}) - -if(NOT ENABLE_SSL) - if (USE_INTERNAL_SSL_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal ssl library with ENABLE_SSL=OFF") - endif() - return() -endif() - -option(USE_INTERNAL_SSL_LIBRARY "Set to FALSE to use system *ssl library instead of bundled" ON) - -if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/boringssl/README.md") - if(USE_INTERNAL_SSL_LIBRARY) - message(WARNING "submodule contrib/boringssl is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal ssl library") - endif() - set(USE_INTERNAL_SSL_LIBRARY 0) - set(MISSING_INTERNAL_SSL_LIBRARY 1) -endif() - -set (OPENSSL_USE_STATIC_LIBS ${USE_STATIC_LIBRARIES}) - -if (NOT USE_INTERNAL_SSL_LIBRARY) - if (APPLE) - set (OPENSSL_ROOT_DIR "/usr/local/opt/openssl" CACHE INTERNAL "") - # https://rt.openssl.org/Ticket/Display.html?user=guest&pass=guest&id=2232 - if (USE_STATIC_LIBRARIES) - message(WARNING "Disable USE_STATIC_LIBRARIES if you have linking problems with OpenSSL on MacOS") - endif () - endif () - find_package (OpenSSL) - - if (NOT OPENSSL_FOUND) - # Try to find manually. - set (OPENSSL_INCLUDE_PATHS "/usr/local/opt/openssl/include") - set (OPENSSL_PATHS "/usr/local/opt/openssl/lib") - find_path (OPENSSL_INCLUDE_DIR NAMES openssl/ssl.h PATHS ${OPENSSL_INCLUDE_PATHS}) - find_library (OPENSSL_SSL_LIBRARY ssl PATHS ${OPENSSL_PATHS}) - find_library (OPENSSL_CRYPTO_LIBRARY crypto PATHS ${OPENSSL_PATHS}) - if (OPENSSL_SSL_LIBRARY AND OPENSSL_CRYPTO_LIBRARY AND OPENSSL_INCLUDE_DIR) - set (OPENSSL_LIBRARIES ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY}) - set (OPENSSL_FOUND 1) - endif () - endif () - - if (NOT OPENSSL_FOUND) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system ssl") - endif() -endif () - -if (NOT OPENSSL_FOUND AND NOT MISSING_INTERNAL_SSL_LIBRARY) - set (USE_INTERNAL_SSL_LIBRARY 1) - set (OPENSSL_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/boringssl") - set (OPENSSL_INCLUDE_DIR "${OPENSSL_ROOT_DIR}/include") - set (OPENSSL_CRYPTO_LIBRARY crypto) - set (OPENSSL_SSL_LIBRARY ssl) - set (OPENSSL_FOUND 1) - set (OPENSSL_LIBRARIES ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY}) -endif () - -if(OPENSSL_FOUND) - # we need keep OPENSSL_FOUND for many libs in contrib - set(USE_SSL 1) -endif() - -# used by new poco -# part from /usr/share/cmake-*/Modules/FindOpenSSL.cmake, with removed all "EXISTS " -if(OPENSSL_FOUND AND NOT USE_INTERNAL_SSL_LIBRARY) - if(NOT TARGET OpenSSL::Crypto AND - (OPENSSL_CRYPTO_LIBRARY OR - LIB_EAY_LIBRARY_DEBUG OR - LIB_EAY_LIBRARY_RELEASE) - ) - add_library(OpenSSL::Crypto UNKNOWN IMPORTED) - set_target_properties(OpenSSL::Crypto PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIR}") - if(OPENSSL_CRYPTO_LIBRARY) - set_target_properties(OpenSSL::Crypto PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES "C" - IMPORTED_LOCATION "${OPENSSL_CRYPTO_LIBRARY}") - endif() - if(LIB_EAY_LIBRARY_RELEASE) - set_property(TARGET OpenSSL::Crypto APPEND PROPERTY - IMPORTED_CONFIGURATIONS RELEASE) - set_target_properties(OpenSSL::Crypto PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "C" - IMPORTED_LOCATION_RELEASE "${LIB_EAY_LIBRARY_RELEASE}") - endif() - if(LIB_EAY_LIBRARY_DEBUG) - set_property(TARGET OpenSSL::Crypto APPEND PROPERTY - IMPORTED_CONFIGURATIONS DEBUG) - set_target_properties(OpenSSL::Crypto PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES_DEBUG "C" - IMPORTED_LOCATION_DEBUG "${LIB_EAY_LIBRARY_DEBUG}") - endif() - endif() - if(NOT TARGET OpenSSL::SSL AND - (OPENSSL_SSL_LIBRARY OR - SSL_EAY_LIBRARY_DEBUG OR - SSL_EAY_LIBRARY_RELEASE) - ) - add_library(OpenSSL::SSL UNKNOWN IMPORTED) - set_target_properties(OpenSSL::SSL PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIR}") - if(OPENSSL_SSL_LIBRARY) - set_target_properties(OpenSSL::SSL PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES "C" - IMPORTED_LOCATION "${OPENSSL_SSL_LIBRARY}") - endif() - if(SSL_EAY_LIBRARY_RELEASE) - set_property(TARGET OpenSSL::SSL APPEND PROPERTY - IMPORTED_CONFIGURATIONS RELEASE) - set_target_properties(OpenSSL::SSL PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "C" - IMPORTED_LOCATION_RELEASE "${SSL_EAY_LIBRARY_RELEASE}") - endif() - if(SSL_EAY_LIBRARY_DEBUG) - set_property(TARGET OpenSSL::SSL APPEND PROPERTY - IMPORTED_CONFIGURATIONS DEBUG) - set_target_properties(OpenSSL::SSL PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES_DEBUG "C" - IMPORTED_LOCATION_DEBUG "${SSL_EAY_LIBRARY_DEBUG}") - endif() - if(TARGET OpenSSL::Crypto) - set_target_properties(OpenSSL::SSL PROPERTIES - INTERFACE_LINK_LIBRARIES OpenSSL::Crypto) - endif() - endif() -endif() - -message (STATUS "Using ssl=${USE_SSL}: ${OPENSSL_INCLUDE_DIR} : ${OPENSSL_LIBRARIES}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 8b8f74bb698..3b80771695d 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -54,6 +54,7 @@ if (USE_INTERNAL_XZ_LIBRARY) add_subdirectory (xz-cmake) endif() +add_subdirectory (boringssl-cmake) add_subdirectory (poco-cmake) add_subdirectory (croaring-cmake) add_subdirectory (zstd-cmake) @@ -91,13 +92,6 @@ if (USE_INTERNAL_H3_LIBRARY) add_subdirectory(h3-cmake) endif () -if (USE_INTERNAL_SSL_LIBRARY) - add_subdirectory (boringssl-cmake) - - add_library(OpenSSL::Crypto ALIAS crypto) - add_library(OpenSSL::SSL ALIAS ssl) -endif () - if (USE_INTERNAL_LDAP_LIBRARY) add_subdirectory (openldap-cmake) endif () diff --git a/contrib/amqpcpp-cmake/CMakeLists.txt b/contrib/amqpcpp-cmake/CMakeLists.txt index faef7bd4a1c..9b61d3120af 100644 --- a/contrib/amqpcpp-cmake/CMakeLists.txt +++ b/contrib/amqpcpp-cmake/CMakeLists.txt @@ -41,4 +41,4 @@ target_compile_options (amqp-cpp ) target_include_directories (amqp-cpp SYSTEM PUBLIC "${LIBRARY_DIR}/include") -target_link_libraries(amqp-cpp PUBLIC ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY}) +target_link_libraries(amqp-cpp PUBLIC OpenSSL::Crypto OpenSSL::SSL) diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 8adae333965..51d8ea5cb81 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -435,16 +435,14 @@ add_library(ch_contrib::parquet ALIAS _parquet) target_include_directories(_parquet SYSTEM BEFORE PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src" - "${CMAKE_CURRENT_SOURCE_DIR}/cpp/src" - PRIVATE - ${OPENSSL_INCLUDE_DIR}) + "${CMAKE_CURRENT_SOURCE_DIR}/cpp/src") target_link_libraries(_parquet PUBLIC _arrow PRIVATE ch_contrib::thrift boost::headers_only boost::regex - ${OPENSSL_LIBRARIES}) + OpenSSL::Crypto OpenSSL::SSL) if (SANITIZE STREQUAL "undefined") target_compile_options(_parquet PRIVATE -fno-sanitize=undefined) diff --git a/contrib/aws-s3-cmake/CMakeLists.txt b/contrib/aws-s3-cmake/CMakeLists.txt index 50f9482ef54..5024763f19d 100644 --- a/contrib/aws-s3-cmake/CMakeLists.txt +++ b/contrib/aws-s3-cmake/CMakeLists.txt @@ -95,9 +95,9 @@ target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_MINOR=7") target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_PATCH=231") target_include_directories(aws_s3 SYSTEM PUBLIC ${S3_INCLUDES}) -if (OPENSSL_FOUND) +if (TARGET OpenSSL::SSL) target_compile_definitions(aws_s3 PUBLIC -DENABLE_OPENSSL_ENCRYPTION) - target_link_libraries(aws_s3 PRIVATE ${OPENSSL_LIBRARIES}) + target_link_libraries(aws_s3 PRIVATE OpenSSL::Crypto OpenSSL::SSL) endif() target_link_libraries(aws_s3 PRIVATE aws_s3_checksums) diff --git a/contrib/azure-cmake/CMakeLists.txt b/contrib/azure-cmake/CMakeLists.txt index 527503b85a2..9310b57dc75 100644 --- a/contrib/azure-cmake/CMakeLists.txt +++ b/contrib/azure-cmake/CMakeLists.txt @@ -60,8 +60,8 @@ if (COMPILER_CLANG) endif() # Originally, on Windows azure-core is built with bcrypt and crypt32 by default -if (OPENSSL_FOUND) - target_link_libraries(azure_sdk PRIVATE ${OPENSSL_LIBRARIES}) +if (TARGET OpenSSL::SSL) + target_link_libraries(azure_sdk PRIVATE OpenSSL::Crypto OpenSSL::SSL) endif() # Originally, on Windows azure-core is built with winhttp by default diff --git a/contrib/boringssl-cmake/CMakeLists.txt b/contrib/boringssl-cmake/CMakeLists.txt index d599351fd5c..3ce78493576 100644 --- a/contrib/boringssl-cmake/CMakeLists.txt +++ b/contrib/boringssl-cmake/CMakeLists.txt @@ -1,3 +1,13 @@ +# Needed for: +# - securely connecting to an external server, e.g. clickhouse-client --host ... --secure +# - lots of thirdparty libraries +option(ENABLE_SSL "Enable ssl" ${ENABLE_LIBRARIES}) + +if(NOT ENABLE_SSL) + message(STATUS "Not using openssl") + return() +endif() + # Copyright (c) 2019 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -677,3 +687,6 @@ target_include_directories(crypto SYSTEM PUBLIC "${BORINGSSL_SOURCE_DIR}/include target_include_directories(ssl SYSTEM PUBLIC "${BORINGSSL_SOURCE_DIR}/include") target_compile_options(crypto PRIVATE -Wno-gnu-anonymous-struct) + +add_library(OpenSSL::Crypto ALIAS crypto) +add_library(OpenSSL::SSL ALIAS ssl) diff --git a/contrib/grpc-cmake/CMakeLists.txt b/contrib/grpc-cmake/CMakeLists.txt index ed8160ab2a1..4824802a5fb 100644 --- a/contrib/grpc-cmake/CMakeLists.txt +++ b/contrib/grpc-cmake/CMakeLists.txt @@ -31,8 +31,8 @@ set(_gRPC_PROTOBUF_PROTOC_LIBRARIES "${Protobuf_PROTOC_LIBRARY}") # Use OpenSSL from ClickHouse contrib, not from gRPC third_party. set(gRPC_SSL_PROVIDER "clickhouse" CACHE STRING "" FORCE) -set(_gRPC_SSL_INCLUDE_DIR ${OPENSSL_INCLUDE_DIR}) -set(_gRPC_SSL_LIBRARIES ${OPENSSL_LIBRARIES}) +set(_gRPC_SSL_INCLUDE_DIR "") +set(_gRPC_SSL_LIBRARIES OpenSSL::Crypto OpenSSL::SSL) # Use abseil-cpp from ClickHouse contrib, not from gRPC third_party. set(gRPC_ABSL_PROVIDER "clickhouse" CACHE STRING "" FORCE) diff --git a/contrib/krb5-cmake/CMakeLists.txt b/contrib/krb5-cmake/CMakeLists.txt index f7318a5bf8a..5bb5e981579 100644 --- a/contrib/krb5-cmake/CMakeLists.txt +++ b/contrib/krb5-cmake/CMakeLists.txt @@ -664,7 +664,6 @@ target_include_directories(${KRB5_LIBRARY} PRIVATE "${KRB5_SOURCE_DIR}/lib/krb5/rcache" "${KRB5_SOURCE_DIR}/lib/krb5/unicode" "${KRB5_SOURCE_DIR}/lib/krb5/os" - # ${OPENSSL_INCLUDE_DIR} ) target_compile_definitions(${KRB5_LIBRARY} PRIVATE @@ -677,6 +676,4 @@ target_compile_definitions(${KRB5_LIBRARY} PRIVATE LIBDIR="/usr/local/lib" ) -target_link_libraries(${KRB5_LIBRARY} - PRIVATE ${OPENSSL_CRYPTO_LIBRARY} -) +target_link_libraries(${KRB5_LIBRARY} PRIVATE OpenSSL::Crypto OpenSSL::SSL) diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index fcc4a15666c..0b8fe361baf 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -116,8 +116,6 @@ target_link_libraries(hdfs3 PRIVATE ${LIBXML2_LIBRARIES}) # inherit from parent cmake target_include_directories(hdfs3 PRIVATE ${Protobuf_INCLUDE_DIR}) target_link_libraries(hdfs3 PRIVATE ${Protobuf_LIBRARY} boost::headers_only) - -if(OPENSSL_INCLUDE_DIR AND OPENSSL_LIBRARIES) - target_include_directories(hdfs3 PRIVATE ${OPENSSL_INCLUDE_DIR}) - target_link_libraries(hdfs3 PRIVATE ${OPENSSL_LIBRARIES}) +if (TARGET OpenSSL::SSL) + target_link_libraries(hdfs3 PRIVATE OpenSSL::Crypto OpenSSL::SSL) endif() diff --git a/contrib/mariadb-connector-c-cmake/CMakeLists.txt b/contrib/mariadb-connector-c-cmake/CMakeLists.txt index b4c163ac4d8..1037f231a38 100644 --- a/contrib/mariadb-connector-c-cmake/CMakeLists.txt +++ b/contrib/mariadb-connector-c-cmake/CMakeLists.txt @@ -93,8 +93,7 @@ set(HAVE_THREADS 1) set(DEFAULT_CHARSET "utf8mb4") add_definitions(-DHAVE_OPENSSL -DHAVE_TLS) -set(SSL_LIBRARIES ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY}) -include_directories(BEFORE ${OPENSSL_INCLUDE_DIR}) +set(SSL_LIBRARIES OpenSSL::Crypto OpenSSL::SSL) set(TLS_LIBRARY_VERSION "OpenSSL ${OPENSSL_VERSION}") set(ENABLED_LOCAL_INFILE OFF) diff --git a/contrib/nuraft-cmake/CMakeLists.txt b/contrib/nuraft-cmake/CMakeLists.txt index d9e0aa6efc7..297cee1abac 100644 --- a/contrib/nuraft-cmake/CMakeLists.txt +++ b/contrib/nuraft-cmake/CMakeLists.txt @@ -32,7 +32,7 @@ set(SRCS add_library(nuraft ${SRCS}) -if (NOT OPENSSL_SSL_LIBRARY OR NOT OPENSSL_CRYPTO_LIBRARY) +if(NOT TARGET OpenSSL::Crypto) target_compile_definitions(nuraft PRIVATE USE_BOOST_ASIO=1 BOOST_ASIO_STANDALONE=1 SSL_LIBRARY_NOT_FOUND=1) else() target_compile_definitions(nuraft PRIVATE USE_BOOST_ASIO=1 BOOST_ASIO_STANDALONE=1) @@ -44,8 +44,8 @@ target_include_directories (nuraft SYSTEM PRIVATE "${ClickHouse_SOURCE_DIR}/cont target_link_libraries (nuraft PRIVATE boost::headers_only boost::coroutine) -if(OPENSSL_SSL_LIBRARY AND OPENSSL_CRYPTO_LIBRARY) - target_link_libraries (nuraft PRIVATE ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY}) +if(TARGET OpenSSL::Crypto) + target_link_libraries (nuraft PRIVATE OpenSSL::Crypto OpenSSL::SSL) endif() target_include_directories (nuraft SYSTEM PUBLIC "${LIBRARY_DIR}/include") diff --git a/contrib/openldap-cmake/CMakeLists.txt b/contrib/openldap-cmake/CMakeLists.txt index 0892403bb62..5d9492450ec 100644 --- a/contrib/openldap-cmake/CMakeLists.txt +++ b/contrib/openldap-cmake/CMakeLists.txt @@ -57,14 +57,13 @@ add_library(lber ${_libs_type} ) target_link_libraries(lber - PRIVATE ${OPENSSL_LIBRARIES} + PRIVATE OpenSSL::Crypto OpenSSL::SSL ) target_include_directories(lber PRIVATE ${_extra_build_dir}/include PRIVATE "${OPENLDAP_SOURCE_DIR}/include" PRIVATE "${OPENLDAP_SOURCE_DIR}/libraries/liblber" - PRIVATE ${OPENSSL_INCLUDE_DIR} ) target_compile_definitions(lber @@ -148,14 +147,13 @@ add_library(ldap ${_libs_type} target_link_libraries(ldap PRIVATE lber - PRIVATE ${OPENSSL_LIBRARIES} + PRIVATE OpenSSL::Crypto OpenSSL::SSL ) target_include_directories(ldap PRIVATE ${_extra_build_dir}/include PRIVATE "${OPENLDAP_SOURCE_DIR}/include" PRIVATE "${OPENLDAP_SOURCE_DIR}/libraries/libldap" - PRIVATE ${OPENSSL_INCLUDE_DIR} ) target_compile_definitions(ldap @@ -185,7 +183,7 @@ add_library(ldap_r ${_libs_type} target_link_libraries(ldap_r PRIVATE lber - PRIVATE ${OPENSSL_LIBRARIES} + PRIVATE OpenSSL::Crypto OpenSSL::SSL ) target_include_directories(ldap_r @@ -193,7 +191,6 @@ target_include_directories(ldap_r PRIVATE "${OPENLDAP_SOURCE_DIR}/include" PRIVATE "${OPENLDAP_SOURCE_DIR}/libraries/libldap_r" PRIVATE "${OPENLDAP_SOURCE_DIR}/libraries/libldap" - PRIVATE ${OPENSSL_INCLUDE_DIR} ) target_compile_definitions(ldap_r diff --git a/contrib/s2geometry-cmake/CMakeLists.txt b/contrib/s2geometry-cmake/CMakeLists.txt index e2b0f20f408..e5177689953 100644 --- a/contrib/s2geometry-cmake/CMakeLists.txt +++ b/contrib/s2geometry-cmake/CMakeLists.txt @@ -111,8 +111,8 @@ set(S2_SRCS add_library(s2 ${S2_SRCS}) set_property(TARGET s2 PROPERTY CXX_STANDARD 17) -if (OPENSSL_FOUND) - target_link_libraries(s2 PRIVATE ${OPENSSL_LIBRARIES}) +if (TARGET OpenSSL::SSL) + target_link_libraries(s2 PRIVATE OpenSSL::Crypto OpenSSL::SSL) endif() # Copied from contrib/s2geometry/CMakeLists diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 991ed315346..d362dbe0b9b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -416,9 +416,9 @@ if (TARGET ch_contrib::avrocpp) dbms_target_link_libraries(PRIVATE ch_contrib::avrocpp) endif () -if (OPENSSL_CRYPTO_LIBRARY) - dbms_target_link_libraries (PRIVATE ${OPENSSL_CRYPTO_LIBRARY}) - target_link_libraries (clickhouse_common_io PRIVATE ${OPENSSL_CRYPTO_LIBRARY}) +if (TARGET OpenSSL::Crypto) + dbms_target_link_libraries (PRIVATE OpenSSL::Crypto) + target_link_libraries (clickhouse_common_io PRIVATE OpenSSL::Crypto) endif () if (USE_LDAP) diff --git a/src/Common/examples/CMakeLists.txt b/src/Common/examples/CMakeLists.txt index 86b88cda24a..7410671bc2c 100644 --- a/src/Common/examples/CMakeLists.txt +++ b/src/Common/examples/CMakeLists.txt @@ -1,7 +1,7 @@ add_executable (hashes_test hashes_test.cpp) target_link_libraries (hashes_test PRIVATE clickhouse_common_io ${CITYHASH_LIBRARIES}) -if(OPENSSL_CRYPTO_LIBRARY) - target_link_libraries (hashes_test PRIVATE ${OPENSSL_CRYPTO_LIBRARY}) +if (TARGET OpenSSL::Crypto) + target_link_libraries (hashes_test PRIVATE OpenSSL::Crypto) endif() add_executable (sip_hash_perf sip_hash_perf.cpp) diff --git a/src/Core/examples/CMakeLists.txt b/src/Core/examples/CMakeLists.txt index c8846eb1743..7d02265967e 100644 --- a/src/Core/examples/CMakeLists.txt +++ b/src/Core/examples/CMakeLists.txt @@ -10,9 +10,6 @@ target_link_libraries (string_ref_hash PRIVATE clickhouse_common_io) add_executable (mysql_protocol mysql_protocol.cpp) target_link_libraries (mysql_protocol PRIVATE dbms) -if(USE_SSL) - target_include_directories (mysql_protocol SYSTEM PRIVATE ${OPENSSL_INCLUDE_DIR}) -endif() add_executable (coro coro.cpp) target_link_libraries (coro PRIVATE clickhouse_common_io) diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 7cb4371bb2f..623e3dfee29 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -30,8 +30,8 @@ target_link_libraries(clickhouse_functions divide_impl ) -if (OPENSSL_CRYPTO_LIBRARY) - target_link_libraries(clickhouse_functions PUBLIC ${OPENSSL_CRYPTO_LIBRARY}) +if (TARGET OpenSSL::Crypto) + target_link_libraries(clickhouse_functions PUBLIC OpenSSL::Crypto) endif() target_include_directories(clickhouse_functions SYSTEM PRIVATE ${SPARSEHASH_INCLUDE_DIR}) diff --git a/src/configure_config.cmake b/src/configure_config.cmake index 2a084d328a2..319802ddf51 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -13,3 +13,6 @@ endif() if (TARGET ch_contrib::rdkafka) set(USE_RDKAFKA 1) endif() +if (TARGET OpenSSL::SSL) + set(USE_SSL 1) +endif() From 94ba901fbd40e2a58cb7e779b3b705dd8a721cbc Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 08:53:49 +0300 Subject: [PATCH 134/403] Remove unbundled s2geometry support --- CMakeLists.txt | 1 - cmake/find/s2geometry.cmake | 24 ------------------------ contrib/s2geometry-cmake/CMakeLists.txt | 23 ++++++++++++++++------- src/CMakeLists.txt | 5 ++--- src/Functions/CMakeLists.txt | 3 --- 5 files changed, 18 insertions(+), 38 deletions(-) delete mode 100644 cmake/find/s2geometry.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 15ab3351a4f..fb845eccb4d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -527,7 +527,6 @@ include (cmake/find/sqlite.cmake) include (cmake/find/libpqxx.cmake) include (cmake/find/nuraft.cmake) include (cmake/find/yaml-cpp.cmake) -include (cmake/find/s2geometry.cmake) include (cmake/find/nlp.cmake) include (cmake/find/filelog.cmake) include (cmake/find/msgpack.cmake) diff --git a/cmake/find/s2geometry.cmake b/cmake/find/s2geometry.cmake deleted file mode 100644 index ccbf72e0c41..00000000000 --- a/cmake/find/s2geometry.cmake +++ /dev/null @@ -1,24 +0,0 @@ - -option(ENABLE_S2_GEOMETRY "Enable S2 geometry library" ${ENABLE_LIBRARIES}) - -if (ENABLE_S2_GEOMETRY) - if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/s2geometry") - message (WARNING "submodule contrib/s2geometry is missing. to fix try run: \n git submodule update --init") - set (ENABLE_S2_GEOMETRY 0) - set (USE_S2_GEOMETRY 0) - else() - #if (TARGET OpenSSL::SSL) - set (S2_GEOMETRY_LIBRARY s2) - set (S2_GEOMETRY_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/s2geometry/src/s2) - set (USE_S2_GEOMETRY 1) - #else() - # message (WARNING "S2 uses OpenSSL, but the latter is absent.") - #endif() - endif() - - if (NOT USE_S2_GEOMETRY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't enable S2 geometry library") - endif() -endif() - -message (STATUS "Using s2geometry=${USE_S2_GEOMETRY} : ${S2_GEOMETRY_INCLUDE_DIR}") diff --git a/contrib/s2geometry-cmake/CMakeLists.txt b/contrib/s2geometry-cmake/CMakeLists.txt index e5177689953..49c80e45b18 100644 --- a/contrib/s2geometry-cmake/CMakeLists.txt +++ b/contrib/s2geometry-cmake/CMakeLists.txt @@ -1,3 +1,10 @@ +option(ENABLE_S2_GEOMETRY "Enable S2 geometry library" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_S2_GEOMETRY) + message(STATUS "Not using S2 geometry") + return() +endif() + set(S2_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/s2geometry/src") set(ABSL_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/abseil-cpp") @@ -108,15 +115,17 @@ set(S2_SRCS ) -add_library(s2 ${S2_SRCS}) -set_property(TARGET s2 PROPERTY CXX_STANDARD 17) +add_library(_s2 ${S2_SRCS}) +add_library(ch_contrib::s2 ALIAS _s2) + +set_property(TARGET _s2 PROPERTY CXX_STANDARD 17) if (TARGET OpenSSL::SSL) - target_link_libraries(s2 PRIVATE OpenSSL::Crypto OpenSSL::SSL) + target_link_libraries(_s2 PRIVATE OpenSSL::Crypto OpenSSL::SSL) endif() # Copied from contrib/s2geometry/CMakeLists -target_link_libraries(s2 PRIVATE +target_link_libraries(_s2 PRIVATE absl::base absl::btree absl::config @@ -138,9 +147,9 @@ target_link_libraries(s2 PRIVATE absl::utility ) -target_include_directories(s2 SYSTEM BEFORE PUBLIC "${S2_SOURCE_DIR}/") -target_include_directories(s2 SYSTEM PUBLIC "${ABSL_SOURCE_DIR}") +target_include_directories(_s2 SYSTEM BEFORE PUBLIC "${S2_SOURCE_DIR}/") +target_include_directories(_s2 SYSTEM PUBLIC "${ABSL_SOURCE_DIR}") if(M_LIBRARY) - target_link_libraries(s2 PRIVATE ${M_LIBRARY}) + target_link_libraries(_s2 PRIVATE ${M_LIBRARY}) endif() diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d362dbe0b9b..9f8e675972a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -457,9 +457,8 @@ if (USE_AZURE_BLOB_STORAGE) target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${AZURE_SDK_INCLUDES}) endif() -if (USE_S2_GEOMETRY) - dbms_target_link_libraries (PUBLIC ${S2_GEOMETRY_LIBRARY}) - dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${S2_GEOMETRY_INCLUDE_DIR}) +if (TARGET ch_contrib::s2) + dbms_target_link_libraries (PUBLIC ch_contrib::s2) endif() if (USE_BROTLI) diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 623e3dfee29..f0620fe4bc8 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -121,9 +121,6 @@ add_subdirectory(JSONPath) # Signed integer overflow on user-provided data inside boost::geometry - ignore. set_source_files_properties("pointInPolygon.cpp" PROPERTIES COMPILE_FLAGS -fno-sanitize=signed-integer-overflow) -# target_link_libraries(clickhouse_functions PRIVATE ${S2_LIBRARY}) -target_include_directories(clickhouse_functions SYSTEM PUBLIC ${S2_GEOMETRY_INCLUDE_DIR}) - if (ENABLE_FUZZING) add_compile_definitions(FUZZING_MODE=1) endif () From e341dadb4c9ce50c85279bff5df09682d0137dcb Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 09:00:50 +0300 Subject: [PATCH 135/403] Remove unbundled openldap support --- CMakeLists.txt | 1 - cmake/Modules/FindOpenLDAP.cmake | 55 -------------- cmake/find/ldap.cmake | 100 -------------------------- contrib/CMakeLists.txt | 5 +- contrib/openldap-cmake/CMakeLists.txt | 77 +++++++++++++------- src/CMakeLists.txt | 5 +- src/configure_config.cmake | 3 + 7 files changed, 58 insertions(+), 188 deletions(-) delete mode 100644 cmake/Modules/FindOpenLDAP.cmake delete mode 100644 cmake/find/ldap.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index fb845eccb4d..b20a97a0ca2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -490,7 +490,6 @@ include (GNUInstallDirs) include (cmake/contrib_finder.cmake) find_contrib_lib(double-conversion) # Must be before parquet -include (cmake/find/ldap.cmake) # after ssl include (cmake/find/icu.cmake) include (cmake/find/xz.cmake) include (cmake/find/ltdl.cmake) # for odbc diff --git a/cmake/Modules/FindOpenLDAP.cmake b/cmake/Modules/FindOpenLDAP.cmake deleted file mode 100644 index 9c6262fa245..00000000000 --- a/cmake/Modules/FindOpenLDAP.cmake +++ /dev/null @@ -1,55 +0,0 @@ -# Find OpenLDAP libraries. -# -# Can be configured with: -# OPENLDAP_ROOT_DIR - path to the OpenLDAP installation prefix -# OPENLDAP_USE_STATIC_LIBS - look for static version of the libraries -# OPENLDAP_USE_REENTRANT_LIBS - look for thread-safe version of the libraries -# -# Sets values of: -# OPENLDAP_FOUND - TRUE if found -# OPENLDAP_INCLUDE_DIRS - paths to the include directories -# OPENLDAP_LIBRARIES - paths to the libldap and liblber libraries -# OPENLDAP_LDAP_LIBRARY - paths to the libldap library -# OPENLDAP_LBER_LIBRARY - paths to the liblber library -# - -if(OPENLDAP_USE_STATIC_LIBS) - set(_orig_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) - if(WIN32) - set(CMAKE_FIND_LIBRARY_SUFFIXES ".lib" ".a" ${CMAKE_FIND_LIBRARY_SUFFIXES}) - else() - set(CMAKE_FIND_LIBRARY_SUFFIXES ".a") - endif() -endif() - -set(_r_suffix) -if(OPENLDAP_USE_REENTRANT_LIBS) - set(_r_suffix "_r") -endif() - -if(OPENLDAP_ROOT_DIR) - find_path(OPENLDAP_INCLUDE_DIRS NAMES "ldap.h" "lber.h" PATHS "${OPENLDAP_ROOT_DIR}" PATH_SUFFIXES "include" NO_DEFAULT_PATH) - find_library(OPENLDAP_LDAP_LIBRARY NAMES "ldap${_r_suffix}" PATHS "${OPENLDAP_ROOT_DIR}" PATH_SUFFIXES "lib" NO_DEFAULT_PATH) - find_library(OPENLDAP_LBER_LIBRARY NAMES "lber" PATHS "${OPENLDAP_ROOT_DIR}" PATH_SUFFIXES "lib" NO_DEFAULT_PATH) -else() - find_path(OPENLDAP_INCLUDE_DIRS NAMES "ldap.h" "lber.h") - find_library(OPENLDAP_LDAP_LIBRARY NAMES "ldap${_r_suffix}") - find_library(OPENLDAP_LBER_LIBRARY NAMES "lber") -endif() - -unset(_r_suffix) - -set(OPENLDAP_LIBRARIES ${OPENLDAP_LDAP_LIBRARY} ${OPENLDAP_LBER_LIBRARY}) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args( - OpenLDAP DEFAULT_MSG - OPENLDAP_INCLUDE_DIRS OPENLDAP_LDAP_LIBRARY OPENLDAP_LBER_LIBRARY -) - -mark_as_advanced(OPENLDAP_INCLUDE_DIRS OPENLDAP_LIBRARIES OPENLDAP_LDAP_LIBRARY OPENLDAP_LBER_LIBRARY) - -if(OPENLDAP_USE_STATIC_LIBS) - set(CMAKE_FIND_LIBRARY_SUFFIXES ${_orig_CMAKE_FIND_LIBRARY_SUFFIXES}) - unset(_orig_CMAKE_FIND_LIBRARY_SUFFIXES) -endif() diff --git a/cmake/find/ldap.cmake b/cmake/find/ldap.cmake deleted file mode 100644 index a5ed2786746..00000000000 --- a/cmake/find/ldap.cmake +++ /dev/null @@ -1,100 +0,0 @@ -option (ENABLE_LDAP "Enable LDAP" ${ENABLE_LIBRARIES}) - -if (NOT ENABLE_LDAP) - if(USE_INTERNAL_LDAP_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal LDAP library with ENABLE_LDAP=OFF") - endif () - return() -endif() - -option (USE_INTERNAL_LDAP_LIBRARY "Set to FALSE to use system *LDAP library instead of bundled" ON) - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/openldap/README") - if (USE_INTERNAL_LDAP_LIBRARY) - message (WARNING "Submodule contrib/openldap is missing. To fix try running:\n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal LDAP library") - endif () - - set (USE_INTERNAL_LDAP_LIBRARY 0) - set (MISSING_INTERNAL_LDAP_LIBRARY 1) -endif () - -set (OPENLDAP_USE_STATIC_LIBS ${USE_STATIC_LIBRARIES}) -set (OPENLDAP_USE_REENTRANT_LIBS 1) - -if (NOT USE_INTERNAL_LDAP_LIBRARY) - if (OPENLDAP_USE_STATIC_LIBS) - message (WARNING "Unable to use external static OpenLDAP libraries, falling back to the bundled version.") - message (${RECONFIGURE_MESSAGE_LEVEL} "Unable to use external OpenLDAP") - set (USE_INTERNAL_LDAP_LIBRARY 1) - else () - if (APPLE AND NOT OPENLDAP_ROOT_DIR) - set (OPENLDAP_ROOT_DIR "/usr/local/opt/openldap") - endif () - - find_package (OpenLDAP) - - if (NOT OPENLDAP_FOUND) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system OpenLDAP") - endif() - endif () -endif () - -if (NOT OPENLDAP_FOUND AND NOT MISSING_INTERNAL_LDAP_LIBRARY) - string (TOLOWER "${CMAKE_SYSTEM_NAME}" _system_name) - string (TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _system_processor) - - if ( - "${_system_processor}" STREQUAL "amd64" OR - "${_system_processor}" STREQUAL "x64" - ) - set (_system_processor "x86_64") - elseif ( - "${_system_processor}" STREQUAL "arm64" - ) - set (_system_processor "aarch64") - endif () - - if ( - ( "${_system_name}" STREQUAL "linux" AND "${_system_processor}" STREQUAL "x86_64" ) OR - ( "${_system_name}" STREQUAL "linux" AND "${_system_processor}" STREQUAL "aarch64" ) OR - ( "${_system_name}" STREQUAL "linux" AND "${_system_processor}" STREQUAL "ppc64le" ) OR - ( "${_system_name}" STREQUAL "freebsd" AND "${_system_processor}" STREQUAL "x86_64" ) OR - ( "${_system_name}" STREQUAL "freebsd" AND "${_system_processor}" STREQUAL "aarch64" ) OR - ( "${_system_name}" STREQUAL "darwin" AND "${_system_processor}" STREQUAL "x86_64" ) OR - ( "${_system_name}" STREQUAL "darwin" AND "${_system_processor}" STREQUAL "aarch64" ) - ) - set (_ldap_supported_platform TRUE) - endif () - - if (NOT _ldap_supported_platform) - message (WARNING "LDAP support using the bundled library is not implemented for ${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_PROCESSOR} platform.") - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot enable LDAP support") - #elseif (NOT TARGET OpenSSL::SSL) - # message (WARNING "LDAP support using the bundled library is not possible if SSL is not used.") - # message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot enable LDAP support") - else () - set (USE_INTERNAL_LDAP_LIBRARY 1) - set (OPENLDAP_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/openldap") - set (OPENLDAP_INCLUDE_DIRS - "${ClickHouse_SOURCE_DIR}/contrib/openldap-cmake/${_system_name}_${_system_processor}/include" - "${ClickHouse_SOURCE_DIR}/contrib/openldap/include" - ) - # Below, 'ldap'/'ldap_r' and 'lber' will be resolved to - # the targets defined in contrib/openldap-cmake/CMakeLists.txt - if (OPENLDAP_USE_REENTRANT_LIBS) - set (OPENLDAP_LDAP_LIBRARY "ldap_r") - else () - set (OPENLDAP_LDAP_LIBRARY "ldap") - endif() - set (OPENLDAP_LBER_LIBRARY "lber") - set (OPENLDAP_LIBRARIES ${OPENLDAP_LDAP_LIBRARY} ${OPENLDAP_LBER_LIBRARY}) - set (OPENLDAP_FOUND 1) - endif () -endif () - -if (OPENLDAP_FOUND) - set (USE_LDAP 1) -endif () - -message (STATUS "Using ldap=${USE_LDAP}: ${OPENLDAP_INCLUDE_DIRS} : ${OPENLDAP_LIBRARIES}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 3b80771695d..3f088c8a2f8 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -69,6 +69,7 @@ add_subdirectory (avro-cmake) # requires: snappy add_subdirectory (hive-metastore-cmake) # requires: thrift/avro/arrow add_subdirectory (librdkafka-cmake) add_subdirectory (cppkafka-cmake) +add_subdirectory (openldap-cmake) # TODO: refactor the contrib libraries below this comment. @@ -92,10 +93,6 @@ if (USE_INTERNAL_H3_LIBRARY) add_subdirectory(h3-cmake) endif () -if (USE_INTERNAL_LDAP_LIBRARY) - add_subdirectory (openldap-cmake) -endif () - if (USE_INTERNAL_MYSQL_LIBRARY) add_subdirectory (mariadb-connector-c-cmake) endif () diff --git a/contrib/openldap-cmake/CMakeLists.txt b/contrib/openldap-cmake/CMakeLists.txt index 5d9492450ec..f5966474b0d 100644 --- a/contrib/openldap-cmake/CMakeLists.txt +++ b/contrib/openldap-cmake/CMakeLists.txt @@ -1,13 +1,37 @@ +option (ENABLE_LDAP "Enable LDAP" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_LDAP) + message(STATUS "Not using ldap") + return() +endif() + +string (TOLOWER "${CMAKE_SYSTEM_NAME}" _system_name) +string (TOLOWER "${CMAKE_SYSTEM_PROCESSOR}" _system_processor) +if ( + "${_system_processor}" STREQUAL "amd64" OR + "${_system_processor}" STREQUAL "x64" +) + set (_system_processor "x86_64") +elseif ("${_system_processor}" STREQUAL "arm64") + set (_system_processor "aarch64") +endif () +if (NOT( + ( "${_system_name}" STREQUAL "linux" AND "${_system_processor}" STREQUAL "x86_64" ) OR + ( "${_system_name}" STREQUAL "linux" AND "${_system_processor}" STREQUAL "aarch64" ) OR + ( "${_system_name}" STREQUAL "linux" AND "${_system_processor}" STREQUAL "ppc64le" ) OR + ( "${_system_name}" STREQUAL "freebsd" AND "${_system_processor}" STREQUAL "x86_64" ) OR + ( "${_system_name}" STREQUAL "freebsd" AND "${_system_processor}" STREQUAL "aarch64" ) OR + ( "${_system_name}" STREQUAL "darwin" AND "${_system_processor}" STREQUAL "x86_64" ) OR + ( "${_system_name}" STREQUAL "darwin" AND "${_system_processor}" STREQUAL "aarch64" ) +)) + message (${RECONFIGURE_MESSAGE_LEVEL} "LDAP support using the bundled library is not implemented for ${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_PROCESSOR} platform.") +endif () + set(OPENLDAP_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/openldap") # How these lists were generated? # I compiled the original OpenLDAP with it's original build system and copied the list of source files from build commands. -set(_libs_type SHARED) -if(OPENLDAP_USE_STATIC_LIBS) - set(_libs_type STATIC) -endif() - set(OPENLDAP_VERSION_STRING "2.5.X") macro(mkversion _lib_name) @@ -51,22 +75,22 @@ set(_lber_srcs mkversion(lber) -add_library(lber ${_libs_type} +add_library(_lber ${_lber_srcs} "${CMAKE_CURRENT_BINARY_DIR}/lber-version.c" ) -target_link_libraries(lber +target_link_libraries(_lber PRIVATE OpenSSL::Crypto OpenSSL::SSL ) -target_include_directories(lber - PRIVATE ${_extra_build_dir}/include - PRIVATE "${OPENLDAP_SOURCE_DIR}/include" +target_include_directories(_lber SYSTEM + PUBLIC ${_extra_build_dir}/include + PUBLIC "${OPENLDAP_SOURCE_DIR}/include" PRIVATE "${OPENLDAP_SOURCE_DIR}/libraries/liblber" ) -target_compile_definitions(lber +target_compile_definitions(_lber PRIVATE LBER_LIBRARY ) @@ -140,23 +164,23 @@ set(_ldap_srcs mkversion(ldap) -add_library(ldap ${_libs_type} +add_library(_ldap ${_ldap_srcs} "${CMAKE_CURRENT_BINARY_DIR}/ldap-version.c" ) -target_link_libraries(ldap - PRIVATE lber +target_link_libraries(_ldap + PRIVATE _lber PRIVATE OpenSSL::Crypto OpenSSL::SSL ) -target_include_directories(ldap - PRIVATE ${_extra_build_dir}/include - PRIVATE "${OPENLDAP_SOURCE_DIR}/include" +target_include_directories(_ldap SYSTEM + PUBLIC ${_extra_build_dir}/include + PUBLIC "${OPENLDAP_SOURCE_DIR}/include" PRIVATE "${OPENLDAP_SOURCE_DIR}/libraries/libldap" ) -target_compile_definitions(ldap +target_compile_definitions(_ldap PRIVATE LDAP_LIBRARY ) @@ -175,25 +199,28 @@ set(_ldap_r_specific_srcs mkversion(ldap_r) -add_library(ldap_r ${_libs_type} +add_library(_ldap_r ${_ldap_r_specific_srcs} ${_ldap_srcs} "${CMAKE_CURRENT_BINARY_DIR}/ldap_r-version.c" ) -target_link_libraries(ldap_r - PRIVATE lber +target_link_libraries(_ldap_r + PRIVATE _lber PRIVATE OpenSSL::Crypto OpenSSL::SSL ) -target_include_directories(ldap_r - PRIVATE ${_extra_build_dir}/include - PRIVATE "${OPENLDAP_SOURCE_DIR}/include" +target_include_directories(_ldap_r SYSTEM + PUBLIC ${_extra_build_dir}/include + PUBLIC "${OPENLDAP_SOURCE_DIR}/include" PRIVATE "${OPENLDAP_SOURCE_DIR}/libraries/libldap_r" PRIVATE "${OPENLDAP_SOURCE_DIR}/libraries/libldap" ) -target_compile_definitions(ldap_r +target_compile_definitions(_ldap_r PRIVATE LDAP_R_COMPILE PRIVATE LDAP_LIBRARY ) + +add_library(ch_contrib::ldap ALIAS _ldap_r) +add_library(ch_contrib::lber ALIAS _lber) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9f8e675972a..ed7c3df5f0c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -421,9 +421,8 @@ if (TARGET OpenSSL::Crypto) target_link_libraries (clickhouse_common_io PRIVATE OpenSSL::Crypto) endif () -if (USE_LDAP) - dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${OPENLDAP_INCLUDE_DIRS}) - dbms_target_link_libraries (PRIVATE ${OPENLDAP_LIBRARIES}) +if (TARGET ch_contrib::ldap) + dbms_target_link_libraries (PRIVATE ch_contrib::ldap ch_contrib::lber) endif () dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) diff --git a/src/configure_config.cmake b/src/configure_config.cmake index 319802ddf51..bd93472afd9 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -16,3 +16,6 @@ endif() if (TARGET OpenSSL::SSL) set(USE_SSL 1) endif() +if (TARGET ch_contrib::ldap) + set(USE_LDAP 1) +endif() From 6acb4d6ac5f3b8a2078ab498fd444393df861cb7 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 10:36:58 +0300 Subject: [PATCH 136/403] Remove unbundled gRPC support --- CMakeLists.txt | 1 - cmake/Modules/FindgRPC.cmake | 337 -------------------------- cmake/find/grpc.cmake | 72 ------ contrib/CMakeLists.txt | 5 +- contrib/grpc-cmake/CMakeLists.txt | 33 +++ src/CMakeLists.txt | 2 +- src/Server/CMakeLists.txt | 2 +- src/Server/grpc_protos/CMakeLists.txt | 4 +- src/configure_config.cmake | 3 + 9 files changed, 41 insertions(+), 418 deletions(-) delete mode 100644 cmake/Modules/FindgRPC.cmake delete mode 100644 cmake/find/grpc.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index b20a97a0ca2..9db7c069691 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -507,7 +507,6 @@ include (cmake/find/h3.cmake) include (cmake/find/libxml2.cmake) include (cmake/find/brotli.cmake) include (cmake/find/protobuf.cmake) -include (cmake/find/grpc.cmake) include (cmake/find/pdqsort.cmake) include (cmake/find/miniselect.cmake) include (cmake/find/hdfs3.cmake) # uses protobuf diff --git a/cmake/Modules/FindgRPC.cmake b/cmake/Modules/FindgRPC.cmake deleted file mode 100644 index 945d307952b..00000000000 --- a/cmake/Modules/FindgRPC.cmake +++ /dev/null @@ -1,337 +0,0 @@ -#[[ -Defines the following variables: -``gRPC_FOUND`` - Whether the gRPC framework is found -``gRPC_INCLUDE_DIRS`` - The include directories of the gRPC framework, including the include directories of the C++ wrapper. -``gRPC_LIBRARIES`` - The libraries of the gRPC framework. -``gRPC_CPP_PLUGIN`` - The plugin for generating gRPC client and server C++ stubs from `.proto` files -``gRPC_PYTHON_PLUGIN`` - The plugin for generating gRPC client and server Python stubs from `.proto` files - -The following :prop_tgt:`IMPORTED` targets are also defined: -``grpc++`` -``grpc++_unsecure`` -``grpc_cpp_plugin`` -``grpc_python_plugin`` - -Set the following variables to adjust the behaviour of this script: -``gRPC_USE_UNSECURE_LIBRARIES`` - if set gRPC_LIBRARIES will be filled with the unsecure version of the libraries (i.e. without SSL) - instead of the secure ones. -``gRPC_DEBUG` - if set the debug message will be printed. - -Add custom commands to process ``.proto`` files to C++:: -protobuf_generate_grpc_cpp( - [DESCRIPTORS ] [EXPORT_MACRO ] [...]) - -``SRCS`` - Variable to define with autogenerated source files -``HDRS`` - Variable to define with autogenerated header files -``DESCRIPTORS`` - Variable to define with autogenerated descriptor files, if requested. -``EXPORT_MACRO`` - is a macro which should expand to ``__declspec(dllexport)`` or - ``__declspec(dllimport)`` depending on what is being compiled. -``ARGN`` - ``.proto`` files -#]] - -# Function to generate C++ files from .proto files. -# This function is a modified version of the function PROTOBUF_GENERATE_CPP() copied from https://github.com/Kitware/CMake/blob/master/Modules/FindProtobuf.cmake. -function(PROTOBUF_GENERATE_GRPC_CPP SRCS HDRS) - cmake_parse_arguments(protobuf_generate_grpc_cpp "" "EXPORT_MACRO;DESCRIPTORS" "" ${ARGN}) - - set(_proto_files "${protobuf_generate_grpc_cpp_UNPARSED_ARGUMENTS}") - if(NOT _proto_files) - message(SEND_ERROR "Error: PROTOBUF_GENERATE_GRPC_CPP() called without any proto files") - return() - endif() - - if(PROTOBUF_GENERATE_GRPC_CPP_APPEND_PATH) - set(_append_arg APPEND_PATH) - endif() - - if(protobuf_generate_grpc_cpp_DESCRIPTORS) - set(_descriptors DESCRIPTORS) - endif() - - if(DEFINED PROTOBUF_IMPORT_DIRS AND NOT DEFINED Protobuf_IMPORT_DIRS) - set(Protobuf_IMPORT_DIRS "${PROTOBUF_IMPORT_DIRS}") - endif() - - if(DEFINED Protobuf_IMPORT_DIRS) - set(_import_arg IMPORT_DIRS ${Protobuf_IMPORT_DIRS}) - endif() - - set(_outvar) - protobuf_generate_grpc(${_append_arg} ${_descriptors} LANGUAGE cpp EXPORT_MACRO ${protobuf_generate_cpp_EXPORT_MACRO} OUT_VAR _outvar ${_import_arg} PROTOS ${_proto_files}) - - set(${SRCS}) - set(${HDRS}) - if(protobuf_generate_grpc_cpp_DESCRIPTORS) - set(${protobuf_generate_grpc_cpp_DESCRIPTORS}) - endif() - - foreach(_file ${_outvar}) - if(_file MATCHES "cc$") - list(APPEND ${SRCS} ${_file}) - elseif(_file MATCHES "desc$") - list(APPEND ${protobuf_generate_grpc_cpp_DESCRIPTORS} ${_file}) - else() - list(APPEND ${HDRS} ${_file}) - endif() - endforeach() - set(${SRCS} ${${SRCS}} PARENT_SCOPE) - set(${HDRS} ${${HDRS}} PARENT_SCOPE) - if(protobuf_generate_grpc_cpp_DESCRIPTORS) - set(${protobuf_generate_grpc_cpp_DESCRIPTORS} "${${protobuf_generate_grpc_cpp_DESCRIPTORS}}" PARENT_SCOPE) - endif() -endfunction() - -# Helper function. -# This function is a modified version of the function protobuf_generate() copied from https://github.com/Kitware/CMake/blob/master/Modules/FindProtobuf.cmake. -function(protobuf_generate_grpc) - set(_options APPEND_PATH DESCRIPTORS) - set(_singleargs LANGUAGE OUT_VAR EXPORT_MACRO PROTOC_OUT_DIR) - if(COMMAND target_sources) - list(APPEND _singleargs TARGET) - endif() - set(_multiargs PROTOS IMPORT_DIRS GENERATE_EXTENSIONS) - - cmake_parse_arguments(protobuf_generate_grpc "${_options}" "${_singleargs}" "${_multiargs}" "${ARGN}") - - if(NOT protobuf_generate_grpc_PROTOS AND NOT protobuf_generate_grpc_TARGET) - message(SEND_ERROR "Error: protobuf_generate_grpc called without any targets or source files") - return() - endif() - - if(NOT protobuf_generate_grpc_OUT_VAR AND NOT protobuf_generate_grpc_TARGET) - message(SEND_ERROR "Error: protobuf_generate_grpc called without a target or output variable") - return() - endif() - - if(NOT protobuf_generate_grpc_LANGUAGE) - set(protobuf_generate_grpc_LANGUAGE cpp) - endif() - string(TOLOWER ${protobuf_generate_grpc_LANGUAGE} protobuf_generate_grpc_LANGUAGE) - - if(NOT protobuf_generate_grpc_PROTOC_OUT_DIR) - set(protobuf_generate_grpc_PROTOC_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}) - endif() - - if(protobuf_generate_grpc_EXPORT_MACRO AND protobuf_generate_grpc_LANGUAGE STREQUAL cpp) - set(_dll_export_decl "dllexport_decl=${protobuf_generate_grpc_EXPORT_MACRO}:") - endif() - - if(NOT protobuf_generate_grpc_GENERATE_EXTENSIONS) - if(protobuf_generate_grpc_LANGUAGE STREQUAL cpp) - set(protobuf_generate_grpc_GENERATE_EXTENSIONS .pb.h .pb.cc .grpc.pb.h .grpc.pb.cc) - elseif(protobuf_generate_grpc_LANGUAGE STREQUAL python) - set(protobuf_generate_grpc_GENERATE_EXTENSIONS _pb2.py) - else() - message(SEND_ERROR "Error: protobuf_generate_grpc given unknown Language ${LANGUAGE}, please provide a value for GENERATE_EXTENSIONS") - return() - endif() - endif() - - if(NOT protobuf_generate_grpc_PLUGIN) - if(protobuf_generate_grpc_LANGUAGE STREQUAL cpp) - set(protobuf_generate_grpc_PLUGIN "grpc_cpp_plugin") - elseif(protobuf_generate_grpc_LANGUAGE STREQUAL python) - set(protobuf_generate_grpc_PLUGIN "grpc_python_plugin") - else() - message(SEND_ERROR "Error: protobuf_generate_grpc given unknown Language ${LANGUAGE}, please provide a value for PLUGIN") - return() - endif() - endif() - - if(protobuf_generate_grpc_TARGET) - get_target_property(_source_list ${protobuf_generate_grpc_TARGET} SOURCES) - foreach(_file ${_source_list}) - if(_file MATCHES "proto$") - list(APPEND protobuf_generate_grpc_PROTOS ${_file}) - endif() - endforeach() - endif() - - if(NOT protobuf_generate_grpc_PROTOS) - message(SEND_ERROR "Error: protobuf_generate_grpc could not find any .proto files") - return() - endif() - - if(protobuf_generate_grpc_APPEND_PATH) - # Create an include path for each file specified - foreach(_file ${protobuf_generate_grpc_PROTOS}) - get_filename_component(_abs_file ${_file} ABSOLUTE) - get_filename_component(_abs_path ${_abs_file} PATH) - list(FIND _protobuf_include_path ${_abs_path} _contains_already) - if(${_contains_already} EQUAL -1) - list(APPEND _protobuf_include_path -I ${_abs_path}) - endif() - endforeach() - else() - set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR}) - endif() - - foreach(DIR ${protobuf_generate_grpc_IMPORT_DIRS}) - get_filename_component(ABS_PATH ${DIR} ABSOLUTE) - list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) - if(${_contains_already} EQUAL -1) - list(APPEND _protobuf_include_path -I ${ABS_PATH}) - endif() - endforeach() - - set(_generated_srcs_all) - foreach(_proto ${protobuf_generate_grpc_PROTOS}) - get_filename_component(_abs_file ${_proto} ABSOLUTE) - get_filename_component(_abs_dir ${_abs_file} DIRECTORY) - get_filename_component(_basename ${_proto} NAME_WE) - file(RELATIVE_PATH _rel_dir ${CMAKE_CURRENT_SOURCE_DIR} ${_abs_dir}) - - set(_possible_rel_dir) - if(NOT protobuf_generate_grpc_APPEND_PATH) - set(_possible_rel_dir ${_rel_dir}/) - endif() - - set(_generated_srcs) - foreach(_ext ${protobuf_generate_grpc_GENERATE_EXTENSIONS}) - list(APPEND _generated_srcs "${protobuf_generate_grpc_PROTOC_OUT_DIR}/${_possible_rel_dir}${_basename}${_ext}") - endforeach() - - if(protobuf_generate_grpc_DESCRIPTORS AND protobuf_generate_grpc_LANGUAGE STREQUAL cpp) - set(_descriptor_file "${CMAKE_CURRENT_BINARY_DIR}/${_basename}.desc") - set(_dll_desc_out "--descriptor_set_out=${_descriptor_file}") - list(APPEND _generated_srcs ${_descriptor_file}) - endif() - list(APPEND _generated_srcs_all ${_generated_srcs}) - - add_custom_command( - OUTPUT ${_generated_srcs} - COMMAND protobuf::protoc - ARGS --${protobuf_generate_grpc_LANGUAGE}_out ${_dll_export_decl}${protobuf_generate_grpc_PROTOC_OUT_DIR} - --grpc_out ${_dll_export_decl}${protobuf_generate_grpc_PROTOC_OUT_DIR} - --plugin=protoc-gen-grpc=$ - ${_dll_desc_out} ${_protobuf_include_path} ${_abs_file} - DEPENDS ${_abs_file} protobuf::protoc ${protobuf_generate_grpc_PLUGIN} - COMMENT "Running ${protobuf_generate_grpc_LANGUAGE} protocol buffer compiler on ${_proto}" - VERBATIM) - endforeach() - - set_source_files_properties(${_generated_srcs_all} PROPERTIES GENERATED TRUE) - if(protobuf_generate_grpc_OUT_VAR) - set(${protobuf_generate_grpc_OUT_VAR} ${_generated_srcs_all} PARENT_SCOPE) - endif() - if(protobuf_generate_grpc_TARGET) - target_sources(${protobuf_generate_grpc_TARGET} PRIVATE ${_generated_srcs_all}) - endif() -endfunction() - - -# Find the libraries. -if(gRPC_USE_STATIC_LIBS) - # Support preference of static libs by adjusting CMAKE_FIND_LIBRARY_SUFFIXES - set(_gRPC_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) - if(WIN32) - set(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a ${CMAKE_FIND_LIBRARY_SUFFIXES}) - else() - set(CMAKE_FIND_LIBRARY_SUFFIXES .a) - endif() -endif() - -find_library(gRPC_LIBRARY NAMES grpc) -find_library(gRPC_CPP_LIBRARY NAMES grpc++) -find_library(gRPC_UNSECURE_LIBRARY NAMES grpc_unsecure) -find_library(gRPC_CPP_UNSECURE_LIBRARY NAMES grpc++_unsecure) -find_library(gRPC_CARES_LIBRARY NAMES cares) - -set(gRPC_LIBRARIES) -if(gRPC_USE_UNSECURE_LIBRARIES) - if(gRPC_UNSECURE_LIBRARY) - set(gRPC_LIBRARIES ${gRPC_LIBRARIES} ${gRPC_UNSECURE_LIBRARY}) - endif() - if(gRPC_CPP_UNSECURE_LIBRARY) - set(gRPC_LIBRARIES ${gRPC_LIBRARIES} ${gRPC_CPP_UNSECURE_LIBRARY}) - endif() -else() - if(gRPC_LIBRARY) - set(gRPC_LIBRARIES ${gRPC_LIBRARIES} ${gRPC_LIBRARY}) - endif() - if(gRPC_CPP_UNSECURE_LIBRARY) - set(gRPC_LIBRARIES ${gRPC_LIBRARIES} ${gRPC_CPP_LIBRARY}) - endif() -endif() -set(gRPC_LIBRARIES ${gRPC_LIBRARIES} ${gRPC_CARES_LIBRARY}) - -# Restore the original find library ordering. -if(gRPC_USE_STATIC_LIBS) - set(CMAKE_FIND_LIBRARY_SUFFIXES ${_gRPC_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES}) -endif() - -# Find the include directories. -find_path(gRPC_INCLUDE_DIR grpc/grpc.h) -find_path(gRPC_CPP_INCLUDE_DIR grpc++/grpc++.h) - -if(gRPC_INCLUDE_DIR AND gRPC_CPP_INCLUDE_DIR AND NOT(gRPC_INCLUDE_DIR STREQUAL gRPC_CPP_INCLUDE_DIR)) - set(gRPC_INCLUDE_DIRS ${gRPC_INCLUDE_DIR} ${gRPC_CPP_INCLUDE_DIR}) -elseif(gRPC_INCLUDE_DIR) - set(gRPC_INCLUDE_DIRS ${gRPC_INCLUDE_DIR}) -else() - set(gRPC_INCLUDE_DIRS ${gRPC_CPP_INCLUDE_DIR}) -endif() - -# Get full path to plugin. -find_program(gRPC_CPP_PLUGIN - NAMES grpc_cpp_plugin - DOC "The plugin for generating gRPC client and server C++ stubs from `.proto` files") - -find_program(gRPC_PYTHON_PLUGIN - NAMES grpc_python_plugin - DOC "The plugin for generating gRPC client and server Python stubs from `.proto` files") - -# Add imported targets. -if(gRPC_CPP_LIBRARY AND NOT TARGET grpc++) - add_library(grpc++ UNKNOWN IMPORTED) - set_target_properties(grpc++ PROPERTIES - IMPORTED_LOCATION "${gRPC_CPP_LIBRARY}") - set_target_properties(grpc++ PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES ${gRPC_INCLUDE_DIRS}) -endif() - -if(gRPC_CPP_UNSECURE_LIBRARY AND NOT TARGET grpc++_unsecure) - add_library(grpc++_unsecure UNKNOWN IMPORTED) - set_target_properties(grpc++_unsecure PROPERTIES - IMPORTED_LOCATION "${gRPC_CPP_UNSECURE_LIBRARY}") - set_target_properties(grpc++_unsecure PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES ${gRPC_INCLUDE_DIRS}) -endif() - -if(gRPC_CPP_PLUGIN AND NOT TARGET grpc_cpp_plugin) - add_executable(grpc_cpp_plugin IMPORTED) - set_target_properties(grpc_cpp_plugin PROPERTIES - IMPORTED_LOCATION "${gRPC_CPP_PLUGIN}") -endif() - -if(gRPC_PYTHON_PLUGIN AND NOT TARGET grpc_python_plugin) - add_executable(grpc_python_plugin IMPORTED) - set_target_properties(grpc_python_plugin PROPERTIES - IMPORTED_LOCATION "${gRPC_PYTHON_PLUGIN}") -endif() - -#include(FindPackageHandleStandardArgs.cmake) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(gRPC - REQUIRED_VARS gRPC_LIBRARY gRPC_CPP_LIBRARY gRPC_UNSECURE_LIBRARY gRPC_CPP_UNSECURE_LIBRARY gRPC_CARES_LIBRARY - gRPC_INCLUDE_DIR gRPC_CPP_INCLUDE_DIR gRPC_CPP_PLUGIN gRPC_PYTHON_PLUGIN) - -if(gRPC_FOUND) - if(gRPC_DEBUG) - message(STATUS "gRPC: INCLUDE_DIRS=${gRPC_INCLUDE_DIRS}") - message(STATUS "gRPC: LIBRARIES=${gRPC_LIBRARIES}") - message(STATUS "gRPC: CPP_PLUGIN=${gRPC_CPP_PLUGIN}") - message(STATUS "gRPC: PYTHON_PLUGIN=${gRPC_PYTHON_PLUGIN}") - endif() -endif() diff --git a/cmake/find/grpc.cmake b/cmake/find/grpc.cmake deleted file mode 100644 index a627f0fc0b6..00000000000 --- a/cmake/find/grpc.cmake +++ /dev/null @@ -1,72 +0,0 @@ -# disable grpc due to conflicts of abseil (required by grpc) dynamic annotations with libtsan.a -if (SANITIZE STREQUAL "thread" AND COMPILER_GCC) - set(ENABLE_GRPC_DEFAULT OFF) -else() - set(ENABLE_GRPC_DEFAULT ${ENABLE_LIBRARIES}) -endif() - -option(ENABLE_GRPC "Use gRPC" ${ENABLE_GRPC_DEFAULT}) - -if(NOT ENABLE_GRPC) - if(USE_INTERNAL_GRPC_LIBRARY) - message(${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal gRPC library with ENABLE_GRPC=OFF") - endif() - return() -endif() - -if(NOT USE_PROTOBUF) - message(WARNING "Cannot use gRPC library without protobuf") -endif() - -# Normally we use the internal gRPC framework. -# You can set USE_INTERNAL_GRPC_LIBRARY to OFF to force using the external gRPC framework, which should be installed in the system in this case. -# The external gRPC framework can be installed in the system by running -# sudo apt-get install libgrpc++-dev protobuf-compiler-grpc -option(USE_INTERNAL_GRPC_LIBRARY "Set to FALSE to use system gRPC library instead of bundled. (Experimental. Set to OFF on your own risk)" ON) - -if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/grpc/CMakeLists.txt") - if(USE_INTERNAL_GRPC_LIBRARY) - message(WARNING "submodule contrib/grpc is missing. to fix try run: \n git submodule update --init") - message(${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal grpc") - set(USE_INTERNAL_GRPC_LIBRARY 0) - endif() - set(MISSING_INTERNAL_GRPC_LIBRARY 1) -endif() - -#if(USE_SSL) - set(gRPC_USE_UNSECURE_LIBRARIES FALSE) -#else() -# set(gRPC_USE_UNSECURE_LIBRARIES TRUE) -#endif() - -if(NOT USE_INTERNAL_GRPC_LIBRARY) - find_package(gRPC) - if(NOT gRPC_INCLUDE_DIRS OR NOT gRPC_LIBRARIES) - message(${RECONFIGURE_MESSAGE_LEVEL} "Can't find system gRPC library") - set(EXTERNAL_GRPC_LIBRARY_FOUND 0) - elseif(NOT gRPC_CPP_PLUGIN) - message(${RECONFIGURE_MESSAGE_LEVEL} "Can't find system grpc_cpp_plugin") - set(EXTERNAL_GRPC_LIBRARY_FOUND 0) - else() - set(EXTERNAL_GRPC_LIBRARY_FOUND 1) - set(USE_GRPC 1) - endif() -endif() - -if(NOT EXTERNAL_GRPC_LIBRARY_FOUND AND NOT MISSING_INTERNAL_GRPC_LIBRARY) - set(gRPC_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/grpc/include") - if(gRPC_USE_UNSECURE_LIBRARIES) - set(gRPC_LIBRARIES grpc_unsecure grpc++_unsecure) - else() - set(gRPC_LIBRARIES grpc grpc++) - endif() - set(gRPC_CPP_PLUGIN $) - set(gRPC_PYTHON_PLUGIN $) - - include("${ClickHouse_SOURCE_DIR}/contrib/grpc-cmake/protobuf_generate_grpc.cmake") - - set(USE_INTERNAL_GRPC_LIBRARY 1) - set(USE_GRPC 1) -endif() - -message(STATUS "Using gRPC=${USE_GRPC}: ${gRPC_INCLUDE_DIRS} : ${gRPC_LIBRARIES} : ${gRPC_CPP_PLUGIN}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 3f088c8a2f8..271a8d17118 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -70,6 +70,7 @@ add_subdirectory (hive-metastore-cmake) # requires: thrift/avro/arrow add_subdirectory (librdkafka-cmake) add_subdirectory (cppkafka-cmake) add_subdirectory (openldap-cmake) +add_subdirectory (grpc-cmake) # TODO: refactor the contrib libraries below this comment. @@ -153,10 +154,6 @@ if (USE_INTERNAL_HDFS3_LIBRARY) add_subdirectory(libhdfs3-cmake) endif () -if (USE_INTERNAL_GRPC_LIBRARY) - add_subdirectory(grpc-cmake) -endif () - if (USE_INTERNAL_AWS_S3_LIBRARY) add_subdirectory(aws-s3-cmake) diff --git a/contrib/grpc-cmake/CMakeLists.txt b/contrib/grpc-cmake/CMakeLists.txt index 4824802a5fb..479ff76bc64 100644 --- a/contrib/grpc-cmake/CMakeLists.txt +++ b/contrib/grpc-cmake/CMakeLists.txt @@ -1,3 +1,16 @@ +# disable grpc due to conflicts of abseil (required by grpc) dynamic annotations with libtsan.a +if (SANITIZE STREQUAL "thread" AND COMPILER_GCC) + set(ENABLE_GRPC_DEFAULT OFF) +else() + set(ENABLE_GRPC_DEFAULT ${ENABLE_LIBRARIES}) +endif() +option(ENABLE_GRPC "Use gRPC" ${ENABLE_GRPC_DEFAULT}) + +if(NOT ENABLE_GRPC) + message(STATUS "Not using gRPC") + return() +endif() + set(_gRPC_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/grpc") set(_gRPC_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/grpc") @@ -29,6 +42,12 @@ set(_gRPC_PROTOBUF_PROTOC "protoc") set(_gRPC_PROTOBUF_PROTOC_EXECUTABLE "${Protobuf_PROTOC_EXECUTABLE}") set(_gRPC_PROTOBUF_PROTOC_LIBRARIES "${Protobuf_PROTOC_LIBRARY}") +if(TARGET OpenSSL::SSL) + set(gRPC_USE_UNSECURE_LIBRARIES FALSE) +else() + set(gRPC_USE_UNSECURE_LIBRARIES TRUE) +endif() + # Use OpenSSL from ClickHouse contrib, not from gRPC third_party. set(gRPC_SSL_PROVIDER "clickhouse" CACHE STRING "" FORCE) set(_gRPC_SSL_INCLUDE_DIR "") @@ -74,3 +93,17 @@ add_subdirectory("${_gRPC_SOURCE_DIR}" "${_gRPC_BINARY_DIR}") # The contrib/grpc/CMakeLists.txt redefined the PROTOBUF_GENERATE_GRPC_CPP() function for its own purposes, # so we need to redefine it back. include("${ClickHouse_SOURCE_DIR}/contrib/grpc-cmake/protobuf_generate_grpc.cmake") + +set(gRPC_CPP_PLUGIN $) +set(gRPC_PYTHON_PLUGIN $) + +set(gRPC_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/grpc/include") +if(gRPC_USE_UNSECURE_LIBRARIES) + set(gRPC_LIBRARIES grpc_unsecure grpc++_unsecure) +else() + set(gRPC_LIBRARIES grpc grpc++) +endif() +add_library(_ch_contrib_grpc INTERFACE) +target_link_libraries(_ch_contrib_grpc INTERFACE ${gRPC_LIBRARIES}) +target_include_directories(_ch_contrib_grpc SYSTEM INTERFACE ${gRPC_INCLUDE_DIRS}) +add_library(ch_contrib::grpc ALIAS _ch_contrib_grpc) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ed7c3df5f0c..0d1ee281aa2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -431,7 +431,7 @@ if (USE_PROTOBUF) dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${Protobuf_INCLUDE_DIR}) endif () -if (USE_GRPC) +if (TARGET clickhouse_grpc_protos) dbms_target_link_libraries (PUBLIC clickhouse_grpc_protos) endif() diff --git a/src/Server/CMakeLists.txt b/src/Server/CMakeLists.txt index 61237d9bfdc..6d279b45dcf 100644 --- a/src/Server/CMakeLists.txt +++ b/src/Server/CMakeLists.txt @@ -1,3 +1,3 @@ -if (USE_GRPC) +if (TARGET ch_contrib::grpc) add_subdirectory (grpc_protos) endif() diff --git a/src/Server/grpc_protos/CMakeLists.txt b/src/Server/grpc_protos/CMakeLists.txt index 584cf015a65..b673ac808e1 100644 --- a/src/Server/grpc_protos/CMakeLists.txt +++ b/src/Server/grpc_protos/CMakeLists.txt @@ -7,5 +7,5 @@ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w") set (CMAKE_CXX_CLANG_TIDY "") add_library(clickhouse_grpc_protos ${clickhouse_grpc_proto_headers} ${clickhouse_grpc_proto_sources}) -target_include_directories(clickhouse_grpc_protos SYSTEM PUBLIC ${gRPC_INCLUDE_DIRS} ${Protobuf_INCLUDE_DIR} ${CMAKE_CURRENT_BINARY_DIR}) -target_link_libraries (clickhouse_grpc_protos PUBLIC ${gRPC_LIBRARIES}) +target_include_directories(clickhouse_grpc_protos SYSTEM PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) +target_link_libraries (clickhouse_grpc_protos PUBLIC ch_contrib::grpc) diff --git a/src/configure_config.cmake b/src/configure_config.cmake index bd93472afd9..18778595287 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -19,3 +19,6 @@ endif() if (TARGET ch_contrib::ldap) set(USE_LDAP 1) endif() +if (TARGET ch_contrib::grpc) + set(USE_GRPC 1) +endif() From 788cb6b2b071492556dde42e1bda3de5cf17ddc1 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 10:46:31 +0300 Subject: [PATCH 137/403] Remove unbundled protobuf support --- CMakeLists.txt | 2 - cmake/find/hdfs3.cmake | 2 +- cmake/find/protobuf.cmake | 62 ------------------- contrib/CMakeLists.txt | 5 +- contrib/arrow-cmake/CMakeLists.txt | 15 ++--- contrib/grpc-cmake/CMakeLists.txt | 14 +---- contrib/libhdfs3-cmake/CMakeLists.txt | 9 +-- .../libprotobuf-mutator-cmake/CMakeLists.txt | 2 +- contrib/protobuf-cmake/CMakeLists.txt | 31 ++++++++++ src/CMakeLists.txt | 5 +- src/Formats/CMakeLists.txt | 3 + .../fuzzers/codegen_fuzzer/CMakeLists.txt | 4 +- 12 files changed, 51 insertions(+), 103 deletions(-) delete mode 100644 cmake/find/protobuf.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 9db7c069691..c24cd2cf155 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -152,7 +152,6 @@ if (ENABLE_FUZZING) # For codegen_select_fuzzer set (ENABLE_PROTOBUF 1) - set (USE_INTERNAL_PROTOBUF_LIBRARY 1) endif() # Global libraries @@ -506,7 +505,6 @@ include (cmake/find/llvm.cmake) include (cmake/find/h3.cmake) include (cmake/find/libxml2.cmake) include (cmake/find/brotli.cmake) -include (cmake/find/protobuf.cmake) include (cmake/find/pdqsort.cmake) include (cmake/find/miniselect.cmake) include (cmake/find/hdfs3.cmake) # uses protobuf diff --git a/cmake/find/hdfs3.cmake b/cmake/find/hdfs3.cmake index aac6b99dfa2..5f3f61254ed 100644 --- a/cmake/find/hdfs3.cmake +++ b/cmake/find/hdfs3.cmake @@ -1,4 +1,4 @@ -if(NOT ARCH_ARM AND NOT OS_FREEBSD AND NOT APPLE AND USE_PROTOBUF AND NOT ARCH_PPC64LE) +if(NOT ARCH_ARM AND NOT OS_FREEBSD AND NOT APPLE AND NOT ARCH_PPC64LE) # USE_PROTOBUF AND option(ENABLE_HDFS "Enable HDFS" ${ENABLE_LIBRARIES}) elseif(ENABLE_HDFS OR USE_INTERNAL_HDFS3_LIBRARY) message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use HDFS3 with current configuration") diff --git a/cmake/find/protobuf.cmake b/cmake/find/protobuf.cmake deleted file mode 100644 index a2ea8ae87fc..00000000000 --- a/cmake/find/protobuf.cmake +++ /dev/null @@ -1,62 +0,0 @@ -option(ENABLE_PROTOBUF "Enable protobuf" ${ENABLE_LIBRARIES}) - -if(NOT ENABLE_PROTOBUF) - if(USE_INTERNAL_PROTOBUF_LIBRARY) - message(${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal protobuf with ENABLE_PROTOBUF=OFF") - endif() - return() -endif() - -# Normally we use the internal protobuf library. -# You can set USE_INTERNAL_PROTOBUF_LIBRARY to OFF to force using the external protobuf library, which should be installed in the system in this case. -# The external protobuf library can be installed in the system by running -# sudo apt-get install libprotobuf-dev protobuf-compiler libprotoc-dev -option(USE_INTERNAL_PROTOBUF_LIBRARY "Set to FALSE to use system protobuf instead of bundled. (Experimental. Set to OFF on your own risk)" ON) - -if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/protobuf/cmake/CMakeLists.txt") - if(USE_INTERNAL_PROTOBUF_LIBRARY) - message(WARNING "submodule contrib/protobuf is missing. to fix try run: \n git submodule update --init") - message(${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal protobuf") - set(USE_INTERNAL_PROTOBUF_LIBRARY 0) - endif() - set(MISSING_INTERNAL_PROTOBUF_LIBRARY 1) -endif() - -if(NOT USE_INTERNAL_PROTOBUF_LIBRARY) - find_package(Protobuf) - if(NOT Protobuf_INCLUDE_DIR OR NOT Protobuf_LIBRARY) - message(${RECONFIGURE_MESSAGE_LEVEL} "Can't find system protobuf library") - set(EXTERNAL_PROTOBUF_LIBRARY_FOUND 0) - elseif(NOT Protobuf_PROTOC_EXECUTABLE) - message(${RECONFIGURE_MESSAGE_LEVEL} "Can't find system protobuf compiler") - set(EXTERNAL_PROTOBUF_LIBRARY_FOUND 0) - else() - set(EXTERNAL_PROTOBUF_LIBRARY_FOUND 1) - set(USE_PROTOBUF 1) - endif() -endif() - -if(NOT EXTERNAL_PROTOBUF_LIBRARY_FOUND AND NOT MISSING_INTERNAL_PROTOBUF_LIBRARY) - set(Protobuf_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/protobuf/src") - set(Protobuf_LIBRARY libprotobuf) - set(Protobuf_PROTOC_EXECUTABLE "$") - set(Protobuf_PROTOC_LIBRARY libprotoc) - - include("${ClickHouse_SOURCE_DIR}/contrib/protobuf-cmake/protobuf_generate.cmake") - - set(USE_INTERNAL_PROTOBUF_LIBRARY 1) - set(USE_PROTOBUF 1) -endif() - -if(OS_FREEBSD AND SANITIZE STREQUAL "address") - # ../contrib/protobuf/src/google/protobuf/arena_impl.h:45:10: fatal error: 'sanitizer/asan_interface.h' file not found - # #include - if(LLVM_INCLUDE_DIRS) - set(Protobuf_INCLUDE_DIR "${Protobuf_INCLUDE_DIR}" ${LLVM_INCLUDE_DIRS}) - else() - message(${RECONFIGURE_MESSAGE_LEVEL} "Can't use protobuf on FreeBSD with address sanitizer without LLVM") - set(USE_PROTOBUF 0) - endif() -endif() - -message(STATUS "Using protobuf=${USE_PROTOBUF}: ${Protobuf_INCLUDE_DIR} : ${Protobuf_LIBRARY} : ${Protobuf_PROTOC_EXECUTABLE} : ${Protobuf_PROTOC_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 271a8d17118..b64983f50a3 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -71,6 +71,7 @@ add_subdirectory (librdkafka-cmake) add_subdirectory (cppkafka-cmake) add_subdirectory (openldap-cmake) add_subdirectory (grpc-cmake) +add_subdirectory (protobuf-cmake) # TODO: refactor the contrib libraries below this comment. @@ -146,10 +147,6 @@ if (USE_INTERNAL_BROTLI_LIBRARY) target_compile_definitions(brotli PRIVATE BROTLI_BUILD_PORTABLE=1) endif () -if (USE_INTERNAL_PROTOBUF_LIBRARY) - add_subdirectory(protobuf-cmake) -endif () - if (USE_INTERNAL_HDFS3_LIBRARY) add_subdirectory(libhdfs3-cmake) endif () diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 51d8ea5cb81..5738b4a53ea 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -24,10 +24,6 @@ if (NOT ENABLE_PARQUET) return() endif() -if (NOT Protobuf_PROTOC_EXECUTABLE) - message (FATAL_ERROR "Can't use parquet without protoc executable") -endif() - # Freebsd: ../contrib/arrow/cpp/src/arrow/util/bit-util.h:27:10: fatal error: endian.h: No such file or directory if (OS_FREEBSD) message (FATAL_ERROR "Using internal parquet library on FreeBSD is not supported") @@ -68,11 +64,10 @@ set(ORC_SOURCE_SRC_DIR "${ORC_SOURCE_DIR}/src") set(ORC_BUILD_SRC_DIR "${CMAKE_CURRENT_BINARY_DIR}/../orc/c++/src") set(ORC_BUILD_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/../orc/c++/include") -set(GOOGLE_PROTOBUF_DIR "${Protobuf_INCLUDE_DIR}/") set(ORC_ADDITION_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}) set(ARROW_SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src") -set(PROTOBUF_EXECUTABLE ${Protobuf_PROTOC_EXECUTABLE}) +set(PROTOBUF_EXECUTABLE $) set(PROTO_DIR "${ORC_SOURCE_DIR}/../proto") @@ -366,13 +361,12 @@ add_dependencies(_arrow ${FLATBUFFERS_LIBRARY}) target_link_libraries(_arrow PRIVATE ${FLATBUFFERS_LIBRARY} boost::filesystem) -if (USE_INTERNAL_PROTOBUF_LIBRARY) - add_dependencies(_arrow protoc) -endif () +add_dependencies(_arrow protoc) target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ARROW_SRC_DIR}) target_include_directories(_arrow SYSTEM BEFORE PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/cpp/src") -target_link_libraries(_arrow PRIVATE ${DOUBLE_CONVERSION_LIBRARIES} ${Protobuf_LIBRARY}) +target_link_libraries(_arrow PRIVATE ${DOUBLE_CONVERSION_LIBRARIES}) +target_link_libraries(_arrow PRIVATE ch_contrib::protobuf) target_link_libraries(_arrow PRIVATE ch_contrib::lz4) target_link_libraries(_arrow PRIVATE ch_contrib::snappy) target_link_libraries(_arrow PRIVATE ch_contrib::zlib) @@ -382,7 +376,6 @@ target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ORC_INCLUDE_DIR}) target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ORC_BUILD_INCLUDE_DIR}) target_include_directories(_arrow SYSTEM PRIVATE ${ORC_SOURCE_SRC_DIR}) target_include_directories(_arrow SYSTEM PRIVATE ${ORC_SOURCE_WRAP_DIR}) -target_include_directories(_arrow SYSTEM PRIVATE ${GOOGLE_PROTOBUF_DIR}) target_include_directories(_arrow SYSTEM PRIVATE ${ORC_BUILD_SRC_DIR}) target_include_directories(_arrow SYSTEM PRIVATE ${ORC_ADDITION_SOURCE_DIR}) target_include_directories(_arrow SYSTEM PRIVATE ${ARROW_SRC_DIR}) diff --git a/contrib/grpc-cmake/CMakeLists.txt b/contrib/grpc-cmake/CMakeLists.txt index 479ff76bc64..64a5be8e8d5 100644 --- a/contrib/grpc-cmake/CMakeLists.txt +++ b/contrib/grpc-cmake/CMakeLists.txt @@ -28,19 +28,11 @@ set(_gRPC_ZLIB_INCLUDE_DIR "") set(_gRPC_ZLIB_LIBRARIES ch_contrib::zlib) # Use protobuf from ClickHouse contrib, not from gRPC third_party. -if(NOT Protobuf_INCLUDE_DIR OR NOT Protobuf_LIBRARY) - message(FATAL_ERROR " grpc: The location of the \"protobuf\" library is unknown") -elseif (NOT Protobuf_PROTOC_EXECUTABLE) - message(FATAL_ERROR " grpc: The location of the protobuf compiler is unknown") -elseif (NOT Protobuf_PROTOC_LIBRARY) - message(FATAL_ERROR " grpc: The location of the protobuf compiler's library is unknown") -endif() set(gRPC_PROTOBUF_PROVIDER "clickhouse" CACHE STRING "" FORCE) -set(_gRPC_PROTOBUF_WELLKNOWN_INCLUDE_DIR "${Protobuf_INCLUDE_DIR}") -set(_gRPC_PROTOBUF_LIBRARIES "${Protobuf_LIBRARY}") +set(_gRPC_PROTOBUF_LIBRARIES ch_contrib::protobuf) set(_gRPC_PROTOBUF_PROTOC "protoc") -set(_gRPC_PROTOBUF_PROTOC_EXECUTABLE "${Protobuf_PROTOC_EXECUTABLE}") -set(_gRPC_PROTOBUF_PROTOC_LIBRARIES "${Protobuf_PROTOC_LIBRARY}") +set(_gRPC_PROTOBUF_PROTOC_EXECUTABLE $) +set(_gRPC_PROTOBUF_PROTOC_LIBRARIES ch_contrib::protoc) if(TARGET OpenSSL::SSL) set(gRPC_USE_UNSECURE_LIBRARIES FALSE) diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index 0b8fe361baf..96a2b5ca50a 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -27,7 +27,7 @@ set(PROTO_FILES "${HDFS3_SOURCE_DIR}/proto/datatransfer.proto" ) -PROTOBUF_GENERATE_CPP(PROTO_SOURCES PROTO_HEADERS ${PROTO_FILES}) +PROTOBUF_GENERATE_CPP(PROTO_SOURCES PROTO_HEADERS ${PROTO_FILES} APPEND_PATH) configure_file("${HDFS3_SOURCE_DIR}/platform.h.in" "${CMAKE_CURRENT_BINARY_DIR}/platform.h") @@ -96,9 +96,7 @@ set_source_files_properties("${HDFS3_SOURCE_DIR}/rpc/RpcClient.cpp" PROPERTIES C # target add_library(hdfs3 ${SRCS}) -if(USE_INTERNAL_PROTOBUF_LIBRARY) - add_dependencies(hdfs3 protoc) -endif() +add_dependencies(hdfs3 protoc) target_include_directories(hdfs3 PRIVATE ${HDFS3_SOURCE_DIR}) target_include_directories(hdfs3 PRIVATE ${HDFS3_COMMON_DIR}) @@ -114,8 +112,7 @@ endif() target_link_libraries(hdfs3 PRIVATE ${LIBXML2_LIBRARIES}) # inherit from parent cmake -target_include_directories(hdfs3 PRIVATE ${Protobuf_INCLUDE_DIR}) -target_link_libraries(hdfs3 PRIVATE ${Protobuf_LIBRARY} boost::headers_only) +target_link_libraries(hdfs3 PRIVATE ch_contrib::protobuf boost::headers_only) if (TARGET OpenSSL::SSL) target_link_libraries(hdfs3 PRIVATE OpenSSL::Crypto OpenSSL::SSL) endif() diff --git a/contrib/libprotobuf-mutator-cmake/CMakeLists.txt b/contrib/libprotobuf-mutator-cmake/CMakeLists.txt index 978b1e732ba..41cb8af8b49 100644 --- a/contrib/libprotobuf-mutator-cmake/CMakeLists.txt +++ b/contrib/libprotobuf-mutator-cmake/CMakeLists.txt @@ -11,4 +11,4 @@ add_library(protobuf-mutator target_include_directories(protobuf-mutator BEFORE PRIVATE "${LIBRARY_DIR}") target_include_directories(protobuf-mutator BEFORE PRIVATE "${ClickHouse_SOURCE_DIR}/contrib/protobuf/src") -target_link_libraries(protobuf-mutator ${Protobuf_LIBRARY}) +target_link_libraries(protobuf-mutator ch_contrib::protobuf) diff --git a/contrib/protobuf-cmake/CMakeLists.txt b/contrib/protobuf-cmake/CMakeLists.txt index 55fde879d07..b4f026ab074 100644 --- a/contrib/protobuf-cmake/CMakeLists.txt +++ b/contrib/protobuf-cmake/CMakeLists.txt @@ -1,3 +1,22 @@ +option(ENABLE_PROTOBUF "Enable protobuf" ${ENABLE_LIBRARIES}) + +if(NOT ENABLE_PROTOBUF) + message(STATUS "Not using protobuf") + return() +endif() + +set(Protobuf_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/protobuf/src") +if(OS_FREEBSD AND SANITIZE STREQUAL "address") + # ../contrib/protobuf/src/google/protobuf/arena_impl.h:45:10: fatal error: 'sanitizer/asan_interface.h' file not found + # #include + if(LLVM_INCLUDE_DIRS) + set(Protobuf_INCLUDE_DIR "${Protobuf_INCLUDE_DIR}" ${LLVM_INCLUDE_DIRS}) + else() + message(${RECONFIGURE_MESSAGE_LEVEL} "Can't use protobuf on FreeBSD with address sanitizer without LLVM") + return() + endif() +endif() + set(protobuf_source_dir "${ClickHouse_SOURCE_DIR}/contrib/protobuf") set(protobuf_binary_dir "${ClickHouse_BINARY_DIR}/contrib/protobuf") @@ -296,3 +315,15 @@ else () set_target_properties (protoc PROPERTIES IMPORTED_LOCATION "${PROTOC_BUILD_DIR}/protoc") add_dependencies(protoc "${PROTOC_BUILD_DIR}/protoc") endif () + +include("${ClickHouse_SOURCE_DIR}/contrib/protobuf-cmake/protobuf_generate.cmake") + +add_library(_protobuf INTERFACE) +target_link_libraries(_protobuf INTERFACE libprotobuf) +target_include_directories(_protobuf INTERFACE "${Protobuf_INCLUDE_DIR}") +add_library(ch_contrib::protobuf ALIAS _protobuf) + +add_library(_protoc INTERFACE) +target_link_libraries(_protoc INTERFACE libprotoc libprotobuf) +target_include_directories(_protoc INTERFACE "${Protobuf_INCLUDE_DIR}") +add_library(ch_contrib::protoc ALIAS _protoc) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0d1ee281aa2..9c90449560b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -426,9 +426,8 @@ if (TARGET ch_contrib::ldap) endif () dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) -if (USE_PROTOBUF) - dbms_target_link_libraries (PRIVATE ${Protobuf_LIBRARY}) - dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${Protobuf_INCLUDE_DIR}) +if (TARGET ch_contrib::protobuf) + dbms_target_link_libraries (PRIVATE ch_contrib::protobuf) endif () if (TARGET clickhouse_grpc_protos) diff --git a/src/Formats/CMakeLists.txt b/src/Formats/CMakeLists.txt index 9734ee1bd9c..1aa79e98476 100644 --- a/src/Formats/CMakeLists.txt +++ b/src/Formats/CMakeLists.txt @@ -9,4 +9,7 @@ endif() if (TARGET ch_contrib::snappy) set(USE_SNAPPY 1) endif() +if (TARGET ch_contrib::protobuf) + set(USE_PROTOBUF 1) +endif() configure_file(config_formats.h.in ${ConfigIncludePath}/config_formats.h) diff --git a/src/Parsers/fuzzers/codegen_fuzzer/CMakeLists.txt b/src/Parsers/fuzzers/codegen_fuzzer/CMakeLists.txt index 99bbd453144..fee439377aa 100644 --- a/src/Parsers/fuzzers/codegen_fuzzer/CMakeLists.txt +++ b/src/Parsers/fuzzers/codegen_fuzzer/CMakeLists.txt @@ -40,7 +40,7 @@ add_executable(codegen_select_fuzzer ${FUZZER_SRCS}) set_source_files_properties("${PROTO_SRCS}" "out.cpp" PROPERTIES COMPILE_FLAGS "-Wno-reserved-identifier") -target_include_directories(codegen_select_fuzzer SYSTEM BEFORE PRIVATE "${Protobuf_INCLUDE_DIR}" "${CMAKE_CURRENT_BINARY_DIR}") +target_include_directories(codegen_select_fuzzer SYSTEM BEFORE PRIVATE "${CMAKE_CURRENT_BINARY_DIR}") target_include_directories(codegen_select_fuzzer SYSTEM BEFORE PRIVATE "${LibProtobufMutator_SOURCE_DIR}") target_include_directories(codegen_select_fuzzer SYSTEM BEFORE PRIVATE "${LibProtobufMutator_SOURCE_DIR}/src") -target_link_libraries(codegen_select_fuzzer PRIVATE protobuf-mutator ${Protobuf_LIBRARY} ${Protobuf_PROTOC_LIBRARY} dbms ${LIB_FUZZING_ENGINE}) +target_link_libraries(codegen_select_fuzzer PRIVATE protobuf-mutator ch_contrib::protoc dbms ${LIB_FUZZING_ENGINE}) From aef6668cb4ac67b38f12c594702e01c795d10e8e Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 21:45:53 +0300 Subject: [PATCH 138/403] Remove unbundled hdfs3 support --- CMakeLists.txt | 1 - cmake/find/hdfs3.cmake | 45 ------------------- contrib/CMakeLists.txt | 5 +-- contrib/libhdfs3-cmake/CMakeLists.txt | 39 +++++++++++----- src/CMakeLists.txt | 7 ++- src/Common/config.h.in | 1 - src/Storages/HDFS/HDFSCommon.cpp | 9 ---- ...StorageSystemBuildOptions.generated.cpp.in | 1 - src/configure_config.cmake | 3 ++ 9 files changed, 34 insertions(+), 77 deletions(-) delete mode 100644 cmake/find/hdfs3.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index c24cd2cf155..6bc27d70219 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -507,7 +507,6 @@ include (cmake/find/libxml2.cmake) include (cmake/find/brotli.cmake) include (cmake/find/pdqsort.cmake) include (cmake/find/miniselect.cmake) -include (cmake/find/hdfs3.cmake) # uses protobuf include (cmake/find/poco.cmake) include (cmake/find/curl.cmake) include (cmake/find/s3.cmake) diff --git a/cmake/find/hdfs3.cmake b/cmake/find/hdfs3.cmake deleted file mode 100644 index 5f3f61254ed..00000000000 --- a/cmake/find/hdfs3.cmake +++ /dev/null @@ -1,45 +0,0 @@ -if(NOT ARCH_ARM AND NOT OS_FREEBSD AND NOT APPLE AND NOT ARCH_PPC64LE) # USE_PROTOBUF AND - option(ENABLE_HDFS "Enable HDFS" ${ENABLE_LIBRARIES}) -elseif(ENABLE_HDFS OR USE_INTERNAL_HDFS3_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use HDFS3 with current configuration") -endif() - -if(NOT ENABLE_HDFS) - if(USE_INTERNAL_HDFS3_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal HDFS3 library with ENABLE_HDFS3=OFF") - endif() - return() -endif() - -option(USE_INTERNAL_HDFS3_LIBRARY "Set to FALSE to use system HDFS3 instead of bundled (experimental - set to OFF on your own risk)" - ON) # We don't know any linux distribution with package for it - -if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include/hdfs/hdfs.h") - if(USE_INTERNAL_HDFS3_LIBRARY) - message(WARNING "submodule contrib/libhdfs3 is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal HDFS3 library") - set(USE_INTERNAL_HDFS3_LIBRARY 0) - endif() - set(MISSING_INTERNAL_HDFS3_LIBRARY 1) -endif() - -if(NOT USE_INTERNAL_HDFS3_LIBRARY) - find_library(HDFS3_LIBRARY hdfs3) - find_path(HDFS3_INCLUDE_DIR NAMES hdfs/hdfs.h PATHS ${HDFS3_INCLUDE_PATHS}) - if(NOT HDFS3_LIBRARY OR NOT HDFS3_INCLUDE_DIR) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot find system HDFS3 library") - endif() -endif() - -if(HDFS3_LIBRARY AND HDFS3_INCLUDE_DIR) - set(USE_HDFS 1) -elseif(NOT MISSING_INTERNAL_HDFS3_LIBRARY AND LIBGSASL_LIBRARY AND LIBXML2_LIBRARIES) - set(HDFS3_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include") - set(HDFS3_LIBRARY hdfs3) - set(USE_INTERNAL_HDFS3_LIBRARY 1) - set(USE_HDFS 1) -else() - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannout enable HDFS3") -endif() - -message(STATUS "Using hdfs3=${USE_HDFS}: ${HDFS3_INCLUDE_DIR} : ${HDFS3_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index b64983f50a3..573ea3317e4 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -72,6 +72,7 @@ add_subdirectory (cppkafka-cmake) add_subdirectory (openldap-cmake) add_subdirectory (grpc-cmake) add_subdirectory (protobuf-cmake) +add_subdirectory (libhdfs3-cmake) # requires: protobuf # TODO: refactor the contrib libraries below this comment. @@ -147,10 +148,6 @@ if (USE_INTERNAL_BROTLI_LIBRARY) target_compile_definitions(brotli PRIVATE BROTLI_BUILD_PORTABLE=1) endif () -if (USE_INTERNAL_HDFS3_LIBRARY) - add_subdirectory(libhdfs3-cmake) -endif () - if (USE_INTERNAL_AWS_S3_LIBRARY) add_subdirectory(aws-s3-cmake) diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index 96a2b5ca50a..d9ed10282cb 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -1,3 +1,14 @@ +if(NOT ARCH_ARM AND NOT OS_FREEBSD AND NOT APPLE AND NOT ARCH_PPC64LE) + option(ENABLE_HDFS "Enable HDFS" ${ENABLE_LIBRARIES}) +elseif(ENABLE_HDFS) + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use HDFS3 with current configuration") +endif() + +if(NOT ENABLE_HDFS) + message(STATUS "Not using hdfs") + return() +endif() + if (${ENABLE_KRB5}) SET(WITH_KERBEROS 1) else() @@ -94,25 +105,29 @@ set(SRCS set_source_files_properties("${HDFS3_SOURCE_DIR}/rpc/RpcClient.cpp" PROPERTIES COMPILE_FLAGS "-DBOOST_UUID_RANDOM_PROVIDER_FORCE_POSIX=1") # target -add_library(hdfs3 ${SRCS}) +add_library(_hdfs3 ${SRCS}) -add_dependencies(hdfs3 protoc) +add_dependencies(_hdfs3 protoc) -target_include_directories(hdfs3 PRIVATE ${HDFS3_SOURCE_DIR}) -target_include_directories(hdfs3 PRIVATE ${HDFS3_COMMON_DIR}) -target_include_directories(hdfs3 PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +target_include_directories(_hdfs3 PRIVATE ${HDFS3_SOURCE_DIR}) +target_include_directories(_hdfs3 PRIVATE ${HDFS3_COMMON_DIR}) +target_include_directories(_hdfs3 PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) -target_include_directories(hdfs3 PRIVATE ${LIBGSASL_INCLUDE_DIR}) -target_include_directories(hdfs3 PRIVATE ${LIBXML2_INCLUDE_DIR}) +target_include_directories(_hdfs3 PRIVATE ${LIBGSASL_INCLUDE_DIR}) +target_include_directories(_hdfs3 PRIVATE ${LIBXML2_INCLUDE_DIR}) -target_link_libraries(hdfs3 PRIVATE ${LIBGSASL_LIBRARY}) +target_include_directories(_hdfs3 SYSTEM BEFORE INTERFACE "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include") + +target_link_libraries(_hdfs3 PRIVATE ${LIBGSASL_LIBRARY}) if (WITH_KERBEROS) - target_link_libraries(hdfs3 PRIVATE ${KRB5_LIBRARY}) + target_link_libraries(_hdfs3 PRIVATE ${KRB5_LIBRARY}) endif() -target_link_libraries(hdfs3 PRIVATE ${LIBXML2_LIBRARIES}) +target_link_libraries(_hdfs3 PRIVATE ${LIBXML2_LIBRARIES}) # inherit from parent cmake -target_link_libraries(hdfs3 PRIVATE ch_contrib::protobuf boost::headers_only) +target_link_libraries(_hdfs3 PRIVATE ch_contrib::protobuf boost::headers_only) if (TARGET OpenSSL::SSL) - target_link_libraries(hdfs3 PRIVATE OpenSSL::Crypto OpenSSL::SSL) + target_link_libraries(_hdfs3 PRIVATE OpenSSL::Crypto OpenSSL::SSL) endif() + +add_library(ch_contrib::hdfs ALIAS _hdfs3) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9c90449560b..edf1809c49d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -114,7 +114,7 @@ if (USE_AZURE_BLOB_STORAGE) add_headers_and_sources(dbms Disks/AzureBlobStorage) endif() -if (USE_HDFS) +if (TARGET ch_contrib::hdfs) add_headers_and_sources(dbms Storages/HDFS) add_headers_and_sources(dbms Disks/HDFS) endif() @@ -434,9 +434,8 @@ if (TARGET clickhouse_grpc_protos) dbms_target_link_libraries (PUBLIC clickhouse_grpc_protos) endif() -if (USE_HDFS) - dbms_target_link_libraries(PRIVATE ${HDFS3_LIBRARY}) - dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${HDFS3_INCLUDE_DIR}) +if (TARGET ch_contrib::hdfs) + dbms_target_link_libraries(PRIVATE ch_contrib::hdfs) endif() if (TARGET ch_contrib::hivemetastore) diff --git a/src/Common/config.h.in b/src/Common/config.h.in index 28506a94581..03ea3ac2e7a 100644 --- a/src/Common/config.h.in +++ b/src/Common/config.h.in @@ -7,7 +7,6 @@ #cmakedefine01 USE_SSL #cmakedefine01 USE_INTERNAL_SSL_LIBRARY #cmakedefine01 USE_HDFS -#cmakedefine01 USE_INTERNAL_HDFS3_LIBRARY #cmakedefine01 USE_AWS_S3 #cmakedefine01 USE_AZURE_BLOB_STORAGE #cmakedefine01 USE_BROTLI diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index c99d100d6d4..85eb16d7de2 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -45,11 +45,7 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration { need_kinit = true; hadoop_kerberos_principal = config.getString(key_path); - -#if USE_INTERNAL_HDFS3_LIBRARY hdfsBuilderSetPrincipal(hdfs_builder, hadoop_kerberos_principal.c_str()); -#endif - continue; } else if (key == "hadoop_kerberos_kinit_command") @@ -170,12 +166,7 @@ HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::A String user_config_prefix = HDFSBuilderWrapper::CONFIG_PREFIX + "_" + user; if (config.has(user_config_prefix)) { -#if USE_INTERNAL_HDFS3_LIBRARY builder.loadFromConfig(config, user_config_prefix, true); -#else - throw Exception("Multi user HDFS configuration required internal libhdfs3", - ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG); -#endif } } diff --git a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in index 01a79e7c48d..01be3f8456d 100644 --- a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in +++ b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in @@ -63,7 +63,6 @@ const char * auto_config_build[] "USE_AWS_S3", "@USE_AWS_S3@", "USE_CASSANDRA", "@USE_CASSANDRA@", "USE_YAML_CPP", "@USE_YAML_CPP@", - "USE_INTERNAL_HDFS3_LIBRARY", "@USE_INTERNAL_HDFS3_LIBRARY@", "CLICKHOUSE_SPLIT_BINARY", "@CLICKHOUSE_SPLIT_BINARY@", "USE_SENTRY", "@USE_SENTRY@", "USE_DATASKETCHES", "@USE_DATASKETCHES@", diff --git a/src/configure_config.cmake b/src/configure_config.cmake index 18778595287..e216eaf088f 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -22,3 +22,6 @@ endif() if (TARGET ch_contrib::grpc) set(USE_GRPC 1) endif() +if (TARGET ch_contrib::hdfs) + set(USE_HDFS 1) +endif() From 6c0fa21025286fa924677ca335a7a77ac7157a0d Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 21:45:53 +0300 Subject: [PATCH 139/403] Remove unbundled libprotobuf-mutator support --- CMakeLists.txt | 1 - cmake/find/libprotobuf-mutator.cmake | 11 ----------- contrib/CMakeLists.txt | 8 ++++---- contrib/libprotobuf-mutator-cmake/CMakeLists.txt | 16 ++++++++++++---- .../fuzzers/codegen_fuzzer/CMakeLists.txt | 4 +--- 5 files changed, 17 insertions(+), 23 deletions(-) delete mode 100644 cmake/find/libprotobuf-mutator.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 6bc27d70219..5dcf6868549 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -528,7 +528,6 @@ include (cmake/find/msgpack.cmake) include (cmake/find/cassandra.cmake) include (cmake/find/sentry.cmake) include (cmake/find/datasketches.cmake) -include (cmake/find/libprotobuf-mutator.cmake) set (USE_INTERNAL_CITYHASH_LIBRARY ON CACHE INTERNAL "") find_contrib_lib(cityhash) diff --git a/cmake/find/libprotobuf-mutator.cmake b/cmake/find/libprotobuf-mutator.cmake deleted file mode 100644 index a308db67c8b..00000000000 --- a/cmake/find/libprotobuf-mutator.cmake +++ /dev/null @@ -1,11 +0,0 @@ -option(USE_LIBPROTOBUF_MUTATOR "Enable libprotobuf-mutator" ${ENABLE_FUZZING}) - -if (NOT USE_LIBPROTOBUF_MUTATOR) - return() -endif() - -set(LibProtobufMutator_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libprotobuf-mutator") - -if (NOT EXISTS "${LibProtobufMutator_SOURCE_DIR}/README.md") - message (ERROR "submodule contrib/libprotobuf-mutator is missing. to fix try run: \n git submodule update --init") -endif() diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 573ea3317e4..7b0aaf088d4 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -42,10 +42,6 @@ if (USE_INTERNAL_CAPNP_LIBRARY AND NOT MISSING_INTERNAL_CAPNP_LIBRARY) add_subdirectory(capnproto-cmake) endif () -if (ENABLE_FUZZING) - add_subdirectory (libprotobuf-mutator-cmake) -endif() - if (USE_YAML_CPP) add_subdirectory (yaml-cpp-cmake) endif() @@ -74,6 +70,10 @@ add_subdirectory (grpc-cmake) add_subdirectory (protobuf-cmake) add_subdirectory (libhdfs3-cmake) # requires: protobuf +if (ENABLE_FUZZING) + add_subdirectory (libprotobuf-mutator-cmake) +endif() + # TODO: refactor the contrib libraries below this comment. if (USE_INTERNAL_RE2_LIBRARY) diff --git a/contrib/libprotobuf-mutator-cmake/CMakeLists.txt b/contrib/libprotobuf-mutator-cmake/CMakeLists.txt index 41cb8af8b49..a623f95c418 100644 --- a/contrib/libprotobuf-mutator-cmake/CMakeLists.txt +++ b/contrib/libprotobuf-mutator-cmake/CMakeLists.txt @@ -1,6 +1,12 @@ +option(USE_LIBPROTOBUF_MUTATOR "Enable libprotobuf-mutator" ${ENABLE_FUZZING}) + +if (NOT USE_LIBPROTOBUF_MUTATOR) + return() +endif() + set(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/libprotobuf-mutator) -add_library(protobuf-mutator +add_library(_protobuf-mutator ${LIBRARY_DIR}/src/libfuzzer/libfuzzer_macro.cc ${LIBRARY_DIR}/src/libfuzzer/libfuzzer_mutator.cc ${LIBRARY_DIR}/src/binary_format.cc @@ -8,7 +14,9 @@ add_library(protobuf-mutator ${LIBRARY_DIR}/src/text_format.cc ${LIBRARY_DIR}/src/utf8_fix.cc) -target_include_directories(protobuf-mutator BEFORE PRIVATE "${LIBRARY_DIR}") -target_include_directories(protobuf-mutator BEFORE PRIVATE "${ClickHouse_SOURCE_DIR}/contrib/protobuf/src") +target_include_directories(_protobuf-mutator BEFORE INTERFACE "${LIBRARY_DIR}") +target_include_directories(_protobuf-mutator BEFORE INTERFACE "${ClickHouse_SOURCE_DIR}/contrib/protobuf/src") -target_link_libraries(protobuf-mutator ch_contrib::protobuf) +target_link_libraries(_protobuf-mutator ch_contrib::protobuf) + +add_library(ch_contrib::protobuf_mutator ALIAS _protobuf-mutator) diff --git a/src/Parsers/fuzzers/codegen_fuzzer/CMakeLists.txt b/src/Parsers/fuzzers/codegen_fuzzer/CMakeLists.txt index fee439377aa..3d416544419 100644 --- a/src/Parsers/fuzzers/codegen_fuzzer/CMakeLists.txt +++ b/src/Parsers/fuzzers/codegen_fuzzer/CMakeLists.txt @@ -41,6 +41,4 @@ add_executable(codegen_select_fuzzer ${FUZZER_SRCS}) set_source_files_properties("${PROTO_SRCS}" "out.cpp" PROPERTIES COMPILE_FLAGS "-Wno-reserved-identifier") target_include_directories(codegen_select_fuzzer SYSTEM BEFORE PRIVATE "${CMAKE_CURRENT_BINARY_DIR}") -target_include_directories(codegen_select_fuzzer SYSTEM BEFORE PRIVATE "${LibProtobufMutator_SOURCE_DIR}") -target_include_directories(codegen_select_fuzzer SYSTEM BEFORE PRIVATE "${LibProtobufMutator_SOURCE_DIR}/src") -target_link_libraries(codegen_select_fuzzer PRIVATE protobuf-mutator ch_contrib::protoc dbms ${LIB_FUZZING_ENGINE}) +target_link_libraries(codegen_select_fuzzer PRIVATE ch_contrib::protobuf_mutator ch_contrib::protoc dbms ${LIB_FUZZING_ENGINE}) From 7c3a3cebb542c0df1fbf68acb989d869b64726f1 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 21:45:54 +0300 Subject: [PATCH 140/403] Remove unbundled double-conversion support --- CMakeLists.txt | 1 - cmake/Modules/Finddouble-conversion.cmake | 44 ------------------- cmake/print_include_directories.cmake | 4 +- contrib/CMakeLists.txt | 7 +-- contrib/arrow-cmake/CMakeLists.txt | 2 +- .../double-conversion-cmake/CMakeLists.txt | 6 ++- src/CMakeLists.txt | 4 +- 7 files changed, 10 insertions(+), 58 deletions(-) delete mode 100644 cmake/Modules/Finddouble-conversion.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 5dcf6868549..4c899521235 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -488,7 +488,6 @@ message (STATUS include (GNUInstallDirs) include (cmake/contrib_finder.cmake) -find_contrib_lib(double-conversion) # Must be before parquet include (cmake/find/icu.cmake) include (cmake/find/xz.cmake) include (cmake/find/ltdl.cmake) # for odbc diff --git a/cmake/Modules/Finddouble-conversion.cmake b/cmake/Modules/Finddouble-conversion.cmake deleted file mode 100644 index cb01be0f25b..00000000000 --- a/cmake/Modules/Finddouble-conversion.cmake +++ /dev/null @@ -1,44 +0,0 @@ -# - Try to find double-conversion headers and libraries. -# -# Usage of this module as follows: -# -# find_package(double-conversion) -# -# Variables used by this module, they can change the default behaviour and need -# to be set before calling find_package: -# -# DOUBLE_CONVERSION_ROOT_DIR Set this variable to the root installation of -# double-conversion if the module has problems finding -# the proper installation path. -# -# Variables defined by this module: -# -# DOUBLE_CONVERSION_FOUND System has double-conversion libs/headers -# DOUBLE_CONVERSION_LIBRARIES The double-conversion library/libraries -# DOUBLE_CONVERSION_INCLUDE_DIR The location of double-conversion headers - -find_path(DOUBLE_CONVERSION_ROOT_DIR - NAMES include/double-conversion/double-conversion.h -) - -find_library(DOUBLE_CONVERSION_LIBRARIES - NAMES double-conversion - PATHS ${DOUBLE_CONVERSION_ROOT_DIR}/lib ${BTRIE_CITYHASH_PATHS} -) - -find_path(DOUBLE_CONVERSION_INCLUDE_DIR - NAMES double-conversion/double-conversion.h - PATHS ${DOUBLE_CONVERSION_ROOT_DIR}/include ${DOUBLE_CONVERSION_INCLUDE_PATHS} -) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(double_conversion DEFAULT_MSG - DOUBLE_CONVERSION_LIBRARIES - DOUBLE_CONVERSION_INCLUDE_DIR -) - -mark_as_advanced( - DOUBLE_CONVERSION_ROOT_DIR - DOUBLE_CONVERSION_LIBRARIES - DOUBLE_CONVERSION_INCLUDE_DIR -) diff --git a/cmake/print_include_directories.cmake b/cmake/print_include_directories.cmake index cc2098cb397..7871d375fcc 100644 --- a/cmake/print_include_directories.cmake +++ b/cmake/print_include_directories.cmake @@ -16,8 +16,8 @@ list(APPEND dirs ${dirs1}) get_property (dirs1 TARGET roaring PROPERTY INCLUDE_DIRECTORIES) list(APPEND dirs ${dirs1}) -if (TARGET double-conversion) - get_property (dirs1 TARGET double-conversion PROPERTY INCLUDE_DIRECTORIES) +if (TARGET ch_contrib::double_conversion) + get_property (dirs1 TARGET ch_contrib::double_conversion PROPERTY INCLUDE_DIRECTORIES) list(APPEND dirs ${dirs1}) endif () diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 7b0aaf088d4..cd95431e505 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -50,6 +50,7 @@ if (USE_INTERNAL_XZ_LIBRARY) add_subdirectory (xz-cmake) endif() +add_subdirectory (double-conversion-cmake) add_subdirectory (boringssl-cmake) add_subdirectory (poco-cmake) add_subdirectory (croaring-cmake) @@ -60,7 +61,7 @@ add_subdirectory (snappy-cmake) add_subdirectory (rocksdb-cmake) add_subdirectory (thrift-cmake) # parquet/arrow/orc -add_subdirectory (arrow-cmake) # requires: snappy, thrift +add_subdirectory (arrow-cmake) # requires: snappy, thrift, double-conversion add_subdirectory (avro-cmake) # requires: snappy add_subdirectory (hive-metastore-cmake) # requires: thrift/avro/arrow add_subdirectory (librdkafka-cmake) @@ -80,10 +81,6 @@ if (USE_INTERNAL_RE2_LIBRARY) add_subdirectory (re2-cmake) endif () -if (USE_INTERNAL_DOUBLE_CONVERSION_LIBRARY) - add_subdirectory (double-conversion-cmake) -endif () - if (USE_INTERNAL_CITYHASH_LIBRARY) add_subdirectory (cityhash102) endif () diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 5738b4a53ea..5e500877f3c 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -365,7 +365,7 @@ add_dependencies(_arrow protoc) target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ARROW_SRC_DIR}) target_include_directories(_arrow SYSTEM BEFORE PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/cpp/src") -target_link_libraries(_arrow PRIVATE ${DOUBLE_CONVERSION_LIBRARIES}) +target_link_libraries(_arrow PRIVATE ch_contrib::double_conversion) target_link_libraries(_arrow PRIVATE ch_contrib::protobuf) target_link_libraries(_arrow PRIVATE ch_contrib::lz4) target_link_libraries(_arrow PRIVATE ch_contrib::snappy) diff --git a/contrib/double-conversion-cmake/CMakeLists.txt b/contrib/double-conversion-cmake/CMakeLists.txt index c8bf1b34b8f..dc5b1719abf 100644 --- a/contrib/double-conversion-cmake/CMakeLists.txt +++ b/contrib/double-conversion-cmake/CMakeLists.txt @@ -1,6 +1,6 @@ SET(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/double-conversion") -add_library(double-conversion +add_library(_double-conversion "${LIBRARY_DIR}/double-conversion/bignum.cc" "${LIBRARY_DIR}/double-conversion/bignum-dtoa.cc" "${LIBRARY_DIR}/double-conversion/cached-powers.cc" @@ -10,4 +10,6 @@ add_library(double-conversion "${LIBRARY_DIR}/double-conversion/fixed-dtoa.cc" "${LIBRARY_DIR}/double-conversion/strtod.cc") -target_include_directories(double-conversion SYSTEM BEFORE PUBLIC "${LIBRARY_DIR}") +target_include_directories(_double-conversion SYSTEM BEFORE PUBLIC "${LIBRARY_DIR}") + +add_library(ch_contrib::double_conversion ALIAS _double-conversion) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index edf1809c49d..0236f6074d5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -303,7 +303,7 @@ target_link_libraries (clickhouse_common_io ${LINK_LIBRARIES_ONLY_ON_X86_64} PUBLIC common - ${DOUBLE_CONVERSION_LIBRARIES} + ch_contrib::double_conversion dragonbox_to_chars ) @@ -481,8 +481,6 @@ if (USE_CASSANDRA) dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${CASS_INCLUDE_DIR}) endif() -target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${DOUBLE_CONVERSION_INCLUDE_DIR}) - if (USE_MSGPACK) target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${MSGPACK_INCLUDE_DIR}) endif() From 96efe17844d6d1f7df0bc25dfea45042729db694 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 21:45:54 +0300 Subject: [PATCH 141/403] Remove unbundled xz support --- CMakeLists.txt | 1 - cmake/find/xz.cmake | 27 --------------------------- contrib/CMakeLists.txt | 5 +---- contrib/xz-cmake/CMakeLists.txt | 20 +++++++++++++++----- src/CMakeLists.txt | 5 ++--- 5 files changed, 18 insertions(+), 40 deletions(-) delete mode 100644 cmake/find/xz.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 4c899521235..4b25661bc77 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -489,7 +489,6 @@ include (GNUInstallDirs) include (cmake/contrib_finder.cmake) include (cmake/find/icu.cmake) -include (cmake/find/xz.cmake) include (cmake/find/ltdl.cmake) # for odbc # openssl, zlib before poco include (cmake/find/sparsehash.cmake) diff --git a/cmake/find/xz.cmake b/cmake/find/xz.cmake deleted file mode 100644 index f25937fe87d..00000000000 --- a/cmake/find/xz.cmake +++ /dev/null @@ -1,27 +0,0 @@ -option (USE_INTERNAL_XZ_LIBRARY "Set to OFF to use system xz (lzma) library instead of bundled" ON) - -if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/xz/src/liblzma/api/lzma.h") - if(USE_INTERNAL_XZ_LIBRARY) - message(WARNING "submodule contrib/xz is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal xz (lzma) library") - set(USE_INTERNAL_XZ_LIBRARY 0) - endif() - set(MISSING_INTERNAL_XZ_LIBRARY 1) -endif() - -if (NOT USE_INTERNAL_XZ_LIBRARY) - find_library (XZ_LIBRARY lzma) - find_path (XZ_INCLUDE_DIR NAMES lzma.h PATHS ${XZ_INCLUDE_PATHS}) - if (NOT XZ_LIBRARY OR NOT XZ_INCLUDE_DIR) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system xz (lzma) library") - endif () -endif () - -if (XZ_LIBRARY AND XZ_INCLUDE_DIR) -elseif (NOT MISSING_INTERNAL_XZ_LIBRARY) - set (USE_INTERNAL_XZ_LIBRARY 1) - set (XZ_LIBRARY liblzma) - set (XZ_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/xz/src/liblzma/api) -endif () - -message (STATUS "Using xz (lzma): ${XZ_INCLUDE_DIR} : ${XZ_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index cd95431e505..ea7c622f7fe 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -46,10 +46,7 @@ if (USE_YAML_CPP) add_subdirectory (yaml-cpp-cmake) endif() -if (USE_INTERNAL_XZ_LIBRARY) - add_subdirectory (xz-cmake) -endif() - +add_subdirectory (xz-cmake) add_subdirectory (double-conversion-cmake) add_subdirectory (boringssl-cmake) add_subdirectory (poco-cmake) diff --git a/contrib/xz-cmake/CMakeLists.txt b/contrib/xz-cmake/CMakeLists.txt index 5d70199413f..879ef192556 100644 --- a/contrib/xz-cmake/CMakeLists.txt +++ b/contrib/xz-cmake/CMakeLists.txt @@ -1,3 +1,10 @@ +option(ENABLE_XZ "Enable xz/liblzma" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_XZ) + message(STATUS "Not using xz") + return() +endif() + set (SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/xz") # Author: Lasse Collin @@ -97,7 +104,7 @@ endif () find_package(Threads REQUIRED) -add_library(liblzma +add_library(_liblzma ${SRC_DIR}/src/common/mythread.h ${SRC_DIR}/src/common/sysdefs.h ${SRC_DIR}/src/common/tuklib_common.h @@ -241,7 +248,7 @@ add_library(liblzma ${SRC_DIR}/src/liblzma/simple/x86.c ) -target_include_directories(liblzma SYSTEM PUBLIC +target_include_directories(_liblzma SYSTEM PRIVATE ${SRC_DIR}/src/liblzma/api ${SRC_DIR}/src/liblzma/common ${SRC_DIR}/src/liblzma/check @@ -252,12 +259,15 @@ target_include_directories(liblzma SYSTEM PUBLIC ${SRC_DIR}/src/liblzma/simple ${SRC_DIR}/src/common ) +target_include_directories(_liblzma SYSTEM BEFORE PUBLIC ${SRC_DIR}/src/liblzma/api) -target_link_libraries(liblzma Threads::Threads) +target_link_libraries(_liblzma Threads::Threads) # Put the tuklib functions under the lzma_ namespace. -target_compile_definitions(liblzma PRIVATE TUKLIB_SYMBOL_PREFIX=lzma_) +target_compile_definitions(_liblzma PRIVATE TUKLIB_SYMBOL_PREFIX=lzma_) if (ENABLE_SSE2) - target_compile_definitions(liblzma PRIVATE HAVE_IMMINTRIN_H HAVE__MM_MOVEMASK_EPI8) + target_compile_definitions(_liblzma PRIVATE HAVE_IMMINTRIN_H HAVE__MM_MOVEMASK_EPI8) endif() + +add_library(ch_contrib::xz ALIAS _liblzma) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0236f6074d5..c6f818d3f52 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -394,9 +394,8 @@ dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${MINISELECT_INCLUDE_DIR}) dbms_target_link_libraries(PRIVATE ch_contrib::zstd) target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::zstd) -if (XZ_LIBRARY) - target_link_libraries (clickhouse_common_io PUBLIC ${XZ_LIBRARY}) - target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${XZ_INCLUDE_DIR}) +if (TARGET ch_contrib::xz) + target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::xz) endif() if (USE_ICU) From 5c32f6dd3e41fbc39d676afdeace0aff960f3a20 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 21:45:54 +0300 Subject: [PATCH 142/403] Remove unbundled nuraft support --- CMakeLists.txt | 1 - cmake/find/nuraft.cmake | 24 ----------------------- contrib/CMakeLists.txt | 5 +---- contrib/nuraft-cmake/CMakeLists.txt | 30 +++++++++++++++++++++-------- programs/CMakeLists.txt | 4 ++-- src/CMakeLists.txt | 6 +++--- src/configure_config.cmake | 3 +++ utils/CMakeLists.txt | 2 +- 8 files changed, 32 insertions(+), 43 deletions(-) delete mode 100644 cmake/find/nuraft.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 4b25661bc77..f613605c4af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -518,7 +518,6 @@ include (cmake/find/odbc.cmake) include (cmake/find/nanodbc.cmake) include (cmake/find/sqlite.cmake) include (cmake/find/libpqxx.cmake) -include (cmake/find/nuraft.cmake) include (cmake/find/yaml-cpp.cmake) include (cmake/find/nlp.cmake) include (cmake/find/filelog.cmake) diff --git a/cmake/find/nuraft.cmake b/cmake/find/nuraft.cmake deleted file mode 100644 index c19f6774e7d..00000000000 --- a/cmake/find/nuraft.cmake +++ /dev/null @@ -1,24 +0,0 @@ -option(ENABLE_NURAFT "Enable NuRaft" ${ENABLE_LIBRARIES}) - -if (NOT ENABLE_NURAFT) - return() -endif() - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/NuRaft/src") - message (WARNING "submodule contrib/NuRaft is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal NuRaft library") - set (USE_NURAFT 0) - return() -endif () - -if (NOT OS_FREEBSD) - set (USE_NURAFT 1) - set (NURAFT_LIBRARY nuraft) - - set (NURAFT_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/NuRaft/include") - - message (STATUS "Using NuRaft=${USE_NURAFT}: ${NURAFT_INCLUDE_DIR} : ${NURAFT_LIBRARY}") -else() - set (USE_NURAFT 0) - message (STATUS "Using internal NuRaft library on FreeBSD and Darwin is not supported") -endif() diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index ea7c622f7fe..42cb8ae70b9 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -200,10 +200,7 @@ if (USE_LIBPQXX) add_subdirectory (libpqxx-cmake) endif() -if (USE_NURAFT) - add_subdirectory(nuraft-cmake) -endif() - +add_subdirectory (nuraft-cmake) add_subdirectory(fast_float-cmake) if (USE_NLP) diff --git a/contrib/nuraft-cmake/CMakeLists.txt b/contrib/nuraft-cmake/CMakeLists.txt index 297cee1abac..eaca00566d6 100644 --- a/contrib/nuraft-cmake/CMakeLists.txt +++ b/contrib/nuraft-cmake/CMakeLists.txt @@ -1,3 +1,15 @@ +set(ENABLE_NURAFT_DEFAULT ${ENABLE_LIBRARIES}) +if (OS_FREEBSD) + set(ENABLE_NURAFT_DEFAULT OFF) + message (STATUS "Using internal NuRaft library on FreeBSD and Darwin is not supported") +endif() +option(ENABLE_NURAFT "Enable NuRaft" ${ENABLE_NURAFT_DEFAULT}) + +if (NOT ENABLE_NURAFT) + message(STATUS "Not using NuRaft") + return() +endif() + set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/NuRaft") set(SRCS @@ -29,23 +41,25 @@ set(SRCS ) -add_library(nuraft ${SRCS}) +add_library(_nuraft ${SRCS}) if(NOT TARGET OpenSSL::Crypto) - target_compile_definitions(nuraft PRIVATE USE_BOOST_ASIO=1 BOOST_ASIO_STANDALONE=1 SSL_LIBRARY_NOT_FOUND=1) + target_compile_definitions(_nuraft PRIVATE USE_BOOST_ASIO=1 BOOST_ASIO_STANDALONE=1 SSL_LIBRARY_NOT_FOUND=1) else() - target_compile_definitions(nuraft PRIVATE USE_BOOST_ASIO=1 BOOST_ASIO_STANDALONE=1) + target_compile_definitions(_nuraft PRIVATE USE_BOOST_ASIO=1 BOOST_ASIO_STANDALONE=1) endif() -target_include_directories (nuraft SYSTEM PRIVATE "${LIBRARY_DIR}/include/libnuraft") +target_include_directories (_nuraft SYSTEM PRIVATE "${LIBRARY_DIR}/include/libnuraft") # for some reason include "asio.h" directly without "boost/" prefix. -target_include_directories (nuraft SYSTEM PRIVATE "${ClickHouse_SOURCE_DIR}/contrib/boost/boost") +target_include_directories (_nuraft SYSTEM PRIVATE "${ClickHouse_SOURCE_DIR}/contrib/boost/boost") -target_link_libraries (nuraft PRIVATE boost::headers_only boost::coroutine) +target_link_libraries (_nuraft PRIVATE boost::headers_only boost::coroutine) if(TARGET OpenSSL::Crypto) - target_link_libraries (nuraft PRIVATE OpenSSL::Crypto OpenSSL::SSL) + target_link_libraries (_nuraft PRIVATE OpenSSL::Crypto OpenSSL::SSL) endif() -target_include_directories (nuraft SYSTEM PUBLIC "${LIBRARY_DIR}/include") +target_include_directories (_nuraft SYSTEM PUBLIC "${LIBRARY_DIR}/include") + +add_library(ch_contrib::nuraft ALIAS _nuraft) diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 4806a7fe46e..68e8f8cba3b 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -53,9 +53,9 @@ option (ENABLE_CLICKHOUSE_KEEPER "ClickHouse alternative to ZooKeeper" ${ENABLE_ option (ENABLE_CLICKHOUSE_KEEPER_CONVERTER "Util allows to convert ZooKeeper logs and snapshots into clickhouse-keeper snapshot" ${ENABLE_CLICKHOUSE_ALL}) -if (NOT USE_NURAFT) +if (NOT ENABLE_NURAFT) # RECONFIGURE_MESSAGE_LEVEL should not be used here, - # since USE_NURAFT is set to OFF for FreeBSD and Darwin. + # since ENABLE_NURAFT is set to OFF for FreeBSD and Darwin. message (STATUS "clickhouse-keeper and clickhouse-keeper-converter will not be built (lack of NuRaft)") set(ENABLE_CLICKHOUSE_KEEPER OFF) set(ENABLE_CLICKHOUSE_KEEPER_CONVERTER OFF) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c6f818d3f52..5856dbb1864 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -241,7 +241,7 @@ add_object_library(clickhouse_processors_merges_algorithms Processors/Merges/Alg add_object_library(clickhouse_processors_queryplan Processors/QueryPlan) add_object_library(clickhouse_processors_queryplan_optimizations Processors/QueryPlan/Optimizations) -if (USE_NURAFT) +if (TARGET ch_contrib::nuraft) add_object_library(clickhouse_coordination Coordination) endif() @@ -352,8 +352,8 @@ if (USE_KRB5) dbms_target_link_libraries(PRIVATE ${KRB5_LIBRARY}) endif() -if (USE_NURAFT) - dbms_target_link_libraries(PUBLIC ${NURAFT_LIBRARY}) +if (TARGET ch_contrib::nuraft) + dbms_target_link_libraries(PUBLIC ch_contrib::nuraft) endif() if(RE2_INCLUDE_DIR) diff --git a/src/configure_config.cmake b/src/configure_config.cmake index e216eaf088f..e2cfb790311 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -25,3 +25,6 @@ endif() if (TARGET ch_contrib::hdfs) set(USE_HDFS 1) endif() +if (TARGET ch_contrib::nuraft) + set(USE_NURAFT 1) +endif() diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index a930e7db3fc..706532e2ac9 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -34,7 +34,7 @@ if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS) add_subdirectory (keeper-bench) add_subdirectory (graphite-rollup) - if (USE_NURAFT) + if (TARGET ch_contrib::nuraft) add_subdirectory (keeper-data-dumper) endif () From cd4d12c443dfe0be89baa3b2b128ee8e413ab13e Mon Sep 17 00:00:00 2001 From: bharatnc Date: Tue, 18 Jan 2022 22:42:16 -0800 Subject: [PATCH 143/403] add degrees and radians funcs --- src/Functions/degrees.cpp | 27 +++++++++++++++++++++++++ src/Functions/radians.cpp | 27 +++++++++++++++++++++++++ src/Functions/registerFunctionsMath.cpp | 4 ++++ 3 files changed, 58 insertions(+) create mode 100644 src/Functions/degrees.cpp create mode 100644 src/Functions/radians.cpp diff --git a/src/Functions/degrees.cpp b/src/Functions/degrees.cpp new file mode 100644 index 00000000000..a37da8c821d --- /dev/null +++ b/src/Functions/degrees.cpp @@ -0,0 +1,27 @@ +#include +#include + +template +T degrees(T r) +{ + return r * (180 / M_PI); +} + +namespace DB +{ +namespace +{ + struct DegreesName + { + static constexpr auto name = "degrees"; + }; + using FunctionDegrees = FunctionMathUnary>; + +} + +void registerFunctionDegrees(FunctionFactory & factory) +{ + factory.registerFunction(FunctionFactory::CaseInsensitive); +} + +} diff --git a/src/Functions/radians.cpp b/src/Functions/radians.cpp new file mode 100644 index 00000000000..88dae8811a5 --- /dev/null +++ b/src/Functions/radians.cpp @@ -0,0 +1,27 @@ +#include +#include + +template +T radians(T d) +{ + return (d * M_PI) / 180; +} + +namespace DB +{ +namespace +{ + struct RadiansName + { + static constexpr auto name = "radians"; + }; + using FunctionRadians= FunctionMathUnary>; + +} + +void registerFunctionRadians(FunctionFactory & factory) +{ + factory.registerFunction(FunctionFactory::CaseInsensitive); +} + +} diff --git a/src/Functions/registerFunctionsMath.cpp b/src/Functions/registerFunctionsMath.cpp index 9df24f4a8eb..865ec23174a 100644 --- a/src/Functions/registerFunctionsMath.cpp +++ b/src/Functions/registerFunctionsMath.cpp @@ -37,6 +37,8 @@ void registerFunctionSign(FunctionFactory & factory); void registerFunctionMax2(FunctionFactory & factory); void registerFunctionMin2(FunctionFactory & factory); void registerVectorFunctions(FunctionFactory &); +void registerFunctionDegrees(FunctionFactory & factory); +void registerFunctionRadians(FunctionFactory & factory); void registerFunctionsMath(FunctionFactory & factory) @@ -76,6 +78,8 @@ void registerFunctionsMath(FunctionFactory & factory) registerFunctionMax2(factory); registerFunctionMin2(factory); registerVectorFunctions(factory); + registerFunctionDegrees(factory); + registerFunctionRadians(factory); } } From aefa9f25ec89d89f67591bcdc3a5e81a2e664b7f Mon Sep 17 00:00:00 2001 From: bharatnc Date: Tue, 18 Jan 2022 23:17:34 -0800 Subject: [PATCH 144/403] add tests for degrees and radians --- .../02179_degrees_radians.reference | 9 +++++++++ .../0_stateless/02179_degrees_radians.sql | 20 +++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 tests/queries/0_stateless/02179_degrees_radians.reference create mode 100644 tests/queries/0_stateless/02179_degrees_radians.sql diff --git a/tests/queries/0_stateless/02179_degrees_radians.reference b/tests/queries/0_stateless/02179_degrees_radians.reference new file mode 100644 index 00000000000..3c26be9d9b2 --- /dev/null +++ b/tests/queries/0_stateless/02179_degrees_radians.reference @@ -0,0 +1,9 @@ +-360 +-180.6 +-180 +-1 +0 +1 +180 +180.5 +360 diff --git a/tests/queries/0_stateless/02179_degrees_radians.sql b/tests/queries/0_stateless/02179_degrees_radians.sql new file mode 100644 index 00000000000..cdb978761af --- /dev/null +++ b/tests/queries/0_stateless/02179_degrees_radians.sql @@ -0,0 +1,20 @@ +DROP TABLE IF EXISTS test_degs_rads; + + +CREATE TABLE test_degs_rads (degrees Float64) ENGINE = Memory; + + +INSERT INTO test_degs_rads VALUES (-1); +INSERT INTO test_degs_rads VALUES (-180); +INSERT INTO test_degs_rads VALUES (-180.6); +INSERT INTO test_degs_rads VALUES (-360); +INSERT INTO test_degs_rads VALUES (0); +INSERT INTO test_degs_rads VALUES (1); +INSERT INTO test_degs_rads VALUES (180); +INSERT INTO test_degs_rads VALUES (180.5); +INSERT INTO test_degs_rads VALUES (360); + + +select DEGREES(RADIANS(degrees)) from test_degs_rads order by degrees; + +DROP TABLE test_degs_rads; From 9c79ba31512c73eb03f2abb48f498b6639a6b931 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Tue, 18 Jan 2022 23:24:55 -0800 Subject: [PATCH 145/403] minor fix --- src/Functions/radians.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Functions/radians.cpp b/src/Functions/radians.cpp index 88dae8811a5..a1e0b9b0458 100644 --- a/src/Functions/radians.cpp +++ b/src/Functions/radians.cpp @@ -4,7 +4,7 @@ template T radians(T d) { - return (d * M_PI) / 180; + return d * (M_PI / 180); } namespace DB From 0483d14bb911f1ba6fcde58aa7022878289c8dff Mon Sep 17 00:00:00 2001 From: bharatnc Date: Wed, 19 Jan 2022 15:31:04 -0800 Subject: [PATCH 146/403] docs for degrees and radians --- .../sql-reference/functions/math-functions.md | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/docs/en/sql-reference/functions/math-functions.md b/docs/en/sql-reference/functions/math-functions.md index 2b3c000bc19..a5fc07cf687 100644 --- a/docs/en/sql-reference/functions/math-functions.md +++ b/docs/en/sql-reference/functions/math-functions.md @@ -477,3 +477,74 @@ Result: └──────────┘ ``` +## degrees(x) {#degreesx} + +Converts the input value in radians to degrees. + +**Syntax** + +``` sql +degrees(x) +``` + +**Arguments** + +- `x` — Input in radians. [Float64](../../sql-reference/data-types/float.md#float32-float64). + +**Returned value** + +- Value in degrees. + +Type: [Float64](../../sql-reference/data-types/float.md#float32-float64). + +**Example** + +Query: + +``` sql +SELECT degrees(3.141592653589793); +``` + +Result: + +``` text +┌─degrees(3.141592653589793)─┐ +│ 180 │ +└────────────────────────────┘ +``` + +## radians(x) {#radiansx} + +Converts the input value in degrees to radians. + +**Syntax** + +``` sql +radians(x) +``` + +**Arguments** + +- `x` — Input in degrees. [Float64](../../sql-reference/data-types/float.md#float32-float64). + +**Returned value** + +- Value in radians. + +Type: [Float64](../../sql-reference/data-types/float.md#float32-float64). + +**Example** + +Query: + +``` sql +SELECT radians(180); +``` + +Result: + +``` text +┌──────radians(180)─┐ +│ 3.141592653589793 │ +└───────────────────┘ +``` From 59f3fe515b81bc3024150c91ccdaf668671ea30d Mon Sep 17 00:00:00 2001 From: liuneng1994 <1398775315@qq.com> Date: Thu, 20 Jan 2022 06:26:34 +0000 Subject: [PATCH 147/403] change null check position Signed-off-by: neng.liu --- src/Storages/HDFS/StorageHDFS.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index c3240672391..74367c81eb9 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -80,10 +80,10 @@ namespace ErrorCodes::ACCESS_DENIED, "Cannot list directory {}: {}", prefix_without_globs, String(hdfsGetLastError())); } Strings result; + if (!ls.file_info && ls.length > 0) + throw Exception(ErrorCodes::LOGICAL_ERROR, "file_info shouldn't be null"); for (int i = 0; i < ls.length; ++i) { - if (ls.file_info == nullptr) - throw Exception(ErrorCodes::LOGICAL_ERROR, "file_info shouldn't be null"); const String full_path = String(ls.file_info[i].mName); const size_t last_slash = full_path.rfind('/'); const String file_name = full_path.substr(last_slash); From 6e8439e764be599ef7083c9ad5e21d066a6ccd77 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Wed, 19 Jan 2022 21:48:32 -0800 Subject: [PATCH 148/403] review fixes --- src/Functions/degrees.cpp | 15 +++++++++------ src/Functions/radians.cpp | 15 +++++++++------ 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/Functions/degrees.cpp b/src/Functions/degrees.cpp index a37da8c821d..481be7c7beb 100644 --- a/src/Functions/degrees.cpp +++ b/src/Functions/degrees.cpp @@ -1,11 +1,7 @@ +#include #include #include -template -T degrees(T r) -{ - return r * (180 / M_PI); -} namespace DB { @@ -15,8 +11,15 @@ namespace { static constexpr auto name = "degrees"; }; - using FunctionDegrees = FunctionMathUnary>; + template + Float64 degrees(T r) + { + Float64 degrees = r * (180 / M_PI); + return degrees; + } + + using FunctionDegrees = FunctionMathUnary>; } void registerFunctionDegrees(FunctionFactory & factory) diff --git a/src/Functions/radians.cpp b/src/Functions/radians.cpp index a1e0b9b0458..bf7a1fe3394 100644 --- a/src/Functions/radians.cpp +++ b/src/Functions/radians.cpp @@ -1,11 +1,7 @@ +#include #include #include -template -T radians(T d) -{ - return d * (M_PI / 180); -} namespace DB { @@ -15,8 +11,15 @@ namespace { static constexpr auto name = "radians"; }; - using FunctionRadians= FunctionMathUnary>; + template + Float64 radians(T d) + { + Float64 radians = d * (M_PI / 180); + return radians; + } + + using FunctionRadians = FunctionMathUnary>; } void registerFunctionRadians(FunctionFactory & factory) From ab1f8b496bd9d2e974d69407032248713dcc9818 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Wed, 19 Jan 2022 22:05:58 -0800 Subject: [PATCH 149/403] more tests --- .../02179_degrees_radians.reference | 33 ++++++++++++ .../0_stateless/02179_degrees_radians.sql | 52 ++++++++++++++----- 2 files changed, 72 insertions(+), 13 deletions(-) diff --git a/tests/queries/0_stateless/02179_degrees_radians.reference b/tests/queries/0_stateless/02179_degrees_radians.reference index 3c26be9d9b2..f9e5b43f6ed 100644 --- a/tests/queries/0_stateless/02179_degrees_radians.reference +++ b/tests/queries/0_stateless/02179_degrees_radians.reference @@ -7,3 +7,36 @@ 180 180.5 360 +-6.283185307179586 +-3.152064629101759 +-3.141592653589793 +-0.017453292519943295 +0 +0.017453292519943295 +3.141592653589793 +3.1503192998497647 +6.283185307179586 +-10 +-6.283185307179586 +-3.152064629101759 +-3.141592653589793 +-0.017453292519943295 +0 +0.017453292519943295 +1 +3.141592653589793 +3.1503192998497647 +6.283185307179586 +10 +-572.9577951308232 +-360 +-180.6 +-180 +-1 +0 +1 +57.29577951308232 +180 +180.5 +360 +572.9577951308232 diff --git a/tests/queries/0_stateless/02179_degrees_radians.sql b/tests/queries/0_stateless/02179_degrees_radians.sql index cdb978761af..ac07b597476 100644 --- a/tests/queries/0_stateless/02179_degrees_radians.sql +++ b/tests/queries/0_stateless/02179_degrees_radians.sql @@ -1,20 +1,46 @@ -DROP TABLE IF EXISTS test_degs_rads; +-- test conversion from degrees to radians +DROP TABLE IF EXISTS test_degs_to_rads; +CREATE TABLE test_degs_to_rads (degrees Float64) ENGINE = Memory; -CREATE TABLE test_degs_rads (degrees Float64) ENGINE = Memory; +INSERT INTO test_degs_to_rads VALUES (-1); +INSERT INTO test_degs_to_rads VALUES (-180); +INSERT INTO test_degs_to_rads VALUES (-180.6); +INSERT INTO test_degs_to_rads VALUES (-360); +INSERT INTO test_degs_to_rads VALUES (0); +INSERT INTO test_degs_to_rads VALUES (1); +INSERT INTO test_degs_to_rads VALUES (180); +INSERT INTO test_degs_to_rads VALUES (180.5); +INSERT INTO test_degs_to_rads VALUES (360); +-- test that converting degrees to radians and back preserves the original value +select DEGREES(RADIANS(degrees)) from test_degs_to_rads order by degrees; +-- test that radians func returns correct value for both int and floats +select RADIANS(degrees) from test_degs_to_rads order by degrees; -INSERT INTO test_degs_rads VALUES (-1); -INSERT INTO test_degs_rads VALUES (-180); -INSERT INTO test_degs_rads VALUES (-180.6); -INSERT INTO test_degs_rads VALUES (-360); -INSERT INTO test_degs_rads VALUES (0); -INSERT INTO test_degs_rads VALUES (1); -INSERT INTO test_degs_rads VALUES (180); -INSERT INTO test_degs_rads VALUES (180.5); -INSERT INTO test_degs_rads VALUES (360); +DROP TABLE test_degs_to_rads; +-- test conversion from radians to degrees +DROP TABLE IF EXISTS test_rads_to_degs; -select DEGREES(RADIANS(degrees)) from test_degs_rads order by degrees; +CREATE TABLE test_rads_to_degs (radians Float64) ENGINE = Memory; -DROP TABLE test_degs_rads; +INSERT INTO test_rads_to_degs VALUES (-6.283185307179586); +INSERT INTO test_rads_to_degs VALUES (-3.152064629101759); +INSERT INTO test_rads_to_degs VALUES (-3.141592653589793); +INSERT INTO test_rads_to_degs VALUES (-0.017453292519943295); +INSERT INTO test_rads_to_degs VALUES(0); +INSERT INTO test_rads_to_degs VALUES(1); +INSERT INTO test_rads_to_degs VALUES(10); +INSERT INTO test_rads_to_degs VALUES(-10); +INSERT INTO test_rads_to_degs VALUES (0.017453292519943295); +INSERT INTO test_rads_to_degs VALUES (3.141592653589793); +INSERT INTO test_rads_to_degs VALUES (3.1503192998497647); +INSERT INTO test_rads_to_degs VALUES (6.283185307179586); + +-- test that converting radians to degrees and back preserves the original value +select RADIANS(DEGREES(radians)) from test_rads_to_degs order by radians; +-- test that degrees func returns correct value for both int and floats +select DEGREES(radians) from test_rads_to_degs order by radians; + +DROP TABLE test_rads_to_degs; From 789dfd9f3bd6c4d29fb6d0c42c4f9cabd0ddc2ff Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 21:45:54 +0300 Subject: [PATCH 150/403] Remove unbundled re2 support v2: preserve re2_st name to make PVS check pass (since docker image update fails) --- CMakeLists.txt | 1 - cmake/find/re2.cmake | 41 ------------------- contrib/CMakeLists.txt | 5 +-- contrib/grpc-cmake/CMakeLists.txt | 7 +--- contrib/re2-cmake/CMakeLists.txt | 13 ++++-- src/CMakeLists.txt | 12 +----- src/Common/OptimizedRegularExpression.h | 7 +--- src/Common/config.h.in | 1 - src/Functions/MatchImpl.h | 8 +--- src/Functions/ReplaceRegexpImpl.h | 8 +--- ...StorageSystemBuildOptions.generated.cpp.in | 1 - 11 files changed, 18 insertions(+), 86 deletions(-) delete mode 100644 cmake/find/re2.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index f613605c4af..19f97009c50 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -492,7 +492,6 @@ include (cmake/find/icu.cmake) include (cmake/find/ltdl.cmake) # for odbc # openssl, zlib before poco include (cmake/find/sparsehash.cmake) -include (cmake/find/re2.cmake) include (cmake/find/krb5.cmake) include (cmake/find/libgsasl.cmake) include (cmake/find/cyrus-sasl.cmake) diff --git a/cmake/find/re2.cmake b/cmake/find/re2.cmake deleted file mode 100644 index ed5c72d13fa..00000000000 --- a/cmake/find/re2.cmake +++ /dev/null @@ -1,41 +0,0 @@ -option (USE_INTERNAL_RE2_LIBRARY "Set to FALSE to use system re2 library instead of bundled [slower]" ON) - -if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/re2/re2") - if(USE_INTERNAL_RE2_LIBRARY) - message(WARNING "submodule contrib/re2 is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal re2 library") - endif() - set(USE_INTERNAL_RE2_LIBRARY 0) - set(MISSING_INTERNAL_RE2_LIBRARY 1) -endif() - -if (NOT USE_INTERNAL_RE2_LIBRARY) - find_library (RE2_LIBRARY re2) - find_path (RE2_INCLUDE_DIR NAMES re2/re2.h PATHS ${RE2_INCLUDE_PATHS}) - if (NOT RE2_LIBRARY OR NOT RE2_INCLUDE_DIR) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system re2 library") - endif () -endif () - -string(FIND ${CMAKE_CURRENT_BINARY_DIR} " " _have_space) -if(_have_space GREATER 0) - message(WARNING "Using spaces in build path [${CMAKE_CURRENT_BINARY_DIR}] highly not recommended. Library re2st will be disabled.") - set (MISSING_INTERNAL_RE2_ST_LIBRARY 1) -endif() - -if (RE2_LIBRARY AND RE2_INCLUDE_DIR) - set (RE2_ST_LIBRARY ${RE2_LIBRARY}) -elseif (NOT MISSING_INTERNAL_RE2_LIBRARY) - set (USE_INTERNAL_RE2_LIBRARY 1) - set (RE2_LIBRARY re2) - set (RE2_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/re2") - if (NOT MISSING_INTERNAL_RE2_ST_LIBRARY) - set (RE2_ST_LIBRARY re2_st) - set (USE_RE2_ST 1) - else () - set (RE2_ST_LIBRARY ${RE2_LIBRARY}) - message (${RECONFIGURE_MESSAGE_LEVEL} "Using internal re2 library instead of re2_st") - endif () -endif () - -message (STATUS "Using re2: ${RE2_INCLUDE_DIR} : ${RE2_LIBRARY}; ${RE2_ST_INCLUDE_DIR} : ${RE2_ST_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 42cb8ae70b9..52a9b5c2b89 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -46,6 +46,7 @@ if (USE_YAML_CPP) add_subdirectory (yaml-cpp-cmake) endif() +add_subdirectory (re2-cmake) add_subdirectory (xz-cmake) add_subdirectory (double-conversion-cmake) add_subdirectory (boringssl-cmake) @@ -74,10 +75,6 @@ endif() # TODO: refactor the contrib libraries below this comment. -if (USE_INTERNAL_RE2_LIBRARY) - add_subdirectory (re2-cmake) -endif () - if (USE_INTERNAL_CITYHASH_LIBRARY) add_subdirectory (cityhash102) endif () diff --git a/contrib/grpc-cmake/CMakeLists.txt b/contrib/grpc-cmake/CMakeLists.txt index 64a5be8e8d5..15b7550e810 100644 --- a/contrib/grpc-cmake/CMakeLists.txt +++ b/contrib/grpc-cmake/CMakeLists.txt @@ -15,12 +15,9 @@ set(_gRPC_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/grpc") set(_gRPC_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/grpc") # Use re2 from ClickHouse contrib, not from gRPC third_party. -if(NOT RE2_INCLUDE_DIR) - message(FATAL_ERROR " grpc: The location of the \"re2\" library is unknown") -endif() set(gRPC_RE2_PROVIDER "clickhouse" CACHE STRING "" FORCE) -set(_gRPC_RE2_INCLUDE_DIR "${RE2_INCLUDE_DIR}") -set(_gRPC_RE2_LIBRARIES "${RE2_LIBRARY}") +set(_gRPC_RE2_INCLUDE_DIR "") +set(_gRPC_RE2_LIBRARIES ch_contrib::re2) # Use zlib from ClickHouse contrib, not from gRPC third_party. set(gRPC_ZLIB_PROVIDER "clickhouse" CACHE STRING "" FORCE) diff --git a/contrib/re2-cmake/CMakeLists.txt b/contrib/re2-cmake/CMakeLists.txt index ff8b3c43472..f348f412f62 100644 --- a/contrib/re2-cmake/CMakeLists.txt +++ b/contrib/re2-cmake/CMakeLists.txt @@ -4,6 +4,11 @@ # This file was edited for ClickHouse +string(FIND ${CMAKE_CURRENT_BINARY_DIR} " " _have_space) +if(_have_space GREATER 0) + message(FATAL_ERROR "Using spaces in build path [${CMAKE_CURRENT_BINARY_DIR}] highly not recommended. Library re2st will be disabled.") +endif() + set(SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/re2") set(RE2_SOURCES @@ -30,11 +35,9 @@ set(RE2_SOURCES ${SRC_DIR}/util/rune.cc ${SRC_DIR}/util/strutil.cc ) - add_library(re2 ${RE2_SOURCES}) target_include_directories(re2 PUBLIC "${SRC_DIR}") - # Building re2 which is thread-safe and re2_st which is not. # re2 changes its state during matching of regular expression, e.g. creates temporary DFA. # It uses RWLock to process the same regular expression object from different threads. @@ -43,7 +46,8 @@ target_include_directories(re2 PUBLIC "${SRC_DIR}") add_library(re2_st ${RE2_SOURCES}) target_compile_definitions (re2_st PRIVATE NDEBUG NO_THREADS re2=re2_st) target_include_directories (re2_st PRIVATE .) -target_include_directories (re2_st SYSTEM PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ${SRC_DIR}) +target_include_directories (re2_st SYSTEM PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) +target_include_directories (re2_st SYSTEM BEFORE PUBLIC ${SRC_DIR}) file (MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/re2_st) foreach (FILENAME filtered_re2.h re2.h set.h stringpiece.h) @@ -66,3 +70,6 @@ foreach (FILENAME mutex.h) add_custom_target (transform_${FILENAME} DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/util/${FILENAME}") add_dependencies (re2_st transform_${FILENAME}) endforeach () + +add_library(ch_contrib::re2 ALIAS re2) +add_library(ch_contrib::re2_st ALIAS re2_st) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5856dbb1864..1f09d8699f5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -315,12 +315,8 @@ set_source_files_properties( Columns/ColumnDecimal.cpp PROPERTIES COMPILE_FLAGS "${X86_INTRINSICS_FLAGS}") -if(RE2_LIBRARY) - target_link_libraries(clickhouse_common_io PUBLIC ${RE2_LIBRARY}) -endif() -if(RE2_ST_LIBRARY) - target_link_libraries(clickhouse_common_io PUBLIC ${RE2_ST_LIBRARY}) -endif() +target_link_libraries(clickhouse_common_io PUBLIC ch_contrib::re2_st) +target_link_libraries(clickhouse_common_io PUBLIC ch_contrib::re2) target_link_libraries(clickhouse_common_io PRIVATE @@ -356,10 +352,6 @@ if (TARGET ch_contrib::nuraft) dbms_target_link_libraries(PUBLIC ch_contrib::nuraft) endif() -if(RE2_INCLUDE_DIR) - target_include_directories(clickhouse_common_io SYSTEM BEFORE PUBLIC ${RE2_INCLUDE_DIR}) -endif() - dbms_target_link_libraries ( PRIVATE boost::filesystem diff --git a/src/Common/OptimizedRegularExpression.h b/src/Common/OptimizedRegularExpression.h index f7223c2efa9..09b0dbe5337 100644 --- a/src/Common/OptimizedRegularExpression.h +++ b/src/Common/OptimizedRegularExpression.h @@ -8,12 +8,7 @@ #include #include - -#if USE_RE2_ST -# include -#else -# define re2_st re2 -#endif +#include /** Uses two ways to optimize a regular expression: diff --git a/src/Common/config.h.in b/src/Common/config.h.in index 03ea3ac2e7a..5fa9b2ee2ba 100644 --- a/src/Common/config.h.in +++ b/src/Common/config.h.in @@ -3,7 +3,6 @@ // .h autogenerated by cmake! #cmakedefine01 USE_BASE64 -#cmakedefine01 USE_RE2_ST #cmakedefine01 USE_SSL #cmakedefine01 USE_INTERNAL_SSL_LIBRARY #cmakedefine01 USE_HDFS diff --git a/src/Functions/MatchImpl.h b/src/Functions/MatchImpl.h index 63a14484fe2..08ac47e692f 100644 --- a/src/Functions/MatchImpl.h +++ b/src/Functions/MatchImpl.h @@ -8,13 +8,7 @@ #include "config_functions.h" #include - -#if USE_RE2_ST -# include -#else -# include -# define re2_st re2 -#endif +#include namespace DB diff --git a/src/Functions/ReplaceRegexpImpl.h b/src/Functions/ReplaceRegexpImpl.h index e6305431d8f..5d2549239c8 100644 --- a/src/Functions/ReplaceRegexpImpl.h +++ b/src/Functions/ReplaceRegexpImpl.h @@ -7,13 +7,7 @@ #include "config_functions.h" #include - -#if USE_RE2_ST -# include -#else -# include -# define re2_st re2 -#endif +#include namespace DB diff --git a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in index 01be3f8456d..90c5c92c576 100644 --- a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in +++ b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in @@ -30,7 +30,6 @@ const char * auto_config_build[] "USE_ICU", "@USE_ICU@", "USE_H3", "@USE_H3@", "USE_MYSQL", "@USE_MYSQL@", - "USE_RE2_ST", "@USE_RE2_ST@", "USE_LIBGSASL", "@USE_LIBGSASL@", "USE_RDKAFKA", "@USE_RDKAFKA@", "USE_CAPNP", "@USE_CAPNP@", From 041b4f363524bb9d6d18eff8e7e5c53f76402b7f Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 21:48:14 +0300 Subject: [PATCH 151/403] Remove unbundled brotli support --- CMakeLists.txt | 1 - cmake/find/brotli.cmake | 42 ----------------------------- contrib/CMakeLists.txt | 6 +---- contrib/brotli-cmake/CMakeLists.txt | 15 ++++++++--- src/CMakeLists.txt | 5 ++-- src/configure_config.cmake | 3 +++ 6 files changed, 18 insertions(+), 54 deletions(-) delete mode 100644 cmake/find/brotli.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 19f97009c50..10370a0b3fb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -501,7 +501,6 @@ include (cmake/find/capnp.cmake) include (cmake/find/llvm.cmake) include (cmake/find/h3.cmake) include (cmake/find/libxml2.cmake) -include (cmake/find/brotli.cmake) include (cmake/find/pdqsort.cmake) include (cmake/find/miniselect.cmake) include (cmake/find/poco.cmake) diff --git a/cmake/find/brotli.cmake b/cmake/find/brotli.cmake deleted file mode 100644 index 6469ec04f45..00000000000 --- a/cmake/find/brotli.cmake +++ /dev/null @@ -1,42 +0,0 @@ -option (ENABLE_BROTLI "Enable brotli" ${ENABLE_LIBRARIES}) - -if (NOT ENABLE_BROTLI) - if (USE_INTERNAL_BROTLI_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal brotly library with ENABLE_BROTLI=OFF") - endif() - return() -endif() - -option (USE_INTERNAL_BROTLI_LIBRARY "Set to FALSE to use system libbrotli library instead of bundled" ON) - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/brotli/c/include/brotli/decode.h") - if (USE_INTERNAL_BROTLI_LIBRARY) - message (WARNING "submodule contrib/brotli is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot find internal brotli") - set (USE_INTERNAL_BROTLI_LIBRARY 0) - endif () - set (MISSING_INTERNAL_BROTLI_LIBRARY 1) -endif () - -if(NOT USE_INTERNAL_BROTLI_LIBRARY) - find_library(BROTLI_LIBRARY_COMMON brotlicommon) - find_library(BROTLI_LIBRARY_DEC brotlidec) - find_library(BROTLI_LIBRARY_ENC brotlienc) - find_path(BROTLI_INCLUDE_DIR NAMES brotli/decode.h brotli/encode.h brotli/port.h brotli/types.h PATHS ${BROTLI_INCLUDE_PATHS}) - if(BROTLI_LIBRARY_DEC AND BROTLI_LIBRARY_ENC AND BROTLI_LIBRARY_COMMON) - set(BROTLI_LIBRARY ${BROTLI_LIBRARY_DEC} ${BROTLI_LIBRARY_ENC} ${BROTLI_LIBRARY_COMMON}) - else() - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use system brotli") - endif() -endif() - -if (BROTLI_LIBRARY AND BROTLI_INCLUDE_DIR) - set (USE_BROTLI 1) -elseif (NOT MISSING_INTERNAL_BROTLI_LIBRARY) - set (BROTLI_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/brotli/c/include") - set (USE_INTERNAL_BROTLI_LIBRARY 1) - set (BROTLI_LIBRARY brotli) - set (USE_BROTLI 1) -endif () - -message (STATUS "Using brotli=${USE_BROTLI}: ${BROTLI_INCLUDE_DIR} : ${BROTLI_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 52a9b5c2b89..37e52ad8acd 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -48,6 +48,7 @@ endif() add_subdirectory (re2-cmake) add_subdirectory (xz-cmake) +add_subdirectory (brotli-cmake) add_subdirectory (double-conversion-cmake) add_subdirectory (boringssl-cmake) add_subdirectory (poco-cmake) @@ -134,11 +135,6 @@ if (USE_INTERNAL_LIBXML2_LIBRARY) add_subdirectory(libxml2-cmake) endif () -if (USE_INTERNAL_BROTLI_LIBRARY) - add_subdirectory(brotli-cmake) - target_compile_definitions(brotli PRIVATE BROTLI_BUILD_PORTABLE=1) -endif () - if (USE_INTERNAL_AWS_S3_LIBRARY) add_subdirectory(aws-s3-cmake) diff --git a/contrib/brotli-cmake/CMakeLists.txt b/contrib/brotli-cmake/CMakeLists.txt index 7293cae0665..c81a6bf9076 100644 --- a/contrib/brotli-cmake/CMakeLists.txt +++ b/contrib/brotli-cmake/CMakeLists.txt @@ -1,3 +1,10 @@ +option (ENABLE_BROTLI "Enable brotli" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_BROTLI) + message(STATUS "Not using brotli") + return() +endif() + set(BROTLI_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/brotli/c") set(BROTLI_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/brotli/c") @@ -33,10 +40,12 @@ set(SRCS "${BROTLI_SOURCE_DIR}/common/constants.c" ) -add_library(brotli ${SRCS}) +add_library(_brotli ${SRCS}) +add_library(ch_contrib::brotli ALIAS _brotli) -target_include_directories(brotli PUBLIC "${BROTLI_SOURCE_DIR}/include") +target_include_directories(_brotli SYSTEM BEFORE PUBLIC "${BROTLI_SOURCE_DIR}/include") if(M_LIBRARY) - target_link_libraries(brotli PRIVATE ${M_LIBRARY}) + target_link_libraries(_brotli PRIVATE ${M_LIBRARY}) endif() +target_compile_definitions(_brotli PRIVATE BROTLI_BUILD_PORTABLE=1) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1f09d8699f5..fce7ff91b3e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -449,9 +449,8 @@ if (TARGET ch_contrib::s2) dbms_target_link_libraries (PUBLIC ch_contrib::s2) endif() -if (USE_BROTLI) - target_link_libraries (clickhouse_common_io PRIVATE ${BROTLI_LIBRARY}) - target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${BROTLI_INCLUDE_DIR}) +if (TARGET ch_contrib::brotli) + target_link_libraries (clickhouse_common_io PRIVATE ch_contrib::brotli) endif() if (TARGET ch_contrib::snappy) diff --git a/src/configure_config.cmake b/src/configure_config.cmake index e2cfb790311..babf8f0d9ef 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -7,6 +7,9 @@ endif() if (TARGET ch_contrib::snappy) set(USE_SNAPPY 1) endif() +if (TARGET ch_contrib::brotli) + set(USE_BROTLI 1) +endif() if (TARGET ch_contrib::hivemetastore) set(USE_HIVE 1) endif() From 8ede97925ec3653b4179fdb6206b796086be08c3 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 22:04:55 +0300 Subject: [PATCH 152/403] Remove unbundled sparsehash support --- CMakeLists.txt | 1 - cmake/find/sparsehash.cmake | 17 ----------------- contrib/CMakeLists.txt | 1 + contrib/sparsehash-c11-cmake/CMakeLists.txt | 3 +++ src/CMakeLists.txt | 2 +- src/Common/examples/CMakeLists.txt | 2 -- src/Core/examples/CMakeLists.txt | 1 - src/Dictionaries/CMakeLists.txt | 3 +-- src/Functions/CMakeLists.txt | 2 -- src/Interpreters/examples/CMakeLists.txt | 5 ----- 10 files changed, 6 insertions(+), 31 deletions(-) delete mode 100644 cmake/find/sparsehash.cmake create mode 100644 contrib/sparsehash-c11-cmake/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 10370a0b3fb..585f5eeb007 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -491,7 +491,6 @@ include (cmake/contrib_finder.cmake) include (cmake/find/icu.cmake) include (cmake/find/ltdl.cmake) # for odbc # openssl, zlib before poco -include (cmake/find/sparsehash.cmake) include (cmake/find/krb5.cmake) include (cmake/find/libgsasl.cmake) include (cmake/find/cyrus-sasl.cmake) diff --git a/cmake/find/sparsehash.cmake b/cmake/find/sparsehash.cmake deleted file mode 100644 index f258f6c1c5b..00000000000 --- a/cmake/find/sparsehash.cmake +++ /dev/null @@ -1,17 +0,0 @@ -option (USE_INTERNAL_SPARSEHASH_LIBRARY "Set to FALSE to use system sparsehash library instead of bundled" - ON) # ON by default as we are not aware of any system providing package for sparsehash-c11 - -if (NOT USE_INTERNAL_SPARSEHASH_LIBRARY) - find_path (SPARSEHASH_INCLUDE_DIR NAMES sparsehash/sparse_hash_map PATHS ${SPARSEHASH_INCLUDE_PATHS}) - if (NOT SPARSEHASH_INCLUDE_DIR) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system sparsehash library") - endif () -endif () - -if (SPARSEHASH_INCLUDE_DIR) -else () - set (USE_INTERNAL_SPARSEHASH_LIBRARY 1) - set (SPARSEHASH_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/sparsehash-c11") -endif () - -message (STATUS "Using sparsehash: ${SPARSEHASH_INCLUDE_DIR}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 37e52ad8acd..fef8a567f98 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -21,6 +21,7 @@ endif() set_property(DIRECTORY PROPERTY EXCLUDE_FROM_ALL 1) +add_subdirectory (sparsehash-c11-cmake) add_subdirectory (abseil-cpp-cmake) add_subdirectory (magic-enum-cmake) add_subdirectory (boost-cmake) diff --git a/contrib/sparsehash-c11-cmake/CMakeLists.txt b/contrib/sparsehash-c11-cmake/CMakeLists.txt new file mode 100644 index 00000000000..af588c9484f --- /dev/null +++ b/contrib/sparsehash-c11-cmake/CMakeLists.txt @@ -0,0 +1,3 @@ +add_library(_sparsehash INTERFACE) +target_include_directories(_sparsehash SYSTEM BEFORE INTERFACE "${ClickHouse_SOURCE_DIR}/contrib/sparsehash-c11") +add_library(ch_contrib::sparsehash ALIAS _sparsehash) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index fce7ff91b3e..2112f15cda9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -415,7 +415,7 @@ endif () if (TARGET ch_contrib::ldap) dbms_target_link_libraries (PRIVATE ch_contrib::ldap ch_contrib::lber) endif () -dbms_target_include_directories (SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) +dbms_target_link_libraries (PRIVATE ch_contrib::sparsehash) if (TARGET ch_contrib::protobuf) dbms_target_link_libraries (PRIVATE ch_contrib::protobuf) diff --git a/src/Common/examples/CMakeLists.txt b/src/Common/examples/CMakeLists.txt index 7410671bc2c..5f6b8b67e49 100644 --- a/src/Common/examples/CMakeLists.txt +++ b/src/Common/examples/CMakeLists.txt @@ -42,11 +42,9 @@ add_executable (space_saving space_saving.cpp) target_link_libraries (space_saving PRIVATE clickhouse_common_io) add_executable (integer_hash_tables_and_hashes integer_hash_tables_and_hashes.cpp) -target_include_directories (integer_hash_tables_and_hashes SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) target_link_libraries (integer_hash_tables_and_hashes PRIVATE dbms abseil_swiss_tables) add_executable (integer_hash_tables_benchmark integer_hash_tables_benchmark.cpp) -target_include_directories (integer_hash_tables_benchmark SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) target_link_libraries (integer_hash_tables_benchmark PRIVATE dbms abseil_swiss_tables) add_executable (cow_columns cow_columns.cpp) diff --git a/src/Core/examples/CMakeLists.txt b/src/Core/examples/CMakeLists.txt index 7d02265967e..1c7eb04c966 100644 --- a/src/Core/examples/CMakeLists.txt +++ b/src/Core/examples/CMakeLists.txt @@ -1,6 +1,5 @@ add_executable (string_pool string_pool.cpp) target_link_libraries (string_pool PRIVATE clickhouse_common_io) -target_include_directories (string_pool SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) add_executable (field field.cpp) target_link_libraries (field PRIVATE dbms) diff --git a/src/Dictionaries/CMakeLists.txt b/src/Dictionaries/CMakeLists.txt index b1b3d6d55e0..bd2aa80f6d8 100644 --- a/src/Dictionaries/CMakeLists.txt +++ b/src/Dictionaries/CMakeLists.txt @@ -42,5 +42,4 @@ if(USE_CASSANDRA) endif() add_subdirectory(Embedded) - -target_include_directories(clickhouse_dictionaries SYSTEM PRIVATE ${SPARSEHASH_INCLUDE_DIR}) +target_link_libraries(clickhouse_dictionaries PRIVATE ch_contrib::sparsehash) diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index f0620fe4bc8..80749a7f189 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -34,8 +34,6 @@ if (TARGET OpenSSL::Crypto) target_link_libraries(clickhouse_functions PUBLIC OpenSSL::Crypto) endif() -target_include_directories(clickhouse_functions SYSTEM PRIVATE ${SPARSEHASH_INCLUDE_DIR}) - if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL") diff --git a/src/Interpreters/examples/CMakeLists.txt b/src/Interpreters/examples/CMakeLists.txt index 85daffc4064..742a114f7c4 100644 --- a/src/Interpreters/examples/CMakeLists.txt +++ b/src/Interpreters/examples/CMakeLists.txt @@ -1,16 +1,13 @@ add_executable (hash_map hash_map.cpp) -target_include_directories (hash_map SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) target_link_libraries (hash_map PRIVATE dbms) add_executable (hash_map_lookup hash_map_lookup.cpp) -target_include_directories (hash_map_lookup SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) target_link_libraries (hash_map_lookup PRIVATE dbms) add_executable (hash_map3 hash_map3.cpp) target_link_libraries (hash_map3 PRIVATE dbms ${FARMHASH_LIBRARIES} metrohash) add_executable (hash_map_string hash_map_string.cpp) -target_include_directories (hash_map_string SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) target_link_libraries (hash_map_string PRIVATE dbms) add_executable (hash_map_string_2 hash_map_string_2.cpp) @@ -20,7 +17,6 @@ add_executable (hash_map_string_3 hash_map_string_3.cpp) target_link_libraries (hash_map_string_3 PRIVATE dbms ${FARMHASH_LIBRARIES} metrohash) add_executable (hash_map_string_small hash_map_string_small.cpp) -target_include_directories (hash_map_string_small SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) target_link_libraries (hash_map_string_small PRIVATE dbms) add_executable (string_hash_map string_hash_map.cpp) @@ -33,7 +29,6 @@ add_executable (string_hash_set string_hash_set.cpp) target_link_libraries (string_hash_set PRIVATE dbms) add_executable (two_level_hash_map two_level_hash_map.cpp) -target_include_directories (two_level_hash_map SYSTEM BEFORE PRIVATE ${SPARSEHASH_INCLUDE_DIR}) target_link_libraries (two_level_hash_map PRIVATE dbms) add_executable (jit_example jit_example.cpp) From f1cc63d9006265e80ae5408495c1ec819dca2bb2 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 22:09:12 +0300 Subject: [PATCH 153/403] Remove unbundled cityhash support --- CMakeLists.txt | 3 -- base/base/CMakeLists.txt | 2 +- cmake/Modules/Findcityhash.cmake | 44 --------------------------- cmake/print_include_directories.cmake | 2 +- contrib/CMakeLists.txt | 4 +-- contrib/cityhash102/CMakeLists.txt | 8 +++-- src/AggregateFunctions/CMakeLists.txt | 2 +- src/CMakeLists.txt | 2 +- src/Common/examples/CMakeLists.txt | 2 +- src/Functions/CMakeLists.txt | 2 +- 10 files changed, 12 insertions(+), 59 deletions(-) delete mode 100644 cmake/Modules/Findcityhash.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 585f5eeb007..1dcaba0ffe9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -523,9 +523,6 @@ include (cmake/find/cassandra.cmake) include (cmake/find/sentry.cmake) include (cmake/find/datasketches.cmake) -set (USE_INTERNAL_CITYHASH_LIBRARY ON CACHE INTERNAL "") -find_contrib_lib(cityhash) - find_contrib_lib(farmhash) if (ENABLE_TESTS) diff --git a/base/base/CMakeLists.txt b/base/base/CMakeLists.txt index bc82e502e79..434205d812e 100644 --- a/base/base/CMakeLists.txt +++ b/base/base/CMakeLists.txt @@ -52,7 +52,7 @@ endif() target_link_libraries (common PUBLIC - ${CITYHASH_LIBRARIES} + ch_contrib::cityhash boost::headers_only boost::system Poco::Net diff --git a/cmake/Modules/Findcityhash.cmake b/cmake/Modules/Findcityhash.cmake deleted file mode 100644 index 5250df2e0a6..00000000000 --- a/cmake/Modules/Findcityhash.cmake +++ /dev/null @@ -1,44 +0,0 @@ -# - Try to find cityhash headers and libraries. -# -# Usage of this module as follows: -# -# find_package(cityhash) -# -# Variables used by this module, they can change the default behaviour and need -# to be set before calling find_package: -# -# CITYHASH_ROOT_DIR Set this variable to the root installation of -# cityhash if the module has problems finding -# the proper installation path. -# -# Variables defined by this module: -# -# CITYHASH_FOUND System has cityhash libs/headers -# CITYHASH_LIBRARIES The cityhash library/libraries -# CITYHASH_INCLUDE_DIR The location of cityhash headers - -find_path(CITYHASH_ROOT_DIR - NAMES include/city.h -) - -find_library(CITYHASH_LIBRARIES - NAMES cityhash - PATHS ${CITYHASH_ROOT_DIR}/lib ${CITYHASH_LIBRARIES_PATHS} -) - -find_path(CITYHASH_INCLUDE_DIR - NAMES city.h - PATHS ${CITYHASH_ROOT_DIR}/include ${CITYHASH_INCLUDE_PATHS} -) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(cityhash DEFAULT_MSG - CITYHASH_LIBRARIES - CITYHASH_INCLUDE_DIR -) - -mark_as_advanced( - CITYHASH_ROOT_DIR - CITYHASH_LIBRARIES - CITYHASH_INCLUDE_DIR -) diff --git a/cmake/print_include_directories.cmake b/cmake/print_include_directories.cmake index 7871d375fcc..bea02b2b2cc 100644 --- a/cmake/print_include_directories.cmake +++ b/cmake/print_include_directories.cmake @@ -10,7 +10,7 @@ list(APPEND dirs ${dirs1}) get_property (dirs1 TARGET common PROPERTY INCLUDE_DIRECTORIES) list(APPEND dirs ${dirs1}) -get_property (dirs1 TARGET cityhash PROPERTY INCLUDE_DIRECTORIES) +get_property (dirs1 TARGET ch_contrib::cityhash PROPERTY INCLUDE_DIRECTORIES) list(APPEND dirs ${dirs1}) get_property (dirs1 TARGET roaring PROPERTY INCLUDE_DIRECTORIES) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index fef8a567f98..a00b1ed7cbd 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -77,9 +77,7 @@ endif() # TODO: refactor the contrib libraries below this comment. -if (USE_INTERNAL_CITYHASH_LIBRARY) - add_subdirectory (cityhash102) -endif () +add_subdirectory (cityhash102) if (USE_INTERNAL_FARMHASH_LIBRARY) add_subdirectory (libfarmhash) diff --git a/contrib/cityhash102/CMakeLists.txt b/contrib/cityhash102/CMakeLists.txt index f40a6d2408b..744fa29f3b0 100644 --- a/contrib/cityhash102/CMakeLists.txt +++ b/contrib/cityhash102/CMakeLists.txt @@ -1,8 +1,10 @@ -add_library(cityhash +add_library(_cityhash src/city.cc include/citycrc.h include/city.h src/config.h) -target_include_directories(cityhash SYSTEM BEFORE PUBLIC include) -target_include_directories(cityhash SYSTEM PRIVATE src) +target_include_directories(_cityhash SYSTEM BEFORE PUBLIC include) +target_include_directories(_cityhash SYSTEM PRIVATE src) + +add_library(ch_contrib::cityhash ALIAS _cityhash) diff --git a/src/AggregateFunctions/CMakeLists.txt b/src/AggregateFunctions/CMakeLists.txt index 64f6eed9a6c..0cb38fc729a 100644 --- a/src/AggregateFunctions/CMakeLists.txt +++ b/src/AggregateFunctions/CMakeLists.txt @@ -23,7 +23,7 @@ list(REMOVE_ITEM clickhouse_aggregate_functions_headers ) add_library(clickhouse_aggregate_functions ${clickhouse_aggregate_functions_sources}) -target_link_libraries(clickhouse_aggregate_functions PRIVATE dbms PUBLIC ${CITYHASH_LIBRARIES}) +target_link_libraries(clickhouse_aggregate_functions PRIVATE dbms PUBLIC ch_contrib::cityhash) if(ENABLE_EXAMPLES) add_subdirectory(examples) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2112f15cda9..189af04dd4e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -325,7 +325,7 @@ target_link_libraries(clickhouse_common_io PUBLIC boost::program_options boost::system - ${CITYHASH_LIBRARIES} + ch_contrib::cityhash ch_contrib::zlib pcg_random Poco::Foundation diff --git a/src/Common/examples/CMakeLists.txt b/src/Common/examples/CMakeLists.txt index 5f6b8b67e49..d4057954969 100644 --- a/src/Common/examples/CMakeLists.txt +++ b/src/Common/examples/CMakeLists.txt @@ -1,5 +1,5 @@ add_executable (hashes_test hashes_test.cpp) -target_link_libraries (hashes_test PRIVATE clickhouse_common_io ${CITYHASH_LIBRARIES}) +target_link_libraries (hashes_test PRIVATE clickhouse_common_io ch_contrib::cityhash) if (TARGET OpenSSL::Crypto) target_link_libraries (hashes_test PRIVATE OpenSSL::Crypto) endif() diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 80749a7f189..f9f2e37cedb 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -13,7 +13,7 @@ add_library(clickhouse_functions ${clickhouse_functions_sources}) target_link_libraries(clickhouse_functions PUBLIC ${BASE64_LIBRARY} - ${CITYHASH_LIBRARIES} + ch_contrib::cityhash ${FARMHASH_LIBRARIES} ${FASTOPS_LIBRARY} clickhouse_dictionaries From 48b5c098dd24de26275ce27d819665db62e4bd26 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 22:16:12 +0300 Subject: [PATCH 154/403] Remove unbundled farmhash support --- CMakeLists.txt | 2 -- cmake/Modules/Findfarmhash.cmake | 44 ------------------------ contrib/CMakeLists.txt | 5 +-- contrib/libfarmhash/CMakeLists.txt | 12 ++++--- src/Functions/CMakeLists.txt | 2 +- src/Interpreters/examples/CMakeLists.txt | 4 +-- 6 files changed, 11 insertions(+), 58 deletions(-) delete mode 100644 cmake/Modules/Findfarmhash.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 1dcaba0ffe9..11d3e82b563 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -523,8 +523,6 @@ include (cmake/find/cassandra.cmake) include (cmake/find/sentry.cmake) include (cmake/find/datasketches.cmake) -find_contrib_lib(farmhash) - if (ENABLE_TESTS) include (cmake/find/gtest.cmake) endif () diff --git a/cmake/Modules/Findfarmhash.cmake b/cmake/Modules/Findfarmhash.cmake deleted file mode 100644 index 2b45fde2c67..00000000000 --- a/cmake/Modules/Findfarmhash.cmake +++ /dev/null @@ -1,44 +0,0 @@ -# - Try to find farmhash headers and libraries. -# -# Usage of this module as follows: -# -# find_package(farmhash) -# -# Variables used by this module, they can change the default behaviour and need -# to be set before calling find_package: -# -# FARMHASH_ROOT_DIR Set this variable to the root installation of -# farmhash if the module has problems finding -# the proper installation path. -# -# Variables defined by this module: -# -# FARMHASH_FOUND System has farmhash libs/headers -# FARMHASH_LIBRARIES The farmhash library/libraries -# FARMHASH_INCLUDE_DIR The location of farmhash headers - -find_path(FARMHASH_ROOT_DIR - NAMES include/farmhash.h -) - -find_library(FARMHASH_LIBRARIES - NAMES farmhash - PATHS ${FARMHASH_ROOT_DIR}/lib ${FARMHASH_LIBRARIES_PATHS} -) - -find_path(FARMHASH_INCLUDE_DIR - NAMES farmhash.h - PATHS ${FARMHASH_ROOT_DIR}/include ${FARMHASH_INCLUDE_PATHS} -) - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(farmhash DEFAULT_MSG - FARMHASH_LIBRARIES - FARMHASH_INCLUDE_DIR -) - -mark_as_advanced( - FARMHASH_ROOT_DIR - FARMHASH_LIBRARIES - FARMHASH_INCLUDE_DIR -) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index a00b1ed7cbd..d8fb16f30ee 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -78,10 +78,7 @@ endif() # TODO: refactor the contrib libraries below this comment. add_subdirectory (cityhash102) - -if (USE_INTERNAL_FARMHASH_LIBRARY) - add_subdirectory (libfarmhash) -endif () +add_subdirectory (libfarmhash) if (USE_INTERNAL_H3_LIBRARY) add_subdirectory(h3-cmake) diff --git a/contrib/libfarmhash/CMakeLists.txt b/contrib/libfarmhash/CMakeLists.txt index 20bba58cde7..a0533a93f17 100644 --- a/contrib/libfarmhash/CMakeLists.txt +++ b/contrib/libfarmhash/CMakeLists.txt @@ -1,9 +1,11 @@ -add_library(farmhash - farmhash.cc - farmhash.h) +add_library(_farmhash + farmhash.cc + farmhash.h) if (MSVC) - target_compile_definitions (farmhash PRIVATE FARMHASH_NO_BUILTIN_EXPECT=1) + target_compile_definitions (_farmhash PRIVATE FARMHASH_NO_BUILTIN_EXPECT=1) endif () -target_include_directories (farmhash PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +target_include_directories (_farmhash BEFORE PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) + +add_library(ch_contrib::farmhash ALIAS _farmhash) diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index f9f2e37cedb..73e153666af 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -14,7 +14,7 @@ target_link_libraries(clickhouse_functions PUBLIC ${BASE64_LIBRARY} ch_contrib::cityhash - ${FARMHASH_LIBRARIES} + ch_contrib::farmhash ${FASTOPS_LIBRARY} clickhouse_dictionaries clickhouse_dictionaries_embedded diff --git a/src/Interpreters/examples/CMakeLists.txt b/src/Interpreters/examples/CMakeLists.txt index 742a114f7c4..262161c62a8 100644 --- a/src/Interpreters/examples/CMakeLists.txt +++ b/src/Interpreters/examples/CMakeLists.txt @@ -5,7 +5,7 @@ add_executable (hash_map_lookup hash_map_lookup.cpp) target_link_libraries (hash_map_lookup PRIVATE dbms) add_executable (hash_map3 hash_map3.cpp) -target_link_libraries (hash_map3 PRIVATE dbms ${FARMHASH_LIBRARIES} metrohash) +target_link_libraries (hash_map3 PRIVATE dbms ch_contrib::farmhash metrohash) add_executable (hash_map_string hash_map_string.cpp) target_link_libraries (hash_map_string PRIVATE dbms) @@ -14,7 +14,7 @@ add_executable (hash_map_string_2 hash_map_string_2.cpp) target_link_libraries (hash_map_string_2 PRIVATE dbms) add_executable (hash_map_string_3 hash_map_string_3.cpp) -target_link_libraries (hash_map_string_3 PRIVATE dbms ${FARMHASH_LIBRARIES} metrohash) +target_link_libraries (hash_map_string_3 PRIVATE dbms ch_contrib::farmhash metrohash) add_executable (hash_map_string_small hash_map_string_small.cpp) target_link_libraries (hash_map_string_small PRIVATE dbms) From e243957763c9340ee06dc64079a99ffd985b8f47 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 22:17:43 +0300 Subject: [PATCH 155/403] Remove cmake/contrib_finder.cmake --- CMakeLists.txt | 1 - cmake/contrib_finder.cmake | 23 ----------------------- 2 files changed, 24 deletions(-) delete mode 100644 cmake/contrib_finder.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 11d3e82b563..644fc3e7a2e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -486,7 +486,6 @@ message (STATUS CCACHE=${CCACHE_FOUND} ${CCACHE_VERSION}") include (GNUInstallDirs) -include (cmake/contrib_finder.cmake) include (cmake/find/icu.cmake) include (cmake/find/ltdl.cmake) # for odbc diff --git a/cmake/contrib_finder.cmake b/cmake/contrib_finder.cmake deleted file mode 100644 index e97fda6a6f3..00000000000 --- a/cmake/contrib_finder.cmake +++ /dev/null @@ -1,23 +0,0 @@ -macro(find_contrib_lib LIB_NAME) - - string(TOLOWER ${LIB_NAME} LIB_NAME_LC) - string(TOUPPER ${LIB_NAME} LIB_NAME_UC) - string(REPLACE "-" "_" LIB_NAME_UC ${LIB_NAME_UC}) - - option (USE_INTERNAL_${LIB_NAME_UC}_LIBRARY "Use bundled library ${LIB_NAME} instead of system" ON) - - if (NOT USE_INTERNAL_${LIB_NAME_UC}_LIBRARY) - find_package ("${LIB_NAME}") - if (NOT ${LIB_NAME_UC}_FOUND) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use system ${LIB_NAME}") - endif() - endif () - - if (NOT ${LIB_NAME_UC}_FOUND) - set (USE_INTERNAL_${LIB_NAME_UC}_LIBRARY 1) - set (${LIB_NAME_UC}_LIBRARIES ${LIB_NAME_LC}) - set (${LIB_NAME_UC}_INCLUDE_DIR ${${LIB_NAME_UC}_CONTRIB_INCLUDE_DIR}) - endif () - - message (STATUS "Using ${LIB_NAME}: ${${LIB_NAME_UC}_INCLUDE_DIR} : ${${LIB_NAME_UC}_LIBRARIES}") -endmacro() From 4524a19391589d0a2477ffa12f479f23f25fb888 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 22:20:39 +0300 Subject: [PATCH 156/403] Remove unbundled ICU support --- CMakeLists.txt | 1 - cmake/Modules/FindICU.cmake | 394 ------------------------------- cmake/find/icu.cmake | 51 ---- contrib/CMakeLists.txt | 9 +- contrib/icu-cmake/CMakeLists.txt | 35 ++- src/CMakeLists.txt | 5 +- src/Functions/CMakeLists.txt | 5 +- src/configure_config.cmake | 3 + 8 files changed, 35 insertions(+), 468 deletions(-) delete mode 100644 cmake/Modules/FindICU.cmake delete mode 100644 cmake/find/icu.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 644fc3e7a2e..4c6dbdeb109 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -487,7 +487,6 @@ message (STATUS include (GNUInstallDirs) -include (cmake/find/icu.cmake) include (cmake/find/ltdl.cmake) # for odbc # openssl, zlib before poco include (cmake/find/krb5.cmake) diff --git a/cmake/Modules/FindICU.cmake b/cmake/Modules/FindICU.cmake deleted file mode 100644 index 0e61b3dcf29..00000000000 --- a/cmake/Modules/FindICU.cmake +++ /dev/null @@ -1,394 +0,0 @@ -# Distributed under the OSI-approved BSD 3-Clause License. See accompanying -# file Copyright.txt or https://cmake.org/licensing for details. - -#.rst: -# FindICU -# ------- -# -# Find the International Components for Unicode (ICU) libraries and -# programs. -# -# This module supports multiple components. -# Components can include any of: ``data``, ``i18n``, ``io``, ``le``, -# ``lx``, ``test``, ``tu`` and ``uc``. -# -# Note that on Windows ``data`` is named ``dt`` and ``i18n`` is named -# ``in``; any of the names may be used, and the appropriate -# platform-specific library name will be automatically selected. -# -# This module reports information about the ICU installation in -# several variables. General variables:: -# -# ICU_VERSION - ICU release version -# ICU_FOUND - true if the main programs and libraries were found -# ICU_LIBRARIES - component libraries to be linked -# ICU_INCLUDE_DIRS - the directories containing the ICU headers -# -# Imported targets:: -# -# ICU:: -# -# Where ```` is the name of an ICU component, for example -# ``ICU::i18n``. -# -# ICU programs are reported in:: -# -# ICU_GENCNVAL_EXECUTABLE - path to gencnval executable -# ICU_ICUINFO_EXECUTABLE - path to icuinfo executable -# ICU_GENBRK_EXECUTABLE - path to genbrk executable -# ICU_ICU-CONFIG_EXECUTABLE - path to icu-config executable -# ICU_GENRB_EXECUTABLE - path to genrb executable -# ICU_GENDICT_EXECUTABLE - path to gendict executable -# ICU_DERB_EXECUTABLE - path to derb executable -# ICU_PKGDATA_EXECUTABLE - path to pkgdata executable -# ICU_UCONV_EXECUTABLE - path to uconv executable -# ICU_GENCFU_EXECUTABLE - path to gencfu executable -# ICU_MAKECONV_EXECUTABLE - path to makeconv executable -# ICU_GENNORM2_EXECUTABLE - path to gennorm2 executable -# ICU_GENCCODE_EXECUTABLE - path to genccode executable -# ICU_GENSPREP_EXECUTABLE - path to gensprep executable -# ICU_ICUPKG_EXECUTABLE - path to icupkg executable -# ICU_GENCMN_EXECUTABLE - path to gencmn executable -# -# ICU component libraries are reported in:: -# -# ICU__FOUND - ON if component was found -# ICU__LIBRARIES - libraries for component -# -# ICU datafiles are reported in:: -# -# ICU_MAKEFILE_INC - Makefile.inc -# ICU_PKGDATA_INC - pkgdata.inc -# -# Note that ```` is the uppercased name of the component. -# -# This module reads hints about search results from:: -# -# ICU_ROOT - the root of the ICU installation -# -# The environment variable ``ICU_ROOT`` may also be used; the -# ICU_ROOT variable takes precedence. -# -# The following cache variables may also be set:: -# -# ICU_

_EXECUTABLE - the path to executable

-# ICU_INCLUDE_DIR - the directory containing the ICU headers -# ICU__LIBRARY - the library for component -# -# .. note:: -# -# In most cases none of the above variables will require setting, -# unless multiple ICU versions are available and a specific version -# is required. -# -# Other variables one may set to control this module are:: -# -# ICU_DEBUG - Set to ON to enable debug output from FindICU. - -# Written by Roger Leigh - -set(icu_programs - gencnval - icuinfo - genbrk - icu-config - genrb - gendict - derb - pkgdata - uconv - gencfu - makeconv - gennorm2 - genccode - gensprep - icupkg - gencmn) - -set(icu_data - Makefile.inc - pkgdata.inc) - -# The ICU checks are contained in a function due to the large number -# of temporary variables needed. -function(_ICU_FIND) - # Set up search paths, taking compiler into account. Search ICU_ROOT, - # with ICU_ROOT in the environment as a fallback if unset. - if(ICU_ROOT) - list(APPEND icu_roots "${ICU_ROOT}") - else() - if(NOT "$ENV{ICU_ROOT}" STREQUAL "") - file(TO_CMAKE_PATH "$ENV{ICU_ROOT}" NATIVE_PATH) - list(APPEND icu_roots "${NATIVE_PATH}") - set(ICU_ROOT "${NATIVE_PATH}" - CACHE PATH "Location of the ICU installation" FORCE) - endif() - endif() - - # Find include directory - list(APPEND icu_include_suffixes "include") - find_path(ICU_INCLUDE_DIR - NAMES "unicode/utypes.h" - HINTS ${icu_roots} - PATH_SUFFIXES ${icu_include_suffixes} - DOC "ICU include directory") - set(ICU_INCLUDE_DIR "${ICU_INCLUDE_DIR}" PARENT_SCOPE) - - # Get version - if(ICU_INCLUDE_DIR AND EXISTS "${ICU_INCLUDE_DIR}/unicode/uvernum.h") - file(STRINGS "${ICU_INCLUDE_DIR}/unicode/uvernum.h" icu_header_str - REGEX "^#define[\t ]+U_ICU_VERSION[\t ]+\".*\".*") - - string(REGEX REPLACE "^#define[\t ]+U_ICU_VERSION[\t ]+\"([^ \\n]*)\".*" - "\\1" icu_version_string "${icu_header_str}") - set(ICU_VERSION "${icu_version_string}") - set(ICU_VERSION "${icu_version_string}" PARENT_SCOPE) - unset(icu_header_str) - unset(icu_version_string) - endif() - - if(CMAKE_SIZEOF_VOID_P EQUAL 8) - # 64-bit binary directory - set(_bin64 "bin64") - # 64-bit library directory - set(_lib64 "lib64") - endif() - - - # Find all ICU programs - list(APPEND icu_binary_suffixes "${_bin64}" "bin") - foreach(program ${icu_programs}) - string(TOUPPER "${program}" program_upcase) - set(cache_var "ICU_${program_upcase}_EXECUTABLE") - set(program_var "ICU_${program_upcase}_EXECUTABLE") - find_program("${cache_var}" "${program}" - HINTS ${icu_roots} - PATH_SUFFIXES ${icu_binary_suffixes} - DOC "ICU ${program} executable") - mark_as_advanced(cache_var) - set("${program_var}" "${${cache_var}}" PARENT_SCOPE) - endforeach() - - # Find all ICU libraries - list(APPEND icu_library_suffixes "${_lib64}" "lib") - set(ICU_REQUIRED_LIBS_FOUND ON) - foreach(component ${ICU_FIND_COMPONENTS}) - string(TOUPPER "${component}" component_upcase) - set(component_cache "ICU_${component_upcase}_LIBRARY") - set(component_cache_release "${component_cache}_RELEASE") - set(component_cache_debug "${component_cache}_DEBUG") - set(component_found "${component_upcase}_FOUND") - set(component_libnames "icu${component}") - set(component_debug_libnames "icu${component}d") - - # Special case deliberate library naming mismatches between Unix - # and Windows builds - unset(component_libnames) - unset(component_debug_libnames) - list(APPEND component_libnames "icu${component}") - list(APPEND component_debug_libnames "icu${component}d") - if(component STREQUAL "data") - list(APPEND component_libnames "icudt") - # Note there is no debug variant at present - list(APPEND component_debug_libnames "icudtd") - endif() - if(component STREQUAL "dt") - list(APPEND component_libnames "icudata") - # Note there is no debug variant at present - list(APPEND component_debug_libnames "icudatad") - endif() - if(component STREQUAL "i18n") - list(APPEND component_libnames "icuin") - list(APPEND component_debug_libnames "icuind") - endif() - if(component STREQUAL "in") - list(APPEND component_libnames "icui18n") - list(APPEND component_debug_libnames "icui18nd") - endif() - - find_library("${component_cache_release}" ${component_libnames} - HINTS ${icu_roots} - PATH_SUFFIXES ${icu_library_suffixes} - DOC "ICU ${component} library (release)") - find_library("${component_cache_debug}" ${component_debug_libnames} - HINTS ${icu_roots} - PATH_SUFFIXES ${icu_library_suffixes} - DOC "ICU ${component} library (debug)") - include(SelectLibraryConfigurations) - select_library_configurations(ICU_${component_upcase}) - mark_as_advanced("${component_cache_release}" "${component_cache_debug}") - if(${component_cache}) - set("${component_found}" ON) - list(APPEND ICU_LIBRARY "${${component_cache}}") - endif() - mark_as_advanced("${component_found}") - set("${component_cache}" "${${component_cache}}" PARENT_SCOPE) - set("${component_found}" "${${component_found}}" PARENT_SCOPE) - if(${component_found}) - if (ICU_FIND_REQUIRED_${component}) - list(APPEND ICU_LIBS_FOUND "${component} (required)") - else() - list(APPEND ICU_LIBS_FOUND "${component} (optional)") - endif() - else() - if (ICU_FIND_REQUIRED_${component}) - set(ICU_REQUIRED_LIBS_FOUND OFF) - list(APPEND ICU_LIBS_NOTFOUND "${component} (required)") - else() - list(APPEND ICU_LIBS_NOTFOUND "${component} (optional)") - endif() - endif() - endforeach() - set(_ICU_REQUIRED_LIBS_FOUND "${ICU_REQUIRED_LIBS_FOUND}" PARENT_SCOPE) - set(ICU_LIBRARY "${ICU_LIBRARY}" PARENT_SCOPE) - - # Find all ICU data files - if(CMAKE_LIBRARY_ARCHITECTURE) - list(APPEND icu_data_suffixes - "${_lib64}/${CMAKE_LIBRARY_ARCHITECTURE}/icu/${ICU_VERSION}" - "lib/${CMAKE_LIBRARY_ARCHITECTURE}/icu/${ICU_VERSION}" - "${_lib64}/${CMAKE_LIBRARY_ARCHITECTURE}/icu" - "lib/${CMAKE_LIBRARY_ARCHITECTURE}/icu") - endif() - list(APPEND icu_data_suffixes - "${_lib64}/icu/${ICU_VERSION}" - "lib/icu/${ICU_VERSION}" - "${_lib64}/icu" - "lib/icu") - foreach(data ${icu_data}) - string(TOUPPER "${data}" data_upcase) - string(REPLACE "." "_" data_upcase "${data_upcase}") - set(cache_var "ICU_${data_upcase}") - set(data_var "ICU_${data_upcase}") - find_file("${cache_var}" "${data}" - HINTS ${icu_roots} - PATH_SUFFIXES ${icu_data_suffixes} - DOC "ICU ${data} data file") - mark_as_advanced(cache_var) - set("${data_var}" "${${cache_var}}" PARENT_SCOPE) - endforeach() - - if(NOT ICU_FIND_QUIETLY) - if(ICU_LIBS_FOUND) - message(STATUS "Found the following ICU libraries:") - foreach(found ${ICU_LIBS_FOUND}) - message(STATUS " ${found}") - endforeach() - endif() - if(ICU_LIBS_NOTFOUND) - message(STATUS "The following ICU libraries were not found:") - foreach(notfound ${ICU_LIBS_NOTFOUND}) - message(STATUS " ${notfound}") - endforeach() - endif() - endif() - - if(ICU_DEBUG) - message(STATUS "--------FindICU.cmake search debug--------") - message(STATUS "ICU binary path search order: ${icu_roots}") - message(STATUS "ICU include path search order: ${icu_roots}") - message(STATUS "ICU library path search order: ${icu_roots}") - message(STATUS "----------------") - endif() -endfunction() - -_ICU_FIND() - -include(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(ICU - FOUND_VAR ICU_FOUND - REQUIRED_VARS ICU_INCLUDE_DIR - ICU_LIBRARY - _ICU_REQUIRED_LIBS_FOUND - VERSION_VAR ICU_VERSION - FAIL_MESSAGE "Failed to find all ICU components") - -unset(_ICU_REQUIRED_LIBS_FOUND) - -if(ICU_FOUND) - set(ICU_INCLUDE_DIRS "${ICU_INCLUDE_DIR}") - set(ICU_LIBRARIES "${ICU_LIBRARY}") - foreach(_ICU_component ${ICU_FIND_COMPONENTS}) - string(TOUPPER "${_ICU_component}" _ICU_component_upcase) - set(_ICU_component_cache "ICU_${_ICU_component_upcase}_LIBRARY") - set(_ICU_component_cache_release "ICU_${_ICU_component_upcase}_LIBRARY_RELEASE") - set(_ICU_component_cache_debug "ICU_${_ICU_component_upcase}_LIBRARY_DEBUG") - set(_ICU_component_lib "ICU_${_ICU_component_upcase}_LIBRARIES") - set(_ICU_component_found "${_ICU_component_upcase}_FOUND") - set(_ICU_imported_target "ICU::${_ICU_component}") - if(${_ICU_component_found}) - set("${_ICU_component_lib}" "${${_ICU_component_cache}}") - if(NOT TARGET ${_ICU_imported_target}) - add_library(${_ICU_imported_target} UNKNOWN IMPORTED) - if(ICU_INCLUDE_DIR) - set_target_properties(${_ICU_imported_target} PROPERTIES - INTERFACE_INCLUDE_DIRECTORIES "${ICU_INCLUDE_DIR}") - endif() - if(EXISTS "${${_ICU_component_cache}}") - set_target_properties(${_ICU_imported_target} PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES "CXX" - IMPORTED_LOCATION "${${_ICU_component_cache}}") - endif() - if(EXISTS "${${_ICU_component_cache_release}}") - set_property(TARGET ${_ICU_imported_target} APPEND PROPERTY - IMPORTED_CONFIGURATIONS RELEASE) - set_target_properties(${_ICU_imported_target} PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "CXX" - IMPORTED_LOCATION_RELEASE "${${_ICU_component_cache_release}}") - endif() - if(EXISTS "${${_ICU_component_cache_debug}}") - set_property(TARGET ${_ICU_imported_target} APPEND PROPERTY - IMPORTED_CONFIGURATIONS DEBUG) - set_target_properties(${_ICU_imported_target} PROPERTIES - IMPORTED_LINK_INTERFACE_LANGUAGES_DEBUG "CXX" - IMPORTED_LOCATION_DEBUG "${${_ICU_component_cache_debug}}") - endif() - endif() - endif() - unset(_ICU_component_upcase) - unset(_ICU_component_cache) - unset(_ICU_component_lib) - unset(_ICU_component_found) - unset(_ICU_imported_target) - endforeach() -endif() - -if(ICU_DEBUG) - message(STATUS "--------FindICU.cmake results debug--------") - message(STATUS "ICU found: ${ICU_FOUND}") - message(STATUS "ICU_VERSION number: ${ICU_VERSION}") - message(STATUS "ICU_ROOT directory: ${ICU_ROOT}") - message(STATUS "ICU_INCLUDE_DIR directory: ${ICU_INCLUDE_DIR}") - message(STATUS "ICU_LIBRARIES: ${ICU_LIBRARIES}") - - foreach(program IN LISTS icu_programs) - string(TOUPPER "${program}" program_upcase) - set(program_lib "ICU_${program_upcase}_EXECUTABLE") - message(STATUS "${program} program: ${${program_lib}}") - unset(program_upcase) - unset(program_lib) - endforeach() - - foreach(data IN LISTS icu_data) - string(TOUPPER "${data}" data_upcase) - string(REPLACE "." "_" data_upcase "${data_upcase}") - set(data_lib "ICU_${data_upcase}") - message(STATUS "${data} data: ${${data_lib}}") - unset(data_upcase) - unset(data_lib) - endforeach() - - foreach(component IN LISTS ICU_FIND_COMPONENTS) - string(TOUPPER "${component}" component_upcase) - set(component_lib "ICU_${component_upcase}_LIBRARIES") - set(component_found "${component_upcase}_FOUND") - message(STATUS "${component} library found: ${${component_found}}") - message(STATUS "${component} library: ${${component_lib}}") - unset(component_upcase) - unset(component_lib) - unset(component_found) - endforeach() - message(STATUS "----------------") -endif() - -unset(icu_programs) diff --git a/cmake/find/icu.cmake b/cmake/find/icu.cmake deleted file mode 100644 index 5ba25e93875..00000000000 --- a/cmake/find/icu.cmake +++ /dev/null @@ -1,51 +0,0 @@ -if (OS_LINUX) - option(ENABLE_ICU "Enable ICU" ${ENABLE_LIBRARIES}) -else () - option(ENABLE_ICU "Enable ICU" 0) -endif () - -if (NOT ENABLE_ICU) - if(USE_INTERNAL_ICU_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal icu library with ENABLE_ICU=OFF") - endif() - message(STATUS "Build without ICU (support for collations and charset conversion functions will be disabled)") - return() -endif() - -option (USE_INTERNAL_ICU_LIBRARY "Set to FALSE to use system ICU library instead of bundled" ON) - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/icu/icu4c/LICENSE") - if (USE_INTERNAL_ICU_LIBRARY) - message (WARNING "submodule contrib/icu is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal ICU") - set (USE_INTERNAL_ICU_LIBRARY 0) - endif () - set (MISSING_INTERNAL_ICU_LIBRARY 1) -endif () - -if(NOT USE_INTERNAL_ICU_LIBRARY) - if (APPLE) - set(ICU_ROOT "/usr/local/opt/icu4c" CACHE STRING "") - endif() - find_package(ICU COMPONENTS i18n uc data) # TODO: remove Modules/FindICU.cmake after cmake 3.7 - #set (ICU_LIBRARIES ${ICU_I18N_LIBRARY} ${ICU_UC_LIBRARY} ${ICU_DATA_LIBRARY} CACHE STRING "") - if(ICU_FOUND) - set(USE_ICU 1) - else() - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system ICU") - endif() -endif() - -if (ICU_LIBRARY AND ICU_INCLUDE_DIR) - set (USE_ICU 1) -elseif (NOT MISSING_INTERNAL_ICU_LIBRARY) - set (USE_INTERNAL_ICU_LIBRARY 1) - set (ICU_LIBRARIES icui18n icuuc icudata) - set (USE_ICU 1) -endif () - -if(USE_ICU) - message(STATUS "Using icu=${USE_ICU}: ${ICU_INCLUDE_DIR} : ${ICU_LIBRARIES}") -else() - message(STATUS "Build without ICU (support for collations and charset conversion functions will be disabled)") -endif() diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index d8fb16f30ee..d4b5ea38bcd 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -75,10 +75,11 @@ if (ENABLE_FUZZING) add_subdirectory (libprotobuf-mutator-cmake) endif() -# TODO: refactor the contrib libraries below this comment. - add_subdirectory (cityhash102) add_subdirectory (libfarmhash) +add_subdirectory (icu-cmake) + +# TODO: refactor the contrib libraries below this comment. if (USE_INTERNAL_H3_LIBRARY) add_subdirectory(h3-cmake) @@ -88,10 +89,6 @@ if (USE_INTERNAL_MYSQL_LIBRARY) add_subdirectory (mariadb-connector-c-cmake) endif () -if (ENABLE_ICU AND USE_INTERNAL_ICU_LIBRARY) - add_subdirectory (icu-cmake) -endif () - if(USE_INTERNAL_GTEST_LIBRARY) add_subdirectory(googletest-cmake) elseif(GTEST_SRC_DIR) diff --git a/contrib/icu-cmake/CMakeLists.txt b/contrib/icu-cmake/CMakeLists.txt index 26f3bb11006..ae19ef20e38 100644 --- a/contrib/icu-cmake/CMakeLists.txt +++ b/contrib/icu-cmake/CMakeLists.txt @@ -1,3 +1,14 @@ +if (OS_LINUX) + option(ENABLE_ICU "Enable ICU" ${ENABLE_LIBRARIES}) +else () + option(ENABLE_ICU "Enable ICU" 0) +endif () + +if (NOT ENABLE_ICU) + message(STATUS "Not using icu") + return() +endif() + set(ICU_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/icu/icu4c/source") set(ICUDATA_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/icudata/") @@ -447,19 +458,23 @@ set(ICUDATA_SOURCES # Note that we don't like any kind of binary plugins (because of runtime dependencies, vulnerabilities, ABI incompatibilities). add_definitions(-D_REENTRANT -DU_HAVE_ELF_H=1 -DU_HAVE_STRTOD_L=1 -DU_HAVE_XLOCALE_H=0 -DDEFAULT_ICU_PLUGINS="/dev/null") -add_library(icuuc ${ICUUC_SOURCES}) -add_library(icui18n ${ICUI18N_SOURCES}) -add_library(icudata ${ICUDATA_SOURCES}) +add_library(_icuuc ${ICUUC_SOURCES}) +add_library(_icui18n ${ICUI18N_SOURCES}) +add_library(_icudata ${ICUDATA_SOURCES}) -target_link_libraries(icuuc PRIVATE icudata) -target_link_libraries(icui18n PRIVATE icuuc) +target_link_libraries(_icuuc PRIVATE _icudata) +target_link_libraries(_icui18n PRIVATE _icuuc) -target_include_directories(icuuc SYSTEM PUBLIC "${ICU_SOURCE_DIR}/common/") -target_include_directories(icui18n SYSTEM PUBLIC "${ICU_SOURCE_DIR}/i18n/") +target_include_directories(_icuuc SYSTEM PUBLIC "${ICU_SOURCE_DIR}/common/") +target_include_directories(_icui18n SYSTEM PUBLIC "${ICU_SOURCE_DIR}/i18n/") -target_compile_definitions(icuuc PRIVATE -DU_COMMON_IMPLEMENTATION) -target_compile_definitions(icui18n PRIVATE -DU_I18N_IMPLEMENTATION) +target_compile_definitions(_icuuc PRIVATE -DU_COMMON_IMPLEMENTATION) +target_compile_definitions(_icui18n PRIVATE -DU_I18N_IMPLEMENTATION) if (COMPILER_CLANG) - target_compile_options(icudata PRIVATE -Wno-unused-command-line-argument) + target_compile_options(_icudata PRIVATE -Wno-unused-command-line-argument) endif () + +add_library(_icu INTERFACE) +target_link_libraries(_icu INTERFACE _icui18n _icuuc _icudata) +add_library(ch_contrib::icu ALIAS _icu) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 189af04dd4e..166dd82ff0f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -390,9 +390,8 @@ if (TARGET ch_contrib::xz) target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::xz) endif() -if (USE_ICU) - dbms_target_link_libraries (PRIVATE ${ICU_LIBRARIES}) - dbms_target_include_directories (SYSTEM PRIVATE ${ICU_INCLUDE_DIRS}) +if (TARGET ch_contrib::icu) + dbms_target_link_libraries (PRIVATE ch_contrib::icu) endif () if (USE_CAPNP) diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 73e153666af..0fc43b65634 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -55,9 +55,8 @@ else() message(STATUS "Generating debugger info for ClickHouse functions") endif() -if (USE_ICU) - target_link_libraries (clickhouse_functions PRIVATE ${ICU_LIBRARIES}) - target_include_directories(clickhouse_functions SYSTEM PRIVATE ${ICU_INCLUDE_DIRS}) +if (TARGET ch_contrib::icu) + target_link_libraries (clickhouse_functions PRIVATE ch_contrib::icu) endif () if (USE_FASTOPS) diff --git a/src/configure_config.cmake b/src/configure_config.cmake index babf8f0d9ef..d18a77d525b 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -31,3 +31,6 @@ endif() if (TARGET ch_contrib::nuraft) set(USE_NURAFT 1) endif() +if (TARGET ch_contrib::icu) + set(USE_ICU 1) +endif() From db468b6fae44698834288793700750a86e0649f1 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 22:29:50 +0300 Subject: [PATCH 157/403] Remove unbundled simdjson support --- CMakeLists.txt | 1 - cmake/find/simdjson.cmake | 11 ----------- contrib/CMakeLists.txt | 4 +--- contrib/simdjson-cmake/CMakeLists.txt | 15 ++++++++++++--- src/CMakeLists.txt | 4 ++-- src/Functions/CMakeLists.txt | 4 ++-- src/configure_config.cmake | 3 +++ 7 files changed, 20 insertions(+), 22 deletions(-) delete mode 100644 cmake/find/simdjson.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 4c6dbdeb109..270c1a95217 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -505,7 +505,6 @@ include (cmake/find/curl.cmake) include (cmake/find/s3.cmake) include (cmake/find/blob_storage.cmake) include (cmake/find/base64.cmake) -include (cmake/find/simdjson.cmake) include (cmake/find/fast_float.cmake) include (cmake/find/rapidjson.cmake) include (cmake/find/fastops.cmake) diff --git a/cmake/find/simdjson.cmake b/cmake/find/simdjson.cmake deleted file mode 100644 index bf22a331f04..00000000000 --- a/cmake/find/simdjson.cmake +++ /dev/null @@ -1,11 +0,0 @@ -option (USE_SIMDJSON "Use simdjson" ${ENABLE_LIBRARIES}) - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/simdjson/include/simdjson.h") - message (WARNING "submodule contrib/simdjson is missing. to fix try run: \n git submodule update --init") - if (USE_SIMDJSON) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal simdjson library") - endif() - return() -endif () - -message(STATUS "Using simdjson=${USE_SIMDJSON}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index d4b5ea38bcd..1d88219accc 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -141,9 +141,7 @@ if (USE_BASE64) add_subdirectory (base64-cmake) endif() -if (USE_SIMDJSON) - add_subdirectory (simdjson-cmake) -endif() +add_subdirectory (simdjson-cmake) if (USE_FASTOPS) add_subdirectory (fastops-cmake) diff --git a/contrib/simdjson-cmake/CMakeLists.txt b/contrib/simdjson-cmake/CMakeLists.txt index bb9a5844def..ab2840f5b7f 100644 --- a/contrib/simdjson-cmake/CMakeLists.txt +++ b/contrib/simdjson-cmake/CMakeLists.txt @@ -1,11 +1,20 @@ +option (ENABLE_SIMDJSON "Use simdjson" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_SIMDJSON) + message(STATUS "Not using simdjson") + return() +endif() + set(SIMDJSON_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/simdjson/include") set(SIMDJSON_SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/simdjson/src") set(SIMDJSON_SRC "${SIMDJSON_SRC_DIR}/simdjson.cpp") -add_library(simdjson ${SIMDJSON_SRC}) -target_include_directories(simdjson SYSTEM PUBLIC "${SIMDJSON_INCLUDE_DIR}" PRIVATE "${SIMDJSON_SRC_DIR}") +add_library(_simdjson ${SIMDJSON_SRC}) +target_include_directories(_simdjson SYSTEM PUBLIC "${SIMDJSON_INCLUDE_DIR}" PRIVATE "${SIMDJSON_SRC_DIR}") # simdjson is using its own CPU dispatching and get confused if we enable AVX/AVX2 flags. if(ARCH_AMD64) - target_compile_options(simdjson PRIVATE -mno-avx -mno-avx2) + target_compile_options(_simdjson PRIVATE -mno-avx -mno-avx2) endif() + +add_library(ch_contrib::simdjson ALIAS _simdjson) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 166dd82ff0f..bbcb7c6c9e9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -504,8 +504,8 @@ if (TARGET ch_contrib::bzip2) target_link_libraries (clickhouse_common_io PRIVATE ch_contrib::bzip2) endif() -if(USE_SIMDJSON) - dbms_target_link_libraries(PRIVATE simdjson) +if (TARGET ch_contrib::simdjson) + dbms_target_link_libraries(PRIVATE ch_contrib::simdjson) endif() if(USE_RAPIDJSON) diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 0fc43b65634..659c1215a9c 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -85,8 +85,8 @@ endif() target_link_libraries(clickhouse_functions PRIVATE hyperscan) -if(USE_SIMDJSON) - target_link_libraries(clickhouse_functions PRIVATE simdjson) +if (TARGET ch_contrib::simdjson) + target_link_libraries(clickhouse_functions PRIVATE ch_contrib::simdjson) endif() if(USE_RAPIDJSON) diff --git a/src/configure_config.cmake b/src/configure_config.cmake index d18a77d525b..56144a3ecf0 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -34,3 +34,6 @@ endif() if (TARGET ch_contrib::icu) set(USE_ICU 1) endif() +if (TARGET ch_contrib::simdjson) + set(USE_SIMDJSON 1) +endif() From 133e00a43d50465afd4420ce18cbccb9bf21cdf5 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 22:37:33 +0300 Subject: [PATCH 158/403] Remove unbundled rapidjson support --- CMakeLists.txt | 1 - cmake/find/rapidjson.cmake | 35 -------------------------- contrib/rapidjson-cmake/CMakeLists.txt | 11 ++++++++ src/CMakeLists.txt | 4 +-- src/Functions/CMakeLists.txt | 4 +-- src/configure_config.cmake | 3 +++ 6 files changed, 18 insertions(+), 40 deletions(-) delete mode 100644 cmake/find/rapidjson.cmake create mode 100644 contrib/rapidjson-cmake/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 270c1a95217..f054693fb19 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -506,7 +506,6 @@ include (cmake/find/s3.cmake) include (cmake/find/blob_storage.cmake) include (cmake/find/base64.cmake) include (cmake/find/fast_float.cmake) -include (cmake/find/rapidjson.cmake) include (cmake/find/fastops.cmake) include (cmake/find/odbc.cmake) include (cmake/find/nanodbc.cmake) diff --git a/cmake/find/rapidjson.cmake b/cmake/find/rapidjson.cmake deleted file mode 100644 index cdf6761446e..00000000000 --- a/cmake/find/rapidjson.cmake +++ /dev/null @@ -1,35 +0,0 @@ -option(ENABLE_RAPIDJSON "Use rapidjson" ${ENABLE_LIBRARIES}) -if(NOT ENABLE_RAPIDJSON) - if(USE_INTERNAL_RAPIDJSON_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal rapidjson library with ENABLE_RAPIDJSON=OFF") - endif() - return() -endif() - -option(USE_INTERNAL_RAPIDJSON_LIBRARY "Set to FALSE to use system rapidjson library instead of bundled" ON) - -if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/rapidjson/include/rapidjson/rapidjson.h") - if(USE_INTERNAL_RAPIDJSON_LIBRARY) - message(WARNING "submodule contrib/rapidjson is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal rapidjson library") - set(USE_INTERNAL_RAPIDJSON_LIBRARY 0) - endif() - set(MISSING_INTERNAL_RAPIDJSON_LIBRARY 1) -endif() - -if(NOT USE_INTERNAL_RAPIDJSON_LIBRARY) - find_path(RAPIDJSON_INCLUDE_DIR NAMES rapidjson/rapidjson.h PATHS ${RAPIDJSON_INCLUDE_PATHS}) - if(NOT RAPIDJSON_INCLUDE_DIR) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system rapidjson") - endif() -endif() - -if(RAPIDJSON_INCLUDE_DIR) - set(USE_RAPIDJSON 1) -elseif(NOT MISSING_INTERNAL_RAPIDJSON_LIBRARY) - set(RAPIDJSON_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/rapidjson/include") - set(USE_INTERNAL_RAPIDJSON_LIBRARY 1) - set(USE_RAPIDJSON 1) -endif() - -message(STATUS "Using rapidjson=${USE_RAPIDJSON}: ${RAPIDJSON_INCLUDE_DIR}") diff --git a/contrib/rapidjson-cmake/CMakeLists.txt b/contrib/rapidjson-cmake/CMakeLists.txt new file mode 100644 index 00000000000..0d7ba74a399 --- /dev/null +++ b/contrib/rapidjson-cmake/CMakeLists.txt @@ -0,0 +1,11 @@ +option(ENABLE_RAPIDJSON "Use rapidjson" ${ENABLE_LIBRARIES}) + +if(NOT ENABLE_RAPIDJSON) + message(STATUS "Not using rapidjson") + return() +endif() + +set(RAPIDJSON_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/rapidjson/include") +add_library(_rapidjson INTERFACE) +target_include_directories(_rapidjson SYSTEM BEFORE INTERFACE ${RAPIDJSON_INCLUDE_DIR}) +add_library(ch_contrib::rapidjson ALIAS _rapidjson) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index bbcb7c6c9e9..bd8e1f70f10 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -508,8 +508,8 @@ if (TARGET ch_contrib::simdjson) dbms_target_link_libraries(PRIVATE ch_contrib::simdjson) endif() -if(USE_RAPIDJSON) - dbms_target_include_directories(SYSTEM PRIVATE ${RAPIDJSON_INCLUDE_DIR}) +if (TARGET ch_contrib::rapidjson) + dbms_target_link_directories(PRIVATE ch_contrib::rapidjson) endif() dbms_target_link_libraries(PUBLIC consistent-hashing) diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 659c1215a9c..74d76eb7cbc 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -89,8 +89,8 @@ if (TARGET ch_contrib::simdjson) target_link_libraries(clickhouse_functions PRIVATE ch_contrib::simdjson) endif() -if(USE_RAPIDJSON) - target_include_directories(clickhouse_functions SYSTEM PRIVATE ${RAPIDJSON_INCLUDE_DIR}) +if (TARGET ch_contrib::rapidjson) + target_link_libraries(clickhouse_functions PRIVATE ch_contrib::rapidjson) endif() # ClickHouse developers may use platform-dependent code under some macro (e.g. `#ifdef ENABLE_MULTITARGET`). diff --git a/src/configure_config.cmake b/src/configure_config.cmake index 56144a3ecf0..32c29e5607f 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -37,3 +37,6 @@ endif() if (TARGET ch_contrib::simdjson) set(USE_SIMDJSON 1) endif() +if (TARGET ch_contrib::rapidjson) + set(USE_RAPIDJSON 1) +endif() From dae6cd35c7b1fc59470e1e4983f873f5be5e0989 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 22:42:00 +0300 Subject: [PATCH 159/403] Remove unbundled azure blob storage support --- CMakeLists.txt | 1 - cmake/find/blob_storage.cmake | 29 ----------------------------- contrib/CMakeLists.txt | 6 ++---- contrib/azure-cmake/CMakeLists.txt | 23 ++++++++++++++++------- src/CMakeLists.txt | 5 ++--- src/configure_config.cmake | 3 +++ 6 files changed, 23 insertions(+), 44 deletions(-) delete mode 100644 cmake/find/blob_storage.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index f054693fb19..1b5e428acac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -503,7 +503,6 @@ include (cmake/find/miniselect.cmake) include (cmake/find/poco.cmake) include (cmake/find/curl.cmake) include (cmake/find/s3.cmake) -include (cmake/find/blob_storage.cmake) include (cmake/find/base64.cmake) include (cmake/find/fast_float.cmake) include (cmake/find/fastops.cmake) diff --git a/cmake/find/blob_storage.cmake b/cmake/find/blob_storage.cmake deleted file mode 100644 index 4ad7296e95e..00000000000 --- a/cmake/find/blob_storage.cmake +++ /dev/null @@ -1,29 +0,0 @@ -option (ENABLE_AZURE_BLOB_STORAGE "Enable Azure blob storage" ${ENABLE_LIBRARIES}) - -if (ENABLE_AZURE_BLOB_STORAGE) - option(USE_INTERNAL_AZURE_BLOB_STORAGE_LIBRARY - "Set to FALSE to use system Azure SDK instead of bundled (OFF currently not implemented)" - ON) - - set(USE_AZURE_BLOB_STORAGE 1) - set(AZURE_BLOB_STORAGE_LIBRARY azure_sdk) - - if ((NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/azure/sdk" - OR NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/azure/cmake-modules") - AND USE_INTERNAL_AZURE_BLOB_STORAGE_LIBRARY) - message (WARNING "submodule contrib/azure is missing. to fix try run: \n git submodule update --init") - set(USE_INTERNAL_AZURE_BLOB_STORAGE_LIBRARY OFF) - set(USE_AZURE_BLOB_STORAGE 0) - endif () - - if (NOT USE_INTERNAL_SSL_LIBRARY AND USE_INTERNAL_AZURE_BLOB_STORAGE_LIBRARY) - message (FATAL_ERROR "Currently Blob Storage support can be built only with internal SSL library") - endif() - - if (NOT USE_INTERNAL_CURL AND USE_INTERNAL_AZURE_BLOB_STORAGE_LIBRARY) - message (FATAL_ERROR "Currently Blob Storage support can be built only with internal curl library") - endif() - -endif() - -message (STATUS "Using Azure Blob Storage - ${USE_AZURE_BLOB_STORAGE}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 1d88219accc..121d9309dfc 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -1,4 +1,4 @@ -# Third-party libraries may have substandard code. +#"${folder}/CMakeLists.txt" Third-party libraries may have substandard code. set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w") @@ -161,9 +161,7 @@ endif() # - sentry-native add_subdirectory (curl-cmake) -if (USE_INTERNAL_AZURE_BLOB_STORAGE_LIBRARY) - add_subdirectory(azure-cmake) -endif() +add_subdirectory (azure-cmake) if (USE_SENTRY) add_subdirectory (sentry-native-cmake) diff --git a/contrib/azure-cmake/CMakeLists.txt b/contrib/azure-cmake/CMakeLists.txt index 9310b57dc75..a9edad30b50 100644 --- a/contrib/azure-cmake/CMakeLists.txt +++ b/contrib/azure-cmake/CMakeLists.txt @@ -1,3 +1,10 @@ +option (ENABLE_AZURE_BLOB_STORAGE "Enable Azure blob storage" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_AZURE_BLOB_STORAGE) + message(STATUS "Not using Azure blob storage") + return() +endif() + set(AZURE_DIR "${ClickHouse_SOURCE_DIR}/contrib/azure") set(AZURE_SDK_LIBRARY_DIR "${AZURE_DIR}/sdk") @@ -43,10 +50,10 @@ set(AZURE_SDK_INCLUDES include("${AZURE_DIR}/cmake-modules/AzureTransportAdapters.cmake") -add_library(azure_sdk ${AZURE_SDK_UNIFIED_SRC}) +add_library(_azure_sdk ${AZURE_SDK_UNIFIED_SRC}) if (COMPILER_CLANG) - target_compile_options(azure_sdk PRIVATE + target_compile_options(_azure_sdk PRIVATE -Wno-deprecated-copy-dtor -Wno-extra-semi -Wno-suggest-destructor-override @@ -55,20 +62,22 @@ if (COMPILER_CLANG) ) if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 13) - target_compile_options(azure_sdk PRIVATE -Wno-reserved-identifier) + target_compile_options(_azure_sdk PRIVATE -Wno-reserved-identifier) endif() endif() # Originally, on Windows azure-core is built with bcrypt and crypt32 by default if (TARGET OpenSSL::SSL) - target_link_libraries(azure_sdk PRIVATE OpenSSL::Crypto OpenSSL::SSL) + target_link_libraries(_azure_sdk PRIVATE OpenSSL::Crypto OpenSSL::SSL) endif() # Originally, on Windows azure-core is built with winhttp by default if (CURL_FOUND) - target_link_libraries(azure_sdk PRIVATE ${CURL_LIBRARY}) + target_link_libraries(_azure_sdk PRIVATE ${CURL_LIBRARY}) endif() -target_link_libraries(azure_sdk PRIVATE ${LIBXML2_LIBRARIES}) +target_link_libraries(_azure_sdk PRIVATE ${LIBXML2_LIBRARIES}) -target_include_directories(azure_sdk SYSTEM PUBLIC ${AZURE_SDK_INCLUDES}) +target_include_directories(_azure_sdk SYSTEM BEFORE PUBLIC ${AZURE_SDK_INCLUDES}) + +add_library(ch_contrib::azure_sdk ALIAS _azure_sdk) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index bd8e1f70f10..9c085a01c47 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -439,9 +439,8 @@ if (USE_AWS_S3) target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${AWS_S3_INCLUDE_DIR}) endif() -if (USE_AZURE_BLOB_STORAGE) - target_link_libraries (clickhouse_common_io PUBLIC ${AZURE_BLOB_STORAGE_LIBRARY}) - target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${AZURE_SDK_INCLUDES}) +if (TARGET ch_contrib::azure_sdk) + target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::azure_sdk) endif() if (TARGET ch_contrib::s2) diff --git a/src/configure_config.cmake b/src/configure_config.cmake index 32c29e5607f..cc26bb37c21 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -40,3 +40,6 @@ endif() if (TARGET ch_contrib::rapidjson) set(USE_RAPIDJSON 1) endif() +if (TARGET ch_contrib::azure_sdk) + set(USE_AZURE_BLOB_STORAGE 1) +endif() From 4e5f93652d003a59ea01a8f8b6f35f57124354df Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 23:05:15 +0300 Subject: [PATCH 160/403] Remove unbundled poco support --- CMakeLists.txt | 2 - cmake/find/poco.cmake | 8 - cmake/find/s3.cmake | 4 - contrib/poco-cmake/Crypto/CMakeLists.txt | 69 ++- contrib/poco-cmake/Data/CMakeLists.txt | 100 ++--- contrib/poco-cmake/Data/ODBC/CMakeLists.txt | 53 +-- contrib/poco-cmake/Foundation/CMakeLists.txt | 443 +++++++++---------- contrib/poco-cmake/JSON/CMakeLists.txt | 61 ++- contrib/poco-cmake/MongoDB/CMakeLists.txt | 65 ++- contrib/poco-cmake/Net/CMakeLists.txt | 251 +++++------ contrib/poco-cmake/Net/SSL/CMakeLists.txt | 77 ++-- contrib/poco-cmake/Redis/CMakeLists.txt | 49 +- contrib/poco-cmake/Util/CMakeLists.txt | 77 ++-- contrib/poco-cmake/XML/CMakeLists.txt | 193 ++++---- 14 files changed, 656 insertions(+), 796 deletions(-) delete mode 100644 cmake/find/poco.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b5e428acac..f0959caf4cd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -488,7 +488,6 @@ message (STATUS include (GNUInstallDirs) include (cmake/find/ltdl.cmake) # for odbc -# openssl, zlib before poco include (cmake/find/krb5.cmake) include (cmake/find/libgsasl.cmake) include (cmake/find/cyrus-sasl.cmake) @@ -500,7 +499,6 @@ include (cmake/find/h3.cmake) include (cmake/find/libxml2.cmake) include (cmake/find/pdqsort.cmake) include (cmake/find/miniselect.cmake) -include (cmake/find/poco.cmake) include (cmake/find/curl.cmake) include (cmake/find/s3.cmake) include (cmake/find/base64.cmake) diff --git a/cmake/find/poco.cmake b/cmake/find/poco.cmake deleted file mode 100644 index 99233728152..00000000000 --- a/cmake/find/poco.cmake +++ /dev/null @@ -1,8 +0,0 @@ -option (USE_INTERNAL_POCO_LIBRARY "Use internal Poco library" ON) - -if (NOT USE_INTERNAL_POCO_LIBRARY) - find_path (ROOT_DIR NAMES Foundation/include/Poco/Poco.h include/Poco/Poco.h) - if (NOT ROOT_DIR) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system poco") - endif() -endif () diff --git a/cmake/find/s3.cmake b/cmake/find/s3.cmake index 9a10c3f13ef..06029ace63a 100644 --- a/cmake/find/s3.cmake +++ b/cmake/find/s3.cmake @@ -14,10 +14,6 @@ endif() option(USE_INTERNAL_AWS_S3_LIBRARY "Set to FALSE to use system S3 instead of bundled (experimental set to OFF on your own risk)" ON) -if (NOT USE_INTERNAL_POCO_LIBRARY AND USE_INTERNAL_AWS_S3_LIBRARY) - message (FATAL_ERROR "Currently S3 support can be built only with internal POCO library") -endif() - if (NOT USE_INTERNAL_AWS_S3_LIBRARY) message (${RECONFIGURE_MESSAGE_LEVEL} "Compilation with external S3 library is not supported yet") endif() diff --git a/contrib/poco-cmake/Crypto/CMakeLists.txt b/contrib/poco-cmake/Crypto/CMakeLists.txt index e93ed5cf17d..3c410095d6e 100644 --- a/contrib/poco-cmake/Crypto/CMakeLists.txt +++ b/contrib/poco-cmake/Crypto/CMakeLists.txt @@ -1,46 +1,35 @@ if (ENABLE_SSL) - if (USE_INTERNAL_POCO_LIBRARY) - set (SRCS - "${LIBRARY_DIR}/Crypto/src/Cipher.cpp" - "${LIBRARY_DIR}/Crypto/src/CipherFactory.cpp" - "${LIBRARY_DIR}/Crypto/src/CipherImpl.cpp" - "${LIBRARY_DIR}/Crypto/src/CipherKey.cpp" - "${LIBRARY_DIR}/Crypto/src/CipherKeyImpl.cpp" - "${LIBRARY_DIR}/Crypto/src/CryptoException.cpp" - "${LIBRARY_DIR}/Crypto/src/CryptoStream.cpp" - "${LIBRARY_DIR}/Crypto/src/CryptoTransform.cpp" - "${LIBRARY_DIR}/Crypto/src/DigestEngine.cpp" - "${LIBRARY_DIR}/Crypto/src/ECDSADigestEngine.cpp" - "${LIBRARY_DIR}/Crypto/src/ECKey.cpp" - "${LIBRARY_DIR}/Crypto/src/ECKeyImpl.cpp" - "${LIBRARY_DIR}/Crypto/src/EVPPKey.cpp" - "${LIBRARY_DIR}/Crypto/src/KeyPair.cpp" - "${LIBRARY_DIR}/Crypto/src/KeyPairImpl.cpp" - "${LIBRARY_DIR}/Crypto/src/OpenSSLInitializer.cpp" - "${LIBRARY_DIR}/Crypto/src/PKCS12Container.cpp" - "${LIBRARY_DIR}/Crypto/src/RSACipherImpl.cpp" - "${LIBRARY_DIR}/Crypto/src/RSADigestEngine.cpp" - "${LIBRARY_DIR}/Crypto/src/RSAKey.cpp" - "${LIBRARY_DIR}/Crypto/src/RSAKeyImpl.cpp" - "${LIBRARY_DIR}/Crypto/src/X509Certificate.cpp" - ) + set (SRCS + "${LIBRARY_DIR}/Crypto/src/Cipher.cpp" + "${LIBRARY_DIR}/Crypto/src/CipherFactory.cpp" + "${LIBRARY_DIR}/Crypto/src/CipherImpl.cpp" + "${LIBRARY_DIR}/Crypto/src/CipherKey.cpp" + "${LIBRARY_DIR}/Crypto/src/CipherKeyImpl.cpp" + "${LIBRARY_DIR}/Crypto/src/CryptoException.cpp" + "${LIBRARY_DIR}/Crypto/src/CryptoStream.cpp" + "${LIBRARY_DIR}/Crypto/src/CryptoTransform.cpp" + "${LIBRARY_DIR}/Crypto/src/DigestEngine.cpp" + "${LIBRARY_DIR}/Crypto/src/ECDSADigestEngine.cpp" + "${LIBRARY_DIR}/Crypto/src/ECKey.cpp" + "${LIBRARY_DIR}/Crypto/src/ECKeyImpl.cpp" + "${LIBRARY_DIR}/Crypto/src/EVPPKey.cpp" + "${LIBRARY_DIR}/Crypto/src/KeyPair.cpp" + "${LIBRARY_DIR}/Crypto/src/KeyPairImpl.cpp" + "${LIBRARY_DIR}/Crypto/src/OpenSSLInitializer.cpp" + "${LIBRARY_DIR}/Crypto/src/PKCS12Container.cpp" + "${LIBRARY_DIR}/Crypto/src/RSACipherImpl.cpp" + "${LIBRARY_DIR}/Crypto/src/RSADigestEngine.cpp" + "${LIBRARY_DIR}/Crypto/src/RSAKey.cpp" + "${LIBRARY_DIR}/Crypto/src/RSAKeyImpl.cpp" + "${LIBRARY_DIR}/Crypto/src/X509Certificate.cpp" + ) - add_library (_poco_crypto ${SRCS}) - add_library (Poco::Crypto ALIAS _poco_crypto) + add_library (_poco_crypto ${SRCS}) + add_library (Poco::Crypto ALIAS _poco_crypto) - target_compile_options (_poco_crypto PRIVATE -Wno-newline-eof) - target_include_directories (_poco_crypto SYSTEM PUBLIC "${LIBRARY_DIR}/Crypto/include") - target_link_libraries (_poco_crypto PUBLIC Poco::Foundation ssl crypto) - else () - add_library (Poco::Crypto UNKNOWN IMPORTED GLOBAL) - - find_library(LIBRARY_POCO_CRYPTO PocoCrypto) - find_path(INCLUDE_POCO_CRYPTO Poco/Crypto/Crypto.h) - set_target_properties (Poco::Crypto PROPERTIES IMPORTED_LOCATION ${LIBRARY_POCO_CRYPTO}) - set_target_properties (Poco::Crypto PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_POCO_CRYPTO}) - - target_link_libraries (Poco::Crypto INTERFACE Poco::Foundation) - endif () + target_compile_options (_poco_crypto PRIVATE -Wno-newline-eof) + target_include_directories (_poco_crypto SYSTEM PUBLIC "${LIBRARY_DIR}/Crypto/include") + target_link_libraries (_poco_crypto PUBLIC Poco::Foundation ssl crypto) message (STATUS "Using Poco::Crypto") else () diff --git a/contrib/poco-cmake/Data/CMakeLists.txt b/contrib/poco-cmake/Data/CMakeLists.txt index 4fdd755b45d..b13c07583ad 100644 --- a/contrib/poco-cmake/Data/CMakeLists.txt +++ b/contrib/poco-cmake/Data/CMakeLists.txt @@ -1,60 +1,46 @@ -if (USE_INTERNAL_POCO_LIBRARY) - set (SRCS - "${LIBRARY_DIR}/Data/src/AbstractBinder.cpp" - "${LIBRARY_DIR}/Data/src/AbstractBinding.cpp" - "${LIBRARY_DIR}/Data/src/AbstractExtraction.cpp" - "${LIBRARY_DIR}/Data/src/AbstractExtractor.cpp" - "${LIBRARY_DIR}/Data/src/AbstractPreparation.cpp" - "${LIBRARY_DIR}/Data/src/AbstractPreparator.cpp" - "${LIBRARY_DIR}/Data/src/ArchiveStrategy.cpp" - "${LIBRARY_DIR}/Data/src/Bulk.cpp" - "${LIBRARY_DIR}/Data/src/Connector.cpp" - "${LIBRARY_DIR}/Data/src/DataException.cpp" - "${LIBRARY_DIR}/Data/src/Date.cpp" - "${LIBRARY_DIR}/Data/src/DynamicLOB.cpp" - "${LIBRARY_DIR}/Data/src/Limit.cpp" - "${LIBRARY_DIR}/Data/src/MetaColumn.cpp" - "${LIBRARY_DIR}/Data/src/PooledSessionHolder.cpp" - "${LIBRARY_DIR}/Data/src/PooledSessionImpl.cpp" - "${LIBRARY_DIR}/Data/src/Position.cpp" - "${LIBRARY_DIR}/Data/src/Range.cpp" - "${LIBRARY_DIR}/Data/src/RecordSet.cpp" - "${LIBRARY_DIR}/Data/src/Row.cpp" - "${LIBRARY_DIR}/Data/src/RowFilter.cpp" - "${LIBRARY_DIR}/Data/src/RowFormatter.cpp" - "${LIBRARY_DIR}/Data/src/RowIterator.cpp" - "${LIBRARY_DIR}/Data/src/Session.cpp" - "${LIBRARY_DIR}/Data/src/SessionFactory.cpp" - "${LIBRARY_DIR}/Data/src/SessionImpl.cpp" - "${LIBRARY_DIR}/Data/src/SessionPool.cpp" - "${LIBRARY_DIR}/Data/src/SessionPoolContainer.cpp" - "${LIBRARY_DIR}/Data/src/SimpleRowFormatter.cpp" - "${LIBRARY_DIR}/Data/src/SQLChannel.cpp" - "${LIBRARY_DIR}/Data/src/Statement.cpp" - "${LIBRARY_DIR}/Data/src/StatementCreator.cpp" - "${LIBRARY_DIR}/Data/src/StatementImpl.cpp" - "${LIBRARY_DIR}/Data/src/Time.cpp" - "${LIBRARY_DIR}/Data/src/Transaction.cpp" - ) +set (SRCS + "${LIBRARY_DIR}/Data/src/AbstractBinder.cpp" + "${LIBRARY_DIR}/Data/src/AbstractBinding.cpp" + "${LIBRARY_DIR}/Data/src/AbstractExtraction.cpp" + "${LIBRARY_DIR}/Data/src/AbstractExtractor.cpp" + "${LIBRARY_DIR}/Data/src/AbstractPreparation.cpp" + "${LIBRARY_DIR}/Data/src/AbstractPreparator.cpp" + "${LIBRARY_DIR}/Data/src/ArchiveStrategy.cpp" + "${LIBRARY_DIR}/Data/src/Bulk.cpp" + "${LIBRARY_DIR}/Data/src/Connector.cpp" + "${LIBRARY_DIR}/Data/src/DataException.cpp" + "${LIBRARY_DIR}/Data/src/Date.cpp" + "${LIBRARY_DIR}/Data/src/DynamicLOB.cpp" + "${LIBRARY_DIR}/Data/src/Limit.cpp" + "${LIBRARY_DIR}/Data/src/MetaColumn.cpp" + "${LIBRARY_DIR}/Data/src/PooledSessionHolder.cpp" + "${LIBRARY_DIR}/Data/src/PooledSessionImpl.cpp" + "${LIBRARY_DIR}/Data/src/Position.cpp" + "${LIBRARY_DIR}/Data/src/Range.cpp" + "${LIBRARY_DIR}/Data/src/RecordSet.cpp" + "${LIBRARY_DIR}/Data/src/Row.cpp" + "${LIBRARY_DIR}/Data/src/RowFilter.cpp" + "${LIBRARY_DIR}/Data/src/RowFormatter.cpp" + "${LIBRARY_DIR}/Data/src/RowIterator.cpp" + "${LIBRARY_DIR}/Data/src/Session.cpp" + "${LIBRARY_DIR}/Data/src/SessionFactory.cpp" + "${LIBRARY_DIR}/Data/src/SessionImpl.cpp" + "${LIBRARY_DIR}/Data/src/SessionPool.cpp" + "${LIBRARY_DIR}/Data/src/SessionPoolContainer.cpp" + "${LIBRARY_DIR}/Data/src/SimpleRowFormatter.cpp" + "${LIBRARY_DIR}/Data/src/SQLChannel.cpp" + "${LIBRARY_DIR}/Data/src/Statement.cpp" + "${LIBRARY_DIR}/Data/src/StatementCreator.cpp" + "${LIBRARY_DIR}/Data/src/StatementImpl.cpp" + "${LIBRARY_DIR}/Data/src/Time.cpp" + "${LIBRARY_DIR}/Data/src/Transaction.cpp" +) - add_library (_poco_data ${SRCS}) - add_library (Poco::Data ALIAS _poco_data) +add_library (_poco_data ${SRCS}) +add_library (Poco::Data ALIAS _poco_data) - if (COMPILER_GCC) - target_compile_options (_poco_data PRIVATE -Wno-deprecated-copy) - endif () - target_include_directories (_poco_data SYSTEM PUBLIC "${LIBRARY_DIR}/Data/include") - target_link_libraries (_poco_data PUBLIC Poco::Foundation) -else () - # NOTE: don't know why, but the GLOBAL is required here. - add_library (Poco::Data UNKNOWN IMPORTED GLOBAL) - - find_library(LIBRARY_POCO_DATA PocoData) - find_path(INCLUDE_POCO_DATA Poco/Data/Data.h) - set_target_properties (Poco::Data PROPERTIES IMPORTED_LOCATION ${LIBRARY_POCO_DATA}) - set_target_properties (Poco::Data PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_POCO_DATA}) - - target_link_libraries (Poco::Data INTERFACE Poco::Foundation) - - message (STATUS "Using Poco::Data: ${LIBRARY_POCO_DATA} ${INCLUDE_POCO_DATA}") +if (COMPILER_GCC) + target_compile_options (_poco_data PRIVATE -Wno-deprecated-copy) endif () +target_include_directories (_poco_data SYSTEM PUBLIC "${LIBRARY_DIR}/Data/include") +target_link_libraries (_poco_data PUBLIC Poco::Foundation) diff --git a/contrib/poco-cmake/Data/ODBC/CMakeLists.txt b/contrib/poco-cmake/Data/ODBC/CMakeLists.txt index a3561304541..93d8b036526 100644 --- a/contrib/poco-cmake/Data/ODBC/CMakeLists.txt +++ b/contrib/poco-cmake/Data/ODBC/CMakeLists.txt @@ -3,40 +3,29 @@ if (ENABLE_ODBC) message(FATAL_ERROR "Configuration error: unixodbc is not a target") endif() - if (USE_INTERNAL_POCO_LIBRARY) - set (SRCS - "${LIBRARY_DIR}/Data/ODBC/src/Binder.cpp" - "${LIBRARY_DIR}/Data/ODBC/src/ConnectionHandle.cpp" - "${LIBRARY_DIR}/Data/ODBC/src/Connector.cpp" - "${LIBRARY_DIR}/Data/ODBC/src/EnvironmentHandle.cpp" - "${LIBRARY_DIR}/Data/ODBC/src/Extractor.cpp" - "${LIBRARY_DIR}/Data/ODBC/src/ODBCException.cpp" - "${LIBRARY_DIR}/Data/ODBC/src/ODBCMetaColumn.cpp" - "${LIBRARY_DIR}/Data/ODBC/src/ODBCStatementImpl.cpp" - "${LIBRARY_DIR}/Data/ODBC/src/Parameter.cpp" - "${LIBRARY_DIR}/Data/ODBC/src/Preparator.cpp" - "${LIBRARY_DIR}/Data/ODBC/src/SessionImpl.cpp" - "${LIBRARY_DIR}/Data/ODBC/src/TypeInfo.cpp" - "${LIBRARY_DIR}/Data/ODBC/src/Unicode.cpp" - "${LIBRARY_DIR}/Data/ODBC/src/Utility.cpp" - ) + set (SRCS + "${LIBRARY_DIR}/Data/ODBC/src/Binder.cpp" + "${LIBRARY_DIR}/Data/ODBC/src/ConnectionHandle.cpp" + "${LIBRARY_DIR}/Data/ODBC/src/Connector.cpp" + "${LIBRARY_DIR}/Data/ODBC/src/EnvironmentHandle.cpp" + "${LIBRARY_DIR}/Data/ODBC/src/Extractor.cpp" + "${LIBRARY_DIR}/Data/ODBC/src/ODBCException.cpp" + "${LIBRARY_DIR}/Data/ODBC/src/ODBCMetaColumn.cpp" + "${LIBRARY_DIR}/Data/ODBC/src/ODBCStatementImpl.cpp" + "${LIBRARY_DIR}/Data/ODBC/src/Parameter.cpp" + "${LIBRARY_DIR}/Data/ODBC/src/Preparator.cpp" + "${LIBRARY_DIR}/Data/ODBC/src/SessionImpl.cpp" + "${LIBRARY_DIR}/Data/ODBC/src/TypeInfo.cpp" + "${LIBRARY_DIR}/Data/ODBC/src/Unicode.cpp" + "${LIBRARY_DIR}/Data/ODBC/src/Utility.cpp" + ) - add_library (_poco_data_odbc ${SRCS}) - add_library (Poco::Data::ODBC ALIAS _poco_data_odbc) + add_library (_poco_data_odbc ${SRCS}) + add_library (Poco::Data::ODBC ALIAS _poco_data_odbc) - target_compile_options (_poco_data_odbc PRIVATE -Wno-unused-variable) - target_include_directories (_poco_data_odbc SYSTEM PUBLIC "${LIBRARY_DIR}/Data/ODBC/include") - target_link_libraries (_poco_data_odbc PUBLIC Poco::Data unixodbc) - else () - add_library (Poco::Data::ODBC UNKNOWN IMPORTED GLOBAL) - - find_library(LIBRARY_POCO_DATA_ODBC PocoDataODBC) - find_path(INCLUDE_POCO_DATA_ODBC Poco/Data/ODBC/ODBC.h) - set_target_properties (Poco::Data::ODBC PROPERTIES IMPORTED_LOCATION ${LIBRARY_POCO_DATA_ODBC}) - set_target_properties (Poco::Data::ODBC PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_POCO_DATA_ODBC}) - - target_link_libraries (Poco::Data::ODBC INTERFACE Poco::Data) - endif () + target_compile_options (_poco_data_odbc PRIVATE -Wno-unused-variable) + target_include_directories (_poco_data_odbc SYSTEM PUBLIC "${LIBRARY_DIR}/Data/ODBC/include") + target_link_libraries (_poco_data_odbc PUBLIC Poco::Data unixodbc) message (STATUS "Using Poco::Data::ODBC") else () diff --git a/contrib/poco-cmake/Foundation/CMakeLists.txt b/contrib/poco-cmake/Foundation/CMakeLists.txt index 8ed4dfa6c66..795ec985cb4 100644 --- a/contrib/poco-cmake/Foundation/CMakeLists.txt +++ b/contrib/poco-cmake/Foundation/CMakeLists.txt @@ -1,243 +1,232 @@ -if (USE_INTERNAL_POCO_LIBRARY) - # Foundation (pcre) +# Foundation (pcre) - set (SRCS_PCRE - "${LIBRARY_DIR}/Foundation/src/pcre_config.c" - "${LIBRARY_DIR}/Foundation/src/pcre_byte_order.c" - "${LIBRARY_DIR}/Foundation/src/pcre_chartables.c" - "${LIBRARY_DIR}/Foundation/src/pcre_compile.c" - "${LIBRARY_DIR}/Foundation/src/pcre_exec.c" - "${LIBRARY_DIR}/Foundation/src/pcre_fullinfo.c" - "${LIBRARY_DIR}/Foundation/src/pcre_globals.c" - "${LIBRARY_DIR}/Foundation/src/pcre_maketables.c" - "${LIBRARY_DIR}/Foundation/src/pcre_newline.c" - "${LIBRARY_DIR}/Foundation/src/pcre_ord2utf8.c" - "${LIBRARY_DIR}/Foundation/src/pcre_study.c" - "${LIBRARY_DIR}/Foundation/src/pcre_tables.c" - "${LIBRARY_DIR}/Foundation/src/pcre_dfa_exec.c" - "${LIBRARY_DIR}/Foundation/src/pcre_get.c" - "${LIBRARY_DIR}/Foundation/src/pcre_jit_compile.c" - "${LIBRARY_DIR}/Foundation/src/pcre_refcount.c" - "${LIBRARY_DIR}/Foundation/src/pcre_string_utils.c" - "${LIBRARY_DIR}/Foundation/src/pcre_version.c" - "${LIBRARY_DIR}/Foundation/src/pcre_ucd.c" - "${LIBRARY_DIR}/Foundation/src/pcre_valid_utf8.c" - "${LIBRARY_DIR}/Foundation/src/pcre_xclass.c" - ) +set (SRCS_PCRE + "${LIBRARY_DIR}/Foundation/src/pcre_config.c" + "${LIBRARY_DIR}/Foundation/src/pcre_byte_order.c" + "${LIBRARY_DIR}/Foundation/src/pcre_chartables.c" + "${LIBRARY_DIR}/Foundation/src/pcre_compile.c" + "${LIBRARY_DIR}/Foundation/src/pcre_exec.c" + "${LIBRARY_DIR}/Foundation/src/pcre_fullinfo.c" + "${LIBRARY_DIR}/Foundation/src/pcre_globals.c" + "${LIBRARY_DIR}/Foundation/src/pcre_maketables.c" + "${LIBRARY_DIR}/Foundation/src/pcre_newline.c" + "${LIBRARY_DIR}/Foundation/src/pcre_ord2utf8.c" + "${LIBRARY_DIR}/Foundation/src/pcre_study.c" + "${LIBRARY_DIR}/Foundation/src/pcre_tables.c" + "${LIBRARY_DIR}/Foundation/src/pcre_dfa_exec.c" + "${LIBRARY_DIR}/Foundation/src/pcre_get.c" + "${LIBRARY_DIR}/Foundation/src/pcre_jit_compile.c" + "${LIBRARY_DIR}/Foundation/src/pcre_refcount.c" + "${LIBRARY_DIR}/Foundation/src/pcre_string_utils.c" + "${LIBRARY_DIR}/Foundation/src/pcre_version.c" + "${LIBRARY_DIR}/Foundation/src/pcre_ucd.c" + "${LIBRARY_DIR}/Foundation/src/pcre_valid_utf8.c" + "${LIBRARY_DIR}/Foundation/src/pcre_xclass.c" +) - add_library (_poco_foundation_pcre ${SRCS_PCRE}) - add_library (Poco::Foundation::PCRE ALIAS _poco_foundation_pcre) +add_library (_poco_foundation_pcre ${SRCS_PCRE}) +add_library (Poco::Foundation::PCRE ALIAS _poco_foundation_pcre) - target_compile_options (_poco_foundation_pcre PRIVATE -Wno-sign-compare) +target_compile_options (_poco_foundation_pcre PRIVATE -Wno-sign-compare) - # Foundation +# Foundation - set (SRCS - "${LIBRARY_DIR}/Foundation/src/AbstractObserver.cpp" - "${LIBRARY_DIR}/Foundation/src/ActiveDispatcher.cpp" - "${LIBRARY_DIR}/Foundation/src/ArchiveStrategy.cpp" - "${LIBRARY_DIR}/Foundation/src/Ascii.cpp" - "${LIBRARY_DIR}/Foundation/src/ASCIIEncoding.cpp" - "${LIBRARY_DIR}/Foundation/src/AsyncChannel.cpp" - "${LIBRARY_DIR}/Foundation/src/AtomicCounter.cpp" - "${LIBRARY_DIR}/Foundation/src/Base32Decoder.cpp" - "${LIBRARY_DIR}/Foundation/src/Base32Encoder.cpp" - "${LIBRARY_DIR}/Foundation/src/Base64Decoder.cpp" - "${LIBRARY_DIR}/Foundation/src/Base64Encoder.cpp" - "${LIBRARY_DIR}/Foundation/src/BinaryReader.cpp" - "${LIBRARY_DIR}/Foundation/src/BinaryWriter.cpp" - "${LIBRARY_DIR}/Foundation/src/Bugcheck.cpp" - "${LIBRARY_DIR}/Foundation/src/ByteOrder.cpp" - "${LIBRARY_DIR}/Foundation/src/Channel.cpp" - "${LIBRARY_DIR}/Foundation/src/Checksum.cpp" - "${LIBRARY_DIR}/Foundation/src/Clock.cpp" - "${LIBRARY_DIR}/Foundation/src/CompressedLogFile.cpp" - "${LIBRARY_DIR}/Foundation/src/Condition.cpp" - "${LIBRARY_DIR}/Foundation/src/Configurable.cpp" - "${LIBRARY_DIR}/Foundation/src/ConsoleChannel.cpp" - "${LIBRARY_DIR}/Foundation/src/CountingStream.cpp" - "${LIBRARY_DIR}/Foundation/src/DateTime.cpp" - "${LIBRARY_DIR}/Foundation/src/DateTimeFormat.cpp" - "${LIBRARY_DIR}/Foundation/src/DateTimeFormatter.cpp" - "${LIBRARY_DIR}/Foundation/src/DateTimeParser.cpp" - "${LIBRARY_DIR}/Foundation/src/Debugger.cpp" - "${LIBRARY_DIR}/Foundation/src/DeflatingStream.cpp" - "${LIBRARY_DIR}/Foundation/src/DigestEngine.cpp" - "${LIBRARY_DIR}/Foundation/src/DigestStream.cpp" - "${LIBRARY_DIR}/Foundation/src/DirectoryIterator.cpp" - "${LIBRARY_DIR}/Foundation/src/DirectoryIteratorStrategy.cpp" - "${LIBRARY_DIR}/Foundation/src/DirectoryWatcher.cpp" - "${LIBRARY_DIR}/Foundation/src/Environment.cpp" - "${LIBRARY_DIR}/Foundation/src/Error.cpp" - "${LIBRARY_DIR}/Foundation/src/ErrorHandler.cpp" - "${LIBRARY_DIR}/Foundation/src/Event.cpp" - "${LIBRARY_DIR}/Foundation/src/EventArgs.cpp" - "${LIBRARY_DIR}/Foundation/src/EventChannel.cpp" - "${LIBRARY_DIR}/Foundation/src/Exception.cpp" - "${LIBRARY_DIR}/Foundation/src/FIFOBufferStream.cpp" - "${LIBRARY_DIR}/Foundation/src/File.cpp" - "${LIBRARY_DIR}/Foundation/src/FileChannel.cpp" - "${LIBRARY_DIR}/Foundation/src/FileStream.cpp" - "${LIBRARY_DIR}/Foundation/src/FileStreamFactory.cpp" - "${LIBRARY_DIR}/Foundation/src/Format.cpp" - "${LIBRARY_DIR}/Foundation/src/Formatter.cpp" - "${LIBRARY_DIR}/Foundation/src/FormattingChannel.cpp" - "${LIBRARY_DIR}/Foundation/src/FPEnvironment.cpp" - "${LIBRARY_DIR}/Foundation/src/Glob.cpp" - "${LIBRARY_DIR}/Foundation/src/Hash.cpp" - "${LIBRARY_DIR}/Foundation/src/HashStatistic.cpp" - "${LIBRARY_DIR}/Foundation/src/HexBinaryDecoder.cpp" - "${LIBRARY_DIR}/Foundation/src/HexBinaryEncoder.cpp" - "${LIBRARY_DIR}/Foundation/src/InflatingStream.cpp" - "${LIBRARY_DIR}/Foundation/src/JSONString.cpp" - "${LIBRARY_DIR}/Foundation/src/Latin1Encoding.cpp" - "${LIBRARY_DIR}/Foundation/src/Latin2Encoding.cpp" - "${LIBRARY_DIR}/Foundation/src/Latin9Encoding.cpp" - "${LIBRARY_DIR}/Foundation/src/LineEndingConverter.cpp" - "${LIBRARY_DIR}/Foundation/src/LocalDateTime.cpp" - "${LIBRARY_DIR}/Foundation/src/LogFile.cpp" - "${LIBRARY_DIR}/Foundation/src/Logger.cpp" - "${LIBRARY_DIR}/Foundation/src/LoggingFactory.cpp" - "${LIBRARY_DIR}/Foundation/src/LoggingRegistry.cpp" - "${LIBRARY_DIR}/Foundation/src/LogStream.cpp" - "${LIBRARY_DIR}/Foundation/src/Manifest.cpp" - "${LIBRARY_DIR}/Foundation/src/MD4Engine.cpp" - "${LIBRARY_DIR}/Foundation/src/MD5Engine.cpp" - "${LIBRARY_DIR}/Foundation/src/MemoryPool.cpp" - "${LIBRARY_DIR}/Foundation/src/MemoryStream.cpp" - "${LIBRARY_DIR}/Foundation/src/Message.cpp" - "${LIBRARY_DIR}/Foundation/src/Mutex.cpp" - "${LIBRARY_DIR}/Foundation/src/NamedEvent.cpp" - "${LIBRARY_DIR}/Foundation/src/NamedMutex.cpp" - "${LIBRARY_DIR}/Foundation/src/NestedDiagnosticContext.cpp" - "${LIBRARY_DIR}/Foundation/src/Notification.cpp" - "${LIBRARY_DIR}/Foundation/src/NotificationCenter.cpp" - "${LIBRARY_DIR}/Foundation/src/NotificationQueue.cpp" - "${LIBRARY_DIR}/Foundation/src/NullChannel.cpp" - "${LIBRARY_DIR}/Foundation/src/NullStream.cpp" - "${LIBRARY_DIR}/Foundation/src/NumberFormatter.cpp" - "${LIBRARY_DIR}/Foundation/src/NumberParser.cpp" - "${LIBRARY_DIR}/Foundation/src/NumericString.cpp" - "${LIBRARY_DIR}/Foundation/src/Path.cpp" - "${LIBRARY_DIR}/Foundation/src/PatternFormatter.cpp" - "${LIBRARY_DIR}/Foundation/src/Pipe.cpp" - "${LIBRARY_DIR}/Foundation/src/PipeImpl.cpp" - "${LIBRARY_DIR}/Foundation/src/PipeStream.cpp" - "${LIBRARY_DIR}/Foundation/src/PriorityNotificationQueue.cpp" - "${LIBRARY_DIR}/Foundation/src/Process.cpp" - "${LIBRARY_DIR}/Foundation/src/PurgeStrategy.cpp" - "${LIBRARY_DIR}/Foundation/src/Random.cpp" - "${LIBRARY_DIR}/Foundation/src/RandomStream.cpp" - "${LIBRARY_DIR}/Foundation/src/RefCountedObject.cpp" - "${LIBRARY_DIR}/Foundation/src/RegularExpression.cpp" - "${LIBRARY_DIR}/Foundation/src/RotateStrategy.cpp" - "${LIBRARY_DIR}/Foundation/src/Runnable.cpp" - "${LIBRARY_DIR}/Foundation/src/RWLock.cpp" - "${LIBRARY_DIR}/Foundation/src/Semaphore.cpp" - "${LIBRARY_DIR}/Foundation/src/SHA1Engine.cpp" - "${LIBRARY_DIR}/Foundation/src/SharedLibrary.cpp" - "${LIBRARY_DIR}/Foundation/src/SharedMemory.cpp" - "${LIBRARY_DIR}/Foundation/src/SignalHandler.cpp" - "${LIBRARY_DIR}/Foundation/src/SimpleFileChannel.cpp" - "${LIBRARY_DIR}/Foundation/src/SortedDirectoryIterator.cpp" - "${LIBRARY_DIR}/Foundation/src/SplitterChannel.cpp" - "${LIBRARY_DIR}/Foundation/src/Stopwatch.cpp" - "${LIBRARY_DIR}/Foundation/src/StreamChannel.cpp" - "${LIBRARY_DIR}/Foundation/src/StreamConverter.cpp" - "${LIBRARY_DIR}/Foundation/src/StreamCopier.cpp" - "${LIBRARY_DIR}/Foundation/src/StreamTokenizer.cpp" - "${LIBRARY_DIR}/Foundation/src/String.cpp" - "${LIBRARY_DIR}/Foundation/src/StringTokenizer.cpp" - "${LIBRARY_DIR}/Foundation/src/SynchronizedObject.cpp" - "${LIBRARY_DIR}/Foundation/src/SyslogChannel.cpp" - "${LIBRARY_DIR}/Foundation/src/Task.cpp" - "${LIBRARY_DIR}/Foundation/src/TaskManager.cpp" - "${LIBRARY_DIR}/Foundation/src/TaskNotification.cpp" - "${LIBRARY_DIR}/Foundation/src/TeeStream.cpp" - "${LIBRARY_DIR}/Foundation/src/TemporaryFile.cpp" - "${LIBRARY_DIR}/Foundation/src/TextBufferIterator.cpp" - "${LIBRARY_DIR}/Foundation/src/TextConverter.cpp" - "${LIBRARY_DIR}/Foundation/src/TextEncoding.cpp" - "${LIBRARY_DIR}/Foundation/src/TextIterator.cpp" - "${LIBRARY_DIR}/Foundation/src/Thread.cpp" - "${LIBRARY_DIR}/Foundation/src/ThreadLocal.cpp" - "${LIBRARY_DIR}/Foundation/src/ThreadPool.cpp" - "${LIBRARY_DIR}/Foundation/src/ThreadTarget.cpp" - "${LIBRARY_DIR}/Foundation/src/TimedNotificationQueue.cpp" - "${LIBRARY_DIR}/Foundation/src/Timer.cpp" - "${LIBRARY_DIR}/Foundation/src/Timespan.cpp" - "${LIBRARY_DIR}/Foundation/src/Timestamp.cpp" - "${LIBRARY_DIR}/Foundation/src/Timezone.cpp" - "${LIBRARY_DIR}/Foundation/src/Token.cpp" - "${LIBRARY_DIR}/Foundation/src/Unicode.cpp" - "${LIBRARY_DIR}/Foundation/src/UnicodeConverter.cpp" - "${LIBRARY_DIR}/Foundation/src/URI.cpp" - "${LIBRARY_DIR}/Foundation/src/URIStreamFactory.cpp" - "${LIBRARY_DIR}/Foundation/src/URIStreamOpener.cpp" - "${LIBRARY_DIR}/Foundation/src/UTF16Encoding.cpp" - "${LIBRARY_DIR}/Foundation/src/UTF32Encoding.cpp" - "${LIBRARY_DIR}/Foundation/src/UTF8Encoding.cpp" - "${LIBRARY_DIR}/Foundation/src/UTF8String.cpp" - "${LIBRARY_DIR}/Foundation/src/UUID.cpp" - "${LIBRARY_DIR}/Foundation/src/UUIDGenerator.cpp" - "${LIBRARY_DIR}/Foundation/src/Var.cpp" - "${LIBRARY_DIR}/Foundation/src/VarHolder.cpp" - "${LIBRARY_DIR}/Foundation/src/VarIterator.cpp" - "${LIBRARY_DIR}/Foundation/src/Void.cpp" - "${LIBRARY_DIR}/Foundation/src/Windows1250Encoding.cpp" - "${LIBRARY_DIR}/Foundation/src/Windows1251Encoding.cpp" - "${LIBRARY_DIR}/Foundation/src/Windows1252Encoding.cpp" - ) +set (SRCS + "${LIBRARY_DIR}/Foundation/src/AbstractObserver.cpp" + "${LIBRARY_DIR}/Foundation/src/ActiveDispatcher.cpp" + "${LIBRARY_DIR}/Foundation/src/ArchiveStrategy.cpp" + "${LIBRARY_DIR}/Foundation/src/Ascii.cpp" + "${LIBRARY_DIR}/Foundation/src/ASCIIEncoding.cpp" + "${LIBRARY_DIR}/Foundation/src/AsyncChannel.cpp" + "${LIBRARY_DIR}/Foundation/src/AtomicCounter.cpp" + "${LIBRARY_DIR}/Foundation/src/Base32Decoder.cpp" + "${LIBRARY_DIR}/Foundation/src/Base32Encoder.cpp" + "${LIBRARY_DIR}/Foundation/src/Base64Decoder.cpp" + "${LIBRARY_DIR}/Foundation/src/Base64Encoder.cpp" + "${LIBRARY_DIR}/Foundation/src/BinaryReader.cpp" + "${LIBRARY_DIR}/Foundation/src/BinaryWriter.cpp" + "${LIBRARY_DIR}/Foundation/src/Bugcheck.cpp" + "${LIBRARY_DIR}/Foundation/src/ByteOrder.cpp" + "${LIBRARY_DIR}/Foundation/src/Channel.cpp" + "${LIBRARY_DIR}/Foundation/src/Checksum.cpp" + "${LIBRARY_DIR}/Foundation/src/Clock.cpp" + "${LIBRARY_DIR}/Foundation/src/CompressedLogFile.cpp" + "${LIBRARY_DIR}/Foundation/src/Condition.cpp" + "${LIBRARY_DIR}/Foundation/src/Configurable.cpp" + "${LIBRARY_DIR}/Foundation/src/ConsoleChannel.cpp" + "${LIBRARY_DIR}/Foundation/src/CountingStream.cpp" + "${LIBRARY_DIR}/Foundation/src/DateTime.cpp" + "${LIBRARY_DIR}/Foundation/src/DateTimeFormat.cpp" + "${LIBRARY_DIR}/Foundation/src/DateTimeFormatter.cpp" + "${LIBRARY_DIR}/Foundation/src/DateTimeParser.cpp" + "${LIBRARY_DIR}/Foundation/src/Debugger.cpp" + "${LIBRARY_DIR}/Foundation/src/DeflatingStream.cpp" + "${LIBRARY_DIR}/Foundation/src/DigestEngine.cpp" + "${LIBRARY_DIR}/Foundation/src/DigestStream.cpp" + "${LIBRARY_DIR}/Foundation/src/DirectoryIterator.cpp" + "${LIBRARY_DIR}/Foundation/src/DirectoryIteratorStrategy.cpp" + "${LIBRARY_DIR}/Foundation/src/DirectoryWatcher.cpp" + "${LIBRARY_DIR}/Foundation/src/Environment.cpp" + "${LIBRARY_DIR}/Foundation/src/Error.cpp" + "${LIBRARY_DIR}/Foundation/src/ErrorHandler.cpp" + "${LIBRARY_DIR}/Foundation/src/Event.cpp" + "${LIBRARY_DIR}/Foundation/src/EventArgs.cpp" + "${LIBRARY_DIR}/Foundation/src/EventChannel.cpp" + "${LIBRARY_DIR}/Foundation/src/Exception.cpp" + "${LIBRARY_DIR}/Foundation/src/FIFOBufferStream.cpp" + "${LIBRARY_DIR}/Foundation/src/File.cpp" + "${LIBRARY_DIR}/Foundation/src/FileChannel.cpp" + "${LIBRARY_DIR}/Foundation/src/FileStream.cpp" + "${LIBRARY_DIR}/Foundation/src/FileStreamFactory.cpp" + "${LIBRARY_DIR}/Foundation/src/Format.cpp" + "${LIBRARY_DIR}/Foundation/src/Formatter.cpp" + "${LIBRARY_DIR}/Foundation/src/FormattingChannel.cpp" + "${LIBRARY_DIR}/Foundation/src/FPEnvironment.cpp" + "${LIBRARY_DIR}/Foundation/src/Glob.cpp" + "${LIBRARY_DIR}/Foundation/src/Hash.cpp" + "${LIBRARY_DIR}/Foundation/src/HashStatistic.cpp" + "${LIBRARY_DIR}/Foundation/src/HexBinaryDecoder.cpp" + "${LIBRARY_DIR}/Foundation/src/HexBinaryEncoder.cpp" + "${LIBRARY_DIR}/Foundation/src/InflatingStream.cpp" + "${LIBRARY_DIR}/Foundation/src/JSONString.cpp" + "${LIBRARY_DIR}/Foundation/src/Latin1Encoding.cpp" + "${LIBRARY_DIR}/Foundation/src/Latin2Encoding.cpp" + "${LIBRARY_DIR}/Foundation/src/Latin9Encoding.cpp" + "${LIBRARY_DIR}/Foundation/src/LineEndingConverter.cpp" + "${LIBRARY_DIR}/Foundation/src/LocalDateTime.cpp" + "${LIBRARY_DIR}/Foundation/src/LogFile.cpp" + "${LIBRARY_DIR}/Foundation/src/Logger.cpp" + "${LIBRARY_DIR}/Foundation/src/LoggingFactory.cpp" + "${LIBRARY_DIR}/Foundation/src/LoggingRegistry.cpp" + "${LIBRARY_DIR}/Foundation/src/LogStream.cpp" + "${LIBRARY_DIR}/Foundation/src/Manifest.cpp" + "${LIBRARY_DIR}/Foundation/src/MD4Engine.cpp" + "${LIBRARY_DIR}/Foundation/src/MD5Engine.cpp" + "${LIBRARY_DIR}/Foundation/src/MemoryPool.cpp" + "${LIBRARY_DIR}/Foundation/src/MemoryStream.cpp" + "${LIBRARY_DIR}/Foundation/src/Message.cpp" + "${LIBRARY_DIR}/Foundation/src/Mutex.cpp" + "${LIBRARY_DIR}/Foundation/src/NamedEvent.cpp" + "${LIBRARY_DIR}/Foundation/src/NamedMutex.cpp" + "${LIBRARY_DIR}/Foundation/src/NestedDiagnosticContext.cpp" + "${LIBRARY_DIR}/Foundation/src/Notification.cpp" + "${LIBRARY_DIR}/Foundation/src/NotificationCenter.cpp" + "${LIBRARY_DIR}/Foundation/src/NotificationQueue.cpp" + "${LIBRARY_DIR}/Foundation/src/NullChannel.cpp" + "${LIBRARY_DIR}/Foundation/src/NullStream.cpp" + "${LIBRARY_DIR}/Foundation/src/NumberFormatter.cpp" + "${LIBRARY_DIR}/Foundation/src/NumberParser.cpp" + "${LIBRARY_DIR}/Foundation/src/NumericString.cpp" + "${LIBRARY_DIR}/Foundation/src/Path.cpp" + "${LIBRARY_DIR}/Foundation/src/PatternFormatter.cpp" + "${LIBRARY_DIR}/Foundation/src/Pipe.cpp" + "${LIBRARY_DIR}/Foundation/src/PipeImpl.cpp" + "${LIBRARY_DIR}/Foundation/src/PipeStream.cpp" + "${LIBRARY_DIR}/Foundation/src/PriorityNotificationQueue.cpp" + "${LIBRARY_DIR}/Foundation/src/Process.cpp" + "${LIBRARY_DIR}/Foundation/src/PurgeStrategy.cpp" + "${LIBRARY_DIR}/Foundation/src/Random.cpp" + "${LIBRARY_DIR}/Foundation/src/RandomStream.cpp" + "${LIBRARY_DIR}/Foundation/src/RefCountedObject.cpp" + "${LIBRARY_DIR}/Foundation/src/RegularExpression.cpp" + "${LIBRARY_DIR}/Foundation/src/RotateStrategy.cpp" + "${LIBRARY_DIR}/Foundation/src/Runnable.cpp" + "${LIBRARY_DIR}/Foundation/src/RWLock.cpp" + "${LIBRARY_DIR}/Foundation/src/Semaphore.cpp" + "${LIBRARY_DIR}/Foundation/src/SHA1Engine.cpp" + "${LIBRARY_DIR}/Foundation/src/SharedLibrary.cpp" + "${LIBRARY_DIR}/Foundation/src/SharedMemory.cpp" + "${LIBRARY_DIR}/Foundation/src/SignalHandler.cpp" + "${LIBRARY_DIR}/Foundation/src/SimpleFileChannel.cpp" + "${LIBRARY_DIR}/Foundation/src/SortedDirectoryIterator.cpp" + "${LIBRARY_DIR}/Foundation/src/SplitterChannel.cpp" + "${LIBRARY_DIR}/Foundation/src/Stopwatch.cpp" + "${LIBRARY_DIR}/Foundation/src/StreamChannel.cpp" + "${LIBRARY_DIR}/Foundation/src/StreamConverter.cpp" + "${LIBRARY_DIR}/Foundation/src/StreamCopier.cpp" + "${LIBRARY_DIR}/Foundation/src/StreamTokenizer.cpp" + "${LIBRARY_DIR}/Foundation/src/String.cpp" + "${LIBRARY_DIR}/Foundation/src/StringTokenizer.cpp" + "${LIBRARY_DIR}/Foundation/src/SynchronizedObject.cpp" + "${LIBRARY_DIR}/Foundation/src/SyslogChannel.cpp" + "${LIBRARY_DIR}/Foundation/src/Task.cpp" + "${LIBRARY_DIR}/Foundation/src/TaskManager.cpp" + "${LIBRARY_DIR}/Foundation/src/TaskNotification.cpp" + "${LIBRARY_DIR}/Foundation/src/TeeStream.cpp" + "${LIBRARY_DIR}/Foundation/src/TemporaryFile.cpp" + "${LIBRARY_DIR}/Foundation/src/TextBufferIterator.cpp" + "${LIBRARY_DIR}/Foundation/src/TextConverter.cpp" + "${LIBRARY_DIR}/Foundation/src/TextEncoding.cpp" + "${LIBRARY_DIR}/Foundation/src/TextIterator.cpp" + "${LIBRARY_DIR}/Foundation/src/Thread.cpp" + "${LIBRARY_DIR}/Foundation/src/ThreadLocal.cpp" + "${LIBRARY_DIR}/Foundation/src/ThreadPool.cpp" + "${LIBRARY_DIR}/Foundation/src/ThreadTarget.cpp" + "${LIBRARY_DIR}/Foundation/src/TimedNotificationQueue.cpp" + "${LIBRARY_DIR}/Foundation/src/Timer.cpp" + "${LIBRARY_DIR}/Foundation/src/Timespan.cpp" + "${LIBRARY_DIR}/Foundation/src/Timestamp.cpp" + "${LIBRARY_DIR}/Foundation/src/Timezone.cpp" + "${LIBRARY_DIR}/Foundation/src/Token.cpp" + "${LIBRARY_DIR}/Foundation/src/Unicode.cpp" + "${LIBRARY_DIR}/Foundation/src/UnicodeConverter.cpp" + "${LIBRARY_DIR}/Foundation/src/URI.cpp" + "${LIBRARY_DIR}/Foundation/src/URIStreamFactory.cpp" + "${LIBRARY_DIR}/Foundation/src/URIStreamOpener.cpp" + "${LIBRARY_DIR}/Foundation/src/UTF16Encoding.cpp" + "${LIBRARY_DIR}/Foundation/src/UTF32Encoding.cpp" + "${LIBRARY_DIR}/Foundation/src/UTF8Encoding.cpp" + "${LIBRARY_DIR}/Foundation/src/UTF8String.cpp" + "${LIBRARY_DIR}/Foundation/src/UUID.cpp" + "${LIBRARY_DIR}/Foundation/src/UUIDGenerator.cpp" + "${LIBRARY_DIR}/Foundation/src/Var.cpp" + "${LIBRARY_DIR}/Foundation/src/VarHolder.cpp" + "${LIBRARY_DIR}/Foundation/src/VarIterator.cpp" + "${LIBRARY_DIR}/Foundation/src/Void.cpp" + "${LIBRARY_DIR}/Foundation/src/Windows1250Encoding.cpp" + "${LIBRARY_DIR}/Foundation/src/Windows1251Encoding.cpp" + "${LIBRARY_DIR}/Foundation/src/Windows1252Encoding.cpp" +) - add_library (_poco_foundation ${SRCS}) - add_library (Poco::Foundation ALIAS _poco_foundation) +add_library (_poco_foundation ${SRCS}) +add_library (Poco::Foundation ALIAS _poco_foundation) - if (COMPILER_GCC) - target_compile_options (_poco_foundation - PRIVATE - -Wno-suggest-override - ) - elseif (COMPILER_CLANG) - target_compile_options (_poco_foundation - PRIVATE - -Wno-atomic-implicit-seq-cst - -Wno-deprecated - -Wno-extra-semi-stmt - -Wno-zero-as-null-pointer-constant - -Wno-implicit-int-float-conversion - -Wno-thread-safety-analysis - -Wno-thread-safety-negative - ) - endif () +if (COMPILER_GCC) target_compile_options (_poco_foundation PRIVATE - -Wno-sign-compare - -Wno-unused-parameter + -Wno-suggest-override ) - target_compile_definitions (_poco_foundation +elseif (COMPILER_CLANG) + target_compile_options (_poco_foundation PRIVATE - POCO_UNBUNDLED - POCO_UNBUNDLED_ZLIB - PUBLIC - POCO_ENABLE_CPP11 - POCO_OS_FAMILY_UNIX + -Wno-atomic-implicit-seq-cst + -Wno-deprecated + -Wno-extra-semi-stmt + -Wno-zero-as-null-pointer-constant + -Wno-implicit-int-float-conversion + -Wno-thread-safety-analysis + -Wno-thread-safety-negative ) - target_include_directories (_poco_foundation SYSTEM PUBLIC "${LIBRARY_DIR}/Foundation/include") - target_link_libraries (_poco_foundation - PRIVATE - Poco::Foundation::PCRE - ch_contrib::zlib - ch_contrib::lz4) -else () - add_library (Poco::Foundation UNKNOWN IMPORTED GLOBAL) - - find_library (LIBRARY_POCO_FOUNDATION PocoFoundation) - find_path (INCLUDE_POCO_FOUNDATION Poco/Foundation.h) - set_target_properties (Poco::Foundation PROPERTIES IMPORTED_LOCATION ${LIBRARY_POCO_FOUNDATION}) - set_target_properties (Poco::Foundation PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_POCO_FOUNDATION}) - - message (STATUS "Using Poco::Foundation: ${LIBRARY_POCO_FOUNDATION} ${INCLUDE_POCO_FOUNDATION}") endif () +target_compile_options (_poco_foundation + PRIVATE + -Wno-sign-compare + -Wno-unused-parameter +) +target_compile_definitions (_poco_foundation + PRIVATE + POCO_UNBUNDLED + POCO_UNBUNDLED_ZLIB + PUBLIC + POCO_ENABLE_CPP11 + POCO_OS_FAMILY_UNIX +) +target_include_directories (_poco_foundation SYSTEM PUBLIC "${LIBRARY_DIR}/Foundation/include") +target_link_libraries (_poco_foundation + PRIVATE + Poco::Foundation::PCRE + ch_contrib::zlib + ch_contrib::lz4) if(OS_DARWIN AND ARCH_AARCH64) target_compile_definitions (_poco_foundation diff --git a/contrib/poco-cmake/JSON/CMakeLists.txt b/contrib/poco-cmake/JSON/CMakeLists.txt index 7033b800d5d..e138dd046a8 100644 --- a/contrib/poco-cmake/JSON/CMakeLists.txt +++ b/contrib/poco-cmake/JSON/CMakeLists.txt @@ -1,42 +1,31 @@ -if (USE_INTERNAL_POCO_LIBRARY) - # Poco::JSON (pdjson) +# Poco::JSON (pdjson) - set (SRCS_PDJSON - "${LIBRARY_DIR}/JSON/src/pdjson.c" - ) +set (SRCS_PDJSON + "${LIBRARY_DIR}/JSON/src/pdjson.c" +) - add_library (_poco_json_pdjson ${SRCS_PDJSON}) - add_library (Poco::JSON::Pdjson ALIAS _poco_json_pdjson) +add_library (_poco_json_pdjson ${SRCS_PDJSON}) +add_library (Poco::JSON::Pdjson ALIAS _poco_json_pdjson) - # Poco::JSON +# Poco::JSON - set (SRCS - "${LIBRARY_DIR}/JSON/src/Array.cpp" - "${LIBRARY_DIR}/JSON/src/Handler.cpp" - "${LIBRARY_DIR}/JSON/src/JSONException.cpp" - "${LIBRARY_DIR}/JSON/src/Object.cpp" - "${LIBRARY_DIR}/JSON/src/ParseHandler.cpp" - "${LIBRARY_DIR}/JSON/src/Parser.cpp" - "${LIBRARY_DIR}/JSON/src/ParserImpl.cpp" - "${LIBRARY_DIR}/JSON/src/PrintHandler.cpp" - "${LIBRARY_DIR}/JSON/src/Query.cpp" - "${LIBRARY_DIR}/JSON/src/Stringifier.cpp" - "${LIBRARY_DIR}/JSON/src/Template.cpp" - "${LIBRARY_DIR}/JSON/src/TemplateCache.cpp" - ) +set (SRCS + "${LIBRARY_DIR}/JSON/src/Array.cpp" + "${LIBRARY_DIR}/JSON/src/Handler.cpp" + "${LIBRARY_DIR}/JSON/src/JSONException.cpp" + "${LIBRARY_DIR}/JSON/src/Object.cpp" + "${LIBRARY_DIR}/JSON/src/ParseHandler.cpp" + "${LIBRARY_DIR}/JSON/src/Parser.cpp" + "${LIBRARY_DIR}/JSON/src/ParserImpl.cpp" + "${LIBRARY_DIR}/JSON/src/PrintHandler.cpp" + "${LIBRARY_DIR}/JSON/src/Query.cpp" + "${LIBRARY_DIR}/JSON/src/Stringifier.cpp" + "${LIBRARY_DIR}/JSON/src/Template.cpp" + "${LIBRARY_DIR}/JSON/src/TemplateCache.cpp" +) - add_library (_poco_json ${SRCS}) - add_library (Poco::JSON ALIAS _poco_json) +add_library (_poco_json ${SRCS}) +add_library (Poco::JSON ALIAS _poco_json) - target_include_directories (_poco_json SYSTEM PUBLIC "${LIBRARY_DIR}/JSON/include") - target_link_libraries (_poco_json PUBLIC Poco::Foundation Poco::JSON::Pdjson) -else () - add_library (Poco::JSON UNKNOWN IMPORTED GLOBAL) - - find_library (LIBRARY_POCO_JSON PocoJSON) - find_path (INCLUDE_POCO_JSON Poco/JSON/JSON.h) - set_target_properties (Poco::JSON PROPERTIES IMPORTED_LOCATION ${LIBRARY_POCO_JSON}) - set_target_properties (Poco::JSON PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_POCO_JSON}) - - message (STATUS "Using Poco::JSON: ${LIBRARY_POCO_JSON} ${INCLUDE_POCO_JSON}") -endif () +target_include_directories (_poco_json SYSTEM PUBLIC "${LIBRARY_DIR}/JSON/include") +target_link_libraries (_poco_json PUBLIC Poco::Foundation Poco::JSON::Pdjson) diff --git a/contrib/poco-cmake/MongoDB/CMakeLists.txt b/contrib/poco-cmake/MongoDB/CMakeLists.txt index e3dce7ac5cd..fec256b4dcd 100644 --- a/contrib/poco-cmake/MongoDB/CMakeLists.txt +++ b/contrib/poco-cmake/MongoDB/CMakeLists.txt @@ -1,40 +1,29 @@ -if (USE_INTERNAL_POCO_LIBRARY) - set (SRCS - "${LIBRARY_DIR}/MongoDB/src/Array.cpp" - "${LIBRARY_DIR}/MongoDB/src/Binary.cpp" - "${LIBRARY_DIR}/MongoDB/src/Connection.cpp" - "${LIBRARY_DIR}/MongoDB/src/Cursor.cpp" - "${LIBRARY_DIR}/MongoDB/src/Database.cpp" - "${LIBRARY_DIR}/MongoDB/src/DeleteRequest.cpp" - "${LIBRARY_DIR}/MongoDB/src/Document.cpp" - "${LIBRARY_DIR}/MongoDB/src/Element.cpp" - "${LIBRARY_DIR}/MongoDB/src/GetMoreRequest.cpp" - "${LIBRARY_DIR}/MongoDB/src/InsertRequest.cpp" - "${LIBRARY_DIR}/MongoDB/src/JavaScriptCode.cpp" - "${LIBRARY_DIR}/MongoDB/src/KillCursorsRequest.cpp" - "${LIBRARY_DIR}/MongoDB/src/Message.cpp" - "${LIBRARY_DIR}/MongoDB/src/MessageHeader.cpp" - "${LIBRARY_DIR}/MongoDB/src/ObjectId.cpp" - "${LIBRARY_DIR}/MongoDB/src/QueryRequest.cpp" - "${LIBRARY_DIR}/MongoDB/src/RegularExpression.cpp" - "${LIBRARY_DIR}/MongoDB/src/ReplicaSet.cpp" - "${LIBRARY_DIR}/MongoDB/src/RequestMessage.cpp" - "${LIBRARY_DIR}/MongoDB/src/ResponseMessage.cpp" - "${LIBRARY_DIR}/MongoDB/src/UpdateRequest.cpp" - ) +set (SRCS + "${LIBRARY_DIR}/MongoDB/src/Array.cpp" + "${LIBRARY_DIR}/MongoDB/src/Binary.cpp" + "${LIBRARY_DIR}/MongoDB/src/Connection.cpp" + "${LIBRARY_DIR}/MongoDB/src/Cursor.cpp" + "${LIBRARY_DIR}/MongoDB/src/Database.cpp" + "${LIBRARY_DIR}/MongoDB/src/DeleteRequest.cpp" + "${LIBRARY_DIR}/MongoDB/src/Document.cpp" + "${LIBRARY_DIR}/MongoDB/src/Element.cpp" + "${LIBRARY_DIR}/MongoDB/src/GetMoreRequest.cpp" + "${LIBRARY_DIR}/MongoDB/src/InsertRequest.cpp" + "${LIBRARY_DIR}/MongoDB/src/JavaScriptCode.cpp" + "${LIBRARY_DIR}/MongoDB/src/KillCursorsRequest.cpp" + "${LIBRARY_DIR}/MongoDB/src/Message.cpp" + "${LIBRARY_DIR}/MongoDB/src/MessageHeader.cpp" + "${LIBRARY_DIR}/MongoDB/src/ObjectId.cpp" + "${LIBRARY_DIR}/MongoDB/src/QueryRequest.cpp" + "${LIBRARY_DIR}/MongoDB/src/RegularExpression.cpp" + "${LIBRARY_DIR}/MongoDB/src/ReplicaSet.cpp" + "${LIBRARY_DIR}/MongoDB/src/RequestMessage.cpp" + "${LIBRARY_DIR}/MongoDB/src/ResponseMessage.cpp" + "${LIBRARY_DIR}/MongoDB/src/UpdateRequest.cpp" +) - add_library (_poco_mongodb ${SRCS}) - add_library (Poco::MongoDB ALIAS _poco_mongodb) +add_library (_poco_mongodb ${SRCS}) +add_library (Poco::MongoDB ALIAS _poco_mongodb) - target_include_directories (_poco_mongodb SYSTEM PUBLIC "${LIBRARY_DIR}/MongoDB/include") - target_link_libraries (_poco_mongodb PUBLIC Poco::Net) -else () - add_library (Poco::MongoDB UNKNOWN IMPORTED GLOBAL) - - find_library (LIBRARY_POCO_MONGODB PocoMongoDB) - find_path (INCLUDE_POCO_MONGODB Poco/MongoDB/MongoDB.h) - set_target_properties (Poco::MongoDB PROPERTIES IMPORTED_LOCATION ${LIBRARY_POCO_MONGODB}) - set_target_properties (Poco::MongoDB PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_POCO_MONGODB}) - - message (STATUS "Using Poco::MongoDB: ${LIBRARY_POCO_MONGODB} ${INCLUDE_POCO_MONGODB}") -endif () +target_include_directories (_poco_mongodb SYSTEM PUBLIC "${LIBRARY_DIR}/MongoDB/include") +target_link_libraries (_poco_mongodb PUBLIC Poco::Net) diff --git a/contrib/poco-cmake/Net/CMakeLists.txt b/contrib/poco-cmake/Net/CMakeLists.txt index 45989af8d45..30ff799ccfc 100644 --- a/contrib/poco-cmake/Net/CMakeLists.txt +++ b/contrib/poco-cmake/Net/CMakeLists.txt @@ -1,139 +1,128 @@ -if (USE_INTERNAL_POCO_LIBRARY) - set (SRCS - "${LIBRARY_DIR}/Net/src/AbstractHTTPRequestHandler.cpp" - "${LIBRARY_DIR}/Net/src/DatagramSocket.cpp" - "${LIBRARY_DIR}/Net/src/DatagramSocketImpl.cpp" - "${LIBRARY_DIR}/Net/src/DialogSocket.cpp" - "${LIBRARY_DIR}/Net/src/DNS.cpp" - "${LIBRARY_DIR}/Net/src/FilePartSource.cpp" - "${LIBRARY_DIR}/Net/src/FTPClientSession.cpp" - "${LIBRARY_DIR}/Net/src/FTPStreamFactory.cpp" - "${LIBRARY_DIR}/Net/src/HostEntry.cpp" - "${LIBRARY_DIR}/Net/src/HTMLForm.cpp" - "${LIBRARY_DIR}/Net/src/HTTPAuthenticationParams.cpp" - "${LIBRARY_DIR}/Net/src/HTTPBasicCredentials.cpp" - "${LIBRARY_DIR}/Net/src/HTTPBufferAllocator.cpp" - "${LIBRARY_DIR}/Net/src/HTTPChunkedStream.cpp" - "${LIBRARY_DIR}/Net/src/HTTPClientSession.cpp" - "${LIBRARY_DIR}/Net/src/HTTPCookie.cpp" - "${LIBRARY_DIR}/Net/src/HTTPCredentials.cpp" - "${LIBRARY_DIR}/Net/src/HTTPDigestCredentials.cpp" - "${LIBRARY_DIR}/Net/src/HTTPFixedLengthStream.cpp" - "${LIBRARY_DIR}/Net/src/HTTPHeaderStream.cpp" - "${LIBRARY_DIR}/Net/src/HTTPIOStream.cpp" - "${LIBRARY_DIR}/Net/src/HTTPMessage.cpp" - "${LIBRARY_DIR}/Net/src/HTTPRequest.cpp" - "${LIBRARY_DIR}/Net/src/HTTPRequestHandler.cpp" - "${LIBRARY_DIR}/Net/src/HTTPRequestHandlerFactory.cpp" - "${LIBRARY_DIR}/Net/src/HTTPResponse.cpp" - "${LIBRARY_DIR}/Net/src/HTTPServer.cpp" - "${LIBRARY_DIR}/Net/src/HTTPServerConnection.cpp" - "${LIBRARY_DIR}/Net/src/HTTPServerConnectionFactory.cpp" - "${LIBRARY_DIR}/Net/src/HTTPServerParams.cpp" - "${LIBRARY_DIR}/Net/src/HTTPServerRequest.cpp" - "${LIBRARY_DIR}/Net/src/HTTPServerRequestImpl.cpp" - "${LIBRARY_DIR}/Net/src/HTTPServerResponse.cpp" - "${LIBRARY_DIR}/Net/src/HTTPServerResponseImpl.cpp" - "${LIBRARY_DIR}/Net/src/HTTPServerSession.cpp" - "${LIBRARY_DIR}/Net/src/HTTPSession.cpp" - "${LIBRARY_DIR}/Net/src/HTTPSessionFactory.cpp" - "${LIBRARY_DIR}/Net/src/HTTPSessionInstantiator.cpp" - "${LIBRARY_DIR}/Net/src/HTTPStream.cpp" - "${LIBRARY_DIR}/Net/src/HTTPStreamFactory.cpp" - "${LIBRARY_DIR}/Net/src/ICMPClient.cpp" - "${LIBRARY_DIR}/Net/src/ICMPEventArgs.cpp" - "${LIBRARY_DIR}/Net/src/ICMPPacket.cpp" - "${LIBRARY_DIR}/Net/src/ICMPPacketImpl.cpp" - "${LIBRARY_DIR}/Net/src/ICMPSocket.cpp" - "${LIBRARY_DIR}/Net/src/ICMPSocketImpl.cpp" - "${LIBRARY_DIR}/Net/src/ICMPv4PacketImpl.cpp" - "${LIBRARY_DIR}/Net/src/IPAddress.cpp" - "${LIBRARY_DIR}/Net/src/IPAddressImpl.cpp" - "${LIBRARY_DIR}/Net/src/MailMessage.cpp" - "${LIBRARY_DIR}/Net/src/MailRecipient.cpp" - "${LIBRARY_DIR}/Net/src/MailStream.cpp" - "${LIBRARY_DIR}/Net/src/MediaType.cpp" - "${LIBRARY_DIR}/Net/src/MessageHeader.cpp" - "${LIBRARY_DIR}/Net/src/MulticastSocket.cpp" - "${LIBRARY_DIR}/Net/src/MultipartReader.cpp" - "${LIBRARY_DIR}/Net/src/MultipartWriter.cpp" - "${LIBRARY_DIR}/Net/src/NameValueCollection.cpp" - "${LIBRARY_DIR}/Net/src/Net.cpp" - "${LIBRARY_DIR}/Net/src/NetException.cpp" - "${LIBRARY_DIR}/Net/src/NetworkInterface.cpp" - "${LIBRARY_DIR}/Net/src/NTPClient.cpp" - "${LIBRARY_DIR}/Net/src/NTPEventArgs.cpp" - "${LIBRARY_DIR}/Net/src/NTPPacket.cpp" - "${LIBRARY_DIR}/Net/src/NullPartHandler.cpp" - "${LIBRARY_DIR}/Net/src/OAuth10Credentials.cpp" - "${LIBRARY_DIR}/Net/src/OAuth20Credentials.cpp" - "${LIBRARY_DIR}/Net/src/PartHandler.cpp" - "${LIBRARY_DIR}/Net/src/PartSource.cpp" - "${LIBRARY_DIR}/Net/src/PartStore.cpp" - "${LIBRARY_DIR}/Net/src/PollSet.cpp" - "${LIBRARY_DIR}/Net/src/POP3ClientSession.cpp" - "${LIBRARY_DIR}/Net/src/QuotedPrintableDecoder.cpp" - "${LIBRARY_DIR}/Net/src/QuotedPrintableEncoder.cpp" - "${LIBRARY_DIR}/Net/src/RawSocket.cpp" - "${LIBRARY_DIR}/Net/src/RawSocketImpl.cpp" - "${LIBRARY_DIR}/Net/src/RemoteSyslogChannel.cpp" - "${LIBRARY_DIR}/Net/src/RemoteSyslogListener.cpp" - "${LIBRARY_DIR}/Net/src/ServerSocket.cpp" - "${LIBRARY_DIR}/Net/src/ServerSocketImpl.cpp" - "${LIBRARY_DIR}/Net/src/SMTPChannel.cpp" - "${LIBRARY_DIR}/Net/src/SMTPClientSession.cpp" - "${LIBRARY_DIR}/Net/src/Socket.cpp" - "${LIBRARY_DIR}/Net/src/SocketAddress.cpp" - "${LIBRARY_DIR}/Net/src/SocketAddressImpl.cpp" - "${LIBRARY_DIR}/Net/src/SocketImpl.cpp" - "${LIBRARY_DIR}/Net/src/SocketNotification.cpp" - "${LIBRARY_DIR}/Net/src/SocketNotifier.cpp" - "${LIBRARY_DIR}/Net/src/SocketReactor.cpp" - "${LIBRARY_DIR}/Net/src/SocketStream.cpp" - "${LIBRARY_DIR}/Net/src/StreamSocket.cpp" - "${LIBRARY_DIR}/Net/src/StreamSocketImpl.cpp" - "${LIBRARY_DIR}/Net/src/StringPartSource.cpp" - "${LIBRARY_DIR}/Net/src/TCPServer.cpp" - "${LIBRARY_DIR}/Net/src/TCPServerConnection.cpp" - "${LIBRARY_DIR}/Net/src/TCPServerConnectionFactory.cpp" - "${LIBRARY_DIR}/Net/src/TCPServerDispatcher.cpp" - "${LIBRARY_DIR}/Net/src/TCPServerParams.cpp" - "${LIBRARY_DIR}/Net/src/WebSocket.cpp" - "${LIBRARY_DIR}/Net/src/WebSocketImpl.cpp" - ) +set (SRCS + "${LIBRARY_DIR}/Net/src/AbstractHTTPRequestHandler.cpp" + "${LIBRARY_DIR}/Net/src/DatagramSocket.cpp" + "${LIBRARY_DIR}/Net/src/DatagramSocketImpl.cpp" + "${LIBRARY_DIR}/Net/src/DialogSocket.cpp" + "${LIBRARY_DIR}/Net/src/DNS.cpp" + "${LIBRARY_DIR}/Net/src/FilePartSource.cpp" + "${LIBRARY_DIR}/Net/src/FTPClientSession.cpp" + "${LIBRARY_DIR}/Net/src/FTPStreamFactory.cpp" + "${LIBRARY_DIR}/Net/src/HostEntry.cpp" + "${LIBRARY_DIR}/Net/src/HTMLForm.cpp" + "${LIBRARY_DIR}/Net/src/HTTPAuthenticationParams.cpp" + "${LIBRARY_DIR}/Net/src/HTTPBasicCredentials.cpp" + "${LIBRARY_DIR}/Net/src/HTTPBufferAllocator.cpp" + "${LIBRARY_DIR}/Net/src/HTTPChunkedStream.cpp" + "${LIBRARY_DIR}/Net/src/HTTPClientSession.cpp" + "${LIBRARY_DIR}/Net/src/HTTPCookie.cpp" + "${LIBRARY_DIR}/Net/src/HTTPCredentials.cpp" + "${LIBRARY_DIR}/Net/src/HTTPDigestCredentials.cpp" + "${LIBRARY_DIR}/Net/src/HTTPFixedLengthStream.cpp" + "${LIBRARY_DIR}/Net/src/HTTPHeaderStream.cpp" + "${LIBRARY_DIR}/Net/src/HTTPIOStream.cpp" + "${LIBRARY_DIR}/Net/src/HTTPMessage.cpp" + "${LIBRARY_DIR}/Net/src/HTTPRequest.cpp" + "${LIBRARY_DIR}/Net/src/HTTPRequestHandler.cpp" + "${LIBRARY_DIR}/Net/src/HTTPRequestHandlerFactory.cpp" + "${LIBRARY_DIR}/Net/src/HTTPResponse.cpp" + "${LIBRARY_DIR}/Net/src/HTTPServer.cpp" + "${LIBRARY_DIR}/Net/src/HTTPServerConnection.cpp" + "${LIBRARY_DIR}/Net/src/HTTPServerConnectionFactory.cpp" + "${LIBRARY_DIR}/Net/src/HTTPServerParams.cpp" + "${LIBRARY_DIR}/Net/src/HTTPServerRequest.cpp" + "${LIBRARY_DIR}/Net/src/HTTPServerRequestImpl.cpp" + "${LIBRARY_DIR}/Net/src/HTTPServerResponse.cpp" + "${LIBRARY_DIR}/Net/src/HTTPServerResponseImpl.cpp" + "${LIBRARY_DIR}/Net/src/HTTPServerSession.cpp" + "${LIBRARY_DIR}/Net/src/HTTPSession.cpp" + "${LIBRARY_DIR}/Net/src/HTTPSessionFactory.cpp" + "${LIBRARY_DIR}/Net/src/HTTPSessionInstantiator.cpp" + "${LIBRARY_DIR}/Net/src/HTTPStream.cpp" + "${LIBRARY_DIR}/Net/src/HTTPStreamFactory.cpp" + "${LIBRARY_DIR}/Net/src/ICMPClient.cpp" + "${LIBRARY_DIR}/Net/src/ICMPEventArgs.cpp" + "${LIBRARY_DIR}/Net/src/ICMPPacket.cpp" + "${LIBRARY_DIR}/Net/src/ICMPPacketImpl.cpp" + "${LIBRARY_DIR}/Net/src/ICMPSocket.cpp" + "${LIBRARY_DIR}/Net/src/ICMPSocketImpl.cpp" + "${LIBRARY_DIR}/Net/src/ICMPv4PacketImpl.cpp" + "${LIBRARY_DIR}/Net/src/IPAddress.cpp" + "${LIBRARY_DIR}/Net/src/IPAddressImpl.cpp" + "${LIBRARY_DIR}/Net/src/MailMessage.cpp" + "${LIBRARY_DIR}/Net/src/MailRecipient.cpp" + "${LIBRARY_DIR}/Net/src/MailStream.cpp" + "${LIBRARY_DIR}/Net/src/MediaType.cpp" + "${LIBRARY_DIR}/Net/src/MessageHeader.cpp" + "${LIBRARY_DIR}/Net/src/MulticastSocket.cpp" + "${LIBRARY_DIR}/Net/src/MultipartReader.cpp" + "${LIBRARY_DIR}/Net/src/MultipartWriter.cpp" + "${LIBRARY_DIR}/Net/src/NameValueCollection.cpp" + "${LIBRARY_DIR}/Net/src/Net.cpp" + "${LIBRARY_DIR}/Net/src/NetException.cpp" + "${LIBRARY_DIR}/Net/src/NetworkInterface.cpp" + "${LIBRARY_DIR}/Net/src/NTPClient.cpp" + "${LIBRARY_DIR}/Net/src/NTPEventArgs.cpp" + "${LIBRARY_DIR}/Net/src/NTPPacket.cpp" + "${LIBRARY_DIR}/Net/src/NullPartHandler.cpp" + "${LIBRARY_DIR}/Net/src/OAuth10Credentials.cpp" + "${LIBRARY_DIR}/Net/src/OAuth20Credentials.cpp" + "${LIBRARY_DIR}/Net/src/PartHandler.cpp" + "${LIBRARY_DIR}/Net/src/PartSource.cpp" + "${LIBRARY_DIR}/Net/src/PartStore.cpp" + "${LIBRARY_DIR}/Net/src/PollSet.cpp" + "${LIBRARY_DIR}/Net/src/POP3ClientSession.cpp" + "${LIBRARY_DIR}/Net/src/QuotedPrintableDecoder.cpp" + "${LIBRARY_DIR}/Net/src/QuotedPrintableEncoder.cpp" + "${LIBRARY_DIR}/Net/src/RawSocket.cpp" + "${LIBRARY_DIR}/Net/src/RawSocketImpl.cpp" + "${LIBRARY_DIR}/Net/src/RemoteSyslogChannel.cpp" + "${LIBRARY_DIR}/Net/src/RemoteSyslogListener.cpp" + "${LIBRARY_DIR}/Net/src/ServerSocket.cpp" + "${LIBRARY_DIR}/Net/src/ServerSocketImpl.cpp" + "${LIBRARY_DIR}/Net/src/SMTPChannel.cpp" + "${LIBRARY_DIR}/Net/src/SMTPClientSession.cpp" + "${LIBRARY_DIR}/Net/src/Socket.cpp" + "${LIBRARY_DIR}/Net/src/SocketAddress.cpp" + "${LIBRARY_DIR}/Net/src/SocketAddressImpl.cpp" + "${LIBRARY_DIR}/Net/src/SocketImpl.cpp" + "${LIBRARY_DIR}/Net/src/SocketNotification.cpp" + "${LIBRARY_DIR}/Net/src/SocketNotifier.cpp" + "${LIBRARY_DIR}/Net/src/SocketReactor.cpp" + "${LIBRARY_DIR}/Net/src/SocketStream.cpp" + "${LIBRARY_DIR}/Net/src/StreamSocket.cpp" + "${LIBRARY_DIR}/Net/src/StreamSocketImpl.cpp" + "${LIBRARY_DIR}/Net/src/StringPartSource.cpp" + "${LIBRARY_DIR}/Net/src/TCPServer.cpp" + "${LIBRARY_DIR}/Net/src/TCPServerConnection.cpp" + "${LIBRARY_DIR}/Net/src/TCPServerConnectionFactory.cpp" + "${LIBRARY_DIR}/Net/src/TCPServerDispatcher.cpp" + "${LIBRARY_DIR}/Net/src/TCPServerParams.cpp" + "${LIBRARY_DIR}/Net/src/WebSocket.cpp" + "${LIBRARY_DIR}/Net/src/WebSocketImpl.cpp" +) - add_library (_poco_net ${SRCS}) - add_library (Poco::Net ALIAS _poco_net) +add_library (_poco_net ${SRCS}) +add_library (Poco::Net ALIAS _poco_net) - if (OS_LINUX) - target_compile_definitions (_poco_net PUBLIC POCO_HAVE_FD_EPOLL) - elseif (OS_DARWIN OR OS_FREEBSD) - target_compile_definitions (_poco_net PUBLIC POCO_HAVE_FD_POLL) - endif () +if (OS_LINUX) + target_compile_definitions (_poco_net PUBLIC POCO_HAVE_FD_EPOLL) +elseif (OS_DARWIN OR OS_FREEBSD) + target_compile_definitions (_poco_net PUBLIC POCO_HAVE_FD_POLL) +endif () - if (COMPILER_CLANG) - # clang-specific warnings - target_compile_options (_poco_net - PRIVATE - -Wno-atomic-implicit-seq-cst - -Wno-extra-semi-stmt - -Wno-extra-semi - ) - endif () +if (COMPILER_CLANG) + # clang-specific warnings target_compile_options (_poco_net PRIVATE - -Wno-deprecated + -Wno-atomic-implicit-seq-cst + -Wno-extra-semi-stmt -Wno-extra-semi ) - target_include_directories (_poco_net SYSTEM PUBLIC "${LIBRARY_DIR}/Net/include") - target_link_libraries (_poco_net PUBLIC Poco::Foundation) -else () - add_library (Poco::Net UNKNOWN IMPORTED GLOBAL) - - find_library (LIBRARY_POCO_NET PocoNet) - find_path (INCLUDE_POCO_NET Poco/Net/Net.h) - set_target_properties (Poco::Net PROPERTIES IMPORTED_LOCATION ${LIBRARY_POCO_NET}) - set_target_properties (Poco::Net PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_POCO_NET}) - - message (STATUS "Using Poco::Net: ${LIBRARY_POCO_NET} ${INCLUDE_POCO_NET}") endif () +target_compile_options (_poco_net + PRIVATE + -Wno-deprecated + -Wno-extra-semi +) +target_include_directories (_poco_net SYSTEM PUBLIC "${LIBRARY_DIR}/Net/include") +target_link_libraries (_poco_net PUBLIC Poco::Foundation) diff --git a/contrib/poco-cmake/Net/SSL/CMakeLists.txt b/contrib/poco-cmake/Net/SSL/CMakeLists.txt index 4b3adacfb8f..de2bb624a8b 100644 --- a/contrib/poco-cmake/Net/SSL/CMakeLists.txt +++ b/contrib/poco-cmake/Net/SSL/CMakeLists.txt @@ -1,50 +1,39 @@ if (ENABLE_SSL) - if (USE_INTERNAL_POCO_LIBRARY) - set (SRCS - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/AcceptCertificateHandler.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/CertificateHandlerFactory.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/CertificateHandlerFactoryMgr.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/ConsoleCertificateHandler.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/Context.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/HTTPSClientSession.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/HTTPSSessionInstantiator.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/HTTPSStreamFactory.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/InvalidCertificateHandler.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/KeyConsoleHandler.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/KeyFileHandler.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/PrivateKeyFactory.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/PrivateKeyFactoryMgr.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/PrivateKeyPassphraseHandler.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/RejectCertificateHandler.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/SecureServerSocket.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/SecureServerSocketImpl.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/SecureSMTPClientSession.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/SecureSocketImpl.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/SecureStreamSocket.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/SecureStreamSocketImpl.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/Session.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/SSLException.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/SSLManager.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/Utility.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/VerificationErrorArgs.cpp" - "${LIBRARY_DIR}/NetSSL_OpenSSL/src/X509Certificate.cpp" - ) + set (SRCS + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/AcceptCertificateHandler.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/CertificateHandlerFactory.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/CertificateHandlerFactoryMgr.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/ConsoleCertificateHandler.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/Context.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/HTTPSClientSession.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/HTTPSSessionInstantiator.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/HTTPSStreamFactory.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/InvalidCertificateHandler.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/KeyConsoleHandler.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/KeyFileHandler.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/PrivateKeyFactory.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/PrivateKeyFactoryMgr.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/PrivateKeyPassphraseHandler.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/RejectCertificateHandler.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/SecureServerSocket.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/SecureServerSocketImpl.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/SecureSMTPClientSession.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/SecureSocketImpl.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/SecureStreamSocket.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/SecureStreamSocketImpl.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/Session.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/SSLException.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/SSLManager.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/Utility.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/VerificationErrorArgs.cpp" + "${LIBRARY_DIR}/NetSSL_OpenSSL/src/X509Certificate.cpp" + ) - add_library (_poco_net_ssl ${SRCS}) - add_library (Poco::Net::SSL ALIAS _poco_net_ssl) + add_library (_poco_net_ssl ${SRCS}) + add_library (Poco::Net::SSL ALIAS _poco_net_ssl) - target_include_directories (_poco_net_ssl SYSTEM PUBLIC "${LIBRARY_DIR}/NetSSL_OpenSSL/include") - target_link_libraries (_poco_net_ssl PUBLIC Poco::Crypto Poco::Net Poco::Util) - else () - add_library (Poco::Net::SSL UNKNOWN IMPORTED GLOBAL) - - find_library (LIBRARY_POCO_NET_SSL PocoNetSSL) - find_path (INCLUDE_POCO_NET_SSL Poco/Net/NetSSL.h) - set_target_properties (Poco::Net::SSL PROPERTIES IMPORTED_LOCATION ${LIBRARY_POCO_NET_SSL}) - set_target_properties (Poco::Net::SSL PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_POCO_NET_SSL}) - - message (STATUS "Using Poco::Net::SSL: ${LIBRARY_POCO_NET_SSL} ${INCLUDE_POCO_NET_SSL}") - endif () + target_include_directories (_poco_net_ssl SYSTEM PUBLIC "${LIBRARY_DIR}/NetSSL_OpenSSL/include") + target_link_libraries (_poco_net_ssl PUBLIC Poco::Crypto Poco::Net Poco::Util) else () add_library (_poco_net_ssl INTERFACE) add_library (Poco::Net::SSL ALIAS _poco_net_ssl) diff --git a/contrib/poco-cmake/Redis/CMakeLists.txt b/contrib/poco-cmake/Redis/CMakeLists.txt index b5892addd85..98e86a8592b 100644 --- a/contrib/poco-cmake/Redis/CMakeLists.txt +++ b/contrib/poco-cmake/Redis/CMakeLists.txt @@ -1,34 +1,21 @@ -if (USE_INTERNAL_POCO_LIBRARY) - set (SRCS - "${LIBRARY_DIR}/Redis/src/Array.cpp" - "${LIBRARY_DIR}/Redis/src/AsyncReader.cpp" - "${LIBRARY_DIR}/Redis/src/Client.cpp" - "${LIBRARY_DIR}/Redis/src/Command.cpp" - "${LIBRARY_DIR}/Redis/src/Error.cpp" - "${LIBRARY_DIR}/Redis/src/Exception.cpp" - "${LIBRARY_DIR}/Redis/src/RedisEventArgs.cpp" - "${LIBRARY_DIR}/Redis/src/RedisStream.cpp" - "${LIBRARY_DIR}/Redis/src/Type.cpp" - ) +set (SRCS + "${LIBRARY_DIR}/Redis/src/Array.cpp" + "${LIBRARY_DIR}/Redis/src/AsyncReader.cpp" + "${LIBRARY_DIR}/Redis/src/Client.cpp" + "${LIBRARY_DIR}/Redis/src/Command.cpp" + "${LIBRARY_DIR}/Redis/src/Error.cpp" + "${LIBRARY_DIR}/Redis/src/Exception.cpp" + "${LIBRARY_DIR}/Redis/src/RedisEventArgs.cpp" + "${LIBRARY_DIR}/Redis/src/RedisStream.cpp" + "${LIBRARY_DIR}/Redis/src/Type.cpp" +) - add_library (_poco_redis ${SRCS}) - add_library (Poco::Redis ALIAS _poco_redis) +add_library (_poco_redis ${SRCS}) +add_library (Poco::Redis ALIAS _poco_redis) - if (COMPILER_GCC) - target_compile_options (_poco_redis PRIVATE -Wno-deprecated-copy) - endif () - target_compile_options (_poco_redis PRIVATE -Wno-shadow) - target_include_directories (_poco_redis SYSTEM PUBLIC "${LIBRARY_DIR}/Redis/include") - target_link_libraries (_poco_redis PUBLIC Poco::Net) -else () - add_library (Poco::Redis UNKNOWN IMPORTED GLOBAL) - - find_library (LIBRARY_POCO_REDIS PocoRedis) - find_path (INCLUDE_POCO_REDIS Poco/Redis/Redis.h) - set_target_properties (Poco::Redis PROPERTIES IMPORTED_LOCATION ${LIBRARY_POCO_REDIS}) - set_target_properties (Poco::Redis PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_POCO_REDIS}) - - target_link_libraries (Poco::Redis INTERFACE Poco::Net) - - message (STATUS "Using Poco::Redis: ${LIBRARY_POCO_REDIS} ${INCLUDE_POCO_REDIS}") +if (COMPILER_GCC) + target_compile_options (_poco_redis PRIVATE -Wno-deprecated-copy) endif () +target_compile_options (_poco_redis PRIVATE -Wno-shadow) +target_include_directories (_poco_redis SYSTEM PUBLIC "${LIBRARY_DIR}/Redis/include") +target_link_libraries (_poco_redis PUBLIC Poco::Net) diff --git a/contrib/poco-cmake/Util/CMakeLists.txt b/contrib/poco-cmake/Util/CMakeLists.txt index e233e65cfea..dc355e47658 100644 --- a/contrib/poco-cmake/Util/CMakeLists.txt +++ b/contrib/poco-cmake/Util/CMakeLists.txt @@ -1,46 +1,35 @@ -if (USE_INTERNAL_POCO_LIBRARY) - set (SRCS - "${LIBRARY_DIR}/Util/src/AbstractConfiguration.cpp" - "${LIBRARY_DIR}/Util/src/Application.cpp" - "${LIBRARY_DIR}/Util/src/ConfigurationMapper.cpp" - "${LIBRARY_DIR}/Util/src/ConfigurationView.cpp" - "${LIBRARY_DIR}/Util/src/FilesystemConfiguration.cpp" - "${LIBRARY_DIR}/Util/src/HelpFormatter.cpp" - "${LIBRARY_DIR}/Util/src/IniFileConfiguration.cpp" - "${LIBRARY_DIR}/Util/src/IntValidator.cpp" - "${LIBRARY_DIR}/Util/src/JSONConfiguration.cpp" - "${LIBRARY_DIR}/Util/src/LayeredConfiguration.cpp" - "${LIBRARY_DIR}/Util/src/LoggingConfigurator.cpp" - "${LIBRARY_DIR}/Util/src/LoggingSubsystem.cpp" - "${LIBRARY_DIR}/Util/src/MapConfiguration.cpp" - "${LIBRARY_DIR}/Util/src/Option.cpp" - "${LIBRARY_DIR}/Util/src/OptionCallback.cpp" - "${LIBRARY_DIR}/Util/src/OptionException.cpp" - "${LIBRARY_DIR}/Util/src/OptionProcessor.cpp" - "${LIBRARY_DIR}/Util/src/OptionSet.cpp" - "${LIBRARY_DIR}/Util/src/PropertyFileConfiguration.cpp" - "${LIBRARY_DIR}/Util/src/RegExpValidator.cpp" - "${LIBRARY_DIR}/Util/src/ServerApplication.cpp" - "${LIBRARY_DIR}/Util/src/Subsystem.cpp" - "${LIBRARY_DIR}/Util/src/SystemConfiguration.cpp" - "${LIBRARY_DIR}/Util/src/Timer.cpp" - "${LIBRARY_DIR}/Util/src/TimerTask.cpp" - "${LIBRARY_DIR}/Util/src/Validator.cpp" - "${LIBRARY_DIR}/Util/src/XMLConfiguration.cpp" - ) +set (SRCS + "${LIBRARY_DIR}/Util/src/AbstractConfiguration.cpp" + "${LIBRARY_DIR}/Util/src/Application.cpp" + "${LIBRARY_DIR}/Util/src/ConfigurationMapper.cpp" + "${LIBRARY_DIR}/Util/src/ConfigurationView.cpp" + "${LIBRARY_DIR}/Util/src/FilesystemConfiguration.cpp" + "${LIBRARY_DIR}/Util/src/HelpFormatter.cpp" + "${LIBRARY_DIR}/Util/src/IniFileConfiguration.cpp" + "${LIBRARY_DIR}/Util/src/IntValidator.cpp" + "${LIBRARY_DIR}/Util/src/JSONConfiguration.cpp" + "${LIBRARY_DIR}/Util/src/LayeredConfiguration.cpp" + "${LIBRARY_DIR}/Util/src/LoggingConfigurator.cpp" + "${LIBRARY_DIR}/Util/src/LoggingSubsystem.cpp" + "${LIBRARY_DIR}/Util/src/MapConfiguration.cpp" + "${LIBRARY_DIR}/Util/src/Option.cpp" + "${LIBRARY_DIR}/Util/src/OptionCallback.cpp" + "${LIBRARY_DIR}/Util/src/OptionException.cpp" + "${LIBRARY_DIR}/Util/src/OptionProcessor.cpp" + "${LIBRARY_DIR}/Util/src/OptionSet.cpp" + "${LIBRARY_DIR}/Util/src/PropertyFileConfiguration.cpp" + "${LIBRARY_DIR}/Util/src/RegExpValidator.cpp" + "${LIBRARY_DIR}/Util/src/ServerApplication.cpp" + "${LIBRARY_DIR}/Util/src/Subsystem.cpp" + "${LIBRARY_DIR}/Util/src/SystemConfiguration.cpp" + "${LIBRARY_DIR}/Util/src/Timer.cpp" + "${LIBRARY_DIR}/Util/src/TimerTask.cpp" + "${LIBRARY_DIR}/Util/src/Validator.cpp" + "${LIBRARY_DIR}/Util/src/XMLConfiguration.cpp" +) - add_library (_poco_util ${SRCS}) - add_library (Poco::Util ALIAS _poco_util) +add_library (_poco_util ${SRCS}) +add_library (Poco::Util ALIAS _poco_util) - target_include_directories (_poco_util SYSTEM PUBLIC "${LIBRARY_DIR}/Util/include") - target_link_libraries (_poco_util PUBLIC Poco::JSON Poco::XML) -else () - add_library (Poco::Util UNKNOWN IMPORTED GLOBAL) - - find_library (LIBRARY_POCO_UTIL PocoUtil) - find_path (INCLUDE_POCO_UTIL Poco/Util/Util.h) - set_target_properties (Poco::Util PROPERTIES IMPORTED_LOCATION ${LIBRARY_POCO_UTIL}) - set_target_properties (Poco::Util PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_POCO_UTIL}) - - message (STATUS "Using Poco::Util: ${LIBRARY_POCO_UTIL} ${INCLUDE_POCO_UTIL}") -endif () +target_include_directories (_poco_util SYSTEM PUBLIC "${LIBRARY_DIR}/Util/include") +target_link_libraries (_poco_util PUBLIC Poco::JSON Poco::XML) diff --git a/contrib/poco-cmake/XML/CMakeLists.txt b/contrib/poco-cmake/XML/CMakeLists.txt index af801a65f03..45100f11eb7 100644 --- a/contrib/poco-cmake/XML/CMakeLists.txt +++ b/contrib/poco-cmake/XML/CMakeLists.txt @@ -1,110 +1,99 @@ -if (USE_INTERNAL_POCO_LIBRARY) - # Poco::XML (expat) +# Poco::XML (expat) - set (SRCS_EXPAT - "${LIBRARY_DIR}/XML/src/xmlrole.c" - "${LIBRARY_DIR}/XML/src/xmltok_impl.c" - "${LIBRARY_DIR}/XML/src/xmltok_ns.c" - "${LIBRARY_DIR}/XML/src/xmltok.c" - ) +set (SRCS_EXPAT + "${LIBRARY_DIR}/XML/src/xmlrole.c" + "${LIBRARY_DIR}/XML/src/xmltok_impl.c" + "${LIBRARY_DIR}/XML/src/xmltok_ns.c" + "${LIBRARY_DIR}/XML/src/xmltok.c" +) - add_library (_poco_xml_expat ${SRCS_EXPAT}) - add_library (Poco::XML::Expat ALIAS _poco_xml_expat) +add_library (_poco_xml_expat ${SRCS_EXPAT}) +add_library (Poco::XML::Expat ALIAS _poco_xml_expat) - target_include_directories (_poco_xml_expat PUBLIC "${LIBRARY_DIR}/XML/include") +target_include_directories (_poco_xml_expat PUBLIC "${LIBRARY_DIR}/XML/include") - # Poco::XML +# Poco::XML - set (SRCS - "${LIBRARY_DIR}/XML/src/AbstractContainerNode.cpp" - "${LIBRARY_DIR}/XML/src/AbstractNode.cpp" - "${LIBRARY_DIR}/XML/src/Attr.cpp" - "${LIBRARY_DIR}/XML/src/Attributes.cpp" - "${LIBRARY_DIR}/XML/src/AttributesImpl.cpp" - "${LIBRARY_DIR}/XML/src/AttrMap.cpp" - "${LIBRARY_DIR}/XML/src/CDATASection.cpp" - "${LIBRARY_DIR}/XML/src/CharacterData.cpp" - "${LIBRARY_DIR}/XML/src/ChildNodesList.cpp" - "${LIBRARY_DIR}/XML/src/Comment.cpp" - "${LIBRARY_DIR}/XML/src/ContentHandler.cpp" - "${LIBRARY_DIR}/XML/src/DeclHandler.cpp" - "${LIBRARY_DIR}/XML/src/DefaultHandler.cpp" - "${LIBRARY_DIR}/XML/src/Document.cpp" - "${LIBRARY_DIR}/XML/src/DocumentEvent.cpp" - "${LIBRARY_DIR}/XML/src/DocumentFragment.cpp" - "${LIBRARY_DIR}/XML/src/DocumentType.cpp" - "${LIBRARY_DIR}/XML/src/DOMBuilder.cpp" - "${LIBRARY_DIR}/XML/src/DOMException.cpp" - "${LIBRARY_DIR}/XML/src/DOMImplementation.cpp" - "${LIBRARY_DIR}/XML/src/DOMObject.cpp" - "${LIBRARY_DIR}/XML/src/DOMParser.cpp" - "${LIBRARY_DIR}/XML/src/DOMSerializer.cpp" - "${LIBRARY_DIR}/XML/src/DOMWriter.cpp" - "${LIBRARY_DIR}/XML/src/DTDHandler.cpp" - "${LIBRARY_DIR}/XML/src/DTDMap.cpp" - "${LIBRARY_DIR}/XML/src/Element.cpp" - "${LIBRARY_DIR}/XML/src/ElementsByTagNameList.cpp" - "${LIBRARY_DIR}/XML/src/Entity.cpp" - "${LIBRARY_DIR}/XML/src/EntityReference.cpp" - "${LIBRARY_DIR}/XML/src/EntityResolver.cpp" - "${LIBRARY_DIR}/XML/src/EntityResolverImpl.cpp" - "${LIBRARY_DIR}/XML/src/ErrorHandler.cpp" - "${LIBRARY_DIR}/XML/src/Event.cpp" - "${LIBRARY_DIR}/XML/src/EventDispatcher.cpp" - "${LIBRARY_DIR}/XML/src/EventException.cpp" - "${LIBRARY_DIR}/XML/src/EventListener.cpp" - "${LIBRARY_DIR}/XML/src/EventTarget.cpp" - "${LIBRARY_DIR}/XML/src/InputSource.cpp" - "${LIBRARY_DIR}/XML/src/LexicalHandler.cpp" - "${LIBRARY_DIR}/XML/src/Locator.cpp" - "${LIBRARY_DIR}/XML/src/LocatorImpl.cpp" - "${LIBRARY_DIR}/XML/src/MutationEvent.cpp" - "${LIBRARY_DIR}/XML/src/Name.cpp" - "${LIBRARY_DIR}/XML/src/NamedNodeMap.cpp" - "${LIBRARY_DIR}/XML/src/NamePool.cpp" - "${LIBRARY_DIR}/XML/src/NamespaceStrategy.cpp" - "${LIBRARY_DIR}/XML/src/NamespaceSupport.cpp" - "${LIBRARY_DIR}/XML/src/Node.cpp" - "${LIBRARY_DIR}/XML/src/NodeAppender.cpp" - "${LIBRARY_DIR}/XML/src/NodeFilter.cpp" - "${LIBRARY_DIR}/XML/src/NodeIterator.cpp" - "${LIBRARY_DIR}/XML/src/NodeList.cpp" - "${LIBRARY_DIR}/XML/src/Notation.cpp" - "${LIBRARY_DIR}/XML/src/ParserEngine.cpp" - "${LIBRARY_DIR}/XML/src/ProcessingInstruction.cpp" - "${LIBRARY_DIR}/XML/src/QName.cpp" - "${LIBRARY_DIR}/XML/src/SAXException.cpp" - "${LIBRARY_DIR}/XML/src/SAXParser.cpp" - "${LIBRARY_DIR}/XML/src/Text.cpp" - "${LIBRARY_DIR}/XML/src/TreeWalker.cpp" - "${LIBRARY_DIR}/XML/src/ValueTraits.cpp" - "${LIBRARY_DIR}/XML/src/WhitespaceFilter.cpp" - "${LIBRARY_DIR}/XML/src/XMLException.cpp" - "${LIBRARY_DIR}/XML/src/XMLFilter.cpp" - "${LIBRARY_DIR}/XML/src/XMLFilterImpl.cpp" - "${LIBRARY_DIR}/XML/src/XMLReader.cpp" - "${LIBRARY_DIR}/XML/src/XMLStreamParser.cpp" - "${LIBRARY_DIR}/XML/src/XMLStreamParserException.cpp" - "${LIBRARY_DIR}/XML/src/XMLString.cpp" - "${LIBRARY_DIR}/XML/src/XMLWriter.cpp" +set (SRCS + "${LIBRARY_DIR}/XML/src/AbstractContainerNode.cpp" + "${LIBRARY_DIR}/XML/src/AbstractNode.cpp" + "${LIBRARY_DIR}/XML/src/Attr.cpp" + "${LIBRARY_DIR}/XML/src/Attributes.cpp" + "${LIBRARY_DIR}/XML/src/AttributesImpl.cpp" + "${LIBRARY_DIR}/XML/src/AttrMap.cpp" + "${LIBRARY_DIR}/XML/src/CDATASection.cpp" + "${LIBRARY_DIR}/XML/src/CharacterData.cpp" + "${LIBRARY_DIR}/XML/src/ChildNodesList.cpp" + "${LIBRARY_DIR}/XML/src/Comment.cpp" + "${LIBRARY_DIR}/XML/src/ContentHandler.cpp" + "${LIBRARY_DIR}/XML/src/DeclHandler.cpp" + "${LIBRARY_DIR}/XML/src/DefaultHandler.cpp" + "${LIBRARY_DIR}/XML/src/Document.cpp" + "${LIBRARY_DIR}/XML/src/DocumentEvent.cpp" + "${LIBRARY_DIR}/XML/src/DocumentFragment.cpp" + "${LIBRARY_DIR}/XML/src/DocumentType.cpp" + "${LIBRARY_DIR}/XML/src/DOMBuilder.cpp" + "${LIBRARY_DIR}/XML/src/DOMException.cpp" + "${LIBRARY_DIR}/XML/src/DOMImplementation.cpp" + "${LIBRARY_DIR}/XML/src/DOMObject.cpp" + "${LIBRARY_DIR}/XML/src/DOMParser.cpp" + "${LIBRARY_DIR}/XML/src/DOMSerializer.cpp" + "${LIBRARY_DIR}/XML/src/DOMWriter.cpp" + "${LIBRARY_DIR}/XML/src/DTDHandler.cpp" + "${LIBRARY_DIR}/XML/src/DTDMap.cpp" + "${LIBRARY_DIR}/XML/src/Element.cpp" + "${LIBRARY_DIR}/XML/src/ElementsByTagNameList.cpp" + "${LIBRARY_DIR}/XML/src/Entity.cpp" + "${LIBRARY_DIR}/XML/src/EntityReference.cpp" + "${LIBRARY_DIR}/XML/src/EntityResolver.cpp" + "${LIBRARY_DIR}/XML/src/EntityResolverImpl.cpp" + "${LIBRARY_DIR}/XML/src/ErrorHandler.cpp" + "${LIBRARY_DIR}/XML/src/Event.cpp" + "${LIBRARY_DIR}/XML/src/EventDispatcher.cpp" + "${LIBRARY_DIR}/XML/src/EventException.cpp" + "${LIBRARY_DIR}/XML/src/EventListener.cpp" + "${LIBRARY_DIR}/XML/src/EventTarget.cpp" + "${LIBRARY_DIR}/XML/src/InputSource.cpp" + "${LIBRARY_DIR}/XML/src/LexicalHandler.cpp" + "${LIBRARY_DIR}/XML/src/Locator.cpp" + "${LIBRARY_DIR}/XML/src/LocatorImpl.cpp" + "${LIBRARY_DIR}/XML/src/MutationEvent.cpp" + "${LIBRARY_DIR}/XML/src/Name.cpp" + "${LIBRARY_DIR}/XML/src/NamedNodeMap.cpp" + "${LIBRARY_DIR}/XML/src/NamePool.cpp" + "${LIBRARY_DIR}/XML/src/NamespaceStrategy.cpp" + "${LIBRARY_DIR}/XML/src/NamespaceSupport.cpp" + "${LIBRARY_DIR}/XML/src/Node.cpp" + "${LIBRARY_DIR}/XML/src/NodeAppender.cpp" + "${LIBRARY_DIR}/XML/src/NodeFilter.cpp" + "${LIBRARY_DIR}/XML/src/NodeIterator.cpp" + "${LIBRARY_DIR}/XML/src/NodeList.cpp" + "${LIBRARY_DIR}/XML/src/Notation.cpp" + "${LIBRARY_DIR}/XML/src/ParserEngine.cpp" + "${LIBRARY_DIR}/XML/src/ProcessingInstruction.cpp" + "${LIBRARY_DIR}/XML/src/QName.cpp" + "${LIBRARY_DIR}/XML/src/SAXException.cpp" + "${LIBRARY_DIR}/XML/src/SAXParser.cpp" + "${LIBRARY_DIR}/XML/src/Text.cpp" + "${LIBRARY_DIR}/XML/src/TreeWalker.cpp" + "${LIBRARY_DIR}/XML/src/ValueTraits.cpp" + "${LIBRARY_DIR}/XML/src/WhitespaceFilter.cpp" + "${LIBRARY_DIR}/XML/src/XMLException.cpp" + "${LIBRARY_DIR}/XML/src/XMLFilter.cpp" + "${LIBRARY_DIR}/XML/src/XMLFilterImpl.cpp" + "${LIBRARY_DIR}/XML/src/XMLReader.cpp" + "${LIBRARY_DIR}/XML/src/XMLStreamParser.cpp" + "${LIBRARY_DIR}/XML/src/XMLStreamParserException.cpp" + "${LIBRARY_DIR}/XML/src/XMLString.cpp" + "${LIBRARY_DIR}/XML/src/XMLWriter.cpp" - # expat - "${LIBRARY_DIR}/XML/src/xmlparse.cpp" - ) + # expat + "${LIBRARY_DIR}/XML/src/xmlparse.cpp" +) - add_library (_poco_xml ${SRCS}) - add_library (Poco::XML ALIAS _poco_xml) +add_library (_poco_xml ${SRCS}) +add_library (Poco::XML ALIAS _poco_xml) - target_compile_options (_poco_xml PRIVATE -Wno-old-style-cast) - target_include_directories (_poco_xml SYSTEM PUBLIC "${LIBRARY_DIR}/XML/include") - target_link_libraries (_poco_xml PUBLIC Poco::Foundation Poco::XML::Expat) -else () - add_library (Poco::XML UNKNOWN IMPORTED GLOBAL) - - find_library (LIBRARY_POCO_XML PocoXML) - find_path (INCLUDE_POCO_XML Poco/XML/XML.h) - set_target_properties (Poco::XML PROPERTIES IMPORTED_LOCATION ${LIBRARY_POCO_XML}) - set_target_properties (Poco::XML PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_POCO_XML}) - - message (STATUS "Using Poco::XML: ${LIBRARY_POCO_XML} ${INCLUDE_POCO_XML}") -endif () +target_compile_options (_poco_xml PRIVATE -Wno-old-style-cast) +target_include_directories (_poco_xml SYSTEM PUBLIC "${LIBRARY_DIR}/XML/include") +target_link_libraries (_poco_xml PUBLIC Poco::Foundation Poco::XML::Expat) From e9ec9175df2638f75ce91821736cc23b7eeaea8c Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 22:49:52 +0300 Subject: [PATCH 161/403] Remove unbundled libuv support v2: fix compatiblity check --- CMakeLists.txt | 1 - cmake/find/amqpcpp.cmake | 1 - cmake/find/libuv.cmake | 22 -------------------- contrib/CMakeLists.txt | 4 +--- contrib/libuv-cmake/CMakeLists.txt | 32 ++++++++++++++++-------------- 5 files changed, 18 insertions(+), 42 deletions(-) delete mode 100644 cmake/find/libuv.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index f0959caf4cd..3df1d4ffd8f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -491,7 +491,6 @@ include (cmake/find/ltdl.cmake) # for odbc include (cmake/find/krb5.cmake) include (cmake/find/libgsasl.cmake) include (cmake/find/cyrus-sasl.cmake) -include (cmake/find/libuv.cmake) # for amqpcpp and cassandra include (cmake/find/amqpcpp.cmake) include (cmake/find/capnp.cmake) include (cmake/find/llvm.cmake) diff --git a/cmake/find/amqpcpp.cmake b/cmake/find/amqpcpp.cmake index 43f88fae6c9..4cbe3469d32 100644 --- a/cmake/find/amqpcpp.cmake +++ b/cmake/find/amqpcpp.cmake @@ -21,7 +21,6 @@ set (AMQPCPP_LIBRARY amqp-cpp OpenSSL::Crypto OpenSSL::SSL) set (AMQPCPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/include") list (APPEND AMQPCPP_INCLUDE_DIR - "${LIBUV_INCLUDE_DIR}" "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP") list (APPEND AMQPCPP_LIBRARY "${LIBUV_LIBRARY}") diff --git a/cmake/find/libuv.cmake b/cmake/find/libuv.cmake deleted file mode 100644 index c94dfd50b76..00000000000 --- a/cmake/find/libuv.cmake +++ /dev/null @@ -1,22 +0,0 @@ -if (OS_DARWIN AND COMPILER_GCC) - message (WARNING "libuv cannot be built with GCC in macOS due to a bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=93082") - SET(MISSING_INTERNAL_LIBUV_LIBRARY 1) - return() -endif() - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libuv") - message (WARNING "submodule contrib/libuv is missing. to fix try run: \n git submodule update --init") - SET(MISSING_INTERNAL_LIBUV_LIBRARY 1) - return() -endif() - -if (MAKE_STATIC_LIBRARIES) - set (LIBUV_LIBRARY uv_a) -else() - set (LIBUV_LIBRARY uv) -endif() - -set (LIBUV_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/libuv") -set (LIBUV_INCLUDE_DIR "${LIBUV_ROOT_DIR}/include") - -message (STATUS "Using libuv: ${LIBUV_ROOT_DIR} : ${LIBUV_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 121d9309dfc..ce4561c7b48 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -147,9 +147,7 @@ if (USE_FASTOPS) add_subdirectory (fastops-cmake) endif() -if (USE_AMQPCPP OR USE_CASSANDRA) - add_subdirectory (libuv-cmake) -endif() +add_subdirectory (libuv-cmake) if (USE_AMQPCPP) add_subdirectory (amqpcpp-cmake) endif() diff --git a/contrib/libuv-cmake/CMakeLists.txt b/contrib/libuv-cmake/CMakeLists.txt index dc47b0bf496..3c06bdcf6d0 100644 --- a/contrib/libuv-cmake/CMakeLists.txt +++ b/contrib/libuv-cmake/CMakeLists.txt @@ -1,3 +1,8 @@ +if (OS_DARWIN AND COMPILER_GCC) + message (WARNING "libuv cannot be built with GCC in macOS due to a bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=93082") + return() +endif() + # This file is a modified version of contrib/libuv/CMakeLists.txt set (SOURCE_DIR "${CMAKE_SOURCE_DIR}/contrib/libuv") @@ -122,19 +127,17 @@ set(uv_sources "${uv_sources_tmp}") list(APPEND uv_defines CLICKHOUSE_GLIBC_COMPATIBILITY) -add_library(uv ${uv_sources}) -target_compile_definitions(uv - INTERFACE USING_UV_SHARED=1 - PRIVATE ${uv_defines} BUILDING_UV_SHARED=1) -target_compile_options(uv PRIVATE ${uv_cflags}) -target_include_directories(uv PUBLIC ${SOURCE_DIR}/include PRIVATE ${SOURCE_DIR}/src) -target_link_libraries(uv ${uv_libraries}) +add_library(_uv ${uv_sources}) +add_library(ch_contrib::uv ALIAS _uv) -add_library(uv_a STATIC ${uv_sources}) -target_compile_definitions(uv_a PRIVATE ${uv_defines}) -target_compile_options(uv_a PRIVATE ${uv_cflags}) -target_include_directories(uv_a PUBLIC ${SOURCE_DIR}/include PRIVATE ${SOURCE_DIR}/src) -target_link_libraries(uv_a ${uv_libraries}) +target_compile_definitions(_uv PRIVATE ${uv_defines}) +target_include_directories(_uv SYSTEM PUBLIC ${SOURCE_DIR}/include PRIVATE ${SOURCE_DIR}/src) +target_link_libraries(_uv ${uv_libraries}) +if (NOT MAKE_STATIC_LIBRARIES) + target_compile_definitions(_uv + INTERFACE USING_UV_SHARED=1 + PRIVATE BUILDING_UV_SHARED=1) +endif() if(UNIX) # Now for some gibbering horrors from beyond the stars... @@ -145,7 +148,6 @@ if(UNIX) string(REGEX MATCH [0-9]+[.][0-9]+[.][0-9]+ PACKAGE_VERSION "${configure_ac}") string(REGEX MATCH ^[0-9]+ UV_VERSION_MAJOR "${PACKAGE_VERSION}") # The version in the filename is mirroring the behaviour of autotools. - set_target_properties(uv PROPERTIES VERSION ${UV_VERSION_MAJOR}.0.0 - SOVERSION ${UV_VERSION_MAJOR}) + set_target_properties(_uv PROPERTIES VERSION ${UV_VERSION_MAJOR}.0.0 + SOVERSION ${UV_VERSION_MAJOR}) endif() - From 3b54dfa9adfbc6428b390090cf52534efaa631e4 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 22:53:36 +0300 Subject: [PATCH 162/403] Remove unbundled amqpcpp support --- CMakeLists.txt | 1 - cmake/find/amqpcpp.cmake | 28 ---------------------------- contrib/CMakeLists.txt | 4 +--- contrib/amqpcpp-cmake/CMakeLists.txt | 16 ++++++++++++---- src/CMakeLists.txt | 7 +++---- src/configure_config.cmake | 3 +++ 6 files changed, 19 insertions(+), 40 deletions(-) delete mode 100644 cmake/find/amqpcpp.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 3df1d4ffd8f..cd627d7d5c5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -491,7 +491,6 @@ include (cmake/find/ltdl.cmake) # for odbc include (cmake/find/krb5.cmake) include (cmake/find/libgsasl.cmake) include (cmake/find/cyrus-sasl.cmake) -include (cmake/find/amqpcpp.cmake) include (cmake/find/capnp.cmake) include (cmake/find/llvm.cmake) include (cmake/find/h3.cmake) diff --git a/cmake/find/amqpcpp.cmake b/cmake/find/amqpcpp.cmake deleted file mode 100644 index 4cbe3469d32..00000000000 --- a/cmake/find/amqpcpp.cmake +++ /dev/null @@ -1,28 +0,0 @@ -if (MISSING_INTERNAL_LIBUV_LIBRARY) - message (WARNING "Can't find internal libuv needed for AMQP-CPP library") - set (ENABLE_AMQPCPP OFF CACHE INTERNAL "") -endif() - -option(ENABLE_AMQPCPP "Enalbe AMQP-CPP" ${ENABLE_LIBRARIES}) - -if (NOT ENABLE_AMQPCPP) - return() -endif() - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/src") - message (WARNING "submodule contrib/AMQP-CPP is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal AMQP-CPP library") - set (USE_AMQPCPP 0) - return() -endif () - -set (USE_AMQPCPP 1) -set (AMQPCPP_LIBRARY amqp-cpp OpenSSL::Crypto OpenSSL::SSL) - -set (AMQPCPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/include") -list (APPEND AMQPCPP_INCLUDE_DIR - "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP") - -list (APPEND AMQPCPP_LIBRARY "${LIBUV_LIBRARY}") - -message (STATUS "Using AMQP-CPP=${USE_AMQPCPP}: ${AMQPCPP_INCLUDE_DIR} : ${AMQPCPP_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index ce4561c7b48..b3555e848dc 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -148,9 +148,7 @@ if (USE_FASTOPS) endif() add_subdirectory (libuv-cmake) -if (USE_AMQPCPP) - add_subdirectory (amqpcpp-cmake) -endif() +add_subdirectory (amqpcpp-cmake) # requires: libuv if (USE_CASSANDRA) add_subdirectory (cassandra-cmake) endif() diff --git a/contrib/amqpcpp-cmake/CMakeLists.txt b/contrib/amqpcpp-cmake/CMakeLists.txt index 9b61d3120af..974d097e06f 100644 --- a/contrib/amqpcpp-cmake/CMakeLists.txt +++ b/contrib/amqpcpp-cmake/CMakeLists.txt @@ -1,3 +1,10 @@ +option(ENABLE_AMQPCPP "Enable AMQP-CPP" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_AMQPCPP) + message(STATUS "Not using AMQP-CPP") + return() +endif() + set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP") set (SRCS @@ -23,9 +30,9 @@ set (SRCS "${LIBRARY_DIR}/src/watchable.cpp" ) -add_library(amqp-cpp ${SRCS}) +add_library(_amqp-cpp ${SRCS}) -target_compile_options (amqp-cpp +target_compile_options (_amqp-cpp PRIVATE -Wno-old-style-cast -Wno-inconsistent-missing-destructor-override @@ -40,5 +47,6 @@ target_compile_options (amqp-cpp -w ) -target_include_directories (amqp-cpp SYSTEM PUBLIC "${LIBRARY_DIR}/include") -target_link_libraries(amqp-cpp PUBLIC OpenSSL::Crypto OpenSSL::SSL) +target_include_directories (_amqp-cpp SYSTEM BEFORE PUBLIC "${LIBRARY_DIR}/include" "${LIBRARY_DIR}") +target_link_libraries (_amqp-cpp PUBLIC OpenSSL::Crypto OpenSSL::SSL ch_contrib::uv) +add_library (ch_contrib::amqp_cpp ALIAS _amqp-cpp) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9c085a01c47..81ece81fd83 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -91,7 +91,7 @@ if (TARGET ch_contrib::rdkafka) add_headers_and_sources(dbms Storages/Kafka) endif() -if (USE_AMQPCPP) +if (TARGET ch_contrib::amqp_cpp) add_headers_and_sources(dbms Storages/RabbitMQ) endif() @@ -455,9 +455,8 @@ if (TARGET ch_contrib::snappy) target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::snappy) endif() -if (USE_AMQPCPP) - dbms_target_link_libraries(PUBLIC ${AMQPCPP_LIBRARY}) - dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${AMQPCPP_INCLUDE_DIR}) +if (TARGET ch_contrib::amqp_cpp) + dbms_target_link_libraries(PUBLIC ch_contrib::amqp_cpp) endif() if (USE_SQLITE) diff --git a/src/configure_config.cmake b/src/configure_config.cmake index cc26bb37c21..a16521ee880 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -43,3 +43,6 @@ endif() if (TARGET ch_contrib::azure_sdk) set(USE_AZURE_BLOB_STORAGE 1) endif() +if (TARGET ch_contrib::amqp_cpp) + set(USE_AMQPCPP 1) +endif() From 16adb8c4d630511d0f1eb896fe9b5a868c712000 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 22:58:31 +0300 Subject: [PATCH 163/403] Remove unbundled cassandra support --- CMakeLists.txt | 1 - cmake/find/cassandra.cmake | 34 -------------------------- contrib/CMakeLists.txt | 4 +-- contrib/cassandra-cmake/CMakeLists.txt | 27 ++++++++++++++------ src/Dictionaries/CMakeLists.txt | 4 +-- src/configure_config.cmake | 3 +++ 6 files changed, 26 insertions(+), 47 deletions(-) delete mode 100644 cmake/find/cassandra.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index cd627d7d5c5..e026a86792a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -510,7 +510,6 @@ include (cmake/find/yaml-cpp.cmake) include (cmake/find/nlp.cmake) include (cmake/find/filelog.cmake) include (cmake/find/msgpack.cmake) -include (cmake/find/cassandra.cmake) include (cmake/find/sentry.cmake) include (cmake/find/datasketches.cmake) diff --git a/cmake/find/cassandra.cmake b/cmake/find/cassandra.cmake deleted file mode 100644 index 7fcbdbb90a5..00000000000 --- a/cmake/find/cassandra.cmake +++ /dev/null @@ -1,34 +0,0 @@ -if (MISSING_INTERNAL_LIBUV_LIBRARY) - message (WARNING "Disabling cassandra due to missing libuv") - set (ENABLE_CASSANDRA OFF CACHE INTERNAL "") -endif() - -option(ENABLE_CASSANDRA "Enable Cassandra" ${ENABLE_LIBRARIES}) - -if (NOT ENABLE_CASSANDRA) - return() -endif() - -if (APPLE) - set(CMAKE_MACOSX_RPATH ON) -endif() - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cassandra") - message (ERROR "submodule contrib/cassandra is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal Cassandra") - set (USE_CASSANDRA 0) - return() -endif() - -set (USE_CASSANDRA 1) -set (CASSANDRA_INCLUDE_DIR - "${ClickHouse_SOURCE_DIR}/contrib/cassandra/include/") -if (MAKE_STATIC_LIBRARIES) - set (CASSANDRA_LIBRARY cassandra_static) -else() - set (CASSANDRA_LIBRARY cassandra) -endif() - -set (CASS_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/cassandra") - -message (STATUS "Using cassandra=${USE_CASSANDRA}: ${CASSANDRA_INCLUDE_DIR} : ${CASSANDRA_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index b3555e848dc..a26c505f161 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -149,9 +149,7 @@ endif() add_subdirectory (libuv-cmake) add_subdirectory (amqpcpp-cmake) # requires: libuv -if (USE_CASSANDRA) - add_subdirectory (cassandra-cmake) -endif() +add_subdirectory (cassandra-cmake) # requires: libuv # Should go before: # - sentry-native diff --git a/contrib/cassandra-cmake/CMakeLists.txt b/contrib/cassandra-cmake/CMakeLists.txt index 15565ec0a8d..cc4f12f2e83 100644 --- a/contrib/cassandra-cmake/CMakeLists.txt +++ b/contrib/cassandra-cmake/CMakeLists.txt @@ -1,3 +1,14 @@ +option(ENABLE_CASSANDRA "Enable Cassandra" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_CASSANDRA) + message(STATUS "Not using cassandra") + return() +endif() + +if (APPLE) + set(CMAKE_MACOSX_RPATH ON) +endif() + # Need to use C++17 since the compilation is not possible with C++20 currently. set (CMAKE_CXX_STANDARD 17) @@ -108,20 +119,22 @@ configure_file( ${CMAKE_CURRENT_BINARY_DIR}/driver_config.hpp) -add_library(cassandra +add_library(_cassandra ${SOURCES} $ $ $ $) -target_link_libraries(cassandra ch_contrib::zlib) -add_library(cassandra_static ALIAS cassandra) -target_include_directories(cassandra PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${INCLUDE_DIRS}) -target_compile_definitions(cassandra PRIVATE CASS_BUILDING) +target_link_libraries(_cassandra ch_contrib::zlib) +target_include_directories(_cassandra PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${INCLUDE_DIRS}) +target_include_directories(_cassandra SYSTEM BEFORE PUBLIC ${CASS_INCLUDE_DIR}) +target_compile_definitions(_cassandra PRIVATE CASS_BUILDING) -target_link_libraries(cassandra uv) +target_link_libraries(_cassandra ch_contrib::uv) if(CASS_USE_OPENSSL) - target_link_libraries(cassandra ssl) + target_link_libraries(_cassandra OpenSSL::SSL) endif() + +add_library(ch_contrib::cassandra ALIAS _cassandra) diff --git a/src/Dictionaries/CMakeLists.txt b/src/Dictionaries/CMakeLists.txt index bd2aa80f6d8..31b1ac67304 100644 --- a/src/Dictionaries/CMakeLists.txt +++ b/src/Dictionaries/CMakeLists.txt @@ -37,8 +37,8 @@ target_link_libraries(clickhouse_dictionaries PUBLIC abseil_swiss_tables) -if(USE_CASSANDRA) - target_include_directories(clickhouse_dictionaries SYSTEM PRIVATE ${CASSANDRA_INCLUDE_DIR}) +if (TARGET ch_contrib::cassandra) + target_link_libraries(clickhouse_dictionaries PRIVATE ch_contrib::cassandra) endif() add_subdirectory(Embedded) diff --git a/src/configure_config.cmake b/src/configure_config.cmake index a16521ee880..d391870f0eb 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -46,3 +46,6 @@ endif() if (TARGET ch_contrib::amqp_cpp) set(USE_AMQPCPP 1) endif() +if (TARGET ch_contrib::cassandra) + set(USE_CASSANDRA 1) +endif() From 7496ed7fde73ad61a4008895cdd4320d12c57ef7 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 23:26:05 +0300 Subject: [PATCH 164/403] Remove unbundled gtest support v2: Fix unit tests (do not rely on USE_GTEST) --- CMakeLists.txt | 4 --- cmake/find/gtest.cmake | 40 ------------------------- contrib/CMakeLists.txt | 7 ++--- contrib/googletest-cmake/CMakeLists.txt | 20 ++++++++----- docker/packager/packager | 2 -- programs/CMakeLists.txt | 2 +- src/CMakeLists.txt | 4 +-- 7 files changed, 17 insertions(+), 62 deletions(-) delete mode 100644 cmake/find/gtest.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index e026a86792a..16716e91d86 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -513,10 +513,6 @@ include (cmake/find/msgpack.cmake) include (cmake/find/sentry.cmake) include (cmake/find/datasketches.cmake) -if (ENABLE_TESTS) - include (cmake/find/gtest.cmake) -endif () - # Need to process before "contrib" dir: include (cmake/find/mysqlclient.cmake) diff --git a/cmake/find/gtest.cmake b/cmake/find/gtest.cmake deleted file mode 100644 index 935744bcbd1..00000000000 --- a/cmake/find/gtest.cmake +++ /dev/null @@ -1,40 +0,0 @@ -# included only if ENABLE_TESTS=1 - -option (USE_INTERNAL_GTEST_LIBRARY "Set to FALSE to use system Google Test instead of bundled" ON) - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest/CMakeLists.txt") - if (USE_INTERNAL_GTEST_LIBRARY) - message (WARNING "submodule contrib/googletest is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal gtest") - set (USE_INTERNAL_GTEST_LIBRARY 0) - endif () - - set (MISSING_INTERNAL_GTEST_LIBRARY 1) -endif () - -if(NOT USE_INTERNAL_GTEST_LIBRARY) - # TODO: autodetect of GTEST_SRC_DIR by EXISTS /usr/src/googletest/CMakeLists.txt - if(NOT GTEST_SRC_DIR) - find_package(GTest) - if (NOT GTEST_INCLUDE_DIRS) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system Google Test") - endif() - endif() -endif() - -if (NOT GTEST_SRC_DIR AND NOT GTEST_INCLUDE_DIRS AND NOT MISSING_INTERNAL_GTEST_LIBRARY) - set (USE_INTERNAL_GTEST_LIBRARY 1) - set (GTEST_MAIN_LIBRARIES gtest_main) - set (GTEST_LIBRARIES gtest) - set (GTEST_BOTH_LIBRARIES ${GTEST_MAIN_LIBRARIES} ${GTEST_LIBRARIES}) - set (GTEST_INCLUDE_DIRS ${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest) -elseif(USE_INTERNAL_GTEST_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Wouldn't use internal Google Test library") - set (USE_INTERNAL_GTEST_LIBRARY 0) -endif () - -if((GTEST_INCLUDE_DIRS AND GTEST_BOTH_LIBRARIES) OR GTEST_SRC_DIR) - set(USE_GTEST 1) -endif() - -message (STATUS "Using gtest=${USE_GTEST}: ${GTEST_INCLUDE_DIRS} : ${GTEST_BOTH_LIBRARIES} : ${GTEST_SRC_DIR}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index a26c505f161..2b19dad672b 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -89,11 +89,8 @@ if (USE_INTERNAL_MYSQL_LIBRARY) add_subdirectory (mariadb-connector-c-cmake) endif () -if(USE_INTERNAL_GTEST_LIBRARY) - add_subdirectory(googletest-cmake) -elseif(GTEST_SRC_DIR) - add_subdirectory(${GTEST_SRC_DIR}/googletest ${CMAKE_CURRENT_BINARY_DIR}/googletest) - target_compile_definitions(gtest INTERFACE GTEST_HAS_POSIX_RE=0) +if (ENABLE_TESTS) + add_subdirectory (googletest-cmake) endif() function(add_llvm) diff --git a/contrib/googletest-cmake/CMakeLists.txt b/contrib/googletest-cmake/CMakeLists.txt index ec7ac91c471..f116eddc337 100644 --- a/contrib/googletest-cmake/CMakeLists.txt +++ b/contrib/googletest-cmake/CMakeLists.txt @@ -1,11 +1,15 @@ set (SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest") -add_library(gtest "${SRC_DIR}/src/gtest-all.cc") -set_target_properties(gtest PROPERTIES VERSION "1.0.0") -target_compile_definitions (gtest INTERFACE GTEST_HAS_POSIX_RE=0) -target_include_directories(gtest SYSTEM PUBLIC "${SRC_DIR}/include") -target_include_directories(gtest PRIVATE "${SRC_DIR}") +add_library(_gtest "${SRC_DIR}/src/gtest-all.cc") +set_target_properties(_gtest PROPERTIES VERSION "1.0.0") +target_compile_definitions (_gtest INTERFACE GTEST_HAS_POSIX_RE=0) +target_include_directories(_gtest SYSTEM PUBLIC "${SRC_DIR}/include") +target_include_directories(_gtest PRIVATE "${SRC_DIR}") -add_library(gtest_main "${SRC_DIR}/src/gtest_main.cc") -set_target_properties(gtest_main PROPERTIES VERSION "1.0.0") -target_link_libraries(gtest_main PUBLIC gtest) +add_library(_gtest_main "${SRC_DIR}/src/gtest_main.cc") +set_target_properties(_gtest_main PROPERTIES VERSION "1.0.0") +target_link_libraries(_gtest_main PUBLIC _gtest) + +add_library(_gtest_all INTERFACE) +target_link_libraries(_gtest_all INTERFACE _gtest _gtest_main) +add_library(ch_contrib::gtest_all ALIAS _gtest_all) diff --git a/docker/packager/packager b/docker/packager/packager index 4e3e26d215f..05b2e02df96 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -156,7 +156,6 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ result.append('ENABLE_TESTS=1') result.append('BINARY_OUTPUT=tests') cmake_flags.append('-DENABLE_TESTS=1') - cmake_flags.append('-DUSE_GTEST=1') if split_binary: cmake_flags.append('-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1') @@ -168,7 +167,6 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ if clang_tidy: cmake_flags.append('-DENABLE_CLANG_TIDY=1') cmake_flags.append('-DENABLE_UTILS=1') - cmake_flags.append('-DUSE_GTEST=1') cmake_flags.append('-DENABLE_TESTS=1') cmake_flags.append('-DENABLE_EXAMPLES=1') # Don't stop on first error to find more clang-tidy errors in one run. diff --git a/programs/CMakeLists.txt b/programs/CMakeLists.txt index 68e8f8cba3b..f897e9812a8 100644 --- a/programs/CMakeLists.txt +++ b/programs/CMakeLists.txt @@ -468,7 +468,7 @@ else () endif() endif () -if (ENABLE_TESTS AND USE_GTEST) +if (ENABLE_TESTS) set (CLICKHOUSE_UNIT_TESTS_TARGETS unit_tests_dbms) add_custom_target (clickhouse-tests ALL DEPENDS ${CLICKHOUSE_UNIT_TESTS_TARGETS}) add_dependencies(clickhouse-bundle clickhouse-tests) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 81ece81fd83..6353a6dee72 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -514,7 +514,7 @@ dbms_target_link_libraries(PUBLIC consistent-hashing) include ("${ClickHouse_SOURCE_DIR}/cmake/add_check.cmake") -if (ENABLE_TESTS AND USE_GTEST) +if (ENABLE_TESTS) macro (grep_gtest_sources BASE_DIR DST_VAR) # Cold match files that are not in tests/ directories file(GLOB_RECURSE "${DST_VAR}" RELATIVE "${BASE_DIR}" "gtest*.cpp") @@ -536,7 +536,7 @@ if (ENABLE_TESTS AND USE_GTEST) ) target_link_libraries(unit_tests_dbms PRIVATE - ${GTEST_BOTH_LIBRARIES} + ch_contrib::gtest_all clickhouse_functions clickhouse_aggregate_functions clickhouse_parsers From a30ef87d659ee4565c816297950f0cf9e33e12e7 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 23:21:37 +0300 Subject: [PATCH 165/403] Remove unbundled msgpack support --- CMakeLists.txt | 1 - cmake/find/msgpack.cmake | 37 -------------------------- contrib/CMakeLists.txt | 1 + contrib/msgpack-c-cmake/CMakeLists.txt | 10 +++++++ src/CMakeLists.txt | 4 +-- src/Formats/CMakeLists.txt | 3 +++ 6 files changed, 16 insertions(+), 40 deletions(-) delete mode 100644 cmake/find/msgpack.cmake create mode 100644 contrib/msgpack-c-cmake/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 16716e91d86..7a453071b86 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -509,7 +509,6 @@ include (cmake/find/libpqxx.cmake) include (cmake/find/yaml-cpp.cmake) include (cmake/find/nlp.cmake) include (cmake/find/filelog.cmake) -include (cmake/find/msgpack.cmake) include (cmake/find/sentry.cmake) include (cmake/find/datasketches.cmake) diff --git a/cmake/find/msgpack.cmake b/cmake/find/msgpack.cmake deleted file mode 100644 index ac52740c774..00000000000 --- a/cmake/find/msgpack.cmake +++ /dev/null @@ -1,37 +0,0 @@ -option (ENABLE_MSGPACK "Enable msgpack library" ${ENABLE_LIBRARIES}) - -if(NOT ENABLE_MSGPACK) - if(USE_INTERNAL_MSGPACK_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal msgpack with ENABLE_MSGPACK=OFF") - endif() - return() -endif() - -option (USE_INTERNAL_MSGPACK_LIBRARY "Set to FALSE to use system msgpack library instead of bundled" ON) - -if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/msgpack-c/include/msgpack.hpp") - if(USE_INTERNAL_MSGPACK_LIBRARY) - message(WARNING "Submodule contrib/msgpack-c is missing. To fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal msgpack") - set(USE_INTERNAL_MSGPACK_LIBRARY 0) - endif() - set(MISSING_INTERNAL_MSGPACK_LIBRARY 1) -endif() - -if(NOT USE_INTERNAL_MSGPACK_LIBRARY) - find_path(MSGPACK_INCLUDE_DIR NAMES msgpack.hpp PATHS ${MSGPACK_INCLUDE_PATHS}) - if(NOT MSGPACK_INCLUDE_DIR) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system msgpack") - endif() -endif() - -if(NOT MSGPACK_INCLUDE_DIR AND NOT MISSING_INTERNAL_MSGPACK_LIBRARY) - set(MSGPACK_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/msgpack-c/include") - set(USE_INTERNAL_MSGPACK_LIBRARY 1) -endif() - -if (MSGPACK_INCLUDE_DIR) - set(USE_MSGPACK 1) -endif() - -message(STATUS "Using msgpack=${USE_MSGPACK}: ${MSGPACK_INCLUDE_DIR}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 2b19dad672b..0a3179620fd 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -70,6 +70,7 @@ add_subdirectory (openldap-cmake) add_subdirectory (grpc-cmake) add_subdirectory (protobuf-cmake) add_subdirectory (libhdfs3-cmake) # requires: protobuf +add_subdirectory (msgpack-c-cmake) if (ENABLE_FUZZING) add_subdirectory (libprotobuf-mutator-cmake) diff --git a/contrib/msgpack-c-cmake/CMakeLists.txt b/contrib/msgpack-c-cmake/CMakeLists.txt new file mode 100644 index 00000000000..3232b0a9534 --- /dev/null +++ b/contrib/msgpack-c-cmake/CMakeLists.txt @@ -0,0 +1,10 @@ +option (ENABLE_MSGPACK "Enable msgpack library" ${ENABLE_LIBRARIES}) + +if(NOT ENABLE_MSGPACK) + message(STATUS "Not using msgpack") + return() +endif() + +add_library(_msgpack INTERFACE) +target_include_directories(_msgpack SYSTEM BEFORE INTERFACE "${ClickHouse_SOURCE_DIR}/contrib/msgpack-c/include") +add_library(ch_contrib::msgpack ALIAS _msgpack) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6353a6dee72..612a998c70b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -468,8 +468,8 @@ if (USE_CASSANDRA) dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${CASS_INCLUDE_DIR}) endif() -if (USE_MSGPACK) - target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${MSGPACK_INCLUDE_DIR}) +if (TARGET ch_contrib::msgpack) + target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::msgpack) endif() target_link_libraries (clickhouse_common_io PUBLIC ${FAST_FLOAT_LIBRARY}) diff --git a/src/Formats/CMakeLists.txt b/src/Formats/CMakeLists.txt index 1aa79e98476..f3a5a8a6025 100644 --- a/src/Formats/CMakeLists.txt +++ b/src/Formats/CMakeLists.txt @@ -12,4 +12,7 @@ endif() if (TARGET ch_contrib::protobuf) set(USE_PROTOBUF 1) endif() +if (TARGET ch_contrib::msgpack) + set(USE_MSGPACK 1) +endif() configure_file(config_formats.h.in ${ConfigIncludePath}/config_formats.h) From 126aa4b65c463435eab260dd33f7d7b3e9628b6f Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 23:32:13 +0300 Subject: [PATCH 166/403] Remove unbundled base64 support --- CMakeLists.txt | 1 - cmake/find/base64.cmake | 25 ------------------------- contrib/CMakeLists.txt | 5 +---- contrib/base64-cmake/CMakeLists.txt | 13 +++++++++++++ programs/server/Server.cpp | 4 ---- src/Functions/CMakeLists.txt | 5 ++--- src/configure_config.cmake | 3 +++ 7 files changed, 19 insertions(+), 37 deletions(-) delete mode 100644 cmake/find/base64.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 7a453071b86..1c5be86f17b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -499,7 +499,6 @@ include (cmake/find/pdqsort.cmake) include (cmake/find/miniselect.cmake) include (cmake/find/curl.cmake) include (cmake/find/s3.cmake) -include (cmake/find/base64.cmake) include (cmake/find/fast_float.cmake) include (cmake/find/fastops.cmake) include (cmake/find/odbc.cmake) diff --git a/cmake/find/base64.cmake b/cmake/find/base64.cmake deleted file mode 100644 index ee12fbb11ba..00000000000 --- a/cmake/find/base64.cmake +++ /dev/null @@ -1,25 +0,0 @@ -if(ARCH_AMD64 OR ARCH_ARM) - option (ENABLE_BASE64 "Enable base64" ${ENABLE_LIBRARIES}) -elseif(ENABLE_BASE64) - message (${RECONFIGURE_MESSAGE_LEVEL} "base64 library is only supported on x86_64 and aarch64") -endif() - -if (NOT ENABLE_BASE64) - return() -endif() - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64/LICENSE") - set (MISSING_INTERNAL_BASE64_LIBRARY 1) - message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init") -endif () - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64") - message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init") -else() - set (BASE64_LIBRARY base64) - set (USE_BASE64 1) -endif() - -if (NOT USE_BASE64) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot enable base64") -endif() diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 0a3179620fd..4117559f084 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -135,10 +135,7 @@ if (USE_INTERNAL_AWS_S3_LIBRARY) endif () -if (USE_BASE64) - add_subdirectory (base64-cmake) -endif() - +add_subdirectory (base64-cmake) add_subdirectory (simdjson-cmake) if (USE_FASTOPS) diff --git a/contrib/base64-cmake/CMakeLists.txt b/contrib/base64-cmake/CMakeLists.txt index 4ebb4e68728..2443c899869 100644 --- a/contrib/base64-cmake/CMakeLists.txt +++ b/contrib/base64-cmake/CMakeLists.txt @@ -1,3 +1,14 @@ +if(ARCH_AMD64 OR ARCH_ARM) + option (ENABLE_BASE64 "Enable base64" ${ENABLE_LIBRARIES}) +elseif(ENABLE_BASE64) + message (${RECONFIGURE_MESSAGE_LEVEL} "base64 library is only supported on x86_64 and aarch64") +endif() + +if (NOT ENABLE_BASE64) + message(STATUS "Not using base64") + return() +endif() + SET(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/base64") add_library(base64_scalar OBJECT "${LIBRARY_DIR}/turbob64c.c" "${LIBRARY_DIR}/turbob64d.c") @@ -41,3 +52,5 @@ if (XCODE OR XCODE_VERSION) endif () target_sources(base64 PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/dummy.c") endif () + +add_library(ch_contrib::base64 ALIAS base64) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 5fc3f9aa967..184855b386e 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -114,10 +114,6 @@ # include #endif -#if USE_BASE64 -# include -#endif - #if USE_JEMALLOC # include #endif diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 74d76eb7cbc..cf29aa5bae3 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -12,7 +12,6 @@ add_library(clickhouse_functions ${clickhouse_functions_sources}) target_link_libraries(clickhouse_functions PUBLIC - ${BASE64_LIBRARY} ch_contrib::cityhash ch_contrib::farmhash ${FASTOPS_LIBRARY} @@ -72,8 +71,8 @@ if (USE_EMBEDDED_COMPILER) target_include_directories(clickhouse_functions SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS}) endif () -if(USE_BASE64) - target_include_directories(clickhouse_functions SYSTEM PRIVATE ${BASE64_INCLUDE_DIR}) +if (TARGET ch_contrib::base64) + target_link_libraries(clickhouse_functions PRIVATE ch_contrib::base64) endif() target_link_libraries(clickhouse_functions PRIVATE ch_contrib::lz4) diff --git a/src/configure_config.cmake b/src/configure_config.cmake index d391870f0eb..024d87f4fe9 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -49,3 +49,6 @@ endif() if (TARGET ch_contrib::cassandra) set(USE_CASSANDRA 1) endif() +if (TARGET ch_contrib::base64) + set(USE_BASE64 1) +endif() From cb1abf5307b0089f4982958b6631be2080c4e34c Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 23:53:49 +0300 Subject: [PATCH 167/403] Remove unbundled sqlite support --- CMakeLists.txt | 1 - cmake/find/sqlite.cmake | 16 ---------------- contrib/CMakeLists.txt | 4 +--- contrib/sqlite-cmake/CMakeLists.txt | 8 ++++++++ src/CMakeLists.txt | 6 +++--- src/configure_config.cmake | 3 +++ 6 files changed, 15 insertions(+), 23 deletions(-) delete mode 100644 cmake/find/sqlite.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 1c5be86f17b..a63b2c3f339 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -503,7 +503,6 @@ include (cmake/find/fast_float.cmake) include (cmake/find/fastops.cmake) include (cmake/find/odbc.cmake) include (cmake/find/nanodbc.cmake) -include (cmake/find/sqlite.cmake) include (cmake/find/libpqxx.cmake) include (cmake/find/yaml-cpp.cmake) include (cmake/find/nlp.cmake) diff --git a/cmake/find/sqlite.cmake b/cmake/find/sqlite.cmake deleted file mode 100644 index 083a9faea59..00000000000 --- a/cmake/find/sqlite.cmake +++ /dev/null @@ -1,16 +0,0 @@ -option(ENABLE_SQLITE "Enable sqlite" ${ENABLE_LIBRARIES}) - -if (NOT ENABLE_SQLITE) - return() -endif() - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/sqlite-amalgamation/sqlite3.c") - message (WARNING "submodule contrib/sqlite3-amalgamation is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal sqlite library") - set (USE_SQLITE 0) - return() -endif() - -set (USE_SQLITE 1) -set (SQLITE_LIBRARY sqlite) -message (STATUS "Using sqlite=${USE_SQLITE}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 4117559f084..e328e34ca5a 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -180,9 +180,7 @@ if (USE_NLP) add_subdirectory(lemmagen-c-cmake) endif() -if (USE_SQLITE) - add_subdirectory(sqlite-cmake) -endif() +add_subdirectory (sqlite-cmake) if (USE_S2_GEOMETRY) add_subdirectory(s2geometry-cmake) diff --git a/contrib/sqlite-cmake/CMakeLists.txt b/contrib/sqlite-cmake/CMakeLists.txt index 495cb63798d..ea4c3b8e497 100644 --- a/contrib/sqlite-cmake/CMakeLists.txt +++ b/contrib/sqlite-cmake/CMakeLists.txt @@ -1,6 +1,14 @@ +option(ENABLE_SQLITE "Enable sqlite" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_SQLITE) + message(STATUS "Not using sqlite") + return() +endif() + set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/sqlite-amalgamation") set(SRCS ${LIBRARY_DIR}/sqlite3.c) add_library(sqlite ${SRCS}) target_include_directories(sqlite SYSTEM PUBLIC "${LIBRARY_DIR}") +add_library(ch_contrib::sqlite ALIAS sqlite) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 612a998c70b..8342d08b82c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -83,7 +83,7 @@ add_headers_and_sources(clickhouse_common_io IO/S3) list (REMOVE_ITEM clickhouse_common_io_sources Common/malloc.cpp Common/new_delete.cpp) add_headers_and_sources(dbms Disks/IO) -if (USE_SQLITE) +if (TARGET ch_contrib::sqlite) add_headers_and_sources(dbms Databases/SQLite) endif() @@ -459,8 +459,8 @@ if (TARGET ch_contrib::amqp_cpp) dbms_target_link_libraries(PUBLIC ch_contrib::amqp_cpp) endif() -if (USE_SQLITE) - dbms_target_link_libraries(PUBLIC sqlite) +if (TARGET ch_contrib::sqlite) + dbms_target_link_libraries(PUBLIC ch_contrib::sqlite) endif() if (USE_CASSANDRA) diff --git a/src/configure_config.cmake b/src/configure_config.cmake index 024d87f4fe9..ad1f774d7f5 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -52,3 +52,6 @@ endif() if (TARGET ch_contrib::base64) set(USE_BASE64 1) endif() +if (TARGET ch_contrib::sqlite) + set(USE_SQLITE 1) +endif() From 91e3ceeea99ee3974beb91232daf46302e00b01a Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 23:57:21 +0300 Subject: [PATCH 168/403] Remove unbundled capnp support --- CMakeLists.txt | 1 - cmake/find/capnp.cmake | 42 -------------------------- contrib/CMakeLists.txt | 5 +-- contrib/capnproto-cmake/CMakeLists.txt | 9 ++++++ src/CMakeLists.txt | 4 +-- src/Formats/CMakeLists.txt | 3 ++ 6 files changed, 15 insertions(+), 49 deletions(-) delete mode 100644 cmake/find/capnp.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index a63b2c3f339..bddd4fe8bf6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -491,7 +491,6 @@ include (cmake/find/ltdl.cmake) # for odbc include (cmake/find/krb5.cmake) include (cmake/find/libgsasl.cmake) include (cmake/find/cyrus-sasl.cmake) -include (cmake/find/capnp.cmake) include (cmake/find/llvm.cmake) include (cmake/find/h3.cmake) include (cmake/find/libxml2.cmake) diff --git a/cmake/find/capnp.cmake b/cmake/find/capnp.cmake deleted file mode 100644 index fa62c64105f..00000000000 --- a/cmake/find/capnp.cmake +++ /dev/null @@ -1,42 +0,0 @@ -option (ENABLE_CAPNP "Enable Cap'n Proto" ${ENABLE_LIBRARIES}) - -if (NOT ENABLE_CAPNP) - if (USE_INTERNAL_CAPNP_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal capnproto library with ENABLE_CAPNP=OFF") - endif() - return() -endif() - -option (USE_INTERNAL_CAPNP_LIBRARY "Set to FALSE to use system capnproto library instead of bundled" ON) - -if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/capnproto/c++") - if(USE_INTERNAL_CAPNP_LIBRARY) - message(WARNING "submodule contrib/capnproto is missing. to fix try run: \n git submodule update --init") - message(${RECONFIGURE_MESSAGE_LEVEL} "cannot find internal capnproto") - set(USE_INTERNAL_CAPNP_LIBRARY 0) - endif() - set(MISSING_INTERNAL_CAPNP_LIBRARY 1) -endif() - -# FIXME: refactor to use `add_library(… IMPORTED)` if possible. -if (NOT USE_INTERNAL_CAPNP_LIBRARY) - find_library (KJ kj) - find_library (CAPNP capnp) - find_library (CAPNPC capnpc) - - if(KJ AND CAPNP AND CAPNPC) - set (CAPNP_LIBRARIES ${CAPNPC} ${CAPNP} ${KJ}) - else() - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system capnproto") - endif() -endif() - -if (CAPNP_LIBRARIES) - set (USE_CAPNP 1) -elseif(NOT MISSING_INTERNAL_CAPNP_LIBRARY) - set (CAPNP_LIBRARIES capnpc) - set (USE_CAPNP 1) - set (USE_INTERNAL_CAPNP_LIBRARY 1) -endif () - -message (STATUS "Using capnp=${USE_CAPNP}: ${CAPNP_LIBRARIES}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index e328e34ca5a..51bef7d5303 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -38,10 +38,7 @@ add_subdirectory (murmurhash) add_subdirectory (replxx-cmake) add_subdirectory (unixodbc-cmake) add_subdirectory (nanodbc-cmake) - -if (USE_INTERNAL_CAPNP_LIBRARY AND NOT MISSING_INTERNAL_CAPNP_LIBRARY) - add_subdirectory(capnproto-cmake) -endif () +add_subdirectory (capnproto-cmake) if (USE_YAML_CPP) add_subdirectory (yaml-cpp-cmake) diff --git a/contrib/capnproto-cmake/CMakeLists.txt b/contrib/capnproto-cmake/CMakeLists.txt index 05446355535..7d34a4002af 100644 --- a/contrib/capnproto-cmake/CMakeLists.txt +++ b/contrib/capnproto-cmake/CMakeLists.txt @@ -1,3 +1,10 @@ +option (ENABLE_CAPNP "Enable Cap'n Proto" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_CAPNP) + message(STATUS "Not using Cap'n Proto library") + return() +endif() + set (CAPNPROTO_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/capnproto/c++/src") set (CMAKE_CXX_STANDARD 17) @@ -85,3 +92,5 @@ endif () target_compile_options(kj PRIVATE ${SUPPRESS_WARNINGS} ${CAPNP_PRIVATE_CXX_FLAGS}) target_compile_options(capnp PRIVATE ${SUPPRESS_WARNINGS} ${CAPNP_PRIVATE_CXX_FLAGS}) target_compile_options(capnpc PRIVATE ${SUPPRESS_WARNINGS} ${CAPNP_PRIVATE_CXX_FLAGS}) + +add_library(ch_contrib::capnp ALIAS capnpc) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8342d08b82c..13040a5358c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -394,8 +394,8 @@ if (TARGET ch_contrib::icu) dbms_target_link_libraries (PRIVATE ch_contrib::icu) endif () -if (USE_CAPNP) - dbms_target_link_libraries (PRIVATE ${CAPNP_LIBRARIES}) +if (TARGET ch_contrib::capnp) + dbms_target_link_libraries (PRIVATE ch_contrib::capnp) endif () if (TARGET ch_contrib::parquet) diff --git a/src/Formats/CMakeLists.txt b/src/Formats/CMakeLists.txt index f3a5a8a6025..6e6aa6d4553 100644 --- a/src/Formats/CMakeLists.txt +++ b/src/Formats/CMakeLists.txt @@ -15,4 +15,7 @@ endif() if (TARGET ch_contrib::msgpack) set(USE_MSGPACK 1) endif() +if (TARGET ch_contrib::capnp) + set(USE_CAPNP 1) +endif() configure_file(config_formats.h.in ${ConfigIncludePath}/config_formats.h) From 887af0a7e9f3d5e4c5899fac2d6d4668cb148606 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 23:59:41 +0300 Subject: [PATCH 169/403] Remove unbundled nanodbc support --- CMakeLists.txt | 1 - cmake/find/nanodbc.cmake | 16 ---------------- cmake/find/odbc.cmake | 2 -- contrib/nanodbc-cmake/CMakeLists.txt | 3 ++- programs/odbc-bridge/CMakeLists.txt | 2 +- 5 files changed, 3 insertions(+), 21 deletions(-) delete mode 100644 cmake/find/nanodbc.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index bddd4fe8bf6..8b6eac8a36b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -501,7 +501,6 @@ include (cmake/find/s3.cmake) include (cmake/find/fast_float.cmake) include (cmake/find/fastops.cmake) include (cmake/find/odbc.cmake) -include (cmake/find/nanodbc.cmake) include (cmake/find/libpqxx.cmake) include (cmake/find/yaml-cpp.cmake) include (cmake/find/nlp.cmake) diff --git a/cmake/find/nanodbc.cmake b/cmake/find/nanodbc.cmake deleted file mode 100644 index 2fa60e71f55..00000000000 --- a/cmake/find/nanodbc.cmake +++ /dev/null @@ -1,16 +0,0 @@ -if (NOT ENABLE_ODBC) - return () -endif () - -if (NOT USE_INTERNAL_NANODBC_LIBRARY) - message (FATAL_ERROR "Only the bundled nanodbc library can be used") -endif () - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/nanodbc/nanodbc") - message (FATAL_ERROR "submodule contrib/nanodbc is missing. to fix try run: \n git submodule update --init") -endif() - -set (NANODBC_LIBRARY nanodbc) -set (NANODBC_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/nanodbc/nanodbc") - -message (STATUS "Using nanodbc: ${NANODBC_INCLUDE_DIR} : ${NANODBC_LIBRARY}") diff --git a/cmake/find/odbc.cmake b/cmake/find/odbc.cmake index 2f06cfed941..0dfa5ea5025 100644 --- a/cmake/find/odbc.cmake +++ b/cmake/find/odbc.cmake @@ -50,6 +50,4 @@ if (NOT EXTERNAL_ODBC_LIBRARY_FOUND) set (USE_INTERNAL_ODBC_LIBRARY 1) endif () -set (USE_INTERNAL_NANODBC_LIBRARY 1) - message (STATUS "Using unixodbc") diff --git a/contrib/nanodbc-cmake/CMakeLists.txt b/contrib/nanodbc-cmake/CMakeLists.txt index 26a030c3995..b1f4d867685 100644 --- a/contrib/nanodbc-cmake/CMakeLists.txt +++ b/contrib/nanodbc-cmake/CMakeLists.txt @@ -1,4 +1,4 @@ -if (NOT USE_INTERNAL_NANODBC_LIBRARY) +if (NOT ENABLE_ODBC) return () endif () @@ -16,3 +16,4 @@ add_library(nanodbc ${SRCS}) target_link_libraries (nanodbc PUBLIC unixodbc) target_include_directories (nanodbc SYSTEM PUBLIC "${LIBRARY_DIR}/") +add_library(ch_contrib::nanodbc ALIAS nanodbc) diff --git a/programs/odbc-bridge/CMakeLists.txt b/programs/odbc-bridge/CMakeLists.txt index 7b232f2b5dc..504524c0539 100644 --- a/programs/odbc-bridge/CMakeLists.txt +++ b/programs/odbc-bridge/CMakeLists.txt @@ -26,7 +26,7 @@ target_link_libraries(clickhouse-odbc-bridge PRIVATE dbms bridge clickhouse_parsers - nanodbc + ch_contrib::nanodbc unixodbc ) From b51bbde7132553b19df85a06882a17398b89c05f Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 00:03:55 +0300 Subject: [PATCH 170/403] Remove unbundled odbc support --- CMakeLists.txt | 2 - cmake/find/ltdl.cmake | 5 -- cmake/find/odbc.cmake | 53 --------------------- contrib/poco-cmake/Data/ODBC/CMakeLists.txt | 8 ++-- contrib/unixodbc-cmake/CMakeLists.txt | 13 ++++- programs/odbc-bridge/CMakeLists.txt | 2 +- 6 files changed, 18 insertions(+), 65 deletions(-) delete mode 100644 cmake/find/ltdl.cmake delete mode 100644 cmake/find/odbc.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b6eac8a36b..db0232d4341 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -487,7 +487,6 @@ message (STATUS include (GNUInstallDirs) -include (cmake/find/ltdl.cmake) # for odbc include (cmake/find/krb5.cmake) include (cmake/find/libgsasl.cmake) include (cmake/find/cyrus-sasl.cmake) @@ -500,7 +499,6 @@ include (cmake/find/curl.cmake) include (cmake/find/s3.cmake) include (cmake/find/fast_float.cmake) include (cmake/find/fastops.cmake) -include (cmake/find/odbc.cmake) include (cmake/find/libpqxx.cmake) include (cmake/find/yaml-cpp.cmake) include (cmake/find/nlp.cmake) diff --git a/cmake/find/ltdl.cmake b/cmake/find/ltdl.cmake deleted file mode 100644 index b48a3630222..00000000000 --- a/cmake/find/ltdl.cmake +++ /dev/null @@ -1,5 +0,0 @@ -if (ENABLE_ODBC AND NOT USE_INTERNAL_ODBC_LIBRARY) - set (LTDL_PATHS "/usr/local/opt/libtool/lib") - find_library (LTDL_LIBRARY ltdl PATHS ${LTDL_PATHS} REQUIRED) - message (STATUS "Using ltdl: ${LTDL_LIBRARY}") -endif () diff --git a/cmake/find/odbc.cmake b/cmake/find/odbc.cmake deleted file mode 100644 index 0dfa5ea5025..00000000000 --- a/cmake/find/odbc.cmake +++ /dev/null @@ -1,53 +0,0 @@ -option (ENABLE_ODBC "Enable ODBC library" ${ENABLE_LIBRARIES}) - -if (NOT OS_LINUX) - if (ENABLE_ODBC) - message(STATUS "ODBC is only supported on Linux") - endif() - set (ENABLE_ODBC OFF CACHE INTERNAL "") -endif () - -if (NOT ENABLE_ODBC) - if (USE_INTERNAL_ODBC_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal ODBC with ENABLE_ODBC=OFF") - endif() - - add_library (unixodbc INTERFACE) - target_compile_definitions (unixodbc INTERFACE USE_ODBC=0) - - message (STATUS "Not using unixodbc") - return() -endif() - -option (USE_INTERNAL_ODBC_LIBRARY "Use internal ODBC library" ON) - -if (NOT USE_INTERNAL_ODBC_LIBRARY) - find_library (LIBRARY_ODBC NAMES unixodbc odbc) - find_path (INCLUDE_ODBC sql.h) - - if(LIBRARY_ODBC AND INCLUDE_ODBC) - add_library (unixodbc INTERFACE) - set_target_properties (unixodbc PROPERTIES INTERFACE_LINK_LIBRARIES ${LIBRARY_ODBC}) - set_target_properties (unixodbc PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_ODBC}) - set_target_properties (unixodbc PROPERTIES INTERFACE_COMPILE_DEFINITIONS USE_ODBC=1) - - if (USE_STATIC_LIBRARIES) - find_library(LTDL_LIBRARY ltdl) - if (LTDL_LIBRARY) - target_link_libraries(unixodbc INTERFACE ${LTDL_LIBRARY}) - endif() - endif() - - set(EXTERNAL_ODBC_LIBRARY_FOUND 1) - message (STATUS "Found odbc: ${LIBRARY_ODBC}") - else() - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system ODBC library") - set(EXTERNAL_ODBC_LIBRARY_FOUND 0) - endif() -endif() - -if (NOT EXTERNAL_ODBC_LIBRARY_FOUND) - set (USE_INTERNAL_ODBC_LIBRARY 1) -endif () - -message (STATUS "Using unixodbc") diff --git a/contrib/poco-cmake/Data/ODBC/CMakeLists.txt b/contrib/poco-cmake/Data/ODBC/CMakeLists.txt index 93d8b036526..7de77cdacf7 100644 --- a/contrib/poco-cmake/Data/ODBC/CMakeLists.txt +++ b/contrib/poco-cmake/Data/ODBC/CMakeLists.txt @@ -1,5 +1,5 @@ if (ENABLE_ODBC) - if (NOT TARGET unixodbc) + if (NOT TARGET ch_contrib::unixodbc) message(FATAL_ERROR "Configuration error: unixodbc is not a target") endif() @@ -25,13 +25,15 @@ if (ENABLE_ODBC) target_compile_options (_poco_data_odbc PRIVATE -Wno-unused-variable) target_include_directories (_poco_data_odbc SYSTEM PUBLIC "${LIBRARY_DIR}/Data/ODBC/include") - target_link_libraries (_poco_data_odbc PUBLIC Poco::Data unixodbc) + target_link_libraries (_poco_data_odbc PUBLIC Poco::Data ch_contrib::unixodbc) message (STATUS "Using Poco::Data::ODBC") else () add_library (_poco_data_odbc INTERFACE) add_library (Poco::Data::ODBC ALIAS _poco_data_odbc) - target_link_libraries (_poco_data_odbc INTERFACE unixodbc) + if (TARGET ch_contrib::unixodbc) + target_link_libraries (_poco_data_odbc INTERFACE ch_contrib::unixodbc) + endif() message (STATUS "Not using Poco::Data::ODBC") endif () diff --git a/contrib/unixodbc-cmake/CMakeLists.txt b/contrib/unixodbc-cmake/CMakeLists.txt index e03f6313a31..ca0742326de 100644 --- a/contrib/unixodbc-cmake/CMakeLists.txt +++ b/contrib/unixodbc-cmake/CMakeLists.txt @@ -1,4 +1,13 @@ -if (NOT USE_INTERNAL_ODBC_LIBRARY) +option (ENABLE_ODBC "Enable ODBC library" ${ENABLE_LIBRARIES}) +if (NOT OS_LINUX) + if (ENABLE_ODBC) + message(STATUS "ODBC is only supported on Linux") + endif() + set (ENABLE_ODBC OFF CACHE INTERNAL "") +endif () + +if (NOT ENABLE_ODBC) + message(STATUS "Not using ODBC") return() endif() @@ -295,3 +304,5 @@ target_compile_options (unixodbc -O2 ) target_compile_definitions (unixodbc INTERFACE USE_ODBC=1) + +add_library (ch_contrib::unixodbc ALIAS unixodbc) diff --git a/programs/odbc-bridge/CMakeLists.txt b/programs/odbc-bridge/CMakeLists.txt index 504524c0539..54f47204259 100644 --- a/programs/odbc-bridge/CMakeLists.txt +++ b/programs/odbc-bridge/CMakeLists.txt @@ -27,7 +27,7 @@ target_link_libraries(clickhouse-odbc-bridge PRIVATE bridge clickhouse_parsers ch_contrib::nanodbc - unixodbc + ch_contrib::unixodbc ) set_target_properties(clickhouse-odbc-bridge PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..) From cbf8867f50757e73210e72e52b291c4d74f3fb0e Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 00:07:59 +0300 Subject: [PATCH 171/403] Remove unbundled fastops support --- CMakeLists.txt | 1 - cmake/find/fastops.cmake | 24 ------------------------ contrib/CMakeLists.txt | 6 +----- contrib/fastops-cmake/CMakeLists.txt | 13 +++++++++++++ src/Functions/CMakeLists.txt | 6 +++--- src/Functions/configure_config.cmake | 3 +++ 6 files changed, 20 insertions(+), 33 deletions(-) delete mode 100644 cmake/find/fastops.cmake create mode 100644 src/Functions/configure_config.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index db0232d4341..c03c1692830 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -498,7 +498,6 @@ include (cmake/find/miniselect.cmake) include (cmake/find/curl.cmake) include (cmake/find/s3.cmake) include (cmake/find/fast_float.cmake) -include (cmake/find/fastops.cmake) include (cmake/find/libpqxx.cmake) include (cmake/find/yaml-cpp.cmake) include (cmake/find/nlp.cmake) diff --git a/cmake/find/fastops.cmake b/cmake/find/fastops.cmake deleted file mode 100644 index 72426eb5912..00000000000 --- a/cmake/find/fastops.cmake +++ /dev/null @@ -1,24 +0,0 @@ -if(ARCH_AMD64 AND NOT OS_FREEBSD AND NOT OS_DARWIN) - option(ENABLE_FASTOPS "Enable fast vectorized mathematical functions library by Mikhail Parakhin" ${ENABLE_LIBRARIES}) -elseif(ENABLE_FASTOPS) - message (${RECONFIGURE_MESSAGE_LEVEL} "Fastops library is supported on x86_64 only, and not FreeBSD or Darwin") -endif() - -if(NOT ENABLE_FASTOPS) - set(USE_FASTOPS 0) - return() -endif() - -if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/fastops/fastops/fastops.h") - message(WARNING "submodule contrib/fastops is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal fastops library") - set(MISSING_INTERNAL_FASTOPS_LIBRARY 1) -endif() - -if(NOT MISSING_INTERNAL_FASTOPS_LIBRARY) - set(USE_FASTOPS 1) - set(FASTOPS_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/fastops/) - set(FASTOPS_LIBRARY fastops) -endif() - -message(STATUS "Using fastops=${USE_FASTOPS}: ${FASTOPS_INCLUDE_DIR} : ${FASTOPS_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 51bef7d5303..26f2bdc699a 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -134,11 +134,7 @@ endif () add_subdirectory (base64-cmake) add_subdirectory (simdjson-cmake) - -if (USE_FASTOPS) - add_subdirectory (fastops-cmake) -endif() - +add_subdirectory (fastops-cmake) add_subdirectory (libuv-cmake) add_subdirectory (amqpcpp-cmake) # requires: libuv add_subdirectory (cassandra-cmake) # requires: libuv diff --git a/contrib/fastops-cmake/CMakeLists.txt b/contrib/fastops-cmake/CMakeLists.txt index fe7293c614b..f0e5030a3aa 100644 --- a/contrib/fastops-cmake/CMakeLists.txt +++ b/contrib/fastops-cmake/CMakeLists.txt @@ -1,3 +1,14 @@ +if(ARCH_AMD64 AND NOT OS_FREEBSD AND NOT OS_DARWIN) + option(ENABLE_FASTOPS "Enable fast vectorized mathematical functions library by Mikhail Parakhin" ${ENABLE_LIBRARIES}) +elseif(ENABLE_FASTOPS) + message (${RECONFIGURE_MESSAGE_LEVEL} "Fastops library is supported on x86_64 only, and not FreeBSD or Darwin") +endif() + +if(NOT ENABLE_FASTOPS) + message(STATUS "Not using fast vectorized mathematical functions library by Mikhail Parakhin") + return() +endif() + set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/fastops") set(SRCS "") @@ -17,3 +28,5 @@ set (SRCS ${SRCS} "${LIBRARY_DIR}/fastops/plain/ops_plain.cpp" "${LIBRARY_DIR}/f add_library(fastops ${SRCS}) target_include_directories(fastops SYSTEM PUBLIC "${LIBRARY_DIR}") + +add_library(ch_contrib::fastops ALIAS fastops) diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index cf29aa5bae3..9f80c0aaf53 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -1,3 +1,4 @@ +include(configure_config.cmake) configure_file(config_functions.h.in ${ConfigIncludePath}/config_functions.h) add_subdirectory(divide) @@ -14,7 +15,6 @@ target_link_libraries(clickhouse_functions PUBLIC ch_contrib::cityhash ch_contrib::farmhash - ${FASTOPS_LIBRARY} clickhouse_dictionaries clickhouse_dictionaries_embedded clickhouse_parsers @@ -58,8 +58,8 @@ if (TARGET ch_contrib::icu) target_link_libraries (clickhouse_functions PRIVATE ch_contrib::icu) endif () -if (USE_FASTOPS) - target_include_directories (clickhouse_functions SYSTEM PRIVATE ${FASTOPS_INCLUDE_DIR}) +if (TARGET ch_contrib::fastops) + target_link_libraries (clickhouse_functions PRIVATE ch_contrib::fastops) endif () if (ENABLE_EXAMPLES) diff --git a/src/Functions/configure_config.cmake b/src/Functions/configure_config.cmake new file mode 100644 index 00000000000..e2f874031bf --- /dev/null +++ b/src/Functions/configure_config.cmake @@ -0,0 +1,3 @@ +if (TARGET ch_contrib::fastops) + set(USE_FASTOPS 1) +endif() From 8cca397fcb2ec3e57da21d0b99311718b094765d Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 00:11:46 +0300 Subject: [PATCH 172/403] Fix config header for functions --- src/Functions/configure_config.cmake | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/Functions/configure_config.cmake b/src/Functions/configure_config.cmake index e2f874031bf..c729de5d1d8 100644 --- a/src/Functions/configure_config.cmake +++ b/src/Functions/configure_config.cmake @@ -1,3 +1,15 @@ if (TARGET ch_contrib::fastops) set(USE_FASTOPS 1) endif() +if (TARGET ch_contrib::base64) + set(USE_BASE64 1) +endif() +if (TARGET ch_contrib::simdjson) + set(USE_SIMDJSON 1) +endif() +if (TARGET ch_contrib::rapidjson) + set(USE_RAPIDJSON 1) +endif() +if (TARGET ch_contrib::s2) + set(USE_S2_GEOMETRY 1) +endif() From cefbcd6b1d50d0f5d117da81898609ade4173fc8 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 00:13:13 +0300 Subject: [PATCH 173/403] Fix rapidjson (add missing inclusion in contrib) --- contrib/CMakeLists.txt | 1 + src/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 26f2bdc699a..e1ffbb9e25d 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -134,6 +134,7 @@ endif () add_subdirectory (base64-cmake) add_subdirectory (simdjson-cmake) +add_subdirectory (rapidjson-cmake) add_subdirectory (fastops-cmake) add_subdirectory (libuv-cmake) add_subdirectory (amqpcpp-cmake) # requires: libuv diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 13040a5358c..b5fb7059433 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -507,7 +507,7 @@ if (TARGET ch_contrib::simdjson) endif() if (TARGET ch_contrib::rapidjson) - dbms_target_link_directories(PRIVATE ch_contrib::rapidjson) + dbms_target_link_libraries(PRIVATE ch_contrib::rapidjson) endif() dbms_target_link_libraries(PUBLIC consistent-hashing) From 3889f79a3d462f8b0f13fdec2e242d05b5f6ed9d Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 00:17:35 +0300 Subject: [PATCH 174/403] Remove unbundled h3 support --- CMakeLists.txt | 1 - cmake/find/h3.cmake | 39 ---------------------------- contrib/CMakeLists.txt | 5 +--- contrib/h3-cmake/CMakeLists.txt | 9 +++++++ src/Functions/CMakeLists.txt | 5 ++-- src/Functions/configure_config.cmake | 3 +++ 6 files changed, 15 insertions(+), 47 deletions(-) delete mode 100644 cmake/find/h3.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index c03c1692830..eaacccca394 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -491,7 +491,6 @@ include (cmake/find/krb5.cmake) include (cmake/find/libgsasl.cmake) include (cmake/find/cyrus-sasl.cmake) include (cmake/find/llvm.cmake) -include (cmake/find/h3.cmake) include (cmake/find/libxml2.cmake) include (cmake/find/pdqsort.cmake) include (cmake/find/miniselect.cmake) diff --git a/cmake/find/h3.cmake b/cmake/find/h3.cmake deleted file mode 100644 index e692b431e90..00000000000 --- a/cmake/find/h3.cmake +++ /dev/null @@ -1,39 +0,0 @@ -option (ENABLE_H3 "Enable H3" ${ENABLE_LIBRARIES}) -if(NOT ENABLE_H3) - if(USE_INTERNAL_H3_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal h3 library with ENABLE_H3=OFF") - endif () - return() -endif() - -option(USE_INTERNAL_H3_LIBRARY "Set to FALSE to use system h3 library instead of bundled" - ON) # we are not aware of any distribution that provides h3 package - -if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/h3/src/h3lib/include/h3Index.h") - if(USE_INTERNAL_H3_LIBRARY) - message(WARNING "submodule contrib/h3 is missing. to fix try run: \n git submodule update --init") - message(${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal h3 library") - set(USE_INTERNAL_H3_LIBRARY 0) - endif() - set(MISSING_INTERNAL_H3_LIBRARY 1) -endif() - -if(NOT USE_INTERNAL_H3_LIBRARY) - find_library(H3_LIBRARY h3) - find_path(H3_INCLUDE_DIR NAMES h3/h3api.h PATHS ${H3_INCLUDE_PATHS}) - - if(NOT H3_LIBRARY OR NOT H3_INCLUDE_DIR) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system h3 library") - endif() -endif() - -if (H3_LIBRARY AND H3_INCLUDE_DIR) - set (USE_H3 1) -elseif(NOT MISSING_INTERNAL_H3_LIBRARY) - set (H3_LIBRARY h3) - set (H3_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/h3/src/h3lib/include") - set (USE_H3 1) - set (USE_INTERNAL_H3_LIBRARY 1) -endif() - -message (STATUS "Using h3=${USE_H3}: ${H3_INCLUDE_DIR} : ${H3_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index e1ffbb9e25d..4d73db77b7e 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -76,13 +76,10 @@ endif() add_subdirectory (cityhash102) add_subdirectory (libfarmhash) add_subdirectory (icu-cmake) +add_subdirectory (h3-cmake) # TODO: refactor the contrib libraries below this comment. -if (USE_INTERNAL_H3_LIBRARY) - add_subdirectory(h3-cmake) -endif () - if (USE_INTERNAL_MYSQL_LIBRARY) add_subdirectory (mariadb-connector-c-cmake) endif () diff --git a/contrib/h3-cmake/CMakeLists.txt b/contrib/h3-cmake/CMakeLists.txt index f4c70dc476f..ea9432bed8f 100644 --- a/contrib/h3-cmake/CMakeLists.txt +++ b/contrib/h3-cmake/CMakeLists.txt @@ -1,3 +1,10 @@ +option (ENABLE_H3 "Enable H3" ${ENABLE_LIBRARIES}) + +if(NOT ENABLE_H3) + message(STATUS "Not using H3") + return() +endif() + set(H3_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/h3/src/h3lib") set(H3_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/h3/src/h3lib") @@ -30,3 +37,5 @@ target_compile_definitions(h3 PRIVATE H3_HAVE_VLA) if(M_LIBRARY) target_link_libraries(h3 PRIVATE ${M_LIBRARY}) endif() + +add_library(ch_contrib::h3 ALIAS h3) diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index 9f80c0aaf53..d4788d2c9be 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -77,9 +77,8 @@ endif() target_link_libraries(clickhouse_functions PRIVATE ch_contrib::lz4) -if (USE_H3) - target_link_libraries(clickhouse_functions PRIVATE ${H3_LIBRARY}) - target_include_directories(clickhouse_functions SYSTEM PRIVATE ${H3_INCLUDE_DIR}) +if (TARGET ch_contrib::h3) + target_link_libraries (clickhouse_functions PRIVATE ch_contrib::h3) endif() target_link_libraries(clickhouse_functions PRIVATE hyperscan) diff --git a/src/Functions/configure_config.cmake b/src/Functions/configure_config.cmake index c729de5d1d8..0bc72e1e158 100644 --- a/src/Functions/configure_config.cmake +++ b/src/Functions/configure_config.cmake @@ -13,3 +13,6 @@ endif() if (TARGET ch_contrib::s2) set(USE_S2_GEOMETRY 1) endif() +if (TARGET ch_contrib::h3) + set(USE_H3 1) +endif() From badfc7a9ad09b1b49122d30a37bef4d774b4a1ab Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 00:18:31 +0300 Subject: [PATCH 175/403] Fix s2 geometry --- contrib/CMakeLists.txt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 4d73db77b7e..fe75d9623f4 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -172,10 +172,7 @@ if (USE_NLP) endif() add_subdirectory (sqlite-cmake) - -if (USE_S2_GEOMETRY) - add_subdirectory(s2geometry-cmake) -endif() +add_subdirectory (s2geometry-cmake) # Put all targets defined here and in subdirectories under "contrib/" folders in GUI-based IDEs. # Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear From dea90967cf8736fc6e2dcc74c2adb641fa7bf149 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 00:20:47 +0300 Subject: [PATCH 176/403] Remove unbundled fastfloat support --- CMakeLists.txt | 1 - cmake/find/fast_float.cmake | 6 ------ contrib/fast_float-cmake/CMakeLists.txt | 3 ++- src/CMakeLists.txt | 3 +-- 4 files changed, 3 insertions(+), 10 deletions(-) delete mode 100644 cmake/find/fast_float.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index eaacccca394..b665865c50e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -496,7 +496,6 @@ include (cmake/find/pdqsort.cmake) include (cmake/find/miniselect.cmake) include (cmake/find/curl.cmake) include (cmake/find/s3.cmake) -include (cmake/find/fast_float.cmake) include (cmake/find/libpqxx.cmake) include (cmake/find/yaml-cpp.cmake) include (cmake/find/nlp.cmake) diff --git a/cmake/find/fast_float.cmake b/cmake/find/fast_float.cmake deleted file mode 100644 index 3e8b7cc5280..00000000000 --- a/cmake/find/fast_float.cmake +++ /dev/null @@ -1,6 +0,0 @@ -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/fast_float/include/fast_float/fast_float.h") - message (FATAL_ERROR "submodule contrib/fast_float is missing. to fix try run: \n git submodule update --init") -endif () - -set(FAST_FLOAT_LIBRARY fast_float) -set(FAST_FLOAT_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/fast_float/include/") diff --git a/contrib/fast_float-cmake/CMakeLists.txt b/contrib/fast_float-cmake/CMakeLists.txt index cd945f79a20..77fad935c9e 100644 --- a/contrib/fast_float-cmake/CMakeLists.txt +++ b/contrib/fast_float-cmake/CMakeLists.txt @@ -1,2 +1,3 @@ add_library(fast_float INTERFACE) -target_include_directories(fast_float INTERFACE "${ClickHouse_SOURCE_DIR}/contrib/fast_float/include/") +target_include_directories(fast_float SYSTEM BEFORE INTERFACE "${ClickHouse_SOURCE_DIR}/contrib/fast_float/include/") +add_library(ch_contrib::fast_float ALIAS fast_float) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b5fb7059433..40152252db0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -472,8 +472,7 @@ if (TARGET ch_contrib::msgpack) target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::msgpack) endif() -target_link_libraries (clickhouse_common_io PUBLIC ${FAST_FLOAT_LIBRARY}) -target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${FAST_FLOAT_INCLUDE_DIR}) +target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::fast_float) if (TARGET ch_contrib::rocksdb) dbms_target_link_libraries(PUBLIC ch_contrib::rocksdb) From 592bc1e97d80ab94b33b62e90c753a26355c2e36 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 00:23:14 +0300 Subject: [PATCH 177/403] Remove unbundled yaml-cpp support --- CMakeLists.txt | 1 - cmake/find/yaml-cpp.cmake | 9 ---- contrib/CMakeLists.txt | 6 +-- contrib/yaml-cpp-cmake/CMakeLists.txt | 68 +++++++++++++++------------ src/Common/Config/CMakeLists.txt | 6 +-- src/configure_config.cmake | 3 ++ 6 files changed, 45 insertions(+), 48 deletions(-) delete mode 100644 cmake/find/yaml-cpp.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index b665865c50e..400dbcb6a6a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -497,7 +497,6 @@ include (cmake/find/miniselect.cmake) include (cmake/find/curl.cmake) include (cmake/find/s3.cmake) include (cmake/find/libpqxx.cmake) -include (cmake/find/yaml-cpp.cmake) include (cmake/find/nlp.cmake) include (cmake/find/filelog.cmake) include (cmake/find/sentry.cmake) diff --git a/cmake/find/yaml-cpp.cmake b/cmake/find/yaml-cpp.cmake deleted file mode 100644 index 2aba6808e31..00000000000 --- a/cmake/find/yaml-cpp.cmake +++ /dev/null @@ -1,9 +0,0 @@ -option(USE_YAML_CPP "Enable yaml-cpp" ${ENABLE_LIBRARIES}) - -if (NOT USE_YAML_CPP) - return() -endif() - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/yaml-cpp/README.md") - message (ERROR "submodule contrib/yaml-cpp is missing. to fix try run: \n git submodule update --init") -endif() diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index fe75d9623f4..6153d2be47e 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -39,11 +39,7 @@ add_subdirectory (replxx-cmake) add_subdirectory (unixodbc-cmake) add_subdirectory (nanodbc-cmake) add_subdirectory (capnproto-cmake) - -if (USE_YAML_CPP) - add_subdirectory (yaml-cpp-cmake) -endif() - +add_subdirectory (yaml-cpp-cmake) add_subdirectory (re2-cmake) add_subdirectory (xz-cmake) add_subdirectory (brotli-cmake) diff --git a/contrib/yaml-cpp-cmake/CMakeLists.txt b/contrib/yaml-cpp-cmake/CMakeLists.txt index ed0287de110..1681bfe4015 100644 --- a/contrib/yaml-cpp-cmake/CMakeLists.txt +++ b/contrib/yaml-cpp-cmake/CMakeLists.txt @@ -1,39 +1,47 @@ +option(ENABLE_YAML_CPP "Enable yaml-cpp" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_YAML_CPP) + message(STATUS "Not using yaml") + return() +endif() + set (LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/yaml-cpp) set (SRCS - ${LIBRARY_DIR}/src/binary.cpp - ${LIBRARY_DIR}/src/emitterutils.cpp - ${LIBRARY_DIR}/src/null.cpp - ${LIBRARY_DIR}/src/scantoken.cpp - ${LIBRARY_DIR}/src/convert.cpp - ${LIBRARY_DIR}/src/exceptions.cpp - ${LIBRARY_DIR}/src/ostream_wrapper.cpp - ${LIBRARY_DIR}/src/simplekey.cpp - ${LIBRARY_DIR}/src/depthguard.cpp - ${LIBRARY_DIR}/src/exp.cpp - ${LIBRARY_DIR}/src/parse.cpp - ${LIBRARY_DIR}/src/singledocparser.cpp - ${LIBRARY_DIR}/src/directives.cpp - ${LIBRARY_DIR}/src/memory.cpp - ${LIBRARY_DIR}/src/parser.cpp - ${LIBRARY_DIR}/src/stream.cpp - ${LIBRARY_DIR}/src/emit.cpp - ${LIBRARY_DIR}/src/nodebuilder.cpp - ${LIBRARY_DIR}/src/regex_yaml.cpp - ${LIBRARY_DIR}/src/tag.cpp - ${LIBRARY_DIR}/src/emitfromevents.cpp - ${LIBRARY_DIR}/src/node.cpp - ${LIBRARY_DIR}/src/scanner.cpp - ${LIBRARY_DIR}/src/emitter.cpp - ${LIBRARY_DIR}/src/node_data.cpp - ${LIBRARY_DIR}/src/scanscalar.cpp - ${LIBRARY_DIR}/src/emitterstate.cpp - ${LIBRARY_DIR}/src/nodeevents.cpp - ${LIBRARY_DIR}/src/scantag.cpp + ${LIBRARY_DIR}/src/binary.cpp + ${LIBRARY_DIR}/src/emitterutils.cpp + ${LIBRARY_DIR}/src/null.cpp + ${LIBRARY_DIR}/src/scantoken.cpp + ${LIBRARY_DIR}/src/convert.cpp + ${LIBRARY_DIR}/src/exceptions.cpp + ${LIBRARY_DIR}/src/ostream_wrapper.cpp + ${LIBRARY_DIR}/src/simplekey.cpp + ${LIBRARY_DIR}/src/depthguard.cpp + ${LIBRARY_DIR}/src/exp.cpp + ${LIBRARY_DIR}/src/parse.cpp + ${LIBRARY_DIR}/src/singledocparser.cpp + ${LIBRARY_DIR}/src/directives.cpp + ${LIBRARY_DIR}/src/memory.cpp + ${LIBRARY_DIR}/src/parser.cpp + ${LIBRARY_DIR}/src/stream.cpp + ${LIBRARY_DIR}/src/emit.cpp + ${LIBRARY_DIR}/src/nodebuilder.cpp + ${LIBRARY_DIR}/src/regex_yaml.cpp + ${LIBRARY_DIR}/src/tag.cpp + ${LIBRARY_DIR}/src/emitfromevents.cpp + ${LIBRARY_DIR}/src/node.cpp + ${LIBRARY_DIR}/src/scanner.cpp + ${LIBRARY_DIR}/src/emitter.cpp + ${LIBRARY_DIR}/src/node_data.cpp + ${LIBRARY_DIR}/src/scanscalar.cpp + ${LIBRARY_DIR}/src/emitterstate.cpp + ${LIBRARY_DIR}/src/nodeevents.cpp + ${LIBRARY_DIR}/src/scantag.cpp ) add_library (yaml-cpp ${SRCS}) - target_include_directories(yaml-cpp PRIVATE ${LIBRARY_DIR}/include/yaml-cpp) target_include_directories(yaml-cpp SYSTEM BEFORE PUBLIC ${LIBRARY_DIR}/include) + +add_library (ch_contrib::yaml_cpp ALIAS yaml-cpp) diff --git a/src/Common/Config/CMakeLists.txt b/src/Common/Config/CMakeLists.txt index cc41a8b2bb2..ec7bdd10196 100644 --- a/src/Common/Config/CMakeLists.txt +++ b/src/Common/Config/CMakeLists.txt @@ -27,7 +27,7 @@ target_link_libraries(clickhouse_common_config_no_zookeeper_log string_utils ) -if (USE_YAML_CPP) - target_link_libraries(clickhouse_common_config PRIVATE yaml-cpp) - target_link_libraries(clickhouse_common_config_no_zookeeper_log PRIVATE yaml-cpp) +if (TARGET ch_contrib::yaml_cpp) + target_link_libraries(clickhouse_common_config PRIVATE ch_contrib::yaml_cpp) + target_link_libraries(clickhouse_common_config_no_zookeeper_log PRIVATE ch_contrib::yaml_cpp) endif() diff --git a/src/configure_config.cmake b/src/configure_config.cmake index ad1f774d7f5..ea2593f95d2 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -52,6 +52,9 @@ endif() if (TARGET ch_contrib::base64) set(USE_BASE64 1) endif() +if (TARGET ch_contrib::yaml_cpp) + set(USE_YAML_CPP 1) +endif() if (TARGET ch_contrib::sqlite) set(USE_SQLITE 1) endif() From a773e7ff01dfc084cf0beae8ff74612872674fe6 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 00:29:27 +0300 Subject: [PATCH 178/403] Remove unbundled libpqxx support --- CMakeLists.txt | 2 -- cmake/find/libpqxx.cmake | 31 ------------------- contrib/CMakeLists.txt | 6 ++-- contrib/libpq-cmake/CMakeLists.txt | 6 ++++ contrib/libpqxx-cmake/CMakeLists.txt | 12 +++++-- programs/server/Server.cpp | 4 +-- src/CMakeLists.txt | 5 ++- src/Common/config.h.in | 1 - src/Compression/CompressionCodecEncrypted.cpp | 8 ++--- ...StorageSystemBuildOptions.generated.cpp.in | 1 - src/configure_config.cmake | 3 ++ 11 files changed, 28 insertions(+), 51 deletions(-) delete mode 100644 cmake/find/libpqxx.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 400dbcb6a6a..6897aace9e1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -139,7 +139,6 @@ if (ENABLE_FUZZING) set (ENABLE_CLICKHOUSE_ODBC_BRIDGE OFF) set (ENABLE_LIBRARIES 0) set (ENABLE_SSL 1) - set (USE_INTERNAL_SSL_LIBRARY 1) set (USE_UNWIND ON) set (ENABLE_EMBEDDED_COMPILER 0) set (ENABLE_EXAMPLES 0) @@ -496,7 +495,6 @@ include (cmake/find/pdqsort.cmake) include (cmake/find/miniselect.cmake) include (cmake/find/curl.cmake) include (cmake/find/s3.cmake) -include (cmake/find/libpqxx.cmake) include (cmake/find/nlp.cmake) include (cmake/find/filelog.cmake) include (cmake/find/sentry.cmake) diff --git a/cmake/find/libpqxx.cmake b/cmake/find/libpqxx.cmake deleted file mode 100644 index 68dddffde70..00000000000 --- a/cmake/find/libpqxx.cmake +++ /dev/null @@ -1,31 +0,0 @@ -option(ENABLE_LIBPQXX "Enalbe libpqxx" ${ENABLE_LIBRARIES}) - -if (NOT ENABLE_LIBPQXX) - return() -endif() - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libpqxx/src") - message (WARNING "submodule contrib/libpqxx is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libpqxx library") - set (USE_LIBPQXX 0) - return() -endif() - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libpq/include") - message (ERROR "submodule contrib/libpq is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libpq needed for libpqxx") - set (USE_LIBPQXX 0) - return() -endif() - -if (NOT USE_INTERNAL_SSL_LIBRARY) - set (USE_LIBPQXX 0) -else () -set (USE_LIBPQXX 1) -set (LIBPQXX_LIBRARY libpqxx) -set (LIBPQ_LIBRARY libpq) -set (LIBPQXX_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libpqxx/include") -set (LIBPQ_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/libpq") -message (STATUS "Using libpqxx=${USE_LIBPQXX}: ${LIBPQXX_INCLUDE_DIR} : ${LIBPQXX_LIBRARY}") -message (STATUS "Using libpq: ${LIBPQ_ROOT_DIR} : ${LIBPQ_INCLUDE_DIR} : ${LIBPQ_LIBRARY}") -endif() diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 6153d2be47e..b44f430fac3 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -153,10 +153,8 @@ if (USE_KRB5) endif() -if (USE_LIBPQXX) - add_subdirectory (libpq-cmake) - add_subdirectory (libpqxx-cmake) -endif() +add_subdirectory (libpqxx-cmake) +add_subdirectory (libpq-cmake) add_subdirectory (nuraft-cmake) add_subdirectory(fast_float-cmake) diff --git a/contrib/libpq-cmake/CMakeLists.txt b/contrib/libpq-cmake/CMakeLists.txt index 2d2e0c428fe..a1ffa632231 100644 --- a/contrib/libpq-cmake/CMakeLists.txt +++ b/contrib/libpq-cmake/CMakeLists.txt @@ -1,3 +1,7 @@ +if (NOT ENABLE_LIBPQXX) + return() +endif() + set(LIBPQ_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libpq") set(SRCS @@ -60,3 +64,5 @@ target_include_directories (libpq SYSTEM PUBLIC "${LIBPQ_SOURCE_DIR}/include") target_include_directories (libpq SYSTEM PRIVATE "${LIBPQ_SOURCE_DIR}/configs") target_link_libraries (libpq PRIVATE ssl) + +add_library(ch_contrib::libpq ALIAS libpq) diff --git a/contrib/libpqxx-cmake/CMakeLists.txt b/contrib/libpqxx-cmake/CMakeLists.txt index 2804a875436..5462c3cdccf 100644 --- a/contrib/libpqxx-cmake/CMakeLists.txt +++ b/contrib/libpqxx-cmake/CMakeLists.txt @@ -1,3 +1,10 @@ +option(ENABLE_LIBPQXX "Enalbe libpqxx" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_LIBPQXX) + message(STATUS "Not using libpqxx") + return() +endif() + set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/libpqxx") set (SRCS @@ -65,6 +72,7 @@ set (HDRS add_library(libpqxx ${SRCS} ${HDRS}) -target_link_libraries(libpqxx PUBLIC ${LIBPQ_LIBRARY}) -target_include_directories (libpqxx SYSTEM PRIVATE "${LIBRARY_DIR}/include") +target_link_libraries(libpqxx PUBLIC ch_contrib::libpq) +target_include_directories (libpqxx SYSTEM BEFORE PUBLIC "${LIBRARY_DIR}/include") +add_library(ch_contrib::libpqxx ALIAS libpqxx) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 184855b386e..a49ccc79b63 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -98,9 +98,7 @@ #endif #if USE_SSL -# if USE_INTERNAL_SSL_LIBRARY -# include -# endif +# include # include # include #endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 40152252db0..17b6bf85056 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -478,9 +478,8 @@ if (TARGET ch_contrib::rocksdb) dbms_target_link_libraries(PUBLIC ch_contrib::rocksdb) endif() -if (USE_LIBPQXX) - dbms_target_link_libraries(PUBLIC ${LIBPQXX_LIBRARY}) - dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${LIBPQXX_INCLUDE_DIR}) +if (TARGET ch_contrib::libpqxx) + dbms_target_link_libraries(PUBLIC ch_contrib::libpqxx) endif() if (USE_DATASKETCHES) diff --git a/src/Common/config.h.in b/src/Common/config.h.in index 5fa9b2ee2ba..ce8f0286aac 100644 --- a/src/Common/config.h.in +++ b/src/Common/config.h.in @@ -4,7 +4,6 @@ #cmakedefine01 USE_BASE64 #cmakedefine01 USE_SSL -#cmakedefine01 USE_INTERNAL_SSL_LIBRARY #cmakedefine01 USE_HDFS #cmakedefine01 USE_AWS_S3 #cmakedefine01 USE_AZURE_BLOB_STORAGE diff --git a/src/Compression/CompressionCodecEncrypted.cpp b/src/Compression/CompressionCodecEncrypted.cpp index 6cb3874f808..ddf2fb26712 100644 --- a/src/Compression/CompressionCodecEncrypted.cpp +++ b/src/Compression/CompressionCodecEncrypted.cpp @@ -9,7 +9,7 @@ #include // This depends on BoringSSL-specific API, notably . -#if USE_SSL && USE_INTERNAL_SSL_LIBRARY +#if USE_SSL #include #include #include @@ -66,7 +66,7 @@ uint8_t getMethodCode(EncryptionMethod Method) } // end of namespace DB -#if USE_SSL && USE_INTERNAL_SSL_LIBRARY +#if USE_SSL namespace DB { @@ -513,7 +513,7 @@ void CompressionCodecEncrypted::doDecompressData(const char * source, UInt32 sou } -#else /* USE_SSL && USE_INTERNAL_SSL_LIBRARY */ +#else /* USE_SSL */ namespace DB { @@ -551,7 +551,7 @@ void CompressionCodecEncrypted::Configuration::load(const Poco::Util::AbstractCo } -#endif /* USE_SSL && USE_INTERNAL_SSL_LIBRARY */ +#endif /* USE_SSL */ namespace DB { diff --git a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in index 90c5c92c576..31a25dec599 100644 --- a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in +++ b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in @@ -58,7 +58,6 @@ const char * auto_config_build[] "USE_OPENCL", "@USE_OPENCL@", "USE_LIBPQXX", "@USE_LIBPQXX@", "USE_AZURE_BLOB_STORAGE", "@USE_AZURE_BLOB_STORAGE@", - "USE_INTERNAL_SSL_LIBRARY", "@USE_INTERNAL_SSL_LIBRARY@", "USE_AWS_S3", "@USE_AWS_S3@", "USE_CASSANDRA", "@USE_CASSANDRA@", "USE_YAML_CPP", "@USE_YAML_CPP@", diff --git a/src/configure_config.cmake b/src/configure_config.cmake index ea2593f95d2..f148b5cf551 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -58,3 +58,6 @@ endif() if (TARGET ch_contrib::sqlite) set(USE_SQLITE 1) endif() +if (TARGET ch_contrib::libpqxx) + set(USE_LIBPQXX 1) +endif() From 2bd688402a69c1c92ba0d6812fe46475f436850f Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 00:34:07 +0300 Subject: [PATCH 179/403] Move miniselect --- CMakeLists.txt | 1 - cmake/find/miniselect.cmake | 2 -- contrib/CMakeLists.txt | 1 + contrib/miniselect-cmake/CMakeLists.txt | 3 +++ src/CMakeLists.txt | 4 ++-- 5 files changed, 6 insertions(+), 5 deletions(-) delete mode 100644 cmake/find/miniselect.cmake create mode 100644 contrib/miniselect-cmake/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 6897aace9e1..4e576c46bb3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -492,7 +492,6 @@ include (cmake/find/cyrus-sasl.cmake) include (cmake/find/llvm.cmake) include (cmake/find/libxml2.cmake) include (cmake/find/pdqsort.cmake) -include (cmake/find/miniselect.cmake) include (cmake/find/curl.cmake) include (cmake/find/s3.cmake) include (cmake/find/nlp.cmake) diff --git a/cmake/find/miniselect.cmake b/cmake/find/miniselect.cmake deleted file mode 100644 index 0a50c9bf4a8..00000000000 --- a/cmake/find/miniselect.cmake +++ /dev/null @@ -1,2 +0,0 @@ -set(MINISELECT_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/miniselect/include) -message(STATUS "Using miniselect: ${MINISELECT_INCLUDE_DIR}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index b44f430fac3..11c13d35770 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -21,6 +21,7 @@ endif() set_property(DIRECTORY PROPERTY EXCLUDE_FROM_ALL 1) +add_subdirectory (miniselect-cmake) add_subdirectory (sparsehash-c11-cmake) add_subdirectory (abseil-cpp-cmake) add_subdirectory (magic-enum-cmake) diff --git a/contrib/miniselect-cmake/CMakeLists.txt b/contrib/miniselect-cmake/CMakeLists.txt new file mode 100644 index 00000000000..f6dda7a1474 --- /dev/null +++ b/contrib/miniselect-cmake/CMakeLists.txt @@ -0,0 +1,3 @@ +add_library(_miniselect INTERFACE) +target_include_directories(_miniselect BEFORE INTERFACE ${ClickHouse_SOURCE_DIR}/contrib/miniselect/include) +add_library(ch_contrib::miniselect ALIAS _miniselect) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 17b6bf85056..10d95dd30f2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -380,8 +380,8 @@ dbms_target_include_directories(PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/Core/include target_include_directories(clickhouse_common_io SYSTEM BEFORE PUBLIC ${PDQSORT_INCLUDE_DIR}) dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${PDQSORT_INCLUDE_DIR}) -target_include_directories(clickhouse_common_io SYSTEM BEFORE PUBLIC ${MINISELECT_INCLUDE_DIR}) -dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${MINISELECT_INCLUDE_DIR}) +target_link_libraries(clickhouse_common_io PUBLIC ch_contrib::miniselect) +dbms_target_link_libraries(PUBLIC ch_contrib::miniselect) dbms_target_link_libraries(PRIVATE ch_contrib::zstd) target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::zstd) From 7420bd9b18cde4ff76d3767ae7be9faf6cea4602 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 00:37:14 +0300 Subject: [PATCH 180/403] Move pdqsort --- CMakeLists.txt | 1 - cmake/find/pdqsort.cmake | 2 -- contrib/CMakeLists.txt | 1 + contrib/pdqsort-cmake/CMakeLists.txt | 3 +++ src/CMakeLists.txt | 10 ++++++---- src/Common/examples/CMakeLists.txt | 3 +-- 6 files changed, 11 insertions(+), 9 deletions(-) delete mode 100644 cmake/find/pdqsort.cmake create mode 100644 contrib/pdqsort-cmake/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e576c46bb3..9ca4e5ed7e5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -491,7 +491,6 @@ include (cmake/find/libgsasl.cmake) include (cmake/find/cyrus-sasl.cmake) include (cmake/find/llvm.cmake) include (cmake/find/libxml2.cmake) -include (cmake/find/pdqsort.cmake) include (cmake/find/curl.cmake) include (cmake/find/s3.cmake) include (cmake/find/nlp.cmake) diff --git a/cmake/find/pdqsort.cmake b/cmake/find/pdqsort.cmake deleted file mode 100644 index 51461044cf9..00000000000 --- a/cmake/find/pdqsort.cmake +++ /dev/null @@ -1,2 +0,0 @@ -set(PDQSORT_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/pdqsort) -message(STATUS "Using pdqsort: ${PDQSORT_INCLUDE_DIR}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 11c13d35770..89f774eede9 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -22,6 +22,7 @@ endif() set_property(DIRECTORY PROPERTY EXCLUDE_FROM_ALL 1) add_subdirectory (miniselect-cmake) +add_subdirectory (pdqsort-cmake) add_subdirectory (sparsehash-c11-cmake) add_subdirectory (abseil-cpp-cmake) add_subdirectory (magic-enum-cmake) diff --git a/contrib/pdqsort-cmake/CMakeLists.txt b/contrib/pdqsort-cmake/CMakeLists.txt new file mode 100644 index 00000000000..485f345807e --- /dev/null +++ b/contrib/pdqsort-cmake/CMakeLists.txt @@ -0,0 +1,3 @@ +add_library(_pdqsort INTERFACE) +target_include_directories(_pdqsort SYSTEM BEFORE INTERFACE ${ClickHouse_SOURCE_DIR}/contrib/pdqsort) +add_library(ch_contrib::pdqsort ALIAS _pdqsort) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 10d95dd30f2..47c178ee3fb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -378,10 +378,12 @@ dbms_target_link_libraries ( target_include_directories(clickhouse_common_io PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/Core/include") # uses some includes from core dbms_target_include_directories(PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/Core/include") -target_include_directories(clickhouse_common_io SYSTEM BEFORE PUBLIC ${PDQSORT_INCLUDE_DIR}) -dbms_target_include_directories(SYSTEM BEFORE PUBLIC ${PDQSORT_INCLUDE_DIR}) -target_link_libraries(clickhouse_common_io PUBLIC ch_contrib::miniselect) -dbms_target_link_libraries(PUBLIC ch_contrib::miniselect) +target_link_libraries(clickhouse_common_io PUBLIC + ch_contrib::miniselect + ch_contrib::pdqsort) +dbms_target_link_libraries(PUBLIC + ch_contrib::miniselect + ch_contrib::pdqsort) dbms_target_link_libraries(PRIVATE ch_contrib::zstd) target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::zstd) diff --git a/src/Common/examples/CMakeLists.txt b/src/Common/examples/CMakeLists.txt index d4057954969..a0628f0540c 100644 --- a/src/Common/examples/CMakeLists.txt +++ b/src/Common/examples/CMakeLists.txt @@ -23,8 +23,7 @@ add_executable (compact_array compact_array.cpp) target_link_libraries (compact_array PRIVATE clickhouse_common_io) add_executable (radix_sort radix_sort.cpp) -target_link_libraries (radix_sort PRIVATE clickhouse_common_io) -target_include_directories(radix_sort SYSTEM PRIVATE ${PDQSORT_INCLUDE_DIR}) +target_link_libraries (radix_sort PRIVATE clickhouse_common_io ch_contrib::pdqsort) add_executable (arena_with_free_lists arena_with_free_lists.cpp) target_link_libraries (arena_with_free_lists PRIVATE dbms) From 748a75644c54bd3557e6c5be2ee0d3aa1dceb429 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 00:38:43 +0300 Subject: [PATCH 181/403] Cleanup filelog --- CMakeLists.txt | 1 - cmake/find/filelog.cmake | 8 -------- src/CMakeLists.txt | 6 ++++-- 3 files changed, 4 insertions(+), 11 deletions(-) delete mode 100644 cmake/find/filelog.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 9ca4e5ed7e5..2cb354fe17b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -494,7 +494,6 @@ include (cmake/find/libxml2.cmake) include (cmake/find/curl.cmake) include (cmake/find/s3.cmake) include (cmake/find/nlp.cmake) -include (cmake/find/filelog.cmake) include (cmake/find/sentry.cmake) include (cmake/find/datasketches.cmake) diff --git a/cmake/find/filelog.cmake b/cmake/find/filelog.cmake deleted file mode 100644 index f545ee9d0ed..00000000000 --- a/cmake/find/filelog.cmake +++ /dev/null @@ -1,8 +0,0 @@ -# StorageFileLog only support Linux platform -if (OS_LINUX) - set (USE_FILELOG 1) - message (STATUS "Using StorageFileLog = 1") -else() - message(STATUS "StorageFileLog is only supported on Linux") -endif () - diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 47c178ee3fb..45d115f36b5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -124,9 +124,11 @@ if (TARGET ch_contrib::hivemetastore AND TARGET ch_contrib::hdfs) add_headers_and_sources(dbms Storages/Hive) endif() -if(USE_FILELOG) +if (OS_LINUX) add_headers_and_sources(dbms Storages/FileLog) -endif() +else() + message(STATUS "StorageFileLog is only supported on Linux") +endif () list (APPEND clickhouse_common_io_sources ${CONFIG_BUILD}) list (APPEND clickhouse_common_io_headers ${CONFIG_VERSION} ${CONFIG_COMMON}) From bc2dbe10d68b5fd3bc80af88c6c6c9fd127ef1e0 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 00:44:15 +0300 Subject: [PATCH 182/403] Remove unbundled krb5 support --- CMakeLists.txt | 1 - cmake/find/krb5.cmake | 25 ------------------------ contrib/CMakeLists.txt | 9 +++------ contrib/krb5-cmake/CMakeLists.txt | 28 ++++++++++++++++++++------- contrib/libgsasl-cmake/CMakeLists.txt | 6 +++--- contrib/libhdfs3-cmake/CMakeLists.txt | 2 +- src/CMakeLists.txt | 5 ++--- src/configure_config.cmake | 3 +++ 8 files changed, 33 insertions(+), 46 deletions(-) delete mode 100644 cmake/find/krb5.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 2cb354fe17b..cf9d651321e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -486,7 +486,6 @@ message (STATUS include (GNUInstallDirs) -include (cmake/find/krb5.cmake) include (cmake/find/libgsasl.cmake) include (cmake/find/cyrus-sasl.cmake) include (cmake/find/llvm.cmake) diff --git a/cmake/find/krb5.cmake b/cmake/find/krb5.cmake deleted file mode 100644 index 24cc51325dc..00000000000 --- a/cmake/find/krb5.cmake +++ /dev/null @@ -1,25 +0,0 @@ -OPTION(ENABLE_KRB5 "Enable krb5" ${ENABLE_LIBRARIES}) - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/krb5/README") - message (WARNING "submodule contrib/krb5 is missing. to fix try run: \n git submodule update --init") - set (ENABLE_KRB5 0) -endif () - -if (NOT CMAKE_SYSTEM_NAME MATCHES "Linux" AND NOT (CMAKE_SYSTEM_NAME MATCHES "Darwin" AND NOT CMAKE_CROSSCOMPILING)) - message (WARNING "krb5 disabled in non-Linux and non-native-Darwin environments") - set (ENABLE_KRB5 0) -endif () - -if (ENABLE_KRB5) - - set (USE_KRB5 1) - set (KRB5_LIBRARY krb5) - - set (KRB5_INCLUDE_DIR - "${ClickHouse_SOURCE_DIR}/contrib/krb5/src/include" - "${ClickHouse_BINARY_DIR}/contrib/krb5-cmake/include" - ) - -endif () - -message (STATUS "Using krb5=${USE_KRB5}: ${KRB5_INCLUDE_DIR} : ${KRB5_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 89f774eede9..b0206b096da 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -147,14 +147,11 @@ endif() add_subdirectory (fmtlib-cmake) -if (USE_KRB5) - add_subdirectory (krb5-cmake) - if (USE_CYRUS_SASL) - add_subdirectory (cyrus-sasl-cmake) - endif() +add_subdirectory (krb5-cmake) +if (USE_CYRUS_SASL) + add_subdirectory (cyrus-sasl-cmake) # for krb5 endif() - add_subdirectory (libpqxx-cmake) add_subdirectory (libpq-cmake) diff --git a/contrib/krb5-cmake/CMakeLists.txt b/contrib/krb5-cmake/CMakeLists.txt index 5bb5e981579..32ceecb889c 100644 --- a/contrib/krb5-cmake/CMakeLists.txt +++ b/contrib/krb5-cmake/CMakeLists.txt @@ -1,3 +1,15 @@ +set (ENABLE_KRB5_DEFAULT 1) +if (NOT CMAKE_SYSTEM_NAME MATCHES "Linux" AND NOT (CMAKE_SYSTEM_NAME MATCHES "Darwin" AND NOT CMAKE_CROSSCOMPILING)) + message (WARNING "krb5 disabled in non-Linux and non-native-Darwin environments") + set (ENABLE_KRB5_DEFAULT 0) +endif () +OPTION(ENABLE_KRB5 "Enable krb5" ${ENABLE_KRB5_DEFAULT}) + +if (NOT ENABLE_KRB5) + message(STATUS "Not using krb5") + return() +endif () + find_program(AWK_PROGRAM awk) if(NOT AWK_PROGRAM) message(FATAL_ERROR "You need the awk program to build ClickHouse with krb5 enabled.") @@ -546,10 +558,10 @@ add_custom_target( VERBATIM ) -add_library(${KRB5_LIBRARY}) +add_library(krb5) add_dependencies( - ${KRB5_LIBRARY} + krb5 ERRMAP_H ERROR_MAP_H KRB_5_H @@ -567,7 +579,7 @@ if(CMAKE_SYSTEM_NAME MATCHES "Darwin") list(APPEND ALL_SRCS "${CMAKE_CURRENT_BINARY_DIR}/include_private/kcmrpc.c") endif() -target_sources(${KRB5_LIBRARY} PRIVATE +target_sources(krb5 PRIVATE ${ALL_SRCS} ) @@ -639,12 +651,12 @@ add_custom_command( -target_include_directories(${KRB5_LIBRARY} PUBLIC +target_include_directories(krb5 SYSTEM BEFORE PUBLIC "${KRB5_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include" ) -target_include_directories(${KRB5_LIBRARY} PRIVATE +target_include_directories(krb5 PRIVATE "${CMAKE_CURRENT_BINARY_DIR}/include_private" # For autoconf.h and other generated headers. ${KRB5_SOURCE_DIR} "${KRB5_SOURCE_DIR}/include" @@ -666,7 +678,7 @@ target_include_directories(${KRB5_LIBRARY} PRIVATE "${KRB5_SOURCE_DIR}/lib/krb5/os" ) -target_compile_definitions(${KRB5_LIBRARY} PRIVATE +target_compile_definitions(krb5 PRIVATE KRB5_PRIVATE _GSS_STATIC_LINK=1 KRB5_DEPRECATED=1 @@ -676,4 +688,6 @@ target_compile_definitions(${KRB5_LIBRARY} PRIVATE LIBDIR="/usr/local/lib" ) -target_link_libraries(${KRB5_LIBRARY} PRIVATE OpenSSL::Crypto OpenSSL::SSL) +target_link_libraries(krb5 PRIVATE OpenSSL::Crypto OpenSSL::SSL) + +add_library(ch_contrib::krb5 ALIAS krb5) diff --git a/contrib/libgsasl-cmake/CMakeLists.txt b/contrib/libgsasl-cmake/CMakeLists.txt index 102ef12b9f5..69228c8de0c 100644 --- a/contrib/libgsasl-cmake/CMakeLists.txt +++ b/contrib/libgsasl-cmake/CMakeLists.txt @@ -84,7 +84,7 @@ set(SRCS ${SRC_DIR}/login/server.c ) -if (USE_KRB5) +if (TARGET ch_contrib::krb5) set(SRCS ${SRCS} ${SRC_DIR}/gssapi/client.c ${SRC_DIR}/gssapi/mechinfo.c @@ -101,7 +101,7 @@ target_include_directories(gsasl PRIVATE "${ClickHouse_SOURCE_DIR}/contrib/libgs target_compile_definitions (gsasl PRIVATE HAVE_CONFIG_H=1) -if (USE_KRB5) - target_link_libraries(gsasl PUBLIC ${KRB5_LIBRARY}) +if (TARGET ch_contrib::krb5) + target_link_libraries(gsasl PUBLIC ch_contrib::krb5) target_compile_definitions (gsasl PRIVATE HAVE_GSSAPI_H=1 USE_GSSAPI=1) endif() diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index d9ed10282cb..94b2bfe5eb3 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -120,7 +120,7 @@ target_include_directories(_hdfs3 SYSTEM BEFORE INTERFACE "${ClickHouse_SOURCE_D target_link_libraries(_hdfs3 PRIVATE ${LIBGSASL_LIBRARY}) if (WITH_KERBEROS) - target_link_libraries(_hdfs3 PRIVATE ${KRB5_LIBRARY}) + target_link_libraries(_hdfs3 PRIVATE ch_contrib::krb5) endif() target_link_libraries(_hdfs3 PRIVATE ${LIBXML2_LIBRARIES}) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 45d115f36b5..bf1dad5933c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -345,9 +345,8 @@ if (USE_CYRUS_SASL) dbms_target_link_libraries(PRIVATE ${CYRUS_SASL_LIBRARY}) endif() -if (USE_KRB5) - dbms_target_include_directories(SYSTEM BEFORE PRIVATE ${KRB5_INCLUDE_DIR}) - dbms_target_link_libraries(PRIVATE ${KRB5_LIBRARY}) +if (TARGET ch_contrib::krb5) + dbms_target_link_libraries(PRIVATE ch_contrib::krb5) endif() if (TARGET ch_contrib::nuraft) diff --git a/src/configure_config.cmake b/src/configure_config.cmake index f148b5cf551..4c599038c4d 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -61,3 +61,6 @@ endif() if (TARGET ch_contrib::libpqxx) set(USE_LIBPQXX 1) endif() +if (TARGET ch_contrib::krb5) + set(USE_KRB5 1) +endif() From 614f86edbb03301634f7a1f9ae4fe5bf8bc5b3bb Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 00:48:09 +0300 Subject: [PATCH 183/403] Remove unbundled gsasl support --- CMakeLists.txt | 1 - cmake/find/libgsasl.cmake | 40 ------------------- contrib/CMakeLists.txt | 4 +- contrib/libgsasl-cmake/CMakeLists.txt | 11 ++++- contrib/libhdfs3-cmake/CMakeLists.txt | 3 +- ...StorageSystemBuildOptions.generated.cpp.in | 1 - 6 files changed, 12 insertions(+), 48 deletions(-) delete mode 100644 cmake/find/libgsasl.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index cf9d651321e..b2c0e054b7f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -486,7 +486,6 @@ message (STATUS include (GNUInstallDirs) -include (cmake/find/libgsasl.cmake) include (cmake/find/cyrus-sasl.cmake) include (cmake/find/llvm.cmake) include (cmake/find/libxml2.cmake) diff --git a/cmake/find/libgsasl.cmake b/cmake/find/libgsasl.cmake deleted file mode 100644 index d4e1ebce629..00000000000 --- a/cmake/find/libgsasl.cmake +++ /dev/null @@ -1,40 +0,0 @@ -option(ENABLE_GSASL_LIBRARY "Enable gsasl library" ${ENABLE_LIBRARIES}) - -if (NOT ENABLE_GSASL_LIBRARY) - if(USE_INTERNAL_LIBGSASL_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal libgsasl library with ENABLE_GSASL_LIBRARY=OFF") - endif() - return() -endif() - -option (USE_INTERNAL_LIBGSASL_LIBRARY "Set to FALSE to use system libgsasl library instead of bundled" ON) - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libgsasl/src/gsasl.h") - if (USE_INTERNAL_LIBGSASL_LIBRARY) - message (WARNING "submodule contrib/libgsasl is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libgsasl") - set (USE_INTERNAL_LIBGSASL_LIBRARY 0) - endif () - set (MISSING_INTERNAL_LIBGSASL_LIBRARY 1) -endif () - -if (NOT USE_INTERNAL_LIBGSASL_LIBRARY) - find_library (LIBGSASL_LIBRARY gsasl) - find_path (LIBGSASL_INCLUDE_DIR NAMES gsasl.h PATHS ${LIBGSASL_INCLUDE_PATHS}) - if (NOT LIBGSASL_LIBRARY OR NOT LIBGSASL_INCLUDE_DIR) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system libgsasl") - endif () -endif () - -if (LIBGSASL_LIBRARY AND LIBGSASL_INCLUDE_DIR) -elseif (NOT MISSING_INTERNAL_LIBGSASL_LIBRARY) - set (LIBGSASL_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libgsasl/src" "${ClickHouse_SOURCE_DIR}/contrib/libgsasl/linux_x86_64/include") - set (USE_INTERNAL_LIBGSASL_LIBRARY 1) - set (LIBGSASL_LIBRARY gsasl) -endif () - -if(LIBGSASL_LIBRARY AND LIBGSASL_INCLUDE_DIR) - set (USE_LIBGSASL 1) -endif() - -message (STATUS "Using libgsasl=${USE_LIBGSASL}: ${LIBGSASL_INCLUDE_DIR} : ${LIBGSASL_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index b0206b096da..2621db9ae9a 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -110,9 +110,7 @@ if (USE_EMBEDDED_COMPILER) add_llvm() endif () -if (USE_INTERNAL_LIBGSASL_LIBRARY) - add_subdirectory(libgsasl-cmake) -endif() +add_subdirectory(libgsasl-cmake) if (USE_INTERNAL_LIBXML2_LIBRARY) add_subdirectory(libxml2-cmake) diff --git a/contrib/libgsasl-cmake/CMakeLists.txt b/contrib/libgsasl-cmake/CMakeLists.txt index 69228c8de0c..2b5baeeff0b 100644 --- a/contrib/libgsasl-cmake/CMakeLists.txt +++ b/contrib/libgsasl-cmake/CMakeLists.txt @@ -1,3 +1,10 @@ +option(ENABLE_GSASL_LIBRARY "Enable gsasl library" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_GSASL_LIBRARY) + message(STATUS "Not using gsasl library") + return() +endif() + set (SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/libgsasl") set(SRCS @@ -97,7 +104,7 @@ target_include_directories(gsasl PUBLIC ${SRC_DIR}) target_include_directories(gsasl PUBLIC ${SRC_DIR}/gl) target_include_directories(gsasl PUBLIC ${SRC_DIR}/src) target_include_directories(gsasl PUBLIC ${SRC_DIR}/digest-md5) -target_include_directories(gsasl PRIVATE "${ClickHouse_SOURCE_DIR}/contrib/libgsasl-cmake/linux_x86_64/include") +target_include_directories(gsasl PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/libgsasl-cmake/linux_x86_64/include") target_compile_definitions (gsasl PRIVATE HAVE_CONFIG_H=1) @@ -105,3 +112,5 @@ if (TARGET ch_contrib::krb5) target_link_libraries(gsasl PUBLIC ch_contrib::krb5) target_compile_definitions (gsasl PRIVATE HAVE_GSSAPI_H=1 USE_GSSAPI=1) endif() + +add_library(ch_contrib::gsasl ALIAS gsasl) diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index 94b2bfe5eb3..f09702e47a3 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -113,12 +113,11 @@ target_include_directories(_hdfs3 PRIVATE ${HDFS3_SOURCE_DIR}) target_include_directories(_hdfs3 PRIVATE ${HDFS3_COMMON_DIR}) target_include_directories(_hdfs3 PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) -target_include_directories(_hdfs3 PRIVATE ${LIBGSASL_INCLUDE_DIR}) target_include_directories(_hdfs3 PRIVATE ${LIBXML2_INCLUDE_DIR}) target_include_directories(_hdfs3 SYSTEM BEFORE INTERFACE "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include") -target_link_libraries(_hdfs3 PRIVATE ${LIBGSASL_LIBRARY}) +target_link_libraries(_hdfs3 PRIVATE ch_contrib::gsasl) if (WITH_KERBEROS) target_link_libraries(_hdfs3 PRIVATE ch_contrib::krb5) endif() diff --git a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in index 31a25dec599..bb2d095aa78 100644 --- a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in +++ b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in @@ -30,7 +30,6 @@ const char * auto_config_build[] "USE_ICU", "@USE_ICU@", "USE_H3", "@USE_H3@", "USE_MYSQL", "@USE_MYSQL@", - "USE_LIBGSASL", "@USE_LIBGSASL@", "USE_RDKAFKA", "@USE_RDKAFKA@", "USE_CAPNP", "@USE_CAPNP@", "USE_BASE64", "@USE_BASE64@", From f2b1f65ce13a1d6d4dcb7c9e3bcf870378843468 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 00:51:17 +0300 Subject: [PATCH 184/403] Remove unbundled libxml2 support --- CMakeLists.txt | 1 - cmake/find/libxml2.cmake | 34 --------------------------- contrib/CMakeLists.txt | 5 +--- contrib/azure-cmake/CMakeLists.txt | 2 +- contrib/libhdfs3-cmake/CMakeLists.txt | 4 +--- contrib/libxml2-cmake/CMakeLists.txt | 6 +++-- 6 files changed, 7 insertions(+), 45 deletions(-) delete mode 100644 cmake/find/libxml2.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index b2c0e054b7f..a0789b8f9b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -488,7 +488,6 @@ include (GNUInstallDirs) include (cmake/find/cyrus-sasl.cmake) include (cmake/find/llvm.cmake) -include (cmake/find/libxml2.cmake) include (cmake/find/curl.cmake) include (cmake/find/s3.cmake) include (cmake/find/nlp.cmake) diff --git a/cmake/find/libxml2.cmake b/cmake/find/libxml2.cmake deleted file mode 100644 index e9fe7780d39..00000000000 --- a/cmake/find/libxml2.cmake +++ /dev/null @@ -1,34 +0,0 @@ -option (USE_INTERNAL_LIBXML2_LIBRARY "Set to FALSE to use system libxml2 library instead of bundled" ON) - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libxml2/libxml.h") - if (USE_INTERNAL_LIBXML2_LIBRARY) - message (WARNING "submodule contrib/libxml2 is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libxml") - set (USE_INTERNAL_LIBXML2_LIBRARY 0) - endif () - set (MISSING_INTERNAL_LIBXML2_LIBRARY 1) -endif () - -if (NOT USE_INTERNAL_LIBXML2_LIBRARY) - find_package (LibXml2) - #find_library (LIBXML2_LIBRARY libxml2) - #find_path (LIBXML2_INCLUDE_DIR NAMES libxml.h PATHS ${LIBXML2_INCLUDE_PATHS}) - - if (NOT LIBXML2_LIBRARY OR NOT LIBXML2_INCLUDE_DIR) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system libxml2") - endif () - - if (USE_STATIC_LIBRARIES) - find_package(LibLZMA) - set (LIBXML2_LIBRARIES ${LIBXML2_LIBRARIES} ${LIBLZMA_LIBRARIES}) - endif () -endif () - -if (LIBXML2_LIBRARY AND LIBXML2_INCLUDE_DIR) -elseif (NOT MISSING_INTERNAL_LIBXML2_LIBRARY) - set (LIBXML2_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libxml2/include ${ClickHouse_SOURCE_DIR}/contrib/libxml2-cmake/linux_x86_64/include) - set (USE_INTERNAL_LIBXML2_LIBRARY 1) - set (LIBXML2_LIBRARIES libxml2) -endif () - -message (STATUS "Using libxml2: ${LIBXML2_INCLUDE_DIR} : ${LIBXML2_LIBRARIES}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 2621db9ae9a..287f6f87046 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -111,10 +111,7 @@ if (USE_EMBEDDED_COMPILER) endif () add_subdirectory(libgsasl-cmake) - -if (USE_INTERNAL_LIBXML2_LIBRARY) - add_subdirectory(libxml2-cmake) -endif () +add_subdirectory(libxml2-cmake) if (USE_INTERNAL_AWS_S3_LIBRARY) add_subdirectory(aws-s3-cmake) diff --git a/contrib/azure-cmake/CMakeLists.txt b/contrib/azure-cmake/CMakeLists.txt index a9edad30b50..d914732956a 100644 --- a/contrib/azure-cmake/CMakeLists.txt +++ b/contrib/azure-cmake/CMakeLists.txt @@ -76,7 +76,7 @@ if (CURL_FOUND) target_link_libraries(_azure_sdk PRIVATE ${CURL_LIBRARY}) endif() -target_link_libraries(_azure_sdk PRIVATE ${LIBXML2_LIBRARIES}) +target_link_libraries(_azure_sdk PRIVATE ch_contrib::libxml2) target_include_directories(_azure_sdk SYSTEM BEFORE PUBLIC ${AZURE_SDK_INCLUDES}) diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index f09702e47a3..180751f21d8 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -113,15 +113,13 @@ target_include_directories(_hdfs3 PRIVATE ${HDFS3_SOURCE_DIR}) target_include_directories(_hdfs3 PRIVATE ${HDFS3_COMMON_DIR}) target_include_directories(_hdfs3 PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) -target_include_directories(_hdfs3 PRIVATE ${LIBXML2_INCLUDE_DIR}) - target_include_directories(_hdfs3 SYSTEM BEFORE INTERFACE "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include") target_link_libraries(_hdfs3 PRIVATE ch_contrib::gsasl) if (WITH_KERBEROS) target_link_libraries(_hdfs3 PRIVATE ch_contrib::krb5) endif() -target_link_libraries(_hdfs3 PRIVATE ${LIBXML2_LIBRARIES}) +target_link_libraries(_hdfs3 PRIVATE ch_contrib::libxml2) # inherit from parent cmake target_link_libraries(_hdfs3 PRIVATE ch_contrib::protobuf boost::headers_only) diff --git a/contrib/libxml2-cmake/CMakeLists.txt b/contrib/libxml2-cmake/CMakeLists.txt index 48bf87e71f6..2db16cf53e7 100644 --- a/contrib/libxml2-cmake/CMakeLists.txt +++ b/contrib/libxml2-cmake/CMakeLists.txt @@ -57,5 +57,7 @@ if(M_LIBRARY) target_link_libraries(libxml2 PRIVATE ${M_LIBRARY}) endif() -target_include_directories(libxml2 PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/include") -target_include_directories(libxml2 PUBLIC "${LIBXML2_SOURCE_DIR}/include") +target_include_directories(libxml2 BEFORE PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/include") +target_include_directories(libxml2 BEFORE PUBLIC "${LIBXML2_SOURCE_DIR}/include") + +add_library(ch_contrib::libxml2 ALIAS libxml2) From e8c2052367541c492d2e81b71ebeb92bc350c0bc Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 00:54:24 +0300 Subject: [PATCH 185/403] Remove unbundled cyrus-sasl support --- CMakeLists.txt | 1 - cmake/find/cyrus-sasl.cmake | 23 -------------------- contrib/CMakeLists.txt | 9 ++------ contrib/cyrus-sasl-cmake/CMakeLists.txt | 28 ++++++++++++++++++------- contrib/librdkafka-cmake/CMakeLists.txt | 2 +- src/CMakeLists.txt | 4 ++-- 6 files changed, 25 insertions(+), 42 deletions(-) delete mode 100644 cmake/find/cyrus-sasl.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index a0789b8f9b9..589988ed2bb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -486,7 +486,6 @@ message (STATUS include (GNUInstallDirs) -include (cmake/find/cyrus-sasl.cmake) include (cmake/find/llvm.cmake) include (cmake/find/curl.cmake) include (cmake/find/s3.cmake) diff --git a/cmake/find/cyrus-sasl.cmake b/cmake/find/cyrus-sasl.cmake deleted file mode 100644 index f0c088995b0..00000000000 --- a/cmake/find/cyrus-sasl.cmake +++ /dev/null @@ -1,23 +0,0 @@ -if (${ENABLE_LIBRARIES} AND ${ENABLE_KRB5}) - set (DEFAULT_ENABLE_CYRUS_SASL 1) -else() - set (DEFAULT_ENABLE_CYRUS_SASL 0) -endif() - -OPTION(ENABLE_CYRUS_SASL "Enable cyrus-sasl" ${DEFAULT_ENABLE_CYRUS_SASL}) -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cyrus-sasl/README") - message (WARNING "submodule contrib/cyrus-sasl is missing. to fix try run: \n git submodule update --init") - set (ENABLE_CYRUS_SASL 0) -endif () - -if (ENABLE_CYRUS_SASL) - - set (USE_CYRUS_SASL 1) - set (CYRUS_SASL_LIBRARY sasl2) - - set (CYRUS_SASL_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/cyrus-sasl/include") - - -endif () - -message (STATUS "Using cyrus-sasl: krb5=${USE_KRB5}: ${CYRUS_SASL_INCLUDE_DIR} : ${CYRUS_SASL_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 287f6f87046..46a6a824585 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -141,17 +141,12 @@ if (USE_SENTRY) endif() add_subdirectory (fmtlib-cmake) - add_subdirectory (krb5-cmake) -if (USE_CYRUS_SASL) - add_subdirectory (cyrus-sasl-cmake) # for krb5 -endif() - +add_subdirectory (cyrus-sasl-cmake) # for krb5 add_subdirectory (libpqxx-cmake) add_subdirectory (libpq-cmake) - add_subdirectory (nuraft-cmake) -add_subdirectory(fast_float-cmake) +add_subdirectory (fast_float-cmake) if (USE_NLP) add_subdirectory(libstemmer-c-cmake) diff --git a/contrib/cyrus-sasl-cmake/CMakeLists.txt b/contrib/cyrus-sasl-cmake/CMakeLists.txt index aa25a078718..377af599d43 100644 --- a/contrib/cyrus-sasl-cmake/CMakeLists.txt +++ b/contrib/cyrus-sasl-cmake/CMakeLists.txt @@ -1,8 +1,20 @@ +if (${ENABLE_LIBRARIES} AND ${ENABLE_KRB5}) + set (DEFAULT_ENABLE_CYRUS_SASL 1) +else() + set (DEFAULT_ENABLE_CYRUS_SASL 0) +endif() +option(ENABLE_CYRUS_SASL "Enable cyrus-sasl" ${DEFAULT_ENABLE_CYRUS_SASL}) + +if (NOT ENABLE_CYRUS_SASL) + message(STATUS "Not using cyrus-sasl") + return() +endif() + set(CYRUS_SASL_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/cyrus-sasl") -add_library(${CYRUS_SASL_LIBRARY}) +add_library(sasl2) -target_sources(${CYRUS_SASL_LIBRARY} PRIVATE +target_sources(sasl2 PRIVATE "${CYRUS_SASL_SOURCE_DIR}/plugins/gssapi.c" # "${CYRUS_SASL_SOURCE_DIR}/plugins/gssapiv2_init.c" "${CYRUS_SASL_SOURCE_DIR}/common/plugin_common.c" @@ -20,11 +32,11 @@ target_sources(${CYRUS_SASL_LIBRARY} PRIVATE "${CYRUS_SASL_SOURCE_DIR}/lib/checkpw.c" ) -target_include_directories(${CYRUS_SASL_LIBRARY} PUBLIC +target_include_directories(sasl2 PUBLIC ${CMAKE_CURRENT_BINARY_DIR} ) -target_include_directories(${CYRUS_SASL_LIBRARY} PRIVATE +target_include_directories(sasl2 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} # for config.h "${CYRUS_SASL_SOURCE_DIR}/plugins" ${CYRUS_SASL_SOURCE_DIR} @@ -38,7 +50,7 @@ target_include_directories(${CYRUS_SASL_LIBRARY} PRIVATE "${CYRUS_SASL_SOURCE_DIR}/tests" ) -target_compile_definitions(${CYRUS_SASL_LIBRARY} PUBLIC +target_compile_definitions(sasl2 PUBLIC HAVE_CONFIG_H # PLUGINDIR="/usr/local/lib/sasl2" PLUGINDIR="" @@ -64,6 +76,6 @@ file(COPY DESTINATION ${CMAKE_CURRENT_BINARY_DIR} ) -target_link_libraries(${CYRUS_SASL_LIBRARY} - PUBLIC ${KRB5_LIBRARY} -) +target_link_libraries(sasl2 PUBLIC ch_contrib::krb5) + +add_library(ch_contrib::sasl2 ALIAS sasl2) diff --git a/contrib/librdkafka-cmake/CMakeLists.txt b/contrib/librdkafka-cmake/CMakeLists.txt index b8135a927e0..45ff79b1377 100644 --- a/contrib/librdkafka-cmake/CMakeLists.txt +++ b/contrib/librdkafka-cmake/CMakeLists.txt @@ -128,7 +128,7 @@ target_link_libraries(_rdkafka OpenSSL::Crypto OpenSSL::SSL ) if(${ENABLE_CYRUS_SASL}) - target_link_libraries(_rdkafka PRIVATE ${CYRUS_SASL_LIBRARY}) + target_link_libraries(_rdkafka PRIVATE ch_contrib::sasl2) endif() file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/auxdir") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index bf1dad5933c..1e8512959cf 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -341,8 +341,8 @@ if (TARGET ch_contrib::rdkafka) dbms_target_link_libraries(PRIVATE ch_contrib::rdkafka ch_contrib::cppkafka) endif() -if (USE_CYRUS_SASL) - dbms_target_link_libraries(PRIVATE ${CYRUS_SASL_LIBRARY}) +if (TARGET ch_contrib::sasl2) + dbms_target_link_libraries(PRIVATE ch_contrib::sasl2) endif() if (TARGET ch_contrib::krb5) From 4f4ec8912bcf8b3ab5404a24b790bfce39afe960 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 01:02:56 +0300 Subject: [PATCH 186/403] Remove unbundled curl support --- CMakeLists.txt | 1 - cmake/find/curl.cmake | 35 ---------------------- contrib/azure-cmake/CMakeLists.txt | 4 +-- contrib/curl-cmake/CMakeLists.txt | 10 ++++--- contrib/sentry-native-cmake/CMakeLists.txt | 2 +- 5 files changed, 9 insertions(+), 43 deletions(-) delete mode 100644 cmake/find/curl.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 589988ed2bb..39e4bc3f20b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -487,7 +487,6 @@ message (STATUS include (GNUInstallDirs) include (cmake/find/llvm.cmake) -include (cmake/find/curl.cmake) include (cmake/find/s3.cmake) include (cmake/find/nlp.cmake) include (cmake/find/sentry.cmake) diff --git a/cmake/find/curl.cmake b/cmake/find/curl.cmake deleted file mode 100644 index 577b13698c2..00000000000 --- a/cmake/find/curl.cmake +++ /dev/null @@ -1,35 +0,0 @@ -option (ENABLE_CURL "Enable curl" ${ENABLE_LIBRARIES}) - -if (NOT ENABLE_CURL) - if (USE_INTERNAL_CURL) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal curl with ENABLE_CURL=OFF") - endif() - return() -endif() - -option (USE_INTERNAL_CURL "Use internal curl library" ON) - -if (NOT USE_INTERNAL_CURL) - find_package (CURL) - if (NOT CURL_FOUND) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system curl") - endif() -endif() - -if (NOT CURL_FOUND) - set (USE_INTERNAL_CURL 1) - set (CURL_LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/curl") - - # find_package(CURL) compatibility for the following packages that uses - # find_package(CURL)/include(FindCURL): - # - sentry-native - set (CURL_FOUND ON CACHE BOOL "") - set (CURL_ROOT_DIR ${CURL_LIBRARY_DIR} CACHE PATH "") - set (CURL_INCLUDE_DIR ${CURL_LIBRARY_DIR}/include CACHE PATH "") - set (CURL_INCLUDE_DIRS ${CURL_LIBRARY_DIR}/include CACHE PATH "") - set (CURL_LIBRARY curl CACHE STRING "") - set (CURL_LIBRARIES ${CURL_LIBRARY} CACHE STRING "") - set (CURL_VERSION_STRING 7.67.0 CACHE STRING "") -endif () - -message (STATUS "Using curl: ${CURL_INCLUDE_DIRS} : ${CURL_LIBRARIES}") diff --git a/contrib/azure-cmake/CMakeLists.txt b/contrib/azure-cmake/CMakeLists.txt index d914732956a..031d8dc9a0b 100644 --- a/contrib/azure-cmake/CMakeLists.txt +++ b/contrib/azure-cmake/CMakeLists.txt @@ -72,8 +72,8 @@ if (TARGET OpenSSL::SSL) endif() # Originally, on Windows azure-core is built with winhttp by default -if (CURL_FOUND) - target_link_libraries(_azure_sdk PRIVATE ${CURL_LIBRARY}) +if (TARGET ch_contrib::curl) + target_link_libraries(_azure_sdk PRIVATE ch_contrib::curl) endif() target_link_libraries(_azure_sdk PRIVATE ch_contrib::libxml2) diff --git a/contrib/curl-cmake/CMakeLists.txt b/contrib/curl-cmake/CMakeLists.txt index 63ac8da24b1..0fe5282b98e 100644 --- a/contrib/curl-cmake/CMakeLists.txt +++ b/contrib/curl-cmake/CMakeLists.txt @@ -1,4 +1,7 @@ -if (NOT USE_INTERNAL_CURL) +option (ENABLE_CURL "Enable curl" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_CURL) + message(STATUS "Not using curl") return() endif() @@ -166,8 +169,6 @@ target_compile_options (curl PRIVATE -g0) # find_package(CURL) compatibility for the following packages that uses # find_package(CURL)/include(FindCURL): -# - mariadb-connector-c -# - aws-s3-cmake # - sentry-native set (CURL_FOUND ON CACHE BOOL "") set (CURL_ROOT_DIR ${LIBRARY_DIR} CACHE PATH "") @@ -176,4 +177,5 @@ set (CURL_INCLUDE_DIRS "${LIBRARY_DIR}/include" CACHE PATH "") set (CURL_LIBRARY curl CACHE STRING "") set (CURL_LIBRARIES ${CURL_LIBRARY} CACHE STRING "") set (CURL_VERSION_STRING 7.67.0 CACHE STRING "") -add_library (CURL::libcurl ALIAS ${CURL_LIBRARY}) +# add_library (CURL::libcurl ALIAS ${CURL_LIBRARY}) +add_library (ch_contrib::curl ALIAS ${CURL_LIBRARY}) diff --git a/contrib/sentry-native-cmake/CMakeLists.txt b/contrib/sentry-native-cmake/CMakeLists.txt index f4e946cf797..c0936137db0 100644 --- a/contrib/sentry-native-cmake/CMakeLists.txt +++ b/contrib/sentry-native-cmake/CMakeLists.txt @@ -47,6 +47,6 @@ else() target_compile_definitions(sentry PUBLIC SENTRY_BUILD_STATIC) endif() -target_link_libraries(sentry PRIVATE curl pthread) +target_link_libraries(sentry PRIVATE ch_contrib::curl pthread) target_include_directories(sentry PUBLIC "${SRC_DIR}/include" PRIVATE "${SRC_DIR}/src") target_compile_definitions(sentry PRIVATE SENTRY_WITH_INPROC_BACKEND SIZEOF_LONG=8) From 3147bbab510bfcab4734b131013b109117119474 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 01:08:35 +0300 Subject: [PATCH 187/403] Cleanup sentry support --- CMakeLists.txt | 1 - base/daemon/CMakeLists.txt | 4 ++-- cmake/find/sentry.cmake | 23 ---------------------- cmake/target.cmake | 6 +++--- contrib/CMakeLists.txt | 10 +--------- contrib/sentry-native-cmake/CMakeLists.txt | 13 ++++++++++++ src/configure_config.cmake | 3 +++ 7 files changed, 22 insertions(+), 38 deletions(-) delete mode 100644 cmake/find/sentry.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 39e4bc3f20b..a1072e3d9e9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -489,7 +489,6 @@ include (GNUInstallDirs) include (cmake/find/llvm.cmake) include (cmake/find/s3.cmake) include (cmake/find/nlp.cmake) -include (cmake/find/sentry.cmake) include (cmake/find/datasketches.cmake) # Need to process before "contrib" dir: diff --git a/base/daemon/CMakeLists.txt b/base/daemon/CMakeLists.txt index 6ef87db6a61..2a4d3d33ff9 100644 --- a/base/daemon/CMakeLists.txt +++ b/base/daemon/CMakeLists.txt @@ -12,6 +12,6 @@ endif() target_link_libraries (daemon PUBLIC loggers PRIVATE clickhouse_common_io clickhouse_common_config common ${EXECINFO_LIBRARIES}) -if (USE_SENTRY) - target_link_libraries (daemon PRIVATE ${SENTRY_LIBRARY}) +if (TARGET ch_contrib::sentry) + target_link_libraries (daemon PRIVATE ch_contrib::sentry) endif () diff --git a/cmake/find/sentry.cmake b/cmake/find/sentry.cmake deleted file mode 100644 index e08cbad1729..00000000000 --- a/cmake/find/sentry.cmake +++ /dev/null @@ -1,23 +0,0 @@ -set (SENTRY_LIBRARY "sentry") - -set (SENTRY_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/sentry-native/include") -if (NOT EXISTS "${SENTRY_INCLUDE_DIR}/sentry.h") - message (WARNING "submodule contrib/sentry-native is missing. to fix try run: \n git submodule update --init") - if (USE_SENTRY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal sentry library") - endif() - return() -endif () - -if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES AND NOT (OS_DARWIN AND COMPILER_CLANG)) - option (USE_SENTRY "Use Sentry" ${ENABLE_LIBRARIES}) - set (SENTRY_TRANSPORT "curl" CACHE STRING "") - set (SENTRY_BACKEND "none" CACHE STRING "") - set (SENTRY_EXPORT_SYMBOLS OFF CACHE BOOL "") - set (SENTRY_LINK_PTHREAD OFF CACHE BOOL "") - set (SENTRY_PIC OFF CACHE BOOL "") - set (BUILD_SHARED_LIBS OFF) - message (STATUS "Using sentry=${USE_SENTRY}: ${SENTRY_LIBRARY}") -elseif (USE_SENTRY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Sentry is not supported in current configuration") -endif () diff --git a/cmake/target.cmake b/cmake/target.cmake index 4b109d165e7..ff216f86618 100644 --- a/cmake/target.cmake +++ b/cmake/target.cmake @@ -27,10 +27,10 @@ if (CMAKE_CROSSCOMPILING) if (ARCH_AARCH64) # FIXME: broken dependencies set (ENABLE_GRPC OFF CACHE INTERNAL "") - set (USE_SENTRY OFF CACHE INTERNAL "") + set (ENABLE_SENTRY OFF CACHE INTERNAL "") elseif (ARCH_PPC64LE) set (ENABLE_GRPC OFF CACHE INTERNAL "") - set (USE_SENTRY OFF CACHE INTERNAL "") + set (ENABLE_SENTRY OFF CACHE INTERNAL "") endif () elseif (OS_FREEBSD) # FIXME: broken dependencies @@ -43,7 +43,7 @@ if (CMAKE_CROSSCOMPILING) endif () if (USE_MUSL) - set (USE_SENTRY OFF CACHE INTERNAL "") + set (ENABLE_SENTRY OFF CACHE INTERNAL "") set (ENABLE_ODBC OFF CACHE INTERNAL "") set (ENABLE_GRPC OFF CACHE INTERNAL "") set (ENABLE_HDFS OFF CACHE INTERNAL "") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 46a6a824585..45c6bb62091 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -129,17 +129,9 @@ add_subdirectory (fastops-cmake) add_subdirectory (libuv-cmake) add_subdirectory (amqpcpp-cmake) # requires: libuv add_subdirectory (cassandra-cmake) # requires: libuv - -# Should go before: -# - sentry-native add_subdirectory (curl-cmake) - add_subdirectory (azure-cmake) - -if (USE_SENTRY) - add_subdirectory (sentry-native-cmake) -endif() - +add_subdirectory (sentry-native-cmake) # requires: curl add_subdirectory (fmtlib-cmake) add_subdirectory (krb5-cmake) add_subdirectory (cyrus-sasl-cmake) # for krb5 diff --git a/contrib/sentry-native-cmake/CMakeLists.txt b/contrib/sentry-native-cmake/CMakeLists.txt index c0936137db0..18cbc9a2df8 100644 --- a/contrib/sentry-native-cmake/CMakeLists.txt +++ b/contrib/sentry-native-cmake/CMakeLists.txt @@ -1,3 +1,14 @@ +if (NOT OS_FREEBSD AND NOT SPLIT_SHARED_LIBRARIES AND NOT (OS_DARWIN AND COMPILER_CLANG)) + option (ENABLE_SENTRY "Enable Sentry" ${ENABLE_LIBRARIES}) +else() + option (ENABLE_SENTRY "Enable Sentry" OFF) +endif() + +if (NOT ENABLE_SENTRY) + message(STATUS "Not using sentry") + return() +endif() + set (SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/sentry-native") set (SRCS @@ -50,3 +61,5 @@ endif() target_link_libraries(sentry PRIVATE ch_contrib::curl pthread) target_include_directories(sentry PUBLIC "${SRC_DIR}/include" PRIVATE "${SRC_DIR}/src") target_compile_definitions(sentry PRIVATE SENTRY_WITH_INPROC_BACKEND SIZEOF_LONG=8) + +add_library(ch_contrib::sentry ALIAS sentry) diff --git a/src/configure_config.cmake b/src/configure_config.cmake index 4c599038c4d..18925ac6be2 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -64,3 +64,6 @@ endif() if (TARGET ch_contrib::krb5) set(USE_KRB5 1) endif() +if (TARGET ch_contrib::sentry) + set(USE_SENTRY 1) +endif() From 6a721baa59ccda14ea24a14aa601656eb5274fd3 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 01:13:47 +0300 Subject: [PATCH 188/403] Remove unbundled datasketches support --- CMakeLists.txt | 1 - cmake/find/datasketches.cmake | 29 ------------------- contrib/CMakeLists.txt | 1 + contrib/datasketches-cpp-cmake/CMakeLists.txt | 14 +++++++++ src/CMakeLists.txt | 4 +-- src/configure_config.cmake | 3 ++ 6 files changed, 20 insertions(+), 32 deletions(-) delete mode 100644 cmake/find/datasketches.cmake create mode 100644 contrib/datasketches-cpp-cmake/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index a1072e3d9e9..d35f2faeec9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -489,7 +489,6 @@ include (GNUInstallDirs) include (cmake/find/llvm.cmake) include (cmake/find/s3.cmake) include (cmake/find/nlp.cmake) -include (cmake/find/datasketches.cmake) # Need to process before "contrib" dir: include (cmake/find/mysqlclient.cmake) diff --git a/cmake/find/datasketches.cmake b/cmake/find/datasketches.cmake deleted file mode 100644 index 3d0bb1d1f95..00000000000 --- a/cmake/find/datasketches.cmake +++ /dev/null @@ -1,29 +0,0 @@ -option (ENABLE_DATASKETCHES "Enable DataSketches" ${ENABLE_LIBRARIES}) - -if (ENABLE_DATASKETCHES) - -option (USE_INTERNAL_DATASKETCHES_LIBRARY "Set to FALSE to use system DataSketches library instead of bundled" ON) - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/datasketches-cpp/theta/CMakeLists.txt") - if (USE_INTERNAL_DATASKETCHES_LIBRARY) - message(WARNING "submodule contrib/datasketches-cpp is missing. to fix try run: \n git submodule update --init") - endif() - set(MISSING_INTERNAL_DATASKETCHES_LIBRARY 1) - set(USE_INTERNAL_DATASKETCHES_LIBRARY 0) -endif() - -if (USE_INTERNAL_DATASKETCHES_LIBRARY) - set(DATASKETCHES_LIBRARY theta) - set(DATASKETCHES_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/datasketches-cpp/common/include" "${ClickHouse_SOURCE_DIR}/contrib/datasketches-cpp/theta/include") -elseif (NOT MISSING_INTERNAL_DATASKETCHES_LIBRARY) - find_library(DATASKETCHES_LIBRARY theta) - find_path(DATASKETCHES_INCLUDE_DIR NAMES theta_sketch.hpp PATHS ${DATASKETCHES_INCLUDE_PATHS}) -endif() - -if (DATASKETCHES_LIBRARY AND DATASKETCHES_INCLUDE_DIR) - set(USE_DATASKETCHES 1) -endif() - -endif() - -message (STATUS "Using datasketches=${USE_DATASKETCHES}: ${DATASKETCHES_INCLUDE_DIR} : ${DATASKETCHES_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 45c6bb62091..e53ea42ddd3 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -139,6 +139,7 @@ add_subdirectory (libpqxx-cmake) add_subdirectory (libpq-cmake) add_subdirectory (nuraft-cmake) add_subdirectory (fast_float-cmake) +add_subdirectory (datasketches-cpp-cmake) if (USE_NLP) add_subdirectory(libstemmer-c-cmake) diff --git a/contrib/datasketches-cpp-cmake/CMakeLists.txt b/contrib/datasketches-cpp-cmake/CMakeLists.txt new file mode 100644 index 00000000000..b12a88ad57b --- /dev/null +++ b/contrib/datasketches-cpp-cmake/CMakeLists.txt @@ -0,0 +1,14 @@ +option (ENABLE_DATASKETCHES "Enable DataSketches" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_DATASKETCHES) + message(STATUS "Not using DataSketches") + return() +endif() + +set(DATASKETCHES_LIBRARY theta) +add_library(_datasketches INTERFACE) +target_include_directories(_datasketches SYSTEM BEFORE INTERFACE + "${ClickHouse_SOURCE_DIR}/contrib/datasketches-cpp/common/include" + "${ClickHouse_SOURCE_DIR}/contrib/datasketches-cpp/theta/include") + +add_library(ch_contrib::datasketches ALIAS _datasketches) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1e8512959cf..ea532e14f95 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -485,8 +485,8 @@ if (TARGET ch_contrib::libpqxx) dbms_target_link_libraries(PUBLIC ch_contrib::libpqxx) endif() -if (USE_DATASKETCHES) - target_include_directories (clickhouse_aggregate_functions SYSTEM BEFORE PRIVATE ${DATASKETCHES_INCLUDE_DIR}) +if (TARGET ch_contrib::datasketches) + target_link_libraries (clickhouse_aggregate_functions PRIVATE ch_contrib::datasketches) endif () target_link_libraries (clickhouse_common_io PRIVATE ch_contrib::lz4) diff --git a/src/configure_config.cmake b/src/configure_config.cmake index 18925ac6be2..7e191478a44 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -67,3 +67,6 @@ endif() if (TARGET ch_contrib::sentry) set(USE_SENTRY 1) endif() +if (TARGET ch_contrib::datasketches) + set(USE_DATASKETCHES 1) +endif() From 50d38c670e5d525b2a4f2c583bd448fa912205ec Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:48:22 +0300 Subject: [PATCH 189/403] Fix azure blob storage --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ea532e14f95..87b9b08180d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -110,7 +110,7 @@ if (USE_AWS_S3) add_headers_and_sources(dbms Disks/S3) endif() -if (USE_AZURE_BLOB_STORAGE) +if (TARGET ch_contrib::azure_sdk) add_headers_and_sources(dbms Disks/AzureBlobStorage) endif() From cad885ab1cbe3af818b52d7462ba2c084a872603 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:48:23 +0300 Subject: [PATCH 190/403] Remove unbundled aws-s3 support --- CMakeLists.txt | 1 - cmake/find/s3.cmake | 41 ----------------------------- contrib/CMakeLists.txt | 15 +++-------- contrib/aws-s3-cmake/CMakeLists.txt | 19 ++++++++++++- src/CMakeLists.txt | 8 +++--- src/configure_config.cmake | 3 +++ 6 files changed, 27 insertions(+), 60 deletions(-) delete mode 100644 cmake/find/s3.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index d35f2faeec9..14fcd0f3023 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -487,7 +487,6 @@ message (STATUS include (GNUInstallDirs) include (cmake/find/llvm.cmake) -include (cmake/find/s3.cmake) include (cmake/find/nlp.cmake) # Need to process before "contrib" dir: diff --git a/cmake/find/s3.cmake b/cmake/find/s3.cmake deleted file mode 100644 index 06029ace63a..00000000000 --- a/cmake/find/s3.cmake +++ /dev/null @@ -1,41 +0,0 @@ -if(NOT OS_FREEBSD) - option(ENABLE_S3 "Enable S3" ${ENABLE_LIBRARIES}) -elseif(ENABLE_S3 OR USE_INTERNAL_AWS_S3_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use S3 on FreeBSD") -endif() - -if(NOT ENABLE_S3) - if(USE_INTERNAL_AWS_S3_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal S3 library with ENABLE_S3=OFF") - endif() - return() -endif() - -option(USE_INTERNAL_AWS_S3_LIBRARY "Set to FALSE to use system S3 instead of bundled (experimental set to OFF on your own risk)" - ON) - -if (NOT USE_INTERNAL_AWS_S3_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Compilation with external S3 library is not supported yet") -endif() - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-s3") - message (WARNING "submodule contrib/aws is missing. to fix try run: \n git submodule update --init") - if (USE_INTERNAL_AWS_S3_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal S3 library") - endif () - set (MISSING_AWS_S3 1) -endif () - -if (USE_INTERNAL_AWS_S3_LIBRARY AND NOT MISSING_AWS_S3) - set(AWS_S3_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-s3/include") - set(AWS_S3_CORE_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-core/include") - set(AWS_S3_LIBRARY aws_s3) - set(USE_INTERNAL_AWS_S3_LIBRARY 1) - set(USE_AWS_S3 1) -else() - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't enable S3") - set(USE_INTERNAL_AWS_S3_LIBRARY 0) - set(USE_AWS_S3 0) -endif () - -message (STATUS "Using aws_s3=${USE_AWS_S3}: ${AWS_S3_INCLUDE_DIR} : ${AWS_S3_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index e53ea42ddd3..f74d5cbc3a1 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -110,18 +110,9 @@ if (USE_EMBEDDED_COMPILER) add_llvm() endif () -add_subdirectory(libgsasl-cmake) -add_subdirectory(libxml2-cmake) - -if (USE_INTERNAL_AWS_S3_LIBRARY) - add_subdirectory(aws-s3-cmake) - - # The library is large - avoid bloat. - target_compile_options (aws_s3 PRIVATE -g0) - target_compile_options (aws_s3_checksums PRIVATE -g0) - -endif () - +add_subdirectory (libgsasl-cmake) +add_subdirectory (libxml2-cmake) +add_subdirectory (aws-s3-cmake) add_subdirectory (base64-cmake) add_subdirectory (simdjson-cmake) add_subdirectory (rapidjson-cmake) diff --git a/contrib/aws-s3-cmake/CMakeLists.txt b/contrib/aws-s3-cmake/CMakeLists.txt index 5024763f19d..9c0b3ce192c 100644 --- a/contrib/aws-s3-cmake/CMakeLists.txt +++ b/contrib/aws-s3-cmake/CMakeLists.txt @@ -1,3 +1,14 @@ +if(NOT OS_FREEBSD) + option(ENABLE_S3 "Enable S3" ${ENABLE_LIBRARIES}) +elseif(ENABLE_S3) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use S3 on FreeBSD") +endif() + +if(NOT ENABLE_S3) + message(STATUS "Not using S3") + return() +endif() + SET(AWS_S3_LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-s3") SET(AWS_CORE_LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws/aws-cpp-sdk-core") SET(AWS_CHECKSUMS_LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/aws-checksums") @@ -93,7 +104,7 @@ add_library(aws_s3 ${S3_UNIFIED_SRC}) target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_MAJOR=1") target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_MINOR=7") target_compile_definitions(aws_s3 PUBLIC "AWS_SDK_VERSION_PATCH=231") -target_include_directories(aws_s3 SYSTEM PUBLIC ${S3_INCLUDES}) +target_include_directories(aws_s3 SYSTEM BEFORE PUBLIC ${S3_INCLUDES}) if (TARGET OpenSSL::SSL) target_compile_definitions(aws_s3 PUBLIC -DENABLE_OPENSSL_ENCRYPTION) @@ -101,3 +112,9 @@ if (TARGET OpenSSL::SSL) endif() target_link_libraries(aws_s3 PRIVATE aws_s3_checksums) + +# The library is large - avoid bloat. +target_compile_options (aws_s3 PRIVATE -g0) +target_compile_options (aws_s3_checksums PRIVATE -g0) + +add_library(ch_contrib::aws_s3 ALIAS aws_s3) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 87b9b08180d..b6b44c4374f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -105,7 +105,7 @@ if (TARGET ch_contrib::rocksdb) add_headers_and_sources(dbms Storages/RocksDB) endif() -if (USE_AWS_S3) +if (TARGET ch_contrib::aws_s3) add_headers_and_sources(dbms Common/S3) add_headers_and_sources(dbms Disks/S3) endif() @@ -436,10 +436,8 @@ if (TARGET ch_contrib::hivemetastore) endif() -if (USE_AWS_S3) - target_link_libraries (clickhouse_common_io PUBLIC ${AWS_S3_LIBRARY}) - target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${AWS_S3_CORE_INCLUDE_DIR}) - target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${AWS_S3_INCLUDE_DIR}) +if (TARGET ch_contrib::aws_s3) + target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::aws_s3) endif() if (TARGET ch_contrib::azure_sdk) diff --git a/src/configure_config.cmake b/src/configure_config.cmake index 7e191478a44..53a2663e35c 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -70,3 +70,6 @@ endif() if (TARGET ch_contrib::datasketches) set(USE_DATASKETCHES 1) endif() +if (TARGET ch_contrib::aws_s3) + set(USE_AWS_S3 1) +endif() From 17c45eb2380cd7bf7c04617011600488a3bf80e2 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:48:23 +0300 Subject: [PATCH 191/403] Fix filelog --- src/configure_config.cmake | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/configure_config.cmake b/src/configure_config.cmake index 53a2663e35c..991c04d4b98 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -55,6 +55,9 @@ endif() if (TARGET ch_contrib::yaml_cpp) set(USE_YAML_CPP 1) endif() +if (OS_LINUX) + set(USE_FILELOG 1) +endif() if (TARGET ch_contrib::sqlite) set(USE_SQLITE 1) endif() From a75b748fee87827ec858fdddfadc1d1ba562843d Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:48:23 +0300 Subject: [PATCH 192/403] Remove unbundled mysql support --- CMakeLists.txt | 3 - cmake/find/mysqlclient.cmake | 78 ------------------- contrib/CMakeLists.txt | 7 +- .../mariadb-connector-c-cmake/CMakeLists.txt | 13 ++++ src/CMakeLists.txt | 5 +- src/Common/CMakeLists.txt | 2 +- src/Common/mysqlxx/CMakeLists.txt | 4 +- src/Dictionaries/Embedded/CMakeLists.txt | 5 +- src/configure_config.cmake | 3 + 9 files changed, 29 insertions(+), 91 deletions(-) delete mode 100644 cmake/find/mysqlclient.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 14fcd0f3023..d8f062d8a45 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -489,9 +489,6 @@ include (GNUInstallDirs) include (cmake/find/llvm.cmake) include (cmake/find/nlp.cmake) -# Need to process before "contrib" dir: -include (cmake/find/mysqlclient.cmake) - # When testing for memory leaks with Valgrind, don't link tcmalloc or jemalloc. if (TARGET global-group) diff --git a/cmake/find/mysqlclient.cmake b/cmake/find/mysqlclient.cmake deleted file mode 100644 index f191d06b17e..00000000000 --- a/cmake/find/mysqlclient.cmake +++ /dev/null @@ -1,78 +0,0 @@ -if(OS_LINUX AND TARGET OpenSSL::SSL) - option(ENABLE_MYSQL "Enable MySQL" ${ENABLE_LIBRARIES}) -else () - option(ENABLE_MYSQL "Enable MySQL" FALSE) -endif () - -if(NOT ENABLE_MYSQL) - if (USE_INTERNAL_MYSQL_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal mysql library with ENABLE_MYSQL=OFF") - endif () - message (STATUS "Build without mysqlclient (support for MYSQL dictionary source will be disabled)") - return() -endif() - -option(USE_INTERNAL_MYSQL_LIBRARY "Set to FALSE to use system mysqlclient library instead of bundled" ON) - -if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/mariadb-connector-c/README") - if(USE_INTERNAL_MYSQL_LIBRARY) - message(WARNING "submodule contrib/mariadb-connector-c is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal mysql library") - set(USE_INTERNAL_MYSQL_LIBRARY 0) - endif() - set(MISSING_INTERNAL_MYSQL_LIBRARY 1) -endif() - -if (NOT USE_INTERNAL_MYSQL_LIBRARY) - set (MYSQL_LIB_PATHS - "/usr/local/opt/mysql/lib" - "/usr/local/lib" - "/usr/local/lib64" - "/usr/local/lib/mariadb" # macos brew mariadb-connector-c - "/usr/mysql/lib" - "/usr/mysql/lib64" - "/usr/lib" - "/usr/lib64" - "/lib" - "/lib64") - - set (MYSQL_INCLUDE_PATHS - "/usr/local/opt/mysql/include" - "/usr/mysql/include" - "/usr/local/include" - "/usr/include/mariadb" - "/usr/include/mysql" - "/usr/include") - - find_path (MYSQL_INCLUDE_DIR NAMES mysql.h mysql/mysql.h mariadb/mysql.h PATHS ${MYSQL_INCLUDE_PATHS} PATH_SUFFIXES mysql) - - if (USE_STATIC_LIBRARIES) - find_library (STATIC_MYSQLCLIENT_LIB NAMES mariadbclient mysqlclient PATHS ${MYSQL_LIB_PATHS} PATH_SUFFIXES mysql) - else () - find_library (MYSQLCLIENT_LIBRARIES NAMES mariadb mariadbclient mysqlclient PATHS ${MYSQL_LIB_PATHS} PATH_SUFFIXES mysql) - endif () - - if (MYSQL_INCLUDE_DIR AND (STATIC_MYSQLCLIENT_LIB OR MYSQLCLIENT_LIBRARIES)) - set (USE_MYSQL 1) - set (MYSQLXX_LIBRARY mysqlxx) - if (APPLE) - # /usr/local/include/mysql/mysql_com.h:1011:10: fatal error: mysql/udf_registration_types.h: No such file or directory - set(MYSQL_INCLUDE_DIR ${MYSQL_INCLUDE_DIR} ${MYSQL_INCLUDE_DIR}/mysql) - endif () - else () - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system mysql library") - endif () -endif () - -if (NOT USE_MYSQL AND NOT MISSING_INTERNAL_MYSQL_LIBRARY) - set (MYSQLCLIENT_LIBRARIES mariadbclient) - set (MYSQLXX_LIBRARY mysqlxx) - set (USE_MYSQL 1) - set (USE_INTERNAL_MYSQL_LIBRARY 1) -endif() - -if (USE_MYSQL) - message (STATUS "Using mysqlclient=${USE_MYSQL}: ${MYSQL_INCLUDE_DIR} : ${MYSQLCLIENT_LIBRARIES}; staticlib=${STATIC_MYSQLCLIENT_LIB}") -else () - message (STATUS "Build without mysqlclient (support for MYSQL dictionary source will be disabled)") -endif () diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index f74d5cbc3a1..714a15601ee 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -75,12 +75,7 @@ add_subdirectory (cityhash102) add_subdirectory (libfarmhash) add_subdirectory (icu-cmake) add_subdirectory (h3-cmake) - -# TODO: refactor the contrib libraries below this comment. - -if (USE_INTERNAL_MYSQL_LIBRARY) - add_subdirectory (mariadb-connector-c-cmake) -endif () +add_subdirectory (mariadb-connector-c-cmake) if (ENABLE_TESTS) add_subdirectory (googletest-cmake) diff --git a/contrib/mariadb-connector-c-cmake/CMakeLists.txt b/contrib/mariadb-connector-c-cmake/CMakeLists.txt index 1037f231a38..5e337bc4621 100644 --- a/contrib/mariadb-connector-c-cmake/CMakeLists.txt +++ b/contrib/mariadb-connector-c-cmake/CMakeLists.txt @@ -1,3 +1,14 @@ +if(OS_LINUX AND TARGET OpenSSL::SSL) + option(ENABLE_MYSQL "Enable MySQL" ${ENABLE_LIBRARIES}) +else () + option(ENABLE_MYSQL "Enable MySQL" FALSE) +endif () + +if(NOT ENABLE_MYSQL) + message (STATUS "Build without mysqlclient (support for MYSQL dictionary source will be disabled)") + return() +endif() + if (GLIBC_COMPATIBILITY) set(LIBM glibc-compatibility) endif() @@ -235,3 +246,5 @@ target_include_directories(mariadbclient PRIVATE ${CC_BINARY_DIR}/include-privat target_include_directories(mariadbclient SYSTEM PUBLIC ${CC_BINARY_DIR}/include-public ${CC_SOURCE_DIR}/include ${CC_SOURCE_DIR}/libmariadb) set_target_properties(mariadbclient PROPERTIES IMPORTED_INTERFACE_LINK_LIBRARIES "${SYSTEM_LIBS}") + +add_library(ch_contrib::mariadbclient ALIAS mariadbclient) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b6b44c4374f..68f09ac727c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -366,11 +366,14 @@ dbms_target_link_libraries ( Poco::MongoDB string_utils PUBLIC - ${MYSQLXX_LIBRARY} boost::system clickhouse_common_io ) +if (TARGET ch::mysqlxx) + dbms_target_link_libraries (PUBLIC ch::mysqlxx) +endif() + dbms_target_link_libraries ( PUBLIC boost::circular_buffer diff --git a/src/Common/CMakeLists.txt b/src/Common/CMakeLists.txt index 1e7d3591a48..490628a2180 100644 --- a/src/Common/CMakeLists.txt +++ b/src/Common/CMakeLists.txt @@ -4,6 +4,6 @@ if (ENABLE_EXAMPLES) add_subdirectory(examples) endif() -if (USE_MYSQL) +if (ENABLE_MYSQL) add_subdirectory (mysqlxx) endif () diff --git a/src/Common/mysqlxx/CMakeLists.txt b/src/Common/mysqlxx/CMakeLists.txt index 8821834eef8..d7292075aae 100644 --- a/src/Common/mysqlxx/CMakeLists.txt +++ b/src/Common/mysqlxx/CMakeLists.txt @@ -15,10 +15,12 @@ target_include_directories (mysqlxx PUBLIC .) target_link_libraries (mysqlxx clickhouse_common_io - ${MYSQLCLIENT_LIBRARIES} ch_contrib::zlib + ch_contrib::mariadbclient ) +add_library(ch::mysqlxx ALIAS mysqlxx) + if (ENABLE_TESTS) add_subdirectory (tests) endif () diff --git a/src/Dictionaries/Embedded/CMakeLists.txt b/src/Dictionaries/Embedded/CMakeLists.txt index 20c7b3c832a..236111bc801 100644 --- a/src/Dictionaries/Embedded/CMakeLists.txt +++ b/src/Dictionaries/Embedded/CMakeLists.txt @@ -2,4 +2,7 @@ include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") add_headers_and_sources(clickhouse_dictionaries_embedded .) add_headers_and_sources(clickhouse_dictionaries_embedded GeodataProviders) add_library(clickhouse_dictionaries_embedded ${clickhouse_dictionaries_embedded_sources}) -target_link_libraries(clickhouse_dictionaries_embedded PRIVATE clickhouse_common_io ${MYSQLXX_LIBRARY}) +target_link_libraries(clickhouse_dictionaries_embedded PRIVATE clickhouse_common_io) +if (TARGET ch::mysqlxx) + target_link_libraries(clickhouse_dictionaries_embedded PRIVATE ch::mysqlxx) +endif() diff --git a/src/configure_config.cmake b/src/configure_config.cmake index 991c04d4b98..eeeb8b11c45 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -76,3 +76,6 @@ endif() if (TARGET ch_contrib::aws_s3) set(USE_AWS_S3 1) endif() +if (TARGET ch_contrib::mariadbclient) # ch::mysqlxx + set(USE_MYSQL 1) +endif() From 323fe70fcd10e3e22a3c17738a79db3349f5bfea Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:48:23 +0300 Subject: [PATCH 193/403] Cleanup NLP support --- CMakeLists.txt | 1 - cmake/find/nlp.cmake | 32 ---------------------- contrib/CMakeLists.txt | 9 +++--- contrib/lemmagen-c-cmake/CMakeLists.txt | 1 + contrib/libstemmer-c-cmake/CMakeLists.txt | 1 + contrib/wordnet-blast-cmake/CMakeLists.txt | 3 +- src/CMakeLists.txt | 8 +++--- src/configure_config.cmake | 3 ++ 8 files changed, 14 insertions(+), 44 deletions(-) delete mode 100644 cmake/find/nlp.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index d8f062d8a45..1af899bdc52 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -487,7 +487,6 @@ message (STATUS include (GNUInstallDirs) include (cmake/find/llvm.cmake) -include (cmake/find/nlp.cmake) # When testing for memory leaks with Valgrind, don't link tcmalloc or jemalloc. diff --git a/cmake/find/nlp.cmake b/cmake/find/nlp.cmake deleted file mode 100644 index 5c10f2f24e7..00000000000 --- a/cmake/find/nlp.cmake +++ /dev/null @@ -1,32 +0,0 @@ -option(ENABLE_NLP "Enable NLP functions support" ${ENABLE_LIBRARIES}) - -if (NOT ENABLE_NLP) - - message (STATUS "NLP functions disabled") - return() -endif() - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libstemmer_c/Makefile") - message (WARNING "submodule contrib/libstemmer_c is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libstemmer_c library, NLP functions will be disabled") - set (USE_NLP 0) - return() -endif () - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/wordnet-blast/wnb") - message (WARNING "submodule contrib/wordnet-blast is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal wordnet-blast library, NLP functions will be disabled") - set (USE_NLP 0) - return() -endif () - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/lemmagen-c/README.md") - message (WARNING "submodule contrib/lemmagen-c is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal lemmagen-c library, NLP functions will be disabled") - set (USE_NLP 0) - return() -endif () - -set (USE_NLP 1) - -message (STATUS "Using Libraries for NLP functions: contrib/wordnet-blast, contrib/libstemmer_c, contrib/lemmagen-c") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 714a15601ee..4fd049c0374 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -127,11 +127,10 @@ add_subdirectory (nuraft-cmake) add_subdirectory (fast_float-cmake) add_subdirectory (datasketches-cpp-cmake) -if (USE_NLP) - add_subdirectory(libstemmer-c-cmake) - add_subdirectory(wordnet-blast-cmake) - add_subdirectory(lemmagen-c-cmake) -endif() +option(ENABLE_NLP "Enable NLP functions support" ${ENABLE_LIBRARIES}) +add_subdirectory(libstemmer-c-cmake) # ENABLE_NLP +add_subdirectory(wordnet-blast-cmake) # ENABLE_NLP +add_subdirectory(lemmagen-c-cmake) # ENABLE_NLP add_subdirectory (sqlite-cmake) add_subdirectory (s2geometry-cmake) diff --git a/contrib/lemmagen-c-cmake/CMakeLists.txt b/contrib/lemmagen-c-cmake/CMakeLists.txt index 3a067916bf6..71abf07acc0 100644 --- a/contrib/lemmagen-c-cmake/CMakeLists.txt +++ b/contrib/lemmagen-c-cmake/CMakeLists.txt @@ -7,3 +7,4 @@ set(SRCS add_library(lemmagen STATIC ${SRCS}) target_include_directories(lemmagen SYSTEM PUBLIC "${LEMMAGEN_INCLUDE_DIR}") +add_library(ch_contrib::lemmagen ALIAS lemmagen) diff --git a/contrib/libstemmer-c-cmake/CMakeLists.txt b/contrib/libstemmer-c-cmake/CMakeLists.txt index b5cd59e4633..f47ff91045e 100644 --- a/contrib/libstemmer-c-cmake/CMakeLists.txt +++ b/contrib/libstemmer-c-cmake/CMakeLists.txt @@ -29,3 +29,4 @@ endforeach () # all the sources parsed. Now just add the lib add_library ( stemmer STATIC ${_SOURCES} ${_HEADERS} ) target_include_directories (stemmer SYSTEM PUBLIC "${STEMMER_INCLUDE_DIR}") +add_library(ch_contrib::stemmer ALIAS stemmer) diff --git a/contrib/wordnet-blast-cmake/CMakeLists.txt b/contrib/wordnet-blast-cmake/CMakeLists.txt index 37e4e9825ca..ec4bdee48e5 100644 --- a/contrib/wordnet-blast-cmake/CMakeLists.txt +++ b/contrib/wordnet-blast-cmake/CMakeLists.txt @@ -7,7 +7,6 @@ set(SRCS ) add_library(wnb ${SRCS}) - target_link_libraries(wnb PRIVATE boost::headers_only boost::graph) - target_include_directories(wnb SYSTEM PUBLIC "${LIBRARY_DIR}") +add_library(ch_contrib::wnb ALIAS wnb) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 68f09ac727c..78f094ceae0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -494,10 +494,10 @@ target_link_libraries (clickhouse_common_io PRIVATE ch_contrib::lz4) dbms_target_link_libraries(PRIVATE _boost_context) -if (USE_NLP) - dbms_target_link_libraries (PUBLIC stemmer) - dbms_target_link_libraries (PUBLIC wnb) - dbms_target_link_libraries (PUBLIC lemmagen) +if (ENABLE_NLP) + dbms_target_link_libraries (PUBLIC ch_contrib::stemmer) + dbms_target_link_libraries (PUBLIC ch_contrib::wnb) + dbms_target_link_libraries (PUBLIC ch_contrib::lemmagen) endif() if (TARGET ch_contrib::bzip2) diff --git a/src/configure_config.cmake b/src/configure_config.cmake index eeeb8b11c45..d6ef5af1256 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -79,3 +79,6 @@ endif() if (TARGET ch_contrib::mariadbclient) # ch::mysqlxx set(USE_MYSQL 1) endif() +if (ENABLE_NLP) + set(USE_NLP 1) +endif() From f22827e658ea8d3a81d049716cd5341d84225975 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:48:23 +0300 Subject: [PATCH 194/403] Remove cmake/find/gperf.cmake --- cmake/find/gperf.cmake | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 cmake/find/gperf.cmake diff --git a/cmake/find/gperf.cmake b/cmake/find/gperf.cmake deleted file mode 100644 index 9b806598c57..00000000000 --- a/cmake/find/gperf.cmake +++ /dev/null @@ -1,16 +0,0 @@ -if(NOT DEFINED ENABLE_GPERF OR ENABLE_GPERF) - # Check if gperf was installed - find_program(GPERF gperf) - if(GPERF) - option(ENABLE_GPERF "Use gperf function hash generator tool" ${ENABLE_LIBRARIES}) - endif() -endif() - -if (ENABLE_GPERF) - if(NOT GPERF) - message(FATAL_ERROR "Could not find the program gperf") - endif() - set(USE_GPERF 1) -endif() - -message(STATUS "Using gperf=${USE_GPERF}: ${GPERF}") From 6dcb09f9e16b8f445285752fbb5a879a1cbf2a82 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:48:23 +0300 Subject: [PATCH 195/403] Cleanup llvm --- CMakeLists.txt | 2 - cmake/find/llvm.cmake | 79 --------------- contrib/CMakeLists.txt | 25 +---- contrib/llvm-cmake/CMakeLists.txt | 98 +++++++++++++++++++ src/CMakeLists.txt | 5 +- src/Core/config_core.h.in | 1 - src/Functions/CMakeLists.txt | 5 +- ...StorageSystemBuildOptions.generated.cpp.in | 1 - src/configure_config.cmake | 3 + 9 files changed, 106 insertions(+), 113 deletions(-) delete mode 100644 cmake/find/llvm.cmake create mode 100644 contrib/llvm-cmake/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 1af899bdc52..3c98693510a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -486,8 +486,6 @@ message (STATUS include (GNUInstallDirs) -include (cmake/find/llvm.cmake) - # When testing for memory leaks with Valgrind, don't link tcmalloc or jemalloc. if (TARGET global-group) diff --git a/cmake/find/llvm.cmake b/cmake/find/llvm.cmake deleted file mode 100644 index 81cafb5eec4..00000000000 --- a/cmake/find/llvm.cmake +++ /dev/null @@ -1,79 +0,0 @@ -if (APPLE OR NOT ARCH_AMD64 OR SANITIZE STREQUAL "undefined") - set (ENABLE_EMBEDDED_COMPILER_DEFAULT OFF) -else() - set (ENABLE_EMBEDDED_COMPILER_DEFAULT ON) -endif() - -option (ENABLE_EMBEDDED_COMPILER "Enable support for 'compile_expressions' option for query execution" ${ENABLE_EMBEDDED_COMPILER_DEFAULT}) - -if (NOT ENABLE_EMBEDDED_COMPILER) - set (USE_EMBEDDED_COMPILER 0) - return() -endif() - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm/CMakeLists.txt") - message (${RECONFIGURE_MESSAGE_LEVEL} "submodule /contrib/llvm is missing. to fix try run: \n git submodule update --init") -endif () - -set (USE_EMBEDDED_COMPILER 1) - -set (LLVM_FOUND 1) -set (LLVM_VERSION "12.0.0bundled") -set (LLVM_INCLUDE_DIRS - "${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm/include" - "${ClickHouse_BINARY_DIR}/contrib/llvm/llvm/include" -) -set (LLVM_LIBRARY_DIRS "${ClickHouse_BINARY_DIR}/contrib/llvm/llvm") - -message(STATUS "LLVM include Directory: ${LLVM_INCLUDE_DIRS}") -message(STATUS "LLVM library Directory: ${LLVM_LIBRARY_DIRS}") -message(STATUS "LLVM C++ compiler flags: ${LLVM_CXXFLAGS}") - -# This list was generated by listing all LLVM libraries, compiling the binary and removing all libraries while it still compiles. -set (REQUIRED_LLVM_LIBRARIES -LLVMExecutionEngine -LLVMRuntimeDyld -LLVMX86CodeGen -LLVMX86Desc -LLVMX86Info -LLVMAsmPrinter -LLVMDebugInfoDWARF -LLVMGlobalISel -LLVMSelectionDAG -LLVMMCDisassembler -LLVMPasses -LLVMCodeGen -LLVMipo -LLVMBitWriter -LLVMInstrumentation -LLVMScalarOpts -LLVMAggressiveInstCombine -LLVMInstCombine -LLVMVectorize -LLVMTransformUtils -LLVMTarget -LLVMAnalysis -LLVMProfileData -LLVMObject -LLVMBitReader -LLVMCore -LLVMRemarks -LLVMBitstreamReader -LLVMMCParser -LLVMMC -LLVMBinaryFormat -LLVMDebugInfoCodeView -LLVMSupport -LLVMDemangle -) - -#function(llvm_libs_all REQUIRED_LLVM_LIBRARIES) -# llvm_map_components_to_libnames (result all) -# if (USE_STATIC_LIBRARIES OR NOT "LLVM" IN_LIST result) -# list (REMOVE_ITEM result "LTO" "LLVM") -# else() -# set (result "LLVM") -# endif () -# list (APPEND result ${CMAKE_DL_LIBS} ch_contrib::zlib) -# set (${REQUIRED_LLVM_LIBRARIES} ${result} PARENT_SCOPE) -#endfunction() diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 4fd049c0374..5207638e19d 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -81,30 +81,7 @@ if (ENABLE_TESTS) add_subdirectory (googletest-cmake) endif() -function(add_llvm) - # ld: unknown option: --color-diagnostics - if (APPLE) - set (LINKER_SUPPORTS_COLOR_DIAGNOSTICS 0 CACHE INTERNAL "") - endif () - - # Do not adjust RPATH in llvm, since then it will not be able to find libcxx/libcxxabi/libunwind - set (CMAKE_INSTALL_RPATH "ON") - set (LLVM_COMPILER_CHECKED 1 CACHE INTERNAL "") - set (LLVM_ENABLE_EH 1 CACHE INTERNAL "") - set (LLVM_ENABLE_RTTI 1 CACHE INTERNAL "") - set (LLVM_ENABLE_PIC 0 CACHE INTERNAL "") - set (LLVM_TARGETS_TO_BUILD "X86;AArch64" CACHE STRING "") - - # Need to use C++17 since the compilation is not possible with C++20 currently, due to ambiguous operator != etc. - # LLVM project will set its default value for the -std=... but our global setting from CMake will override it. - set (CMAKE_CXX_STANDARD 17) - - add_subdirectory (llvm/llvm) -endfunction() -if (USE_EMBEDDED_COMPILER) - add_llvm() -endif () - +add_subdirectory (llvm-cmake) add_subdirectory (libgsasl-cmake) add_subdirectory (libxml2-cmake) add_subdirectory (aws-s3-cmake) diff --git a/contrib/llvm-cmake/CMakeLists.txt b/contrib/llvm-cmake/CMakeLists.txt new file mode 100644 index 00000000000..d240924cac3 --- /dev/null +++ b/contrib/llvm-cmake/CMakeLists.txt @@ -0,0 +1,98 @@ +if (APPLE OR NOT ARCH_AMD64 OR SANITIZE STREQUAL "undefined") + set (ENABLE_EMBEDDED_COMPILER_DEFAULT OFF) +else() + set (ENABLE_EMBEDDED_COMPILER_DEFAULT ON) +endif() +option (ENABLE_EMBEDDED_COMPILER "Enable support for 'compile_expressions' option for query execution" ${ENABLE_EMBEDDED_COMPILER_DEFAULT}) + +if (NOT ENABLE_EMBEDDED_COMPILER) + set (USE_EMBEDDED_COMPILER 0) + return() +endif() + +set (LLVM_FOUND 1) +set (LLVM_VERSION "12.0.0bundled") +set (LLVM_INCLUDE_DIRS + "${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm/include" + "${ClickHouse_BINARY_DIR}/contrib/llvm/llvm/include" +) +set (LLVM_LIBRARY_DIRS "${ClickHouse_BINARY_DIR}/contrib/llvm/llvm") + +# This list was generated by listing all LLVM libraries, compiling the binary and removing all libraries while it still compiles. +set (REQUIRED_LLVM_LIBRARIES + LLVMExecutionEngine + LLVMRuntimeDyld + LLVMX86CodeGen + LLVMX86Desc + LLVMX86Info + LLVMAsmPrinter + LLVMDebugInfoDWARF + LLVMGlobalISel + LLVMSelectionDAG + LLVMMCDisassembler + LLVMPasses + LLVMCodeGen + LLVMipo + LLVMBitWriter + LLVMInstrumentation + LLVMScalarOpts + LLVMAggressiveInstCombine + LLVMInstCombine + LLVMVectorize + LLVMTransformUtils + LLVMTarget + LLVMAnalysis + LLVMProfileData + LLVMObject + LLVMBitReader + LLVMCore + LLVMRemarks + LLVMBitstreamReader + LLVMMCParser + LLVMMC + LLVMBinaryFormat + LLVMDebugInfoCodeView + LLVMSupport + LLVMDemangle +) + +#function(llvm_libs_all REQUIRED_LLVM_LIBRARIES) +# llvm_map_components_to_libnames (result all) +# if (USE_STATIC_LIBRARIES OR NOT "LLVM" IN_LIST result) +# list (REMOVE_ITEM result "LTO" "LLVM") +# else() +# set (result "LLVM") +# endif () +# list (APPEND result ${CMAKE_DL_LIBS} ch_contrib::zlib) +# set (${REQUIRED_LLVM_LIBRARIES} ${result} PARENT_SCOPE) +#endfunction() + +message (STATUS "LLVM include Directory: ${LLVM_INCLUDE_DIRS}") +message (STATUS "LLVM library Directory: ${LLVM_LIBRARY_DIRS}") +message (STATUS "LLVM C++ compiler flags: ${LLVM_CXXFLAGS}") + +# ld: unknown option: --color-diagnostics +if (APPLE) + set (LINKER_SUPPORTS_COLOR_DIAGNOSTICS 0 CACHE INTERNAL "") +endif () + +# Do not adjust RPATH in llvm, since then it will not be able to find libcxx/libcxxabi/libunwind +set (CMAKE_INSTALL_RPATH "ON") +set (LLVM_COMPILER_CHECKED 1 CACHE INTERNAL "") +set (LLVM_ENABLE_EH 1 CACHE INTERNAL "") +set (LLVM_ENABLE_RTTI 1 CACHE INTERNAL "") +set (LLVM_ENABLE_PIC 0 CACHE INTERNAL "") +set (LLVM_TARGETS_TO_BUILD "X86;AArch64" CACHE STRING "") + +# Need to use C++17 since the compilation is not possible with C++20 currently, due to ambiguous operator != etc. +# LLVM project will set its default value for the -std=... but our global setting from CMake will override it. +set (CMAKE_CXX_STANDARD 17) + +set (LLVM_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm") +set (LLVM_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/llvm/llvm") +add_subdirectory ("${LLVM_SOURCE_DIR}" "${LLVM_BINARY_DIR}") + +add_library (_llvm INTERFACE) +target_link_libraries (_llvm INTERFACE ${REQUIRED_LLVM_LIBRARIES}) +target_include_directories (_llvm SYSTEM BEFORE INTERFACE ${LLVM_INCLUDE_DIRS}) +add_library(ch_contrib::llvm ALIAS _llvm) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 78f094ceae0..4e98f598c12 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -286,9 +286,8 @@ endmacro () dbms_target_include_directories (PUBLIC "${ClickHouse_SOURCE_DIR}/src" "${ClickHouse_BINARY_DIR}/src") target_include_directories (clickhouse_common_io PUBLIC "${ClickHouse_SOURCE_DIR}/src" "${ClickHouse_BINARY_DIR}/src") -if (USE_EMBEDDED_COMPILER) - dbms_target_link_libraries (PUBLIC ${REQUIRED_LLVM_LIBRARIES}) - dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS}) +if (TARGET ch_contrib::llvm) + dbms_target_link_libraries (PUBLIC ch_contrib::llvm) endif () # Otherwise it will slow down stack traces printing too much. diff --git a/src/Core/config_core.h.in b/src/Core/config_core.h.in index 11dd9bf96f1..adf0f2d4b2d 100644 --- a/src/Core/config_core.h.in +++ b/src/Core/config_core.h.in @@ -7,7 +7,6 @@ #cmakedefine01 USE_RDKAFKA #cmakedefine01 USE_AMQPCPP #cmakedefine01 USE_EMBEDDED_COMPILER -#cmakedefine01 USE_INTERNAL_LLVM_LIBRARY #cmakedefine01 USE_SSL #cmakedefine01 USE_OPENCL #cmakedefine01 USE_LDAP diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index d4788d2c9be..d79ab5f32a1 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -66,9 +66,8 @@ if (ENABLE_EXAMPLES) add_subdirectory(examples) endif () -if (USE_EMBEDDED_COMPILER) - target_link_libraries(clickhouse_functions PRIVATE ${REQUIRED_LLVM_LIBRARIES}) - target_include_directories(clickhouse_functions SYSTEM BEFORE PUBLIC ${LLVM_INCLUDE_DIRS}) +if (TARGET ch_contrib::llvm) + target_link_libraries(clickhouse_functions PRIVATE ch_contrib::llvm) endif () if (TARGET ch_contrib::base64) diff --git a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in index bb2d095aa78..c10f5759d6a 100644 --- a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in +++ b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in @@ -53,7 +53,6 @@ const char * auto_config_build[] "USE_NURAFT", "@USE_NURAFT@", "USE_NLP", "@USE_NLP@", "USE_SQLITE", "@USE_SQLITE@", - "USE_INTERNAL_LLVM_LIBRARY", "@USE_INTERNAL_LLVM_LIBRARY@", "USE_OPENCL", "@USE_OPENCL@", "USE_LIBPQXX", "@USE_LIBPQXX@", "USE_AZURE_BLOB_STORAGE", "@USE_AZURE_BLOB_STORAGE@", diff --git a/src/configure_config.cmake b/src/configure_config.cmake index d6ef5af1256..278148d2c1c 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -82,3 +82,6 @@ endif() if (ENABLE_NLP) set(USE_NLP 1) endif() +if (TARGET ch_contrib::llvm) + set(USE_EMBEDDED_COMPILER 1) +endif() From 08efa1e75d7d5cca805912790c0678b099de56bd Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:48:24 +0300 Subject: [PATCH 196/403] Remove unbundled cxx support --- cmake/find/cxx.cmake | 40 ---------------------------------------- 1 file changed, 40 deletions(-) diff --git a/cmake/find/cxx.cmake b/cmake/find/cxx.cmake index d1f62f0ecea..4a8620930e8 100644 --- a/cmake/find/cxx.cmake +++ b/cmake/find/cxx.cmake @@ -1,52 +1,13 @@ option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++" ON) if (NOT USE_LIBCXX) - if (USE_INTERNAL_LIBCXX_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal libcxx with USE_LIBCXX=OFF") - endif() - target_link_libraries(global-libs INTERFACE -l:libstdc++.a -l:libstdc++fs.a) # Always link these libraries as static target_link_libraries(global-libs INTERFACE ${EXCEPTION_HANDLING_LIBRARY}) return() endif() -set(USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT ON) - -option (USE_INTERNAL_LIBCXX_LIBRARY "Disable to use system libcxx and libcxxabi libraries instead of bundled" - ${USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT}) - -if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxx/src") - if (USE_INTERNAL_LIBCXX_LIBRARY) - message(WARNING "submodule contrib/libcxx is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libcxx") - set(USE_INTERNAL_LIBCXX_LIBRARY 0) - endif() - set(USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT 0) - set(MISSING_INTERNAL_LIBCXX_LIBRARY 1) -endif() - set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=0") # More checks in debug build. -if (NOT USE_INTERNAL_LIBCXX_LIBRARY) - find_library (LIBCXX_LIBRARY c++) - find_library (LIBCXXFS_LIBRARY c++fs) - find_library (LIBCXXABI_LIBRARY c++abi) - - if(LIBCXX_LIBRARY AND LIBCXXABI_LIBRARY) # c++fs is now a part of the libc++ - set (HAVE_LIBCXX 1) - else () - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system libcxx") - endif() - - if(NOT LIBCXXFS_LIBRARY) - set(LIBCXXFS_LIBRARY ${LIBCXX_LIBRARY}) - endif() - - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") - - target_link_libraries(global-libs INTERFACE ${EXCEPTION_HANDLING_LIBRARY}) -endif () - if (NOT HAVE_LIBCXX AND NOT MISSING_INTERNAL_LIBCXX_LIBRARY) set (LIBCXX_LIBRARY cxx) set (LIBCXXABI_LIBRARY cxxabi) @@ -56,7 +17,6 @@ if (NOT HAVE_LIBCXX AND NOT MISSING_INTERNAL_LIBCXX_LIBRARY) # Exception handling library is embedded into libcxxabi. set (HAVE_LIBCXX 1) - set(USE_INTERNAL_LIBCXX_LIBRARY 1) endif () if (HAVE_LIBCXX) From 8b692b607cbacf2c7f3bb94fc15ba3f17222dfc6 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:48:24 +0300 Subject: [PATCH 197/403] Remove unbundled cctz support --- base/base/CMakeLists.txt | 6 +- contrib/cctz-cmake/CMakeLists.txt | 159 +++++++++++------------------- 2 files changed, 59 insertions(+), 106 deletions(-) diff --git a/base/base/CMakeLists.txt b/base/base/CMakeLists.txt index 434205d812e..a5d32522594 100644 --- a/base/base/CMakeLists.txt +++ b/base/base/CMakeLists.txt @@ -40,10 +40,6 @@ else () target_compile_definitions(common PUBLIC WITH_COVERAGE=0) endif () -if (USE_INTERNAL_CCTZ) - set_source_files_properties(DateLUTImpl.cpp PROPERTIES COMPILE_DEFINITIONS USE_INTERNAL_CCTZ) -endif() - target_include_directories(common PUBLIC .. "${CMAKE_CURRENT_BINARY_DIR}/..") if (OS_DARWIN AND NOT MAKE_STATIC_LIBRARIES) @@ -60,7 +56,7 @@ target_link_libraries (common Poco::Util Poco::Foundation replxx - cctz + ch_contrib::cctz fmt magic_enum ) diff --git a/contrib/cctz-cmake/CMakeLists.txt b/contrib/cctz-cmake/CMakeLists.txt index 2248ba8b612..d252ddc864f 100644 --- a/contrib/cctz-cmake/CMakeLists.txt +++ b/contrib/cctz-cmake/CMakeLists.txt @@ -1,106 +1,63 @@ -option (USE_INTERNAL_CCTZ_LIBRARY "Use internal cctz library" ON) +include(${ClickHouse_SOURCE_DIR}/cmake/embed_binary.cmake) +set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/cctz") -if (NOT USE_INTERNAL_CCTZ_LIBRARY) - find_library (LIBRARY_CCTZ cctz) - find_path (INCLUDE_CCTZ NAMES cctz/civil_time.h) +set (SRCS + "${LIBRARY_DIR}/src/civil_time_detail.cc" + "${LIBRARY_DIR}/src/time_zone_fixed.cc" + "${LIBRARY_DIR}/src/time_zone_format.cc" + "${LIBRARY_DIR}/src/time_zone_if.cc" + "${LIBRARY_DIR}/src/time_zone_impl.cc" + "${LIBRARY_DIR}/src/time_zone_info.cc" + "${LIBRARY_DIR}/src/time_zone_libc.cc" + "${LIBRARY_DIR}/src/time_zone_lookup.cc" + "${LIBRARY_DIR}/src/time_zone_posix.cc" + "${LIBRARY_DIR}/src/zone_info_source.cc" +) - if (LIBRARY_CCTZ AND INCLUDE_CCTZ) - set (EXTERNAL_CCTZ_LIBRARY_FOUND 1) +add_library (cctz ${SRCS}) +target_include_directories (cctz PUBLIC "${LIBRARY_DIR}/include") - set(CMAKE_REQUIRED_LIBRARIES ${LIBRARY_CCTZ}) - set(CMAKE_REQUIRED_INCLUDES ${INCLUDE_CCTZ}) - check_cxx_source_compiles( - " - #include - int main() { - cctz::civil_day date; - } - " - EXTERNAL_CCTZ_LIBRARY_WORKS - ) - - if (NOT EXTERNAL_CCTZ_LIBRARY_WORKS) - message (${RECONFIGURE_MESSAGE_LEVEL} "External cctz is not working: ${LIBRARY_CCTZ} ${INCLUDE_CCTZ}") - else() - add_library (cctz UNKNOWN IMPORTED) - set_property (TARGET cctz PROPERTY IMPORTED_LOCATION ${LIBRARY_CCTZ}) - set_property (TARGET cctz PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_CCTZ}) - endif() - - set(SYSTEM_STORAGE_TZ_FILE "${CMAKE_BINARY_DIR}/src/Storages/System/StorageSystemTimeZones.generated.cpp") - file(REMOVE ${SYSTEM_STORAGE_TZ_FILE}) - file(APPEND ${SYSTEM_STORAGE_TZ_FILE} "// autogenerated by ClickHouse/contrib/cctz-cmake/CMakeLists.txt\n") - file(APPEND ${SYSTEM_STORAGE_TZ_FILE} "const char * auto_time_zones[] {nullptr};\n" ) - - else() - set (EXTERNAL_CCTZ_LIBRARY_FOUND 0) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system cctz") - endif() -endif() - -if (NOT EXTERNAL_CCTZ_LIBRARY_FOUND OR NOT EXTERNAL_CCTZ_LIBRARY_WORKS) - include(${ClickHouse_SOURCE_DIR}/cmake/embed_binary.cmake) - set(USE_INTERNAL_CCTZ_LIBRARY 1) - set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/cctz") - - set (SRCS - "${LIBRARY_DIR}/src/civil_time_detail.cc" - "${LIBRARY_DIR}/src/time_zone_fixed.cc" - "${LIBRARY_DIR}/src/time_zone_format.cc" - "${LIBRARY_DIR}/src/time_zone_if.cc" - "${LIBRARY_DIR}/src/time_zone_impl.cc" - "${LIBRARY_DIR}/src/time_zone_info.cc" - "${LIBRARY_DIR}/src/time_zone_libc.cc" - "${LIBRARY_DIR}/src/time_zone_lookup.cc" - "${LIBRARY_DIR}/src/time_zone_posix.cc" - "${LIBRARY_DIR}/src/zone_info_source.cc" - ) - - add_library (cctz ${SRCS}) - target_include_directories (cctz SYSTEM PUBLIC "${LIBRARY_DIR}/include") - - if (OS_FREEBSD) - # yes, need linux, because bsd check inside linux in time_zone_libc.cc:24 - target_compile_definitions (cctz PRIVATE __USE_BSD linux _XOPEN_SOURCE=600) - endif () - - # Related to time_zones table: - # StorageSystemTimeZones.generated.cpp is autogenerated each time during a build - # data in this file will be used to populate the system.time_zones table, this is specific to OS_LINUX - # as the library that's built using embedded tzdata is also specific to OS_LINUX - set(SYSTEM_STORAGE_TZ_FILE "${CMAKE_BINARY_DIR}/src/Storages/System/StorageSystemTimeZones.generated.cpp") - # remove existing copies so that its generated fresh on each build. - file(REMOVE ${SYSTEM_STORAGE_TZ_FILE}) - - # get the list of timezones from tzdata shipped with cctz - set(TZDIR "${LIBRARY_DIR}/testdata/zoneinfo") - file(STRINGS "${LIBRARY_DIR}/testdata/version" TZDATA_VERSION) - set_property(GLOBAL PROPERTY TZDATA_VERSION_PROP "${TZDATA_VERSION}") - message(STATUS "Packaging with tzdata version: ${TZDATA_VERSION}") - - set(TIMEZONE_RESOURCE_FILES) - - # each file in that dir (except of tab and localtime) store the info about timezone - execute_process(COMMAND - bash -c "cd ${TZDIR} && find * -type f -and ! -name '*.tab' -and ! -name 'localtime' | LC_ALL=C sort | paste -sd ';' -" - OUTPUT_STRIP_TRAILING_WHITESPACE - OUTPUT_VARIABLE TIMEZONES) - - file(APPEND ${SYSTEM_STORAGE_TZ_FILE} "// autogenerated by ClickHouse/contrib/cctz-cmake/CMakeLists.txt\n") - file(APPEND ${SYSTEM_STORAGE_TZ_FILE} "const char * auto_time_zones[] {\n" ) - - foreach(TIMEZONE ${TIMEZONES}) - file(APPEND ${SYSTEM_STORAGE_TZ_FILE} " \"${TIMEZONE}\",\n") - list(APPEND TIMEZONE_RESOURCE_FILES "${TIMEZONE}") - endforeach(TIMEZONE) - file(APPEND ${SYSTEM_STORAGE_TZ_FILE} " nullptr};\n") - clickhouse_embed_binaries( - TARGET tzdata - RESOURCE_DIR "${TZDIR}" - RESOURCES ${TIMEZONE_RESOURCE_FILES} - ) - add_dependencies(cctz tzdata) - target_link_libraries(cctz INTERFACE "-Wl,${WHOLE_ARCHIVE} $ -Wl,${NO_WHOLE_ARCHIVE}") +if (OS_FREEBSD) + # yes, need linux, because bsd check inside linux in time_zone_libc.cc:24 + target_compile_definitions (cctz PRIVATE __USE_BSD linux _XOPEN_SOURCE=600) endif () -message (STATUS "Using cctz") +# Related to time_zones table: +# StorageSystemTimeZones.generated.cpp is autogenerated each time during a build +# data in this file will be used to populate the system.time_zones table, this is specific to OS_LINUX +# as the library that's built using embedded tzdata is also specific to OS_LINUX +set(SYSTEM_STORAGE_TZ_FILE "${CMAKE_BINARY_DIR}/src/Storages/System/StorageSystemTimeZones.generated.cpp") +# remove existing copies so that its generated fresh on each build. +file(REMOVE ${SYSTEM_STORAGE_TZ_FILE}) + +# get the list of timezones from tzdata shipped with cctz +set(TZDIR "${LIBRARY_DIR}/testdata/zoneinfo") +file(STRINGS "${LIBRARY_DIR}/testdata/version" TZDATA_VERSION) +set_property(GLOBAL PROPERTY TZDATA_VERSION_PROP "${TZDATA_VERSION}") +message(STATUS "Packaging with tzdata version: ${TZDATA_VERSION}") + +set(TIMEZONE_RESOURCE_FILES) + +# each file in that dir (except of tab and localtime) store the info about timezone +execute_process(COMMAND + bash -c "cd ${TZDIR} && find * -type f -and ! -name '*.tab' -and ! -name 'localtime' | LC_ALL=C sort | paste -sd ';' -" + OUTPUT_STRIP_TRAILING_WHITESPACE + OUTPUT_VARIABLE TIMEZONES) + +file(APPEND ${SYSTEM_STORAGE_TZ_FILE} "// autogenerated by ClickHouse/contrib/cctz-cmake/CMakeLists.txt\n") +file(APPEND ${SYSTEM_STORAGE_TZ_FILE} "const char * auto_time_zones[] {\n" ) + +foreach(TIMEZONE ${TIMEZONES}) + file(APPEND ${SYSTEM_STORAGE_TZ_FILE} " \"${TIMEZONE}\",\n") + list(APPEND TIMEZONE_RESOURCE_FILES "${TIMEZONE}") +endforeach(TIMEZONE) +file(APPEND ${SYSTEM_STORAGE_TZ_FILE} " nullptr};\n") +clickhouse_embed_binaries( + TARGET tzdata + RESOURCE_DIR "${TZDIR}" + RESOURCES ${TIMEZONE_RESOURCE_FILES} +) +add_dependencies(cctz tzdata) +target_link_libraries(cctz INTERFACE "-Wl,${WHOLE_ARCHIVE} $ -Wl,${NO_WHOLE_ARCHIVE}") + +add_library(ch_contrib::cctz ALIAS cctz) From ea68b07c9ee4c7d2550f6b36219d7cca2788e940 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:48:24 +0300 Subject: [PATCH 198/403] Remove unbundled replxx support --- base/base/CMakeLists.txt | 2 +- contrib/replxx-cmake/CMakeLists.txt | 71 +++++++---------------------- 2 files changed, 17 insertions(+), 56 deletions(-) diff --git a/base/base/CMakeLists.txt b/base/base/CMakeLists.txt index a5d32522594..ddb9279aa3c 100644 --- a/base/base/CMakeLists.txt +++ b/base/base/CMakeLists.txt @@ -55,7 +55,7 @@ target_link_libraries (common Poco::Net::SSL Poco::Util Poco::Foundation - replxx + ch_contrib::replxx ch_contrib::cctz fmt magic_enum diff --git a/contrib/replxx-cmake/CMakeLists.txt b/contrib/replxx-cmake/CMakeLists.txt index 222a38095cb..f8625913821 100644 --- a/contrib/replxx-cmake/CMakeLists.txt +++ b/contrib/replxx-cmake/CMakeLists.txt @@ -1,10 +1,6 @@ option (ENABLE_REPLXX "Enable replxx support" ${ENABLE_LIBRARIES}) if (NOT ENABLE_REPLXX) - if (USE_INTERNAL_REPLXX_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal replxx with ENABLE_REPLXX=OFF") - endif() - add_library(replxx INTERFACE) target_compile_definitions(replxx INTERFACE USE_REPLXX=0) @@ -12,58 +8,23 @@ if (NOT ENABLE_REPLXX) return() endif() -option (USE_INTERNAL_REPLXX_LIBRARY "Use internal replxx library (Experimental: set to OFF on your own risk)" ON) +set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/replxx") -if (NOT USE_INTERNAL_REPLXX_LIBRARY) - find_library(LIBRARY_REPLXX NAMES replxx replxx-static) - find_path(INCLUDE_REPLXX replxx.hxx) +set(SRCS + "${LIBRARY_DIR}/src/conversion.cxx" + "${LIBRARY_DIR}/src/ConvertUTF.cpp" + "${LIBRARY_DIR}/src/escape.cxx" + "${LIBRARY_DIR}/src/history.cxx" + "${LIBRARY_DIR}/src/terminal.cxx" + "${LIBRARY_DIR}/src/prompt.cxx" + "${LIBRARY_DIR}/src/replxx_impl.cxx" + "${LIBRARY_DIR}/src/replxx.cxx" + "${LIBRARY_DIR}/src/util.cxx" + "${LIBRARY_DIR}/src/wcwidth.cpp" +) - if (LIBRARY_REPLXX AND INCLUDE_REPLXX) - set(CMAKE_REQUIRED_LIBRARIES ${LIBRARY_REPLXX}) - set(CMAKE_REQUIRED_INCLUDES ${INCLUDE_REPLXX}) - check_cxx_source_compiles( - " - #include - int main() { - replxx::Replxx rx; - } - " - EXTERNAL_REPLXX_WORKS - ) - - if (NOT EXTERNAL_REPLXX_WORKS) - message (${RECONFIGURE_MESSAGE_LEVEL} "replxx is unusable: ${LIBRARY_REPLXX} ${INCLUDE_REPLXX}") - else() - add_library(replxx UNKNOWN IMPORTED) - set_property(TARGET replxx PROPERTY IMPORTED_LOCATION ${LIBRARY_REPLXX}) - target_include_directories(replxx SYSTEM PUBLIC ${INCLUDE_REPLXX}) - endif() - else() - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system replxx") - endif() -endif() - - -if (NOT LIBRARY_REPLXX OR NOT INCLUDE_REPLXX OR NOT EXTERNAL_REPLXX_WORKS) - set(USE_INTERNAL_REPLXX_LIBRARY 1) - set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/replxx") - - set(SRCS - "${LIBRARY_DIR}/src/conversion.cxx" - "${LIBRARY_DIR}/src/ConvertUTF.cpp" - "${LIBRARY_DIR}/src/escape.cxx" - "${LIBRARY_DIR}/src/history.cxx" - "${LIBRARY_DIR}/src/terminal.cxx" - "${LIBRARY_DIR}/src/prompt.cxx" - "${LIBRARY_DIR}/src/replxx_impl.cxx" - "${LIBRARY_DIR}/src/replxx.cxx" - "${LIBRARY_DIR}/src/util.cxx" - "${LIBRARY_DIR}/src/wcwidth.cpp" - ) - - add_library (replxx ${SRCS}) - target_include_directories(replxx SYSTEM PUBLIC "${LIBRARY_DIR}/include") -endif () +add_library (replxx ${SRCS}) +target_include_directories(replxx SYSTEM PUBLIC "${LIBRARY_DIR}/include") if (COMPILER_CLANG) target_compile_options(replxx PRIVATE -Wno-documentation) @@ -71,4 +32,4 @@ endif () target_compile_definitions(replxx PUBLIC USE_REPLXX=1) -message (STATUS "Using replxx") +add_library(ch_contrib::replxx ALIAS replxx) From c79c9d41c0eb1e29fd5b225e47225a7004d00644 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:48:24 +0300 Subject: [PATCH 199/403] Remove unbundled hyperscan support --- contrib/hyperscan-cmake/CMakeLists.txt | 476 ++++++++++++------------- 1 file changed, 222 insertions(+), 254 deletions(-) diff --git a/contrib/hyperscan-cmake/CMakeLists.txt b/contrib/hyperscan-cmake/CMakeLists.txt index 248551d0b0c..9afed2a0732 100644 --- a/contrib/hyperscan-cmake/CMakeLists.txt +++ b/contrib/hyperscan-cmake/CMakeLists.txt @@ -6,10 +6,6 @@ elseif(ENABLE_HYPERSCAN) endif () if (NOT ENABLE_HYPERSCAN) - if (USE_INTERNAL_HYPERSCAN_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal hyperscan with ENABLE_HYPERSCAN=OFF") - endif() - add_library (hyperscan INTERFACE) target_compile_definitions (hyperscan INTERFACE USE_HYPERSCAN=0) @@ -17,257 +13,229 @@ if (NOT ENABLE_HYPERSCAN) return() endif() -option (USE_INTERNAL_HYPERSCAN_LIBRARY "Use internal hyperscan library" ON) +set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/hyperscan") -if (NOT USE_INTERNAL_HYPERSCAN_LIBRARY) - find_library (LIBRARY_HYPERSCAN hs) - find_path (INCLUDE_HYPERSCAN NAMES hs.h HINTS /usr/include/hs) # Ubuntu puts headers in this folder +set (SRCS + "${LIBRARY_DIR}/src/alloc.c" + "${LIBRARY_DIR}/src/compiler/asserts.cpp" + "${LIBRARY_DIR}/src/compiler/compiler.cpp" + "${LIBRARY_DIR}/src/compiler/error.cpp" + "${LIBRARY_DIR}/src/crc32.c" + "${LIBRARY_DIR}/src/database.c" + "${LIBRARY_DIR}/src/fdr/engine_description.cpp" + "${LIBRARY_DIR}/src/fdr/fdr_compile_util.cpp" + "${LIBRARY_DIR}/src/fdr/fdr_compile.cpp" + "${LIBRARY_DIR}/src/fdr/fdr_confirm_compile.cpp" + "${LIBRARY_DIR}/src/fdr/fdr_engine_description.cpp" + "${LIBRARY_DIR}/src/fdr/fdr.c" + "${LIBRARY_DIR}/src/fdr/flood_compile.cpp" + "${LIBRARY_DIR}/src/fdr/teddy_compile.cpp" + "${LIBRARY_DIR}/src/fdr/teddy_engine_description.cpp" + "${LIBRARY_DIR}/src/fdr/teddy.c" + "${LIBRARY_DIR}/src/grey.cpp" + "${LIBRARY_DIR}/src/hs_valid_platform.c" + "${LIBRARY_DIR}/src/hs_version.c" + "${LIBRARY_DIR}/src/hs.cpp" + "${LIBRARY_DIR}/src/hwlm/hwlm_build.cpp" + "${LIBRARY_DIR}/src/hwlm/hwlm_literal.cpp" + "${LIBRARY_DIR}/src/hwlm/hwlm.c" + "${LIBRARY_DIR}/src/hwlm/noodle_build.cpp" + "${LIBRARY_DIR}/src/hwlm/noodle_engine.c" + "${LIBRARY_DIR}/src/nfa/accel_dfa_build_strat.cpp" + "${LIBRARY_DIR}/src/nfa/accel.c" + "${LIBRARY_DIR}/src/nfa/accelcompile.cpp" + "${LIBRARY_DIR}/src/nfa/castle.c" + "${LIBRARY_DIR}/src/nfa/castlecompile.cpp" + "${LIBRARY_DIR}/src/nfa/dfa_build_strat.cpp" + "${LIBRARY_DIR}/src/nfa/dfa_min.cpp" + "${LIBRARY_DIR}/src/nfa/gough.c" + "${LIBRARY_DIR}/src/nfa/goughcompile_accel.cpp" + "${LIBRARY_DIR}/src/nfa/goughcompile_reg.cpp" + "${LIBRARY_DIR}/src/nfa/goughcompile.cpp" + "${LIBRARY_DIR}/src/nfa/lbr.c" + "${LIBRARY_DIR}/src/nfa/limex_64.c" + "${LIBRARY_DIR}/src/nfa/limex_accel.c" + "${LIBRARY_DIR}/src/nfa/limex_compile.cpp" + "${LIBRARY_DIR}/src/nfa/limex_native.c" + "${LIBRARY_DIR}/src/nfa/limex_simd128.c" + "${LIBRARY_DIR}/src/nfa/limex_simd256.c" + "${LIBRARY_DIR}/src/nfa/limex_simd384.c" + "${LIBRARY_DIR}/src/nfa/limex_simd512.c" + "${LIBRARY_DIR}/src/nfa/mcclellan.c" + "${LIBRARY_DIR}/src/nfa/mcclellancompile_util.cpp" + "${LIBRARY_DIR}/src/nfa/mcclellancompile.cpp" + "${LIBRARY_DIR}/src/nfa/mcsheng_compile.cpp" + "${LIBRARY_DIR}/src/nfa/mcsheng_data.c" + "${LIBRARY_DIR}/src/nfa/mcsheng.c" + "${LIBRARY_DIR}/src/nfa/mpv.c" + "${LIBRARY_DIR}/src/nfa/mpvcompile.cpp" + "${LIBRARY_DIR}/src/nfa/nfa_api_dispatch.c" + "${LIBRARY_DIR}/src/nfa/nfa_build_util.cpp" + "${LIBRARY_DIR}/src/nfa/rdfa_graph.cpp" + "${LIBRARY_DIR}/src/nfa/rdfa_merge.cpp" + "${LIBRARY_DIR}/src/nfa/rdfa.cpp" + "${LIBRARY_DIR}/src/nfa/repeat.c" + "${LIBRARY_DIR}/src/nfa/repeatcompile.cpp" + "${LIBRARY_DIR}/src/nfa/sheng.c" + "${LIBRARY_DIR}/src/nfa/shengcompile.cpp" + "${LIBRARY_DIR}/src/nfa/shufti.c" + "${LIBRARY_DIR}/src/nfa/shufticompile.cpp" + "${LIBRARY_DIR}/src/nfa/tamarama.c" + "${LIBRARY_DIR}/src/nfa/tamaramacompile.cpp" + "${LIBRARY_DIR}/src/nfa/truffle.c" + "${LIBRARY_DIR}/src/nfa/trufflecompile.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_anchored_acyclic.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_anchored_dots.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_asserts.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_builder.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_calc_components.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_cyclic_redundancy.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_depth.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_dominators.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_edge_redundancy.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_equivalence.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_execute.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_expr_info.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_extparam.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_fixed_width.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_fuzzy.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_haig.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_holder.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_is_equal.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_lbr.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_limex_accel.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_limex.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_literal_analysis.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_literal_component.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_literal_decorated.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_mcclellan.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_misc_opt.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_netflow.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_prefilter.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_prune.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_puff.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_redundancy.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_region_redundancy.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_region.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_repeat.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_reports.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_restructuring.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_revacc.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_sep.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_small_literal_set.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_som_add_redundancy.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_som_util.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_som.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_split.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_squash.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_stop.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_uncalc_components.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_utf8.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_util.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_vacuous.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_violet.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng_width.cpp" + "${LIBRARY_DIR}/src/nfagraph/ng.cpp" + "${LIBRARY_DIR}/src/parser/AsciiComponentClass.cpp" + "${LIBRARY_DIR}/src/parser/buildstate.cpp" + "${LIBRARY_DIR}/src/parser/check_refs.cpp" + "${LIBRARY_DIR}/src/parser/Component.cpp" + "${LIBRARY_DIR}/src/parser/ComponentAlternation.cpp" + "${LIBRARY_DIR}/src/parser/ComponentAssertion.cpp" + "${LIBRARY_DIR}/src/parser/ComponentAtomicGroup.cpp" + "${LIBRARY_DIR}/src/parser/ComponentBackReference.cpp" + "${LIBRARY_DIR}/src/parser/ComponentBoundary.cpp" + "${LIBRARY_DIR}/src/parser/ComponentByte.cpp" + "${LIBRARY_DIR}/src/parser/ComponentClass.cpp" + "${LIBRARY_DIR}/src/parser/ComponentCondReference.cpp" + "${LIBRARY_DIR}/src/parser/ComponentEmpty.cpp" + "${LIBRARY_DIR}/src/parser/ComponentEUS.cpp" + "${LIBRARY_DIR}/src/parser/ComponentRepeat.cpp" + "${LIBRARY_DIR}/src/parser/ComponentSequence.cpp" + "${LIBRARY_DIR}/src/parser/ComponentVisitor.cpp" + "${LIBRARY_DIR}/src/parser/ComponentWordBoundary.cpp" + "${LIBRARY_DIR}/src/parser/ConstComponentVisitor.cpp" + "${LIBRARY_DIR}/src/parser/control_verbs.cpp" + "${LIBRARY_DIR}/src/parser/logical_combination.cpp" + "${LIBRARY_DIR}/src/parser/parse_error.cpp" + "${LIBRARY_DIR}/src/parser/parser_util.cpp" + "${LIBRARY_DIR}/src/parser/Parser.cpp" + "${LIBRARY_DIR}/src/parser/prefilter.cpp" + "${LIBRARY_DIR}/src/parser/shortcut_literal.cpp" + "${LIBRARY_DIR}/src/parser/ucp_table.cpp" + "${LIBRARY_DIR}/src/parser/unsupported.cpp" + "${LIBRARY_DIR}/src/parser/utf8_validate.cpp" + "${LIBRARY_DIR}/src/parser/Utf8ComponentClass.cpp" + "${LIBRARY_DIR}/src/rose/block.c" + "${LIBRARY_DIR}/src/rose/catchup.c" + "${LIBRARY_DIR}/src/rose/init.c" + "${LIBRARY_DIR}/src/rose/match.c" + "${LIBRARY_DIR}/src/rose/program_runtime.c" + "${LIBRARY_DIR}/src/rose/rose_build_add_mask.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_add.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_anchored.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_bytecode.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_castle.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_compile.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_convert.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_dedupe.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_engine_blob.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_exclusive.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_groups.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_infix.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_instructions.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_lit_accel.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_long_lit.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_lookaround.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_matchers.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_merge.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_misc.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_program.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_role_aliasing.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_scatter.cpp" + "${LIBRARY_DIR}/src/rose/rose_build_width.cpp" + "${LIBRARY_DIR}/src/rose/rose_in_util.cpp" + "${LIBRARY_DIR}/src/rose/stream.c" + "${LIBRARY_DIR}/src/runtime.c" + "${LIBRARY_DIR}/src/scratch.c" + "${LIBRARY_DIR}/src/smallwrite/smallwrite_build.cpp" + "${LIBRARY_DIR}/src/som/slot_manager.cpp" + "${LIBRARY_DIR}/src/som/som_runtime.c" + "${LIBRARY_DIR}/src/som/som_stream.c" + "${LIBRARY_DIR}/src/stream_compress.c" + "${LIBRARY_DIR}/src/util/alloc.cpp" + "${LIBRARY_DIR}/src/util/charreach.cpp" + "${LIBRARY_DIR}/src/util/clique.cpp" + "${LIBRARY_DIR}/src/util/compile_context.cpp" + "${LIBRARY_DIR}/src/util/compile_error.cpp" + "${LIBRARY_DIR}/src/util/cpuid_flags.c" + "${LIBRARY_DIR}/src/util/depth.cpp" + "${LIBRARY_DIR}/src/util/fatbit_build.cpp" + "${LIBRARY_DIR}/src/util/multibit_build.cpp" + "${LIBRARY_DIR}/src/util/multibit.c" + "${LIBRARY_DIR}/src/util/report_manager.cpp" + "${LIBRARY_DIR}/src/util/simd_utils.c" + "${LIBRARY_DIR}/src/util/state_compress.c" + "${LIBRARY_DIR}/src/util/target_info.cpp" + "${LIBRARY_DIR}/src/util/ue2string.cpp" +) - if (LIBRARY_HYPERSCAN AND INCLUDE_HYPERSCAN) - set (EXTERNAL_HYPERSCAN_LIBRARY_FOUND 1) - - add_library (hyperscan INTERFACE) - set_target_properties (hyperscan PROPERTIES INTERFACE_LINK_LIBRARIES ${LIBRARY_HYPERSCAN}) - set_target_properties (hyperscan PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_HYPERSCAN}) - set_property(TARGET hyperscan APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS USE_HYPERSCAN=1) - else () - set (EXTERNAL_HYPERSCAN_LIBRARY_FOUND 0) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system hyperscan library") - endif () +add_library (hyperscan ${SRCS}) +target_compile_definitions (hyperscan PUBLIC USE_HYPERSCAN=1) +target_compile_options (hyperscan + PRIVATE -g0 # Library has too much debug information + -mno-avx -mno-avx2 # The library is using dynamic dispatch and is confused if AVX is enabled globally + -march=corei7 -O2 -fno-strict-aliasing -fno-omit-frame-pointer -fvisibility=hidden # The options from original build system + -fno-sanitize=undefined # Assume the library takes care of itself +) +target_include_directories (hyperscan + PRIVATE + common + "${LIBRARY_DIR}/include" +) +target_include_directories (hyperscan SYSTEM PUBLIC "${LIBRARY_DIR}/src") +if (ARCH_AMD64) + target_include_directories (hyperscan PRIVATE x86_64) endif () - -if (NOT EXTERNAL_HYPERSCAN_LIBRARY_FOUND) - set (USE_INTERNAL_HYPERSCAN_LIBRARY 1) - - set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/hyperscan") - - set (SRCS - "${LIBRARY_DIR}/src/alloc.c" - "${LIBRARY_DIR}/src/compiler/asserts.cpp" - "${LIBRARY_DIR}/src/compiler/compiler.cpp" - "${LIBRARY_DIR}/src/compiler/error.cpp" - "${LIBRARY_DIR}/src/crc32.c" - "${LIBRARY_DIR}/src/database.c" - "${LIBRARY_DIR}/src/fdr/engine_description.cpp" - "${LIBRARY_DIR}/src/fdr/fdr_compile_util.cpp" - "${LIBRARY_DIR}/src/fdr/fdr_compile.cpp" - "${LIBRARY_DIR}/src/fdr/fdr_confirm_compile.cpp" - "${LIBRARY_DIR}/src/fdr/fdr_engine_description.cpp" - "${LIBRARY_DIR}/src/fdr/fdr.c" - "${LIBRARY_DIR}/src/fdr/flood_compile.cpp" - "${LIBRARY_DIR}/src/fdr/teddy_compile.cpp" - "${LIBRARY_DIR}/src/fdr/teddy_engine_description.cpp" - "${LIBRARY_DIR}/src/fdr/teddy.c" - "${LIBRARY_DIR}/src/grey.cpp" - "${LIBRARY_DIR}/src/hs_valid_platform.c" - "${LIBRARY_DIR}/src/hs_version.c" - "${LIBRARY_DIR}/src/hs.cpp" - "${LIBRARY_DIR}/src/hwlm/hwlm_build.cpp" - "${LIBRARY_DIR}/src/hwlm/hwlm_literal.cpp" - "${LIBRARY_DIR}/src/hwlm/hwlm.c" - "${LIBRARY_DIR}/src/hwlm/noodle_build.cpp" - "${LIBRARY_DIR}/src/hwlm/noodle_engine.c" - "${LIBRARY_DIR}/src/nfa/accel_dfa_build_strat.cpp" - "${LIBRARY_DIR}/src/nfa/accel.c" - "${LIBRARY_DIR}/src/nfa/accelcompile.cpp" - "${LIBRARY_DIR}/src/nfa/castle.c" - "${LIBRARY_DIR}/src/nfa/castlecompile.cpp" - "${LIBRARY_DIR}/src/nfa/dfa_build_strat.cpp" - "${LIBRARY_DIR}/src/nfa/dfa_min.cpp" - "${LIBRARY_DIR}/src/nfa/gough.c" - "${LIBRARY_DIR}/src/nfa/goughcompile_accel.cpp" - "${LIBRARY_DIR}/src/nfa/goughcompile_reg.cpp" - "${LIBRARY_DIR}/src/nfa/goughcompile.cpp" - "${LIBRARY_DIR}/src/nfa/lbr.c" - "${LIBRARY_DIR}/src/nfa/limex_64.c" - "${LIBRARY_DIR}/src/nfa/limex_accel.c" - "${LIBRARY_DIR}/src/nfa/limex_compile.cpp" - "${LIBRARY_DIR}/src/nfa/limex_native.c" - "${LIBRARY_DIR}/src/nfa/limex_simd128.c" - "${LIBRARY_DIR}/src/nfa/limex_simd256.c" - "${LIBRARY_DIR}/src/nfa/limex_simd384.c" - "${LIBRARY_DIR}/src/nfa/limex_simd512.c" - "${LIBRARY_DIR}/src/nfa/mcclellan.c" - "${LIBRARY_DIR}/src/nfa/mcclellancompile_util.cpp" - "${LIBRARY_DIR}/src/nfa/mcclellancompile.cpp" - "${LIBRARY_DIR}/src/nfa/mcsheng_compile.cpp" - "${LIBRARY_DIR}/src/nfa/mcsheng_data.c" - "${LIBRARY_DIR}/src/nfa/mcsheng.c" - "${LIBRARY_DIR}/src/nfa/mpv.c" - "${LIBRARY_DIR}/src/nfa/mpvcompile.cpp" - "${LIBRARY_DIR}/src/nfa/nfa_api_dispatch.c" - "${LIBRARY_DIR}/src/nfa/nfa_build_util.cpp" - "${LIBRARY_DIR}/src/nfa/rdfa_graph.cpp" - "${LIBRARY_DIR}/src/nfa/rdfa_merge.cpp" - "${LIBRARY_DIR}/src/nfa/rdfa.cpp" - "${LIBRARY_DIR}/src/nfa/repeat.c" - "${LIBRARY_DIR}/src/nfa/repeatcompile.cpp" - "${LIBRARY_DIR}/src/nfa/sheng.c" - "${LIBRARY_DIR}/src/nfa/shengcompile.cpp" - "${LIBRARY_DIR}/src/nfa/shufti.c" - "${LIBRARY_DIR}/src/nfa/shufticompile.cpp" - "${LIBRARY_DIR}/src/nfa/tamarama.c" - "${LIBRARY_DIR}/src/nfa/tamaramacompile.cpp" - "${LIBRARY_DIR}/src/nfa/truffle.c" - "${LIBRARY_DIR}/src/nfa/trufflecompile.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_anchored_acyclic.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_anchored_dots.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_asserts.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_builder.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_calc_components.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_cyclic_redundancy.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_depth.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_dominators.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_edge_redundancy.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_equivalence.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_execute.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_expr_info.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_extparam.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_fixed_width.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_fuzzy.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_haig.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_holder.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_is_equal.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_lbr.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_limex_accel.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_limex.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_literal_analysis.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_literal_component.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_literal_decorated.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_mcclellan.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_misc_opt.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_netflow.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_prefilter.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_prune.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_puff.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_redundancy.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_region_redundancy.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_region.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_repeat.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_reports.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_restructuring.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_revacc.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_sep.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_small_literal_set.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_som_add_redundancy.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_som_util.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_som.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_split.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_squash.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_stop.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_uncalc_components.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_utf8.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_util.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_vacuous.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_violet.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng_width.cpp" - "${LIBRARY_DIR}/src/nfagraph/ng.cpp" - "${LIBRARY_DIR}/src/parser/AsciiComponentClass.cpp" - "${LIBRARY_DIR}/src/parser/buildstate.cpp" - "${LIBRARY_DIR}/src/parser/check_refs.cpp" - "${LIBRARY_DIR}/src/parser/Component.cpp" - "${LIBRARY_DIR}/src/parser/ComponentAlternation.cpp" - "${LIBRARY_DIR}/src/parser/ComponentAssertion.cpp" - "${LIBRARY_DIR}/src/parser/ComponentAtomicGroup.cpp" - "${LIBRARY_DIR}/src/parser/ComponentBackReference.cpp" - "${LIBRARY_DIR}/src/parser/ComponentBoundary.cpp" - "${LIBRARY_DIR}/src/parser/ComponentByte.cpp" - "${LIBRARY_DIR}/src/parser/ComponentClass.cpp" - "${LIBRARY_DIR}/src/parser/ComponentCondReference.cpp" - "${LIBRARY_DIR}/src/parser/ComponentEmpty.cpp" - "${LIBRARY_DIR}/src/parser/ComponentEUS.cpp" - "${LIBRARY_DIR}/src/parser/ComponentRepeat.cpp" - "${LIBRARY_DIR}/src/parser/ComponentSequence.cpp" - "${LIBRARY_DIR}/src/parser/ComponentVisitor.cpp" - "${LIBRARY_DIR}/src/parser/ComponentWordBoundary.cpp" - "${LIBRARY_DIR}/src/parser/ConstComponentVisitor.cpp" - "${LIBRARY_DIR}/src/parser/control_verbs.cpp" - "${LIBRARY_DIR}/src/parser/logical_combination.cpp" - "${LIBRARY_DIR}/src/parser/parse_error.cpp" - "${LIBRARY_DIR}/src/parser/parser_util.cpp" - "${LIBRARY_DIR}/src/parser/Parser.cpp" - "${LIBRARY_DIR}/src/parser/prefilter.cpp" - "${LIBRARY_DIR}/src/parser/shortcut_literal.cpp" - "${LIBRARY_DIR}/src/parser/ucp_table.cpp" - "${LIBRARY_DIR}/src/parser/unsupported.cpp" - "${LIBRARY_DIR}/src/parser/utf8_validate.cpp" - "${LIBRARY_DIR}/src/parser/Utf8ComponentClass.cpp" - "${LIBRARY_DIR}/src/rose/block.c" - "${LIBRARY_DIR}/src/rose/catchup.c" - "${LIBRARY_DIR}/src/rose/init.c" - "${LIBRARY_DIR}/src/rose/match.c" - "${LIBRARY_DIR}/src/rose/program_runtime.c" - "${LIBRARY_DIR}/src/rose/rose_build_add_mask.cpp" - "${LIBRARY_DIR}/src/rose/rose_build_add.cpp" - "${LIBRARY_DIR}/src/rose/rose_build_anchored.cpp" - "${LIBRARY_DIR}/src/rose/rose_build_bytecode.cpp" - "${LIBRARY_DIR}/src/rose/rose_build_castle.cpp" - "${LIBRARY_DIR}/src/rose/rose_build_compile.cpp" - "${LIBRARY_DIR}/src/rose/rose_build_convert.cpp" - "${LIBRARY_DIR}/src/rose/rose_build_dedupe.cpp" - "${LIBRARY_DIR}/src/rose/rose_build_engine_blob.cpp" - "${LIBRARY_DIR}/src/rose/rose_build_exclusive.cpp" - "${LIBRARY_DIR}/src/rose/rose_build_groups.cpp" - "${LIBRARY_DIR}/src/rose/rose_build_infix.cpp" - "${LIBRARY_DIR}/src/rose/rose_build_instructions.cpp" - "${LIBRARY_DIR}/src/rose/rose_build_lit_accel.cpp" - "${LIBRARY_DIR}/src/rose/rose_build_long_lit.cpp" - "${LIBRARY_DIR}/src/rose/rose_build_lookaround.cpp" - "${LIBRARY_DIR}/src/rose/rose_build_matchers.cpp" - "${LIBRARY_DIR}/src/rose/rose_build_merge.cpp" - "${LIBRARY_DIR}/src/rose/rose_build_misc.cpp" - "${LIBRARY_DIR}/src/rose/rose_build_program.cpp" - "${LIBRARY_DIR}/src/rose/rose_build_role_aliasing.cpp" - "${LIBRARY_DIR}/src/rose/rose_build_scatter.cpp" - "${LIBRARY_DIR}/src/rose/rose_build_width.cpp" - "${LIBRARY_DIR}/src/rose/rose_in_util.cpp" - "${LIBRARY_DIR}/src/rose/stream.c" - "${LIBRARY_DIR}/src/runtime.c" - "${LIBRARY_DIR}/src/scratch.c" - "${LIBRARY_DIR}/src/smallwrite/smallwrite_build.cpp" - "${LIBRARY_DIR}/src/som/slot_manager.cpp" - "${LIBRARY_DIR}/src/som/som_runtime.c" - "${LIBRARY_DIR}/src/som/som_stream.c" - "${LIBRARY_DIR}/src/stream_compress.c" - "${LIBRARY_DIR}/src/util/alloc.cpp" - "${LIBRARY_DIR}/src/util/charreach.cpp" - "${LIBRARY_DIR}/src/util/clique.cpp" - "${LIBRARY_DIR}/src/util/compile_context.cpp" - "${LIBRARY_DIR}/src/util/compile_error.cpp" - "${LIBRARY_DIR}/src/util/cpuid_flags.c" - "${LIBRARY_DIR}/src/util/depth.cpp" - "${LIBRARY_DIR}/src/util/fatbit_build.cpp" - "${LIBRARY_DIR}/src/util/multibit_build.cpp" - "${LIBRARY_DIR}/src/util/multibit.c" - "${LIBRARY_DIR}/src/util/report_manager.cpp" - "${LIBRARY_DIR}/src/util/simd_utils.c" - "${LIBRARY_DIR}/src/util/state_compress.c" - "${LIBRARY_DIR}/src/util/target_info.cpp" - "${LIBRARY_DIR}/src/util/ue2string.cpp" - ) - - add_library (hyperscan ${SRCS}) - - target_compile_definitions (hyperscan PUBLIC USE_HYPERSCAN=1) - target_compile_options (hyperscan - PRIVATE -g0 # Library has too much debug information - -mno-avx -mno-avx2 # The library is using dynamic dispatch and is confused if AVX is enabled globally - -march=corei7 -O2 -fno-strict-aliasing -fno-omit-frame-pointer -fvisibility=hidden # The options from original build system - -fno-sanitize=undefined # Assume the library takes care of itself - ) - target_include_directories (hyperscan - PRIVATE - common - "${LIBRARY_DIR}/include" - ) - target_include_directories (hyperscan SYSTEM PUBLIC "${LIBRARY_DIR}/src") - if (ARCH_AMD64) - target_include_directories (hyperscan PRIVATE x86_64) - endif () - target_link_libraries (hyperscan PRIVATE boost::headers_only) - - set (USE_INTERNAL_HYPERSCAN_LIBRARY 1) -endif () - -message (STATUS "Using hyperscan") +target_link_libraries (hyperscan PRIVATE boost::headers_only) From fc90640e90435ad36ceb79b8b6a57245278f8761 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:48:24 +0300 Subject: [PATCH 200/403] Remove unbundled boost support --- contrib/boost-cmake/CMakeLists.txt | 388 ++++++++++++----------------- 1 file changed, 163 insertions(+), 225 deletions(-) diff --git a/contrib/boost-cmake/CMakeLists.txt b/contrib/boost-cmake/CMakeLists.txt index b0cf461a52a..0215c68e683 100644 --- a/contrib/boost-cmake/CMakeLists.txt +++ b/contrib/boost-cmake/CMakeLists.txt @@ -1,243 +1,181 @@ -option (USE_INTERNAL_BOOST_LIBRARY "Use internal Boost library" ON) +set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/boost") -if (NOT USE_INTERNAL_BOOST_LIBRARY) - set(BOOST_VERSION 1.78) +# filesystem - find_package(Boost ${BOOST_VERSION} COMPONENTS - system - filesystem - iostreams - program_options - regex - context - coroutine - graph - ) +set (SRCS_FILESYSTEM + "${LIBRARY_DIR}/libs/filesystem/src/codecvt_error_category.cpp" + "${LIBRARY_DIR}/libs/filesystem/src/directory.cpp" + "${LIBRARY_DIR}/libs/filesystem/src/exception.cpp" + "${LIBRARY_DIR}/libs/filesystem/src/operations.cpp" + "${LIBRARY_DIR}/libs/filesystem/src/path.cpp" + "${LIBRARY_DIR}/libs/filesystem/src/path_traits.cpp" + "${LIBRARY_DIR}/libs/filesystem/src/portability.cpp" + "${LIBRARY_DIR}/libs/filesystem/src/unique_path.cpp" + "${LIBRARY_DIR}/libs/filesystem/src/utf8_codecvt_facet.cpp" + "${LIBRARY_DIR}/libs/filesystem/src/windows_file_codecvt.cpp" +) - if(Boost_INCLUDE_DIR AND Boost_FILESYSTEM_LIBRARY AND - Boost_PROGRAM_OPTIONS_LIBRARY AND Boost_REGEX_LIBRARY AND Boost_SYSTEM_LIBRARY AND Boost_CONTEXT_LIBRARY AND - Boost_COROUTINE_LIBRARY AND Boost_GRAPH_LIBRARY) +add_library (_boost_filesystem ${SRCS_FILESYSTEM}) +add_library (boost::filesystem ALIAS _boost_filesystem) +target_include_directories (_boost_filesystem SYSTEM BEFORE PUBLIC ${LIBRARY_DIR}) - set(EXTERNAL_BOOST_FOUND 1) +# headers-only - add_library (_boost_headers_only INTERFACE) - add_library (boost::headers_only ALIAS _boost_headers_only) - target_include_directories (_boost_headers_only SYSTEM BEFORE INTERFACE ${Boost_INCLUDE_DIR}) +add_library (_boost_headers_only INTERFACE) +add_library (boost::headers_only ALIAS _boost_headers_only) +target_include_directories (_boost_headers_only SYSTEM BEFORE INTERFACE ${LIBRARY_DIR}) - add_library (_boost_filesystem INTERFACE) - add_library (_boost_iostreams INTERFACE) - add_library (_boost_program_options INTERFACE) - add_library (_boost_regex INTERFACE) - add_library (_boost_system INTERFACE) - add_library (_boost_context INTERFACE) - add_library (_boost_coroutine INTERFACE) - add_library (_boost_graph INTERFACE) +# asio - target_link_libraries (_boost_filesystem INTERFACE ${Boost_FILESYSTEM_LIBRARY}) - target_link_libraries (_boost_iostreams INTERFACE ${Boost_IOSTREAMS_LIBRARY}) - target_link_libraries (_boost_program_options INTERFACE ${Boost_PROGRAM_OPTIONS_LIBRARY}) - target_link_libraries (_boost_regex INTERFACE ${Boost_REGEX_LIBRARY}) - target_link_libraries (_boost_system INTERFACE ${Boost_SYSTEM_LIBRARY}) - target_link_libraries (_boost_context INTERFACE ${Boost_CONTEXT_LIBRARY}) - target_link_libraries (_boost_coroutine INTERFACE ${Boost_COROUTINE_LIBRARY}) - target_link_libraries (_boost_graph INTERFACE ${Boost_GRAPH_LIBRARY}) +target_compile_definitions (_boost_headers_only INTERFACE BOOST_ASIO_STANDALONE=1) - add_library (boost::filesystem ALIAS _boost_filesystem) - add_library (boost::iostreams ALIAS _boost_iostreams) - add_library (boost::program_options ALIAS _boost_program_options) - add_library (boost::regex ALIAS _boost_regex) - add_library (boost::system ALIAS _boost_system) - add_library (boost::context ALIAS _boost_context) - add_library (boost::coroutine ALIAS _boost_coroutine) - add_library (boost::graph ALIAS _boost_graph) - else() - set(EXTERNAL_BOOST_FOUND 0) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system boost") +# iostreams + +set (SRCS_IOSTREAMS + "${LIBRARY_DIR}/libs/iostreams/src/file_descriptor.cpp" + "${LIBRARY_DIR}/libs/iostreams/src/gzip.cpp" + "${LIBRARY_DIR}/libs/iostreams/src/mapped_file.cpp" + "${LIBRARY_DIR}/libs/iostreams/src/zlib.cpp" +) + +add_library (_boost_iostreams ${SRCS_IOSTREAMS}) +add_library (boost::iostreams ALIAS _boost_iostreams) +target_include_directories (_boost_iostreams PRIVATE ${LIBRARY_DIR}) +target_link_libraries (_boost_iostreams PRIVATE ch_contrib::zlib) + +# program_options + +set (SRCS_PROGRAM_OPTIONS + "${LIBRARY_DIR}/libs/program_options/src/cmdline.cpp" + "${LIBRARY_DIR}/libs/program_options/src/config_file.cpp" + "${LIBRARY_DIR}/libs/program_options/src/convert.cpp" + "${LIBRARY_DIR}/libs/program_options/src/options_description.cpp" + "${LIBRARY_DIR}/libs/program_options/src/parsers.cpp" + "${LIBRARY_DIR}/libs/program_options/src/positional_options.cpp" + "${LIBRARY_DIR}/libs/program_options/src/split.cpp" + "${LIBRARY_DIR}/libs/program_options/src/utf8_codecvt_facet.cpp" + "${LIBRARY_DIR}/libs/program_options/src/value_semantic.cpp" + "${LIBRARY_DIR}/libs/program_options/src/variables_map.cpp" + "${LIBRARY_DIR}/libs/program_options/src/winmain.cpp" +) + +add_library (_boost_program_options ${SRCS_PROGRAM_OPTIONS}) +add_library (boost::program_options ALIAS _boost_program_options) +target_include_directories (_boost_program_options SYSTEM BEFORE PUBLIC ${LIBRARY_DIR}) + +# regex + +set (SRCS_REGEX + "${LIBRARY_DIR}/libs/regex/src/posix_api.cpp" + "${LIBRARY_DIR}/libs/regex/src/regex_debug.cpp" + "${LIBRARY_DIR}/libs/regex/src/regex.cpp" + "${LIBRARY_DIR}/libs/regex/src/static_mutex.cpp" + "${LIBRARY_DIR}/libs/regex/src/wide_posix_api.cpp" +) + +add_library (_boost_regex ${SRCS_REGEX}) +add_library (boost::regex ALIAS _boost_regex) +target_include_directories (_boost_regex PRIVATE ${LIBRARY_DIR}) + +# system + +set (SRCS_SYSTEM + "${LIBRARY_DIR}/libs/system/src/error_code.cpp" +) + +add_library (_boost_system ${SRCS_SYSTEM}) +add_library (boost::system ALIAS _boost_system) +target_include_directories (_boost_system PRIVATE ${LIBRARY_DIR}) + +# context +enable_language(ASM) +SET(ASM_OPTIONS "-x assembler-with-cpp") + +set (SRCS_CONTEXT + "${LIBRARY_DIR}/libs/context/src/dummy.cpp" + "${LIBRARY_DIR}/libs/context/src/posix/stack_traits.cpp" +) + +if (SANITIZE AND (SANITIZE STREQUAL "address" OR SANITIZE STREQUAL "thread")) + add_compile_definitions(BOOST_USE_UCONTEXT) + + if (SANITIZE STREQUAL "address") + add_compile_definitions(BOOST_USE_ASAN) + elseif (SANITIZE STREQUAL "thread") + add_compile_definitions(BOOST_USE_TSAN) endif() + + set (SRCS_CONTEXT ${SRCS_CONTEXT} + "${LIBRARY_DIR}/libs/context/src/fiber.cpp" + "${LIBRARY_DIR}/libs/context/src/continuation.cpp" + ) +endif() +if (ARCH_ARM) + set (SRCS_CONTEXT ${SRCS_CONTEXT} + "${LIBRARY_DIR}/libs/context/src/asm/jump_arm64_aapcs_elf_gas.S" + "${LIBRARY_DIR}/libs/context/src/asm/make_arm64_aapcs_elf_gas.S" + "${LIBRARY_DIR}/libs/context/src/asm/ontop_arm64_aapcs_elf_gas.S" + ) +elseif (ARCH_PPC64LE) + set (SRCS_CONTEXT ${SRCS_CONTEXT} + "${LIBRARY_DIR}/libs/context/src/asm/jump_ppc64_sysv_elf_gas.S" + "${LIBRARY_DIR}/libs/context/src/asm/make_ppc64_sysv_elf_gas.S" + "${LIBRARY_DIR}/libs/context/src/asm/ontop_ppc64_sysv_elf_gas.S" + ) +elseif (ARCH_RISCV64) + set (SRCS_CONTEXT ${SRCS_CONTEXT} + "${LIBRARY_DIR}/libs/context/src/asm/jump_riscv64_sysv_elf_gas.S" + "${LIBRARY_DIR}/libs/context/src/asm/make_riscv64_sysv_elf_gas.S" + "${LIBRARY_DIR}/libs/context/src/asm/ontop_riscv64_sysv_elf_gas.S" + ) +elseif(OS_DARWIN) + set (SRCS_CONTEXT ${SRCS_CONTEXT} + "${LIBRARY_DIR}/libs/context/src/asm/jump_x86_64_sysv_macho_gas.S" + "${LIBRARY_DIR}/libs/context/src/asm/make_x86_64_sysv_macho_gas.S" + "${LIBRARY_DIR}/libs/context/src/asm/ontop_x86_64_sysv_macho_gas.S" + ) +else() + set (SRCS_CONTEXT ${SRCS_CONTEXT} + "${LIBRARY_DIR}/libs/context/src/asm/jump_x86_64_sysv_elf_gas.S" + "${LIBRARY_DIR}/libs/context/src/asm/make_x86_64_sysv_elf_gas.S" + "${LIBRARY_DIR}/libs/context/src/asm/ontop_x86_64_sysv_elf_gas.S" + ) endif() -if (NOT EXTERNAL_BOOST_FOUND) - set (USE_INTERNAL_BOOST_LIBRARY 1) - set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/boost") +add_library (_boost_context ${SRCS_CONTEXT}) +add_library (boost::context ALIAS _boost_context) +target_include_directories (_boost_context PRIVATE ${LIBRARY_DIR}) - # filesystem +# coroutine - set (SRCS_FILESYSTEM - "${LIBRARY_DIR}/libs/filesystem/src/codecvt_error_category.cpp" - "${LIBRARY_DIR}/libs/filesystem/src/directory.cpp" - "${LIBRARY_DIR}/libs/filesystem/src/exception.cpp" - "${LIBRARY_DIR}/libs/filesystem/src/operations.cpp" - "${LIBRARY_DIR}/libs/filesystem/src/path.cpp" - "${LIBRARY_DIR}/libs/filesystem/src/path_traits.cpp" - "${LIBRARY_DIR}/libs/filesystem/src/portability.cpp" - "${LIBRARY_DIR}/libs/filesystem/src/unique_path.cpp" - "${LIBRARY_DIR}/libs/filesystem/src/utf8_codecvt_facet.cpp" - "${LIBRARY_DIR}/libs/filesystem/src/windows_file_codecvt.cpp" - ) +set (SRCS_COROUTINE + "${LIBRARY_DIR}/libs/coroutine/detail/coroutine_context.cpp" + "${LIBRARY_DIR}/libs/coroutine/exceptions.cpp" + "${LIBRARY_DIR}/libs/coroutine/posix/stack_traits.cpp" +) +add_library (_boost_coroutine ${SRCS_COROUTINE}) +add_library (boost::coroutine ALIAS _boost_coroutine) +target_include_directories (_boost_coroutine PRIVATE ${LIBRARY_DIR}) +target_link_libraries(_boost_coroutine PRIVATE _boost_context) - add_library (_boost_filesystem ${SRCS_FILESYSTEM}) - add_library (boost::filesystem ALIAS _boost_filesystem) - target_include_directories (_boost_filesystem SYSTEM BEFORE PUBLIC ${LIBRARY_DIR}) +# graph - # headers-only +set (SRCS_GRAPH + "${LIBRARY_DIR}/libs/graph/src/graphml.cpp" + "${LIBRARY_DIR}/libs/graph/src/read_graphviz_new.cpp" +) - add_library (_boost_headers_only INTERFACE) - add_library (boost::headers_only ALIAS _boost_headers_only) - target_include_directories (_boost_headers_only SYSTEM BEFORE INTERFACE ${LIBRARY_DIR}) +add_library (_boost_graph ${SRCS_GRAPH}) +add_library (boost::graph ALIAS _boost_graph) +target_include_directories (_boost_graph PRIVATE ${LIBRARY_DIR}) +target_link_libraries(_boost_graph PRIVATE _boost_regex) - # asio +# circular buffer +add_library(_boost_circular_buffer INTERFACE) +add_library(boost::circular_buffer ALIAS _boost_circular_buffer) +target_include_directories(_boost_circular_buffer SYSTEM BEFORE INTERFACE ${LIBRARY_DIR}) - target_compile_definitions (_boost_headers_only INTERFACE BOOST_ASIO_STANDALONE=1) - - # iostreams - - set (SRCS_IOSTREAMS - "${LIBRARY_DIR}/libs/iostreams/src/file_descriptor.cpp" - "${LIBRARY_DIR}/libs/iostreams/src/gzip.cpp" - "${LIBRARY_DIR}/libs/iostreams/src/mapped_file.cpp" - "${LIBRARY_DIR}/libs/iostreams/src/zlib.cpp" - ) - - add_library (_boost_iostreams ${SRCS_IOSTREAMS}) - add_library (boost::iostreams ALIAS _boost_iostreams) - target_include_directories (_boost_iostreams PRIVATE ${LIBRARY_DIR}) - target_link_libraries (_boost_iostreams PRIVATE ch_contrib::zlib) - - # program_options - - set (SRCS_PROGRAM_OPTIONS - "${LIBRARY_DIR}/libs/program_options/src/cmdline.cpp" - "${LIBRARY_DIR}/libs/program_options/src/config_file.cpp" - "${LIBRARY_DIR}/libs/program_options/src/convert.cpp" - "${LIBRARY_DIR}/libs/program_options/src/options_description.cpp" - "${LIBRARY_DIR}/libs/program_options/src/parsers.cpp" - "${LIBRARY_DIR}/libs/program_options/src/positional_options.cpp" - "${LIBRARY_DIR}/libs/program_options/src/split.cpp" - "${LIBRARY_DIR}/libs/program_options/src/utf8_codecvt_facet.cpp" - "${LIBRARY_DIR}/libs/program_options/src/value_semantic.cpp" - "${LIBRARY_DIR}/libs/program_options/src/variables_map.cpp" - "${LIBRARY_DIR}/libs/program_options/src/winmain.cpp" - ) - - add_library (_boost_program_options ${SRCS_PROGRAM_OPTIONS}) - add_library (boost::program_options ALIAS _boost_program_options) - target_include_directories (_boost_program_options SYSTEM BEFORE PUBLIC ${LIBRARY_DIR}) - - # regex - - set (SRCS_REGEX - "${LIBRARY_DIR}/libs/regex/src/posix_api.cpp" - "${LIBRARY_DIR}/libs/regex/src/regex_debug.cpp" - "${LIBRARY_DIR}/libs/regex/src/regex.cpp" - "${LIBRARY_DIR}/libs/regex/src/static_mutex.cpp" - "${LIBRARY_DIR}/libs/regex/src/wide_posix_api.cpp" - ) - - add_library (_boost_regex ${SRCS_REGEX}) - add_library (boost::regex ALIAS _boost_regex) - target_include_directories (_boost_regex PRIVATE ${LIBRARY_DIR}) - - # system - - set (SRCS_SYSTEM - "${LIBRARY_DIR}/libs/system/src/error_code.cpp" - ) - - add_library (_boost_system ${SRCS_SYSTEM}) - add_library (boost::system ALIAS _boost_system) - target_include_directories (_boost_system PRIVATE ${LIBRARY_DIR}) - - # context - enable_language(ASM) - SET(ASM_OPTIONS "-x assembler-with-cpp") - - set (SRCS_CONTEXT - "${LIBRARY_DIR}/libs/context/src/dummy.cpp" - "${LIBRARY_DIR}/libs/context/src/posix/stack_traits.cpp" - ) - - if (SANITIZE AND (SANITIZE STREQUAL "address" OR SANITIZE STREQUAL "thread")) - add_compile_definitions(BOOST_USE_UCONTEXT) - - if (SANITIZE STREQUAL "address") - add_compile_definitions(BOOST_USE_ASAN) - elseif (SANITIZE STREQUAL "thread") - add_compile_definitions(BOOST_USE_TSAN) - endif() - - set (SRCS_CONTEXT ${SRCS_CONTEXT} - "${LIBRARY_DIR}/libs/context/src/fiber.cpp" - "${LIBRARY_DIR}/libs/context/src/continuation.cpp" - ) - endif() - if (ARCH_ARM) - set (SRCS_CONTEXT ${SRCS_CONTEXT} - "${LIBRARY_DIR}/libs/context/src/asm/jump_arm64_aapcs_elf_gas.S" - "${LIBRARY_DIR}/libs/context/src/asm/make_arm64_aapcs_elf_gas.S" - "${LIBRARY_DIR}/libs/context/src/asm/ontop_arm64_aapcs_elf_gas.S" - ) - elseif (ARCH_PPC64LE) - set (SRCS_CONTEXT ${SRCS_CONTEXT} - "${LIBRARY_DIR}/libs/context/src/asm/jump_ppc64_sysv_elf_gas.S" - "${LIBRARY_DIR}/libs/context/src/asm/make_ppc64_sysv_elf_gas.S" - "${LIBRARY_DIR}/libs/context/src/asm/ontop_ppc64_sysv_elf_gas.S" - ) - elseif (ARCH_RISCV64) - set (SRCS_CONTEXT ${SRCS_CONTEXT} - "${LIBRARY_DIR}/libs/context/src/asm/jump_riscv64_sysv_elf_gas.S" - "${LIBRARY_DIR}/libs/context/src/asm/make_riscv64_sysv_elf_gas.S" - "${LIBRARY_DIR}/libs/context/src/asm/ontop_riscv64_sysv_elf_gas.S" - ) - elseif(OS_DARWIN) - set (SRCS_CONTEXT ${SRCS_CONTEXT} - "${LIBRARY_DIR}/libs/context/src/asm/jump_x86_64_sysv_macho_gas.S" - "${LIBRARY_DIR}/libs/context/src/asm/make_x86_64_sysv_macho_gas.S" - "${LIBRARY_DIR}/libs/context/src/asm/ontop_x86_64_sysv_macho_gas.S" - ) - else() - set (SRCS_CONTEXT ${SRCS_CONTEXT} - "${LIBRARY_DIR}/libs/context/src/asm/jump_x86_64_sysv_elf_gas.S" - "${LIBRARY_DIR}/libs/context/src/asm/make_x86_64_sysv_elf_gas.S" - "${LIBRARY_DIR}/libs/context/src/asm/ontop_x86_64_sysv_elf_gas.S" - ) - endif() - - add_library (_boost_context ${SRCS_CONTEXT}) - add_library (boost::context ALIAS _boost_context) - target_include_directories (_boost_context PRIVATE ${LIBRARY_DIR}) - - # coroutine - - set (SRCS_COROUTINE - "${LIBRARY_DIR}/libs/coroutine/detail/coroutine_context.cpp" - "${LIBRARY_DIR}/libs/coroutine/exceptions.cpp" - "${LIBRARY_DIR}/libs/coroutine/posix/stack_traits.cpp" - ) - add_library (_boost_coroutine ${SRCS_COROUTINE}) - add_library (boost::coroutine ALIAS _boost_coroutine) - target_include_directories (_boost_coroutine PRIVATE ${LIBRARY_DIR}) - target_link_libraries(_boost_coroutine PRIVATE _boost_context) - - # graph - - set (SRCS_GRAPH - "${LIBRARY_DIR}/libs/graph/src/graphml.cpp" - "${LIBRARY_DIR}/libs/graph/src/read_graphviz_new.cpp" - ) - - add_library (_boost_graph ${SRCS_GRAPH}) - add_library (boost::graph ALIAS _boost_graph) - target_include_directories (_boost_graph PRIVATE ${LIBRARY_DIR}) - target_link_libraries(_boost_graph PRIVATE _boost_regex) - - # circular buffer - add_library(_boost_circular_buffer INTERFACE) - add_library(boost::circular_buffer ALIAS _boost_circular_buffer) - target_include_directories(_boost_circular_buffer SYSTEM BEFORE INTERFACE ${LIBRARY_DIR}) - - # heap - add_library(_boost_heap INTERFACE) - add_library(boost::heap ALIAS _boost_heap) - target_include_directories(_boost_heap SYSTEM BEFORE INTERFACE ${LIBRARY_DIR}) - -endif () +# heap +add_library(_boost_heap INTERFACE) +add_library(boost::heap ALIAS _boost_heap) +target_include_directories(_boost_heap SYSTEM BEFORE INTERFACE ${LIBRARY_DIR}) From 47092871e8e16c88d3b1685820a7823808376fae Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:48:24 +0300 Subject: [PATCH 201/403] Cleanup cmake_in_clickhouse_generator --- docs/tools/cmake_in_clickhouse_generator.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/docs/tools/cmake_in_clickhouse_generator.py b/docs/tools/cmake_in_clickhouse_generator.py index e66915d4a39..aa4cbbddd18 100644 --- a/docs/tools/cmake_in_clickhouse_generator.py +++ b/docs/tools/cmake_in_clickhouse_generator.py @@ -39,11 +39,6 @@ def build_entity(path: str, entity: Entity, line_comment: Tuple[int, str]) -> No if name in entities: return - # cannot escape the { in macro option description -> invalid AMP html - # Skipping "USE_INTERNAL_${LIB_NAME_UC}_LIBRARY" - if "LIB_NAME_UC" in name: - return - if len(default) == 0: formatted_default: str = "`OFF`" elif default[0] == "$": @@ -140,13 +135,6 @@ def generate_cmake_flags_files() -> None: f.write(entities[k][1] + "\n") ignored_keys.append(k) - f.write("\n\n### External libraries system/bundled mode\n" + table_header) - - for k in sorted_keys: - if k.startswith("USE_INTERNAL_"): - f.write(entities[k][1] + "\n") - ignored_keys.append(k) - f.write("\n\n### Other flags\n" + table_header) for k in sorted(set(sorted_keys).difference(set(ignored_keys))): From 616fd5dda988231a1bb76c0fd834808105c0923b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:48:24 +0300 Subject: [PATCH 202/403] docker/packager/other/fuzzer: cleanup USE_INTERNAL_XXX --- docker/packager/other/fuzzer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/other/fuzzer.sh b/docker/packager/other/fuzzer.sh index 431352f1126..ac820d9e689 100755 --- a/docker/packager/other/fuzzer.sh +++ b/docker/packager/other/fuzzer.sh @@ -14,7 +14,7 @@ read -ra CMAKE_FLAGS <<< "${CMAKE_FLAGS:-}" # Hope, that the most part of files will be in cache, so we just link new executables # Please, add or change flags directly in cmake cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_C_COMPILER="$CC" -DCMAKE_CXX_COMPILER="$CXX" \ - -DSANITIZE="$SANITIZER" -DENABLE_FUZZING=1 -DFUZZER='libfuzzer' -DENABLE_PROTOBUF=1 -DUSE_INTERNAL_PROTOBUF_LIBRARY=1 "${CMAKE_FLAGS[@]}" .. + -DSANITIZE="$SANITIZER" -DENABLE_FUZZING=1 -DFUZZER='libfuzzer' -DENABLE_PROTOBUF=1 "${CMAKE_FLAGS[@]}" .. FUZZER_TARGETS=$(find ../src -name '*_fuzzer.cpp' -execdir basename {} .cpp ';' | tr '\n' ' ') From c3ac1f976305e5ab0494cdbe86b8009143e45ca8 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:48:25 +0300 Subject: [PATCH 203/403] docs/en/development/build-cross-riscv.md: cleanup USE_INTERNAL_XXX --- docs/en/development/build-cross-riscv.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/build-cross-riscv.md b/docs/en/development/build-cross-riscv.md index 977387af207..5cdce710b41 100644 --- a/docs/en/development/build-cross-riscv.md +++ b/docs/en/development/build-cross-riscv.md @@ -23,7 +23,7 @@ sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ``` bash cd ClickHouse mkdir build-riscv64 -CC=clang-13 CXX=clang++-13 cmake . -Bbuild-riscv64 -G Ninja -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-riscv64.cmake -DGLIBC_COMPATIBILITY=OFF -DENABLE_LDAP=OFF -DOPENSSL_NO_ASM=ON -DENABLE_JEMALLOC=ON -DENABLE_PARQUET=OFF -DUSE_INTERNAL_PARQUET_LIBRARY=OFF -DENABLE_ORC=OFF -DUSE_INTERNAL_ORC_LIBRARY=OFF -DUSE_UNWIND=OFF -DUSE_INTERNAL_PROTOBUF_LIBRARY=ON -DENABLE_GRPC=OFF -DUSE_INTERNAL_GRPC_LIBRARY=OFF -DENABLE_HDFS=OFF -DUSE_INTERNAL_HDFS3_LIBRARY=OFF -DENABLE_MYSQL=OFF -DUSE_INTERNAL_MYSQL_LIBRARY=OFF +CC=clang-13 CXX=clang++-13 cmake . -Bbuild-riscv64 -G Ninja -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-riscv64.cmake -DGLIBC_COMPATIBILITY=OFF -DENABLE_LDAP=OFF -DOPENSSL_NO_ASM=ON -DENABLE_JEMALLOC=ON -DENABLE_PARQUET=OFF -DENABLE_ORC=OFF -DUSE_UNWIND=OFF -DENABLE_GRPC=OFF -DENABLE_HDFS=OFF -DENABLE_MYSQL=OFF ninja -C build-riscv64 ``` From eeb0c83a2d503483d9c09087120f4635ea2fa232 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:48:25 +0300 Subject: [PATCH 204/403] tests/instructions/sanitizers.md: cleanup USE_INTERNAL_XXX --- tests/instructions/sanitizers.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/instructions/sanitizers.md b/tests/instructions/sanitizers.md index 07a33d45b4d..6de789bed34 100644 --- a/tests/instructions/sanitizers.md +++ b/tests/instructions/sanitizers.md @@ -67,5 +67,5 @@ sudo -u clickhouse UBSAN_OPTIONS='print_stacktrace=1' ./clickhouse-ubsan server # How to use Memory Sanitizer ``` -CC=clang-8 CXX=clang++-8 cmake -D ENABLE_HDFS=0 -D ENABLE_CAPNP=0 -D ENABLE_RDKAFKA=0 -D ENABLE_ICU=0 -D ENABLE_POCO_MONGODB=0 -D ENABLE_POCO_NETSSL=0 -D ENABLE_ODBC=0 -D ENABLE_MYSQL=0 -D ENABLE_EMBEDDED_COMPILER=0 -D USE_INTERNAL_CAPNP_LIBRARY=0 -D USE_SIMDJSON=0 -D SANITIZE=memory .. +CC=clang-8 CXX=clang++-8 cmake -D ENABLE_HDFS=0 -D ENABLE_CAPNP=0 -D ENABLE_RDKAFKA=0 -D ENABLE_ICU=0 -D ENABLE_POCO_MONGODB=0 -D ENABLE_POCO_NETSSL=0 -D ENABLE_ODBC=0 -D ENABLE_MYSQL=0 -D ENABLE_EMBEDDED_COMPILER=0 -D USE_SIMDJSON=0 -D SANITIZE=memory .. ``` From c6f95bcde3c1514f89389c0707c0c54c8eefdc07 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:48:25 +0300 Subject: [PATCH 205/403] Add add_contrib() macro --- contrib/CMakeLists.txt | 168 ++++++++++++++++++++++------------------- 1 file changed, 90 insertions(+), 78 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 5207638e19d..332385f3577 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -21,96 +21,108 @@ endif() set_property(DIRECTORY PROPERTY EXCLUDE_FROM_ALL 1) -add_subdirectory (miniselect-cmake) -add_subdirectory (pdqsort-cmake) -add_subdirectory (sparsehash-c11-cmake) -add_subdirectory (abseil-cpp-cmake) -add_subdirectory (magic-enum-cmake) -add_subdirectory (boost-cmake) -add_subdirectory (cctz-cmake) -add_subdirectory (consistent-hashing) -add_subdirectory (dragonbox-cmake) -add_subdirectory (hyperscan-cmake) -add_subdirectory (jemalloc-cmake) -add_subdirectory (libcpuid-cmake) -add_subdirectory (libdivide) -add_subdirectory (libmetrohash) -add_subdirectory (lz4-cmake) -add_subdirectory (murmurhash) -add_subdirectory (replxx-cmake) -add_subdirectory (unixodbc-cmake) -add_subdirectory (nanodbc-cmake) -add_subdirectory (capnproto-cmake) -add_subdirectory (yaml-cpp-cmake) -add_subdirectory (re2-cmake) -add_subdirectory (xz-cmake) -add_subdirectory (brotli-cmake) -add_subdirectory (double-conversion-cmake) -add_subdirectory (boringssl-cmake) -add_subdirectory (poco-cmake) -add_subdirectory (croaring-cmake) -add_subdirectory (zstd-cmake) -add_subdirectory (zlib-ng-cmake) -add_subdirectory (bzip2-cmake) -add_subdirectory (snappy-cmake) -add_subdirectory (rocksdb-cmake) -add_subdirectory (thrift-cmake) +macro(add_contrib folder) + if (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${folder}/CMakeLists.txt") + message(STATUS "submodule ${folder} is missing (no CMakeLists.txt). to fix try run:") + message(STATUS " git submodule update --init") + else() + message(STATUS "Adding contrib module ${folder}") + add_subdirectory (${folder}) + endif() +endmacro() + +add_contrib (miniselect-cmake) +add_contrib (pdqsort-cmake) +add_contrib (sparsehash-c11-cmake) +add_contrib (abseil-cpp-cmake) +add_contrib (magic-enum-cmake) +add_contrib (boost-cmake) +add_contrib (cctz-cmake) +add_contrib (consistent-hashing) +add_contrib (dragonbox-cmake) +add_contrib (hyperscan-cmake) +add_contrib (jemalloc-cmake) +add_contrib (libcpuid-cmake) +add_contrib (libdivide) +add_contrib (libmetrohash) +add_contrib (lz4-cmake) +add_contrib (murmurhash) +add_contrib (replxx-cmake) +add_contrib (unixodbc-cmake) +add_contrib (nanodbc-cmake) +add_contrib (capnproto-cmake) +add_contrib (yaml-cpp-cmake) +add_contrib (re2-cmake) +add_contrib (xz-cmake) +add_contrib (brotli-cmake) +add_contrib (double-conversion-cmake) +add_contrib (boringssl-cmake) +add_contrib (poco-cmake) +add_contrib (croaring-cmake) +add_contrib (zstd-cmake) +add_contrib (zlib-ng-cmake) +add_contrib (bzip2-cmake) +add_contrib (snappy-cmake) +add_contrib (rocksdb-cmake) +add_contrib (thrift-cmake) # parquet/arrow/orc -add_subdirectory (arrow-cmake) # requires: snappy, thrift, double-conversion -add_subdirectory (avro-cmake) # requires: snappy -add_subdirectory (hive-metastore-cmake) # requires: thrift/avro/arrow -add_subdirectory (librdkafka-cmake) -add_subdirectory (cppkafka-cmake) -add_subdirectory (openldap-cmake) -add_subdirectory (grpc-cmake) -add_subdirectory (protobuf-cmake) -add_subdirectory (libhdfs3-cmake) # requires: protobuf -add_subdirectory (msgpack-c-cmake) +add_contrib (arrow-cmake) # requires: snappy, thrift, double-conversion +add_contrib (avro-cmake) # requires: snappy +add_contrib (hive-metastore-cmake) # requires: thrift/avro/arrow +add_contrib (librdkafka-cmake) +add_contrib (cppkafka-cmake) +add_contrib (openldap-cmake) +add_contrib (grpc-cmake) +add_contrib (protobuf-cmake) +add_contrib (libhdfs3-cmake) # requires: protobuf +add_contrib (msgpack-c-cmake) if (ENABLE_FUZZING) - add_subdirectory (libprotobuf-mutator-cmake) + add_contrib (libprotobuf-mutator-cmake) endif() -add_subdirectory (cityhash102) -add_subdirectory (libfarmhash) -add_subdirectory (icu-cmake) -add_subdirectory (h3-cmake) -add_subdirectory (mariadb-connector-c-cmake) +add_contrib (cityhash102) +add_contrib (libfarmhash) +add_contrib (icu-cmake) +add_contrib (h3-cmake) +add_contrib (mariadb-connector-c-cmake) if (ENABLE_TESTS) - add_subdirectory (googletest-cmake) + add_contrib (googletest-cmake) endif() -add_subdirectory (llvm-cmake) -add_subdirectory (libgsasl-cmake) -add_subdirectory (libxml2-cmake) -add_subdirectory (aws-s3-cmake) -add_subdirectory (base64-cmake) -add_subdirectory (simdjson-cmake) -add_subdirectory (rapidjson-cmake) -add_subdirectory (fastops-cmake) -add_subdirectory (libuv-cmake) -add_subdirectory (amqpcpp-cmake) # requires: libuv -add_subdirectory (cassandra-cmake) # requires: libuv -add_subdirectory (curl-cmake) -add_subdirectory (azure-cmake) -add_subdirectory (sentry-native-cmake) # requires: curl -add_subdirectory (fmtlib-cmake) -add_subdirectory (krb5-cmake) -add_subdirectory (cyrus-sasl-cmake) # for krb5 -add_subdirectory (libpqxx-cmake) -add_subdirectory (libpq-cmake) -add_subdirectory (nuraft-cmake) -add_subdirectory (fast_float-cmake) -add_subdirectory (datasketches-cpp-cmake) +add_contrib (llvm-cmake) +add_contrib (libgsasl-cmake) +add_contrib (libxml2-cmake) +add_contrib (aws-s3-cmake) +add_contrib (base64-cmake) +add_contrib (simdjson-cmake) +add_contrib (rapidjson-cmake) +add_contrib (fastops-cmake) +add_contrib (libuv-cmake) +add_contrib (amqpcpp-cmake) # requires: libuv +add_contrib (cassandra-cmake) # requires: libuv +add_contrib (curl-cmake) +add_contrib (azure-cmake) +add_contrib (sentry-native-cmake) # requires: curl +add_contrib (fmtlib-cmake) +add_contrib (krb5-cmake) +add_contrib (cyrus-sasl-cmake) # for krb5 +add_contrib (libpqxx-cmake) +add_contrib (libpq-cmake) +add_contrib (nuraft-cmake) +add_contrib (fast_float-cmake) +add_contrib (datasketches-cpp-cmake) option(ENABLE_NLP "Enable NLP functions support" ${ENABLE_LIBRARIES}) -add_subdirectory(libstemmer-c-cmake) # ENABLE_NLP -add_subdirectory(wordnet-blast-cmake) # ENABLE_NLP -add_subdirectory(lemmagen-c-cmake) # ENABLE_NLP +if (ENABLE_NLP) + add_contrib (libstemmer-c-cmake) # ENABLE_NLP + add_contrib (wordnet-blast-cmake) # ENABLE_NLP + add_contrib (lemmagen-c-cmake) # ENABLE_NLP +endif() -add_subdirectory (sqlite-cmake) -add_subdirectory (s2geometry-cmake) +add_contrib (sqlite-cmake) +add_contrib (s2geometry-cmake) # Put all targets defined here and in subdirectories under "contrib/" folders in GUI-based IDEs. # Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear From ad67aa527cb2b280fbe8fe96ec44a26e9780233e Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:48:25 +0300 Subject: [PATCH 206/403] Add alias library for magic_enum --- base/base/CMakeLists.txt | 4 ++-- contrib/magic-enum-cmake/CMakeLists.txt | 1 + src/CMakeLists.txt | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/base/base/CMakeLists.txt b/base/base/CMakeLists.txt index ddb9279aa3c..d0698725dd0 100644 --- a/base/base/CMakeLists.txt +++ b/base/base/CMakeLists.txt @@ -24,7 +24,7 @@ if (ENABLE_REPLXX) endif () if (USE_DEBUG_HELPERS) - get_target_property(MAGIC_ENUM_INCLUDE_DIR magic_enum INTERFACE_INCLUDE_DIRECTORIES) + get_target_property(MAGIC_ENUM_INCLUDE_DIR ch_contrib::magic_enum INTERFACE_INCLUDE_DIRECTORIES) # CMake generator expression will do insane quoting when it encounters special character like quotes, spaces, etc. # Prefixing "SHELL:" will force it to use the original text. set (INCLUDE_DEBUG_HELPERS "SHELL:-I\"${MAGIC_ENUM_INCLUDE_DIR}\" -include \"${ClickHouse_SOURCE_DIR}/base/base/iostream_debug_helpers.h\"") @@ -58,7 +58,7 @@ target_link_libraries (common ch_contrib::replxx ch_contrib::cctz fmt - magic_enum + ch_contrib::magic_enum ) if (ENABLE_TESTS) diff --git a/contrib/magic-enum-cmake/CMakeLists.txt b/contrib/magic-enum-cmake/CMakeLists.txt index fae2c9c2d05..86f92d6c2b4 100644 --- a/contrib/magic-enum-cmake/CMakeLists.txt +++ b/contrib/magic-enum-cmake/CMakeLists.txt @@ -1,3 +1,4 @@ set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/magic_enum") add_library (magic_enum INTERFACE) target_include_directories(magic_enum SYSTEM INTERFACE ${LIBRARY_DIR}/include) +add_library(ch_contrib::magic_enum ALIAS magic_enum) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4e98f598c12..ac86a5e167a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -29,7 +29,7 @@ configure_file (Common/config_version.h.in ${CONFIG_VERSION}) configure_file (Core/config_core.h.in "${CMAKE_CURRENT_BINARY_DIR}/Core/include/config_core.h") if (USE_DEBUG_HELPERS) - get_target_property(MAGIC_ENUM_INCLUDE_DIR magic_enum INTERFACE_INCLUDE_DIRECTORIES) + get_target_property(MAGIC_ENUM_INCLUDE_DIR ch_contrib::magic_enum INTERFACE_INCLUDE_DIRECTORIES) # CMake generator expression will do insane quoting when it encounters special character like quotes, spaces, etc. # Prefixing "SHELL:" will force it to use the original text. set (INCLUDE_DEBUG_HELPERS "SHELL:-I\"${ClickHouse_SOURCE_DIR}/base\" -I\"${MAGIC_ENUM_INCLUDE_DIR}\" -include \"${ClickHouse_SOURCE_DIR}/src/Core/iostream_debug_helpers.h\"") From e3b140a3873093008454b3199bec28cbb2330bd6 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:48:25 +0300 Subject: [PATCH 207/403] Add alias library for fmt --- base/base/CMakeLists.txt | 2 +- contrib/fmtlib-cmake/CMakeLists.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/base/base/CMakeLists.txt b/base/base/CMakeLists.txt index d0698725dd0..9201a852373 100644 --- a/base/base/CMakeLists.txt +++ b/base/base/CMakeLists.txt @@ -57,7 +57,7 @@ target_link_libraries (common Poco::Foundation ch_contrib::replxx ch_contrib::cctz - fmt + ch_contrib::fmt ch_contrib::magic_enum ) diff --git a/contrib/fmtlib-cmake/CMakeLists.txt b/contrib/fmtlib-cmake/CMakeLists.txt index f3bf73d7dbc..d279360692d 100644 --- a/contrib/fmtlib-cmake/CMakeLists.txt +++ b/contrib/fmtlib-cmake/CMakeLists.txt @@ -18,3 +18,4 @@ set (SRCS add_library(fmt ${SRCS}) target_include_directories(fmt SYSTEM PUBLIC ../fmtlib/include) +add_library(ch_contrib::fmt ALIAS fmt) From eda71823813f96c86171d746896de5923115fcdf Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:48:25 +0300 Subject: [PATCH 208/403] Remove extra cmake modules from libhdfs3-cmake (everything is bundled) --- contrib/libhdfs3-cmake/CMake/FindGSasl.cmake | 26 -------- .../libhdfs3-cmake/CMake/FindGoogleTest.cmake | 65 ------------------- .../libhdfs3-cmake/CMake/FindKERBEROS.cmake | 23 ------- contrib/libhdfs3-cmake/CMake/FindSSL.cmake | 26 -------- 4 files changed, 140 deletions(-) delete mode 100644 contrib/libhdfs3-cmake/CMake/FindGSasl.cmake delete mode 100644 contrib/libhdfs3-cmake/CMake/FindGoogleTest.cmake delete mode 100644 contrib/libhdfs3-cmake/CMake/FindKERBEROS.cmake delete mode 100644 contrib/libhdfs3-cmake/CMake/FindSSL.cmake diff --git a/contrib/libhdfs3-cmake/CMake/FindGSasl.cmake b/contrib/libhdfs3-cmake/CMake/FindGSasl.cmake deleted file mode 100644 index 19ca7c30d1e..00000000000 --- a/contrib/libhdfs3-cmake/CMake/FindGSasl.cmake +++ /dev/null @@ -1,26 +0,0 @@ -# - Try to find the GNU sasl library (gsasl) -# -# Once done this will define -# -# GSASL_FOUND - System has gnutls -# GSASL_INCLUDE_DIR - The gnutls include directory -# GSASL_LIBRARIES - The libraries needed to use gnutls -# GSASL_DEFINITIONS - Compiler switches required for using gnutls - - -IF (GSASL_INCLUDE_DIR AND GSASL_LIBRARIES) - # in cache already - SET(GSasl_FIND_QUIETLY TRUE) -ENDIF (GSASL_INCLUDE_DIR AND GSASL_LIBRARIES) - -FIND_PATH(GSASL_INCLUDE_DIR gsasl.h) - -FIND_LIBRARY(GSASL_LIBRARIES gsasl) - -INCLUDE(FindPackageHandleStandardArgs) - -# handle the QUIETLY and REQUIRED arguments and set GSASL_FOUND to TRUE if -# all listed variables are TRUE -FIND_PACKAGE_HANDLE_STANDARD_ARGS(GSASL DEFAULT_MSG GSASL_LIBRARIES GSASL_INCLUDE_DIR) - -MARK_AS_ADVANCED(GSASL_INCLUDE_DIR GSASL_LIBRARIES) \ No newline at end of file diff --git a/contrib/libhdfs3-cmake/CMake/FindGoogleTest.cmake b/contrib/libhdfs3-cmake/CMake/FindGoogleTest.cmake deleted file mode 100644 index fd57c1e2abd..00000000000 --- a/contrib/libhdfs3-cmake/CMake/FindGoogleTest.cmake +++ /dev/null @@ -1,65 +0,0 @@ -include(CheckCXXSourceRuns) - -find_path(GTest_INCLUDE_DIR gtest/gtest.h - NO_DEFAULT_PATH - PATHS - "${PROJECT_SOURCE_DIR}/../thirdparty/googletest/googletest/include" - "/usr/local/include" - "/usr/include") - -find_path(GMock_INCLUDE_DIR gmock/gmock.h - NO_DEFAULT_PATH - PATHS - "${PROJECT_SOURCE_DIR}/../thirdparty/googletest/googlemock/include" - "/usr/local/include" - "/usr/include") - -find_library(Gtest_LIBRARY - NAMES libgtest.a - HINTS - "${PROJECT_SOURCE_DIR}/../thirdparty/googletest/build/googlemock/gtest" - "/usr/local/lib" - "/usr/lib") - -find_library(Gmock_LIBRARY - NAMES libgmock.a - HINTS - "${PROJECT_SOURCE_DIR}/../thirdparty/googletest/build/googlemock" - "/usr/local/lib" - "/usr/lib") - -message(STATUS "Find GoogleTest include path: ${GTest_INCLUDE_DIR}") -message(STATUS "Find GoogleMock include path: ${GMock_INCLUDE_DIR}") -message(STATUS "Find Gtest library path: ${Gtest_LIBRARY}") -message(STATUS "Find Gmock library path: ${Gmock_LIBRARY}") - -set(CMAKE_REQUIRED_INCLUDES ${GTest_INCLUDE_DIR} ${GMock_INCLUDE_DIR}) -set(CMAKE_REQUIRED_LIBRARIES ${Gtest_LIBRARY} ${Gmock_LIBRARY} -lpthread) -set(CMAKE_REQUIRED_FLAGS) -check_cxx_source_runs(" -#include -#include -int main(int argc, char *argv[]) -{ - double pi = 3.14; - EXPECT_EQ(pi, 3.14); - return 0; -} -" GoogleTest_CHECK_FINE) -message(STATUS "GoogleTest check: ${GoogleTest_CHECK_FINE}") - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args( - GoogleTest - REQUIRED_VARS - GTest_INCLUDE_DIR - GMock_INCLUDE_DIR - Gtest_LIBRARY - Gmock_LIBRARY - GoogleTest_CHECK_FINE) - -set(GoogleTest_INCLUDE_DIR ${GTest_INCLUDE_DIR} ${GMock_INCLUDE_DIR}) -set(GoogleTest_LIBRARIES ${Gtest_LIBRARY} ${Gmock_LIBRARY}) -mark_as_advanced( - GoogleTest_INCLUDE_DIR - GoogleTest_LIBRARIES) diff --git a/contrib/libhdfs3-cmake/CMake/FindKERBEROS.cmake b/contrib/libhdfs3-cmake/CMake/FindKERBEROS.cmake deleted file mode 100644 index 5fc58235a3f..00000000000 --- a/contrib/libhdfs3-cmake/CMake/FindKERBEROS.cmake +++ /dev/null @@ -1,23 +0,0 @@ -# - Find kerberos -# Find the native KERBEROS includes and library -# -# KERBEROS_INCLUDE_DIRS - where to find krb5.h, etc. -# KERBEROS_LIBRARIES - List of libraries when using krb5. -# KERBEROS_FOUND - True if krb5 found. - -IF (KERBEROS_INCLUDE_DIRS) - # Already in cache, be silent - SET(KERBEROS_FIND_QUIETLY TRUE) -ENDIF (KERBEROS_INCLUDE_DIRS) - -FIND_PATH(KERBEROS_INCLUDE_DIRS krb5.h) - -SET(KERBEROS_NAMES krb5 k5crypto com_err) -FIND_LIBRARY(KERBEROS_LIBRARIES NAMES ${KERBEROS_NAMES}) - -# handle the QUIETLY and REQUIRED arguments and set KERBEROS_FOUND to TRUE if -# all listed variables are TRUE -INCLUDE(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(KERBEROS DEFAULT_MSG KERBEROS_LIBRARIES KERBEROS_INCLUDE_DIRS) - -MARK_AS_ADVANCED(KERBEROS_LIBRARIES KERBEROS_INCLUDE_DIRS) diff --git a/contrib/libhdfs3-cmake/CMake/FindSSL.cmake b/contrib/libhdfs3-cmake/CMake/FindSSL.cmake deleted file mode 100644 index bcbc5d89653..00000000000 --- a/contrib/libhdfs3-cmake/CMake/FindSSL.cmake +++ /dev/null @@ -1,26 +0,0 @@ -# - Try to find the Open ssl library (ssl) -# -# Once done this will define -# -# SSL_FOUND - System has gnutls -# SSL_INCLUDE_DIR - The gnutls include directory -# SSL_LIBRARIES - The libraries needed to use gnutls -# SSL_DEFINITIONS - Compiler switches required for using gnutls - - -IF (SSL_INCLUDE_DIR AND SSL_LIBRARIES) - # in cache already - SET(SSL_FIND_QUIETLY TRUE) -ENDIF (SSL_INCLUDE_DIR AND SSL_LIBRARIES) - -FIND_PATH(SSL_INCLUDE_DIR openssl/opensslv.h) - -FIND_LIBRARY(SSL_LIBRARIES crypto) - -INCLUDE(FindPackageHandleStandardArgs) - -# handle the QUIETLY and REQUIRED arguments and set SSL_FOUND to TRUE if -# all listed variables are TRUE -FIND_PACKAGE_HANDLE_STANDARD_ARGS(SSL DEFAULT_MSG SSL_LIBRARIES SSL_INCLUDE_DIR) - -MARK_AS_ADVANCED(SSL_INCLUDE_DIR SSL_LIBRARIES) \ No newline at end of file From 7dcc03729636111aa456c508d104d08d827afa9f Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:48:25 +0300 Subject: [PATCH 209/403] check-style: add a check that there is no system-wide library in use --- utils/check-style/check-style | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/utils/check-style/check-style b/utils/check-style/check-style index 6170f2ee28a..d71ead57477 100755 --- a/utils/check-style/check-style +++ b/utils/check-style/check-style @@ -326,5 +326,10 @@ then echo "ErrorCodes.cpp contains non-unique error codes" fi +# Check that there is no system-wide libraries/headers in use. +if git grep -e find_path -e find_library -- :**CMakeLists.txt; then + echo "There is find_path/find_library usage. ClickHouse should use everything bundled. Consider adding one more contrib module." +fi + # Forbid files that differ only by character case find $ROOT_PATH | sort -f | uniq -i -c | awk '{ if ($1 > 1) print }' From e160a20769705cc4d690e08b145c9461b1b749b1 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:48:26 +0300 Subject: [PATCH 210/403] Remove USE_OPENCL (removed in #31744) --- src/Common/config.h.in | 1 - src/Core/config_core.h.in | 1 - src/Storages/System/StorageSystemBuildOptions.generated.cpp.in | 1 - 3 files changed, 3 deletions(-) diff --git a/src/Common/config.h.in b/src/Common/config.h.in index ce8f0286aac..47d56ca6abd 100644 --- a/src/Common/config.h.in +++ b/src/Common/config.h.in @@ -9,7 +9,6 @@ #cmakedefine01 USE_AZURE_BLOB_STORAGE #cmakedefine01 USE_BROTLI #cmakedefine01 USE_UNWIND -#cmakedefine01 USE_OPENCL #cmakedefine01 USE_CASSANDRA #cmakedefine01 USE_SENTRY #cmakedefine01 USE_GRPC diff --git a/src/Core/config_core.h.in b/src/Core/config_core.h.in index adf0f2d4b2d..de37190b394 100644 --- a/src/Core/config_core.h.in +++ b/src/Core/config_core.h.in @@ -8,7 +8,6 @@ #cmakedefine01 USE_AMQPCPP #cmakedefine01 USE_EMBEDDED_COMPILER #cmakedefine01 USE_SSL -#cmakedefine01 USE_OPENCL #cmakedefine01 USE_LDAP #cmakedefine01 USE_ROCKSDB #cmakedefine01 USE_LIBPQXX diff --git a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in index c10f5759d6a..618482eb89e 100644 --- a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in +++ b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in @@ -53,7 +53,6 @@ const char * auto_config_build[] "USE_NURAFT", "@USE_NURAFT@", "USE_NLP", "@USE_NLP@", "USE_SQLITE", "@USE_SQLITE@", - "USE_OPENCL", "@USE_OPENCL@", "USE_LIBPQXX", "@USE_LIBPQXX@", "USE_AZURE_BLOB_STORAGE", "@USE_AZURE_BLOB_STORAGE@", "USE_AWS_S3", "@USE_AWS_S3@", From 9426e6305bc4a893f5d90d1e231a1e802f14cc55 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:48:26 +0300 Subject: [PATCH 211/403] Fix USE_XXHASH in StorageSystemBuildOptions.generated.cpp.in --- src/Storages/System/StorageSystemBuildOptions.generated.cpp.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in index 618482eb89e..6e3163c7641 100644 --- a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in +++ b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in @@ -33,7 +33,7 @@ const char * auto_config_build[] "USE_RDKAFKA", "@USE_RDKAFKA@", "USE_CAPNP", "@USE_CAPNP@", "USE_BASE64", "@USE_BASE64@", - "USE_XXHASH", + "USE_XXHASH", "@USE_XXHASH@", "USE_HDFS", "@USE_HDFS@", "USE_SNAPPY", "@USE_SNAPPY@", "USE_PARQUET", "@USE_PARQUET@", From 70b3f1de76bbdbbcbdbc534a639e46e84e1528c4 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 10:52:32 +0300 Subject: [PATCH 212/403] Improve add_contrib() (teach it about other dependencies) --- contrib/CMakeLists.txt | 179 +++++++++++++++++++++++------------------ 1 file changed, 100 insertions(+), 79 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 332385f3577..8fd1a11f558 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -21,108 +21,129 @@ endif() set_property(DIRECTORY PROPERTY EXCLUDE_FROM_ALL 1) -macro(add_contrib folder) - if (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/${folder}/CMakeLists.txt") - message(STATUS "submodule ${folder} is missing (no CMakeLists.txt). to fix try run:") - message(STATUS " git submodule update --init") +# add_contrib cmake_folder[ base_folder1[, ...base_folderN]] +function(add_contrib cmake_folder) + if (ARGN) + set(base_folders ${ARGN}) else() - message(STATUS "Adding contrib module ${folder}") - add_subdirectory (${folder}) + set(base_folders ${cmake_folder}) endif() -endmacro() -add_contrib (miniselect-cmake) -add_contrib (pdqsort-cmake) -add_contrib (sparsehash-c11-cmake) -add_contrib (abseil-cpp-cmake) -add_contrib (magic-enum-cmake) -add_contrib (boost-cmake) -add_contrib (cctz-cmake) + foreach (base_folder ${base_folders}) + # some typos in the code + if (NOT IS_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/${base_folder}") + message(FATAL_ERROR "No such base folder '${base_folder}' (for '${cmake_folder}' cmake folder). Typo in the base folder name?") + endif() + + file(GLOB contrib_files "${base_folder}/*") + if (NOT contrib_files) + message(STATUS "submodule ${base_folder} is missing or empty. to fix try run:") + message(STATUS " git submodule update --init") + return() + endif() + endforeach() + + message(STATUS "Adding contrib module ${base_folders} (configuring with ${cmake_folder})") + add_subdirectory (${cmake_folder}) +endfunction() + +add_contrib (miniselect-cmake miniselect) +add_contrib (pdqsort-cmake pdqsort) +add_contrib (sparsehash-c11-cmake sparsehash-c11) +add_contrib (abseil-cpp-cmake abseil-cpp) +add_contrib (magic-enum-cmake magic_enum) +add_contrib (boost-cmake boost) +add_contrib (cctz-cmake cctz) add_contrib (consistent-hashing) -add_contrib (dragonbox-cmake) -add_contrib (hyperscan-cmake) -add_contrib (jemalloc-cmake) -add_contrib (libcpuid-cmake) +add_contrib (dragonbox-cmake dragonbox) +add_contrib (hyperscan-cmake hyperscan) +add_contrib (jemalloc-cmake jemalloc) +add_contrib (libcpuid-cmake libcpuid) add_contrib (libdivide) add_contrib (libmetrohash) -add_contrib (lz4-cmake) +add_contrib (lz4-cmake lz4) add_contrib (murmurhash) -add_contrib (replxx-cmake) -add_contrib (unixodbc-cmake) -add_contrib (nanodbc-cmake) -add_contrib (capnproto-cmake) -add_contrib (yaml-cpp-cmake) -add_contrib (re2-cmake) -add_contrib (xz-cmake) -add_contrib (brotli-cmake) -add_contrib (double-conversion-cmake) -add_contrib (boringssl-cmake) -add_contrib (poco-cmake) -add_contrib (croaring-cmake) -add_contrib (zstd-cmake) -add_contrib (zlib-ng-cmake) -add_contrib (bzip2-cmake) -add_contrib (snappy-cmake) -add_contrib (rocksdb-cmake) -add_contrib (thrift-cmake) +add_contrib (replxx-cmake replxx) +add_contrib (unixodbc-cmake unixodbc) +add_contrib (nanodbc-cmake nanodbc) +add_contrib (capnproto-cmake capnproto) +add_contrib (yaml-cpp-cmake yaml-cpp) +add_contrib (re2-cmake re2) +add_contrib (xz-cmake xz) +add_contrib (brotli-cmake brotli) +add_contrib (double-conversion-cmake double-conversion) +add_contrib (boringssl-cmake boringssl) +add_contrib (poco-cmake poco) +add_contrib (croaring-cmake croaring) +add_contrib (zstd-cmake zstd) +add_contrib (zlib-ng-cmake zlib-ng) +add_contrib (bzip2-cmake bzip2) +add_contrib (snappy-cmake snappy) +add_contrib (rocksdb-cmake rocksdb) +add_contrib (thrift-cmake thrift) # parquet/arrow/orc -add_contrib (arrow-cmake) # requires: snappy, thrift, double-conversion -add_contrib (avro-cmake) # requires: snappy -add_contrib (hive-metastore-cmake) # requires: thrift/avro/arrow -add_contrib (librdkafka-cmake) -add_contrib (cppkafka-cmake) -add_contrib (openldap-cmake) -add_contrib (grpc-cmake) -add_contrib (protobuf-cmake) -add_contrib (libhdfs3-cmake) # requires: protobuf -add_contrib (msgpack-c-cmake) +add_contrib (arrow-cmake arrow) # requires: snappy, thrift, double-conversion +add_contrib (avro-cmake avro) # requires: snappy +add_contrib (hive-metastore-cmake hive-metastore) # requires: thrift/avro/arrow +add_contrib (librdkafka-cmake librdkafka) +add_contrib (cppkafka-cmake cppkafka) +add_contrib (openldap-cmake openldap) +add_contrib (grpc-cmake grpc) +add_contrib (protobuf-cmake protobuf) +add_contrib (libhdfs3-cmake libhdfs3) # requires: protobuf +add_contrib (msgpack-c-cmake msgpack-c) if (ENABLE_FUZZING) - add_contrib (libprotobuf-mutator-cmake) + add_contrib (libprotobuf-mutator-cmake libprotobuf-mutator) endif() add_contrib (cityhash102) add_contrib (libfarmhash) -add_contrib (icu-cmake) -add_contrib (h3-cmake) -add_contrib (mariadb-connector-c-cmake) +add_contrib (icu-cmake icu) +add_contrib (h3-cmake h3) +add_contrib (mariadb-connector-c-cmake mariadb-connector-c) if (ENABLE_TESTS) - add_contrib (googletest-cmake) + add_contrib (googletest-cmake googletest) endif() -add_contrib (llvm-cmake) -add_contrib (libgsasl-cmake) -add_contrib (libxml2-cmake) -add_contrib (aws-s3-cmake) -add_contrib (base64-cmake) -add_contrib (simdjson-cmake) -add_contrib (rapidjson-cmake) -add_contrib (fastops-cmake) -add_contrib (libuv-cmake) -add_contrib (amqpcpp-cmake) # requires: libuv -add_contrib (cassandra-cmake) # requires: libuv -add_contrib (curl-cmake) -add_contrib (azure-cmake) -add_contrib (sentry-native-cmake) # requires: curl -add_contrib (fmtlib-cmake) -add_contrib (krb5-cmake) -add_contrib (cyrus-sasl-cmake) # for krb5 -add_contrib (libpqxx-cmake) -add_contrib (libpq-cmake) -add_contrib (nuraft-cmake) -add_contrib (fast_float-cmake) -add_contrib (datasketches-cpp-cmake) +add_contrib (llvm-cmake llvm) +add_contrib (libgsasl-cmake libgsasl) +add_contrib (libxml2-cmake libxml2) +add_contrib (aws-s3-cmake + aws + aws-c-common + aws-c-event-stream + aws-checksums +) +add_contrib (base64-cmake base64) +add_contrib (simdjson-cmake simdjson) +add_contrib (rapidjson-cmake rapidjson) +add_contrib (fastops-cmake fastops) +add_contrib (libuv-cmake libuv) +add_contrib (amqpcpp-cmake AMQP-CPP) # requires: libuv +add_contrib (cassandra-cmake cassandra) # requires: libuv +add_contrib (curl-cmake curl) +add_contrib (azure-cmake azure) +add_contrib (sentry-native-cmake sentry-native) # requires: curl +add_contrib (fmtlib-cmake fmtlib) +add_contrib (krb5-cmake krb5) +add_contrib (cyrus-sasl-cmake cyrus-sasl) # for krb5 +add_contrib (libpqxx-cmake libpqxx) +add_contrib (libpq-cmake libpq) +add_contrib (nuraft-cmake NuRaft) +add_contrib (fast_float-cmake fast_float) +add_contrib (datasketches-cpp-cmake datasketches-cpp) option(ENABLE_NLP "Enable NLP functions support" ${ENABLE_LIBRARIES}) if (ENABLE_NLP) - add_contrib (libstemmer-c-cmake) # ENABLE_NLP - add_contrib (wordnet-blast-cmake) # ENABLE_NLP - add_contrib (lemmagen-c-cmake) # ENABLE_NLP + add_contrib (libstemmer-c-cmake libstemmer_c) + add_contrib (wordnet-blast-cmake wordnet-blast) + add_contrib (lemmagen-c-cmake lemmagen-c) endif() -add_contrib (sqlite-cmake) -add_contrib (s2geometry-cmake) +add_contrib (sqlite-cmake sqlite-amalgamation) +add_contrib (s2geometry-cmake s2geometry) # Put all targets defined here and in subdirectories under "contrib/" folders in GUI-based IDEs. # Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear From e05a75f85035f0ab47535d8d7651cbfe8c724e4a Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 10:18:46 +0300 Subject: [PATCH 213/403] LZMA/xz cannot be disabled --- contrib/xz-cmake/CMakeLists.txt | 7 ------- src/CMakeLists.txt | 5 +---- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/contrib/xz-cmake/CMakeLists.txt b/contrib/xz-cmake/CMakeLists.txt index 879ef192556..9d08adc9c7a 100644 --- a/contrib/xz-cmake/CMakeLists.txt +++ b/contrib/xz-cmake/CMakeLists.txt @@ -1,10 +1,3 @@ -option(ENABLE_XZ "Enable xz/liblzma" ${ENABLE_LIBRARIES}) - -if (NOT ENABLE_XZ) - message(STATUS "Not using xz") - return() -endif() - set (SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/xz") # Author: Lasse Collin diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ac86a5e167a..a59ea743093 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -390,10 +390,7 @@ dbms_target_link_libraries(PUBLIC dbms_target_link_libraries(PRIVATE ch_contrib::zstd) target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::zstd) - -if (TARGET ch_contrib::xz) - target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::xz) -endif() +target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::xz) if (TARGET ch_contrib::icu) dbms_target_link_libraries (PRIVATE ch_contrib::icu) From e0e81b340d33157a054c0e5258a48950b3885599 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:49:10 +0300 Subject: [PATCH 214/403] Fix w/o ODBC build --- base/bridge/IBridge.cpp | 1 + contrib/nanodbc-cmake/CMakeLists.txt | 4 ++-- contrib/unixodbc-cmake/CMakeLists.txt | 1 - programs/odbc-bridge/ColumnInfoHandler.h | 3 ++- programs/odbc-bridge/HandlerFactory.cpp | 1 + programs/odbc-bridge/IdentifierQuoteHandler.h | 2 +- programs/odbc-bridge/MainHandler.cpp | 1 + programs/odbc-bridge/SchemaAllowedHandler.h | 1 + programs/odbc-bridge/getIdentifierQuote.h | 2 ++ src/Common/config.h.in | 1 + src/Core/config_core.h.in | 1 + src/Dictionaries/XDBCDictionarySource.cpp | 1 + .../System/StorageSystemBuildOptions.generated.cpp.in | 1 + src/configure_config.cmake | 3 +++ 14 files changed, 18 insertions(+), 5 deletions(-) diff --git a/base/bridge/IBridge.cpp b/base/bridge/IBridge.cpp index 553973b645d..4c808278ed0 100644 --- a/base/bridge/IBridge.cpp +++ b/base/bridge/IBridge.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include #include diff --git a/contrib/nanodbc-cmake/CMakeLists.txt b/contrib/nanodbc-cmake/CMakeLists.txt index b1f4d867685..8b5a70e65df 100644 --- a/contrib/nanodbc-cmake/CMakeLists.txt +++ b/contrib/nanodbc-cmake/CMakeLists.txt @@ -4,7 +4,7 @@ endif () set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/nanodbc") -if (NOT TARGET unixodbc) +if (NOT TARGET ch_contrib::unixodbc) message(FATAL_ERROR "Configuration error: unixodbc is not a target") endif() @@ -14,6 +14,6 @@ set (SRCS add_library(nanodbc ${SRCS}) -target_link_libraries (nanodbc PUBLIC unixodbc) +target_link_libraries (nanodbc PUBLIC ch_contrib::unixodbc) target_include_directories (nanodbc SYSTEM PUBLIC "${LIBRARY_DIR}/") add_library(ch_contrib::nanodbc ALIAS nanodbc) diff --git a/contrib/unixodbc-cmake/CMakeLists.txt b/contrib/unixodbc-cmake/CMakeLists.txt index ca0742326de..9adf9ce6d79 100644 --- a/contrib/unixodbc-cmake/CMakeLists.txt +++ b/contrib/unixodbc-cmake/CMakeLists.txt @@ -303,6 +303,5 @@ target_compile_options (unixodbc -Wno-reserved-id-macro -O2 ) -target_compile_definitions (unixodbc INTERFACE USE_ODBC=1) add_library (ch_contrib::unixodbc ALIAS unixodbc) diff --git a/programs/odbc-bridge/ColumnInfoHandler.h b/programs/odbc-bridge/ColumnInfoHandler.h index bc976f54aee..76c0103d604 100644 --- a/programs/odbc-bridge/ColumnInfoHandler.h +++ b/programs/odbc-bridge/ColumnInfoHandler.h @@ -1,11 +1,12 @@ #pragma once +#include + #if USE_ODBC #include #include #include -#include #include diff --git a/programs/odbc-bridge/HandlerFactory.cpp b/programs/odbc-bridge/HandlerFactory.cpp index 6a5ef89ab8b..1a6df287a5c 100644 --- a/programs/odbc-bridge/HandlerFactory.cpp +++ b/programs/odbc-bridge/HandlerFactory.cpp @@ -1,6 +1,7 @@ #include "HandlerFactory.h" #include "PingHandler.h" #include "ColumnInfoHandler.h" +#include #include #include #include diff --git a/programs/odbc-bridge/IdentifierQuoteHandler.h b/programs/odbc-bridge/IdentifierQuoteHandler.h index ef3806fd802..23ffd84663b 100644 --- a/programs/odbc-bridge/IdentifierQuoteHandler.h +++ b/programs/odbc-bridge/IdentifierQuoteHandler.h @@ -2,7 +2,7 @@ #include #include - +#include #include #if USE_ODBC diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index 82d1bd61c24..1252d1ae70a 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include diff --git a/programs/odbc-bridge/SchemaAllowedHandler.h b/programs/odbc-bridge/SchemaAllowedHandler.h index d7b922ed05b..7afa77ca091 100644 --- a/programs/odbc-bridge/SchemaAllowedHandler.h +++ b/programs/odbc-bridge/SchemaAllowedHandler.h @@ -2,6 +2,7 @@ #include #include +#include #include #if USE_ODBC diff --git a/programs/odbc-bridge/getIdentifierQuote.h b/programs/odbc-bridge/getIdentifierQuote.h index f4227af5c07..a7620da2291 100644 --- a/programs/odbc-bridge/getIdentifierQuote.h +++ b/programs/odbc-bridge/getIdentifierQuote.h @@ -1,5 +1,7 @@ #pragma once +#include + #if USE_ODBC #include diff --git a/src/Common/config.h.in b/src/Common/config.h.in index 47d56ca6abd..1ac07af574c 100644 --- a/src/Common/config.h.in +++ b/src/Common/config.h.in @@ -18,3 +18,4 @@ #cmakedefine01 USE_BZIP2 #cmakedefine01 USE_SNAPPY #cmakedefine01 USE_HIVE +#cmakedefine01 USE_ODBC diff --git a/src/Core/config_core.h.in b/src/Core/config_core.h.in index de37190b394..481849f6977 100644 --- a/src/Core/config_core.h.in +++ b/src/Core/config_core.h.in @@ -16,3 +16,4 @@ #cmakedefine01 USE_NLP #cmakedefine01 USE_KRB5 #cmakedefine01 USE_FILELOG +#cmakedefine01 USE_ODBC diff --git a/src/Dictionaries/XDBCDictionarySource.cpp b/src/Dictionaries/XDBCDictionarySource.cpp index ab7cf65eb8b..e95094cac47 100644 --- a/src/Dictionaries/XDBCDictionarySource.cpp +++ b/src/Dictionaries/XDBCDictionarySource.cpp @@ -18,6 +18,7 @@ #include #include #include +#include namespace DB diff --git a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in index 6e3163c7641..a14f4996d52 100644 --- a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in +++ b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in @@ -42,6 +42,7 @@ const char * auto_config_build[] "USE_SSL", "@USE_SSL@", "USE_HYPERSCAN", "@ENABLE_HYPERSCAN@", "USE_SIMDJSON", "@USE_SIMDJSON@", + "USE_ODBC", "@USE_ODBC@", "USE_GRPC", "@USE_GRPC@", "USE_LDAP", "@USE_LDAP@", "TZDATA_VERSION", "@TZDATA_VERSION@", diff --git a/src/configure_config.cmake b/src/configure_config.cmake index 278148d2c1c..af19c24b594 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -85,3 +85,6 @@ endif() if (TARGET ch_contrib::llvm) set(USE_EMBEDDED_COMPILER 1) endif() +if (TARGET ch_contrib::unixodbc) + set(USE_ODBC 1) +endif() From 66a210410f35827fc0c6abb72b55e94a49cf8f58 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 10:21:41 +0300 Subject: [PATCH 215/403] Fix build w/o hyperscan --- contrib/hyperscan-cmake/CMakeLists.txt | 6 ++---- src/Functions/CMakeLists.txt | 4 +++- src/Functions/Regexps.h | 1 + src/Functions/URL/CMakeLists.txt | 4 +++- src/Functions/config_functions.h.in | 1 + src/Functions/configure_config.cmake | 3 +++ 6 files changed, 13 insertions(+), 6 deletions(-) diff --git a/contrib/hyperscan-cmake/CMakeLists.txt b/contrib/hyperscan-cmake/CMakeLists.txt index 9afed2a0732..19978687750 100644 --- a/contrib/hyperscan-cmake/CMakeLists.txt +++ b/contrib/hyperscan-cmake/CMakeLists.txt @@ -6,9 +6,6 @@ elseif(ENABLE_HYPERSCAN) endif () if (NOT ENABLE_HYPERSCAN) - add_library (hyperscan INTERFACE) - target_compile_definitions (hyperscan INTERFACE USE_HYPERSCAN=0) - message (STATUS "Not using hyperscan") return() endif() @@ -222,7 +219,6 @@ set (SRCS add_library (hyperscan ${SRCS}) -target_compile_definitions (hyperscan PUBLIC USE_HYPERSCAN=1) target_compile_options (hyperscan PRIVATE -g0 # Library has too much debug information -mno-avx -mno-avx2 # The library is using dynamic dispatch and is confused if AVX is enabled globally @@ -239,3 +235,5 @@ if (ARCH_AMD64) target_include_directories (hyperscan PRIVATE x86_64) endif () target_link_libraries (hyperscan PRIVATE boost::headers_only) + +add_library (ch_contrib::hyperscan ALIAS hyperscan) diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index d79ab5f32a1..a5746275b87 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -80,7 +80,9 @@ if (TARGET ch_contrib::h3) target_link_libraries (clickhouse_functions PRIVATE ch_contrib::h3) endif() -target_link_libraries(clickhouse_functions PRIVATE hyperscan) +if (TARGET ch_contrib::hyperscan) + target_link_libraries(clickhouse_functions PRIVATE ch_contrib::hyperscan) +endif() if (TARGET ch_contrib::simdjson) target_link_libraries(clickhouse_functions PRIVATE ch_contrib::simdjson) diff --git a/src/Functions/Regexps.h b/src/Functions/Regexps.h index b3477f6f32c..8e6d30c8e14 100644 --- a/src/Functions/Regexps.h +++ b/src/Functions/Regexps.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include "config_functions.h" diff --git a/src/Functions/URL/CMakeLists.txt b/src/Functions/URL/CMakeLists.txt index a523384f030..c6d59b0fae4 100644 --- a/src/Functions/URL/CMakeLists.txt +++ b/src/Functions/URL/CMakeLists.txt @@ -8,7 +8,9 @@ if (STRIP_DEBUG_SYMBOLS_FUNCTIONS) endif() # TODO: move Functions/Regexps.h to some lib and use here -target_link_libraries(clickhouse_functions_url PRIVATE hyperscan) +if (TARGET ch_contrib::hyperscan) + target_link_libraries(clickhouse_functions_url PRIVATE ch_contrib::hyperscan) +endif() if (USE_GPERF) # Only for regenerate diff --git a/src/Functions/config_functions.h.in b/src/Functions/config_functions.h.in index 3e1c862300c..89274bcbfa3 100644 --- a/src/Functions/config_functions.h.in +++ b/src/Functions/config_functions.h.in @@ -8,3 +8,4 @@ #cmakedefine01 USE_H3 #cmakedefine01 USE_S2_GEOMETRY #cmakedefine01 USE_FASTOPS +#cmakedefine01 USE_HYPERSCAN diff --git a/src/Functions/configure_config.cmake b/src/Functions/configure_config.cmake index 0bc72e1e158..7615a2eeeaf 100644 --- a/src/Functions/configure_config.cmake +++ b/src/Functions/configure_config.cmake @@ -16,3 +16,6 @@ endif() if (TARGET ch_contrib::h3) set(USE_H3 1) endif() +if (TARGET ch_contrib::hyperscan) + set(USE_HYPERSCAN 1) +endif() From 3e58094bcb6fabc43835052c1a9de1597e6772d1 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:51:13 +0300 Subject: [PATCH 216/403] Fix build w/o replxx --- contrib/replxx-cmake/CMakeLists.txt | 5 ----- src/Client/ClientBase.cpp | 19 ++++++++++--------- src/Client/ClientBaseHelpers.h | 1 + src/Common/config.h.in | 1 + src/Core/config_core.h.in | 1 + src/configure_config.cmake | 3 +++ 6 files changed, 16 insertions(+), 14 deletions(-) diff --git a/contrib/replxx-cmake/CMakeLists.txt b/contrib/replxx-cmake/CMakeLists.txt index f8625913821..575d044b1d7 100644 --- a/contrib/replxx-cmake/CMakeLists.txt +++ b/contrib/replxx-cmake/CMakeLists.txt @@ -1,9 +1,6 @@ option (ENABLE_REPLXX "Enable replxx support" ${ENABLE_LIBRARIES}) if (NOT ENABLE_REPLXX) - add_library(replxx INTERFACE) - target_compile_definitions(replxx INTERFACE USE_REPLXX=0) - message (STATUS "Not using replxx") return() endif() @@ -30,6 +27,4 @@ if (COMPILER_CLANG) target_compile_options(replxx PRIVATE -Wno-documentation) endif () -target_compile_definitions(replxx PUBLIC USE_REPLXX=1) - add_library(ch_contrib::replxx ALIAS replxx) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index eb00ee349ee..956f337a967 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -13,15 +13,16 @@ #include #include #include -#include "Common/Exception.h" -#include "Common/getNumberOfPhysicalCPUCores.h" -#include "Common/tests/gtest_global_context.h" -#include "Common/typeid_cast.h" -#include "Columns/ColumnString.h" -#include "Columns/ColumnsNumber.h" -#include "Core/Block.h" -#include "Core/Protocol.h" -#include "Formats/FormatFactory.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include diff --git a/src/Client/ClientBaseHelpers.h b/src/Client/ClientBaseHelpers.h index 048a4c17f10..3fb2863082a 100644 --- a/src/Client/ClientBaseHelpers.h +++ b/src/Client/ClientBaseHelpers.h @@ -1,6 +1,7 @@ #pragma once #include +#include #if USE_REPLXX # include diff --git a/src/Common/config.h.in b/src/Common/config.h.in index 1ac07af574c..629f5d3566c 100644 --- a/src/Common/config.h.in +++ b/src/Common/config.h.in @@ -19,3 +19,4 @@ #cmakedefine01 USE_SNAPPY #cmakedefine01 USE_HIVE #cmakedefine01 USE_ODBC +#cmakedefine01 USE_REPLXX diff --git a/src/Core/config_core.h.in b/src/Core/config_core.h.in index 481849f6977..43f58396f47 100644 --- a/src/Core/config_core.h.in +++ b/src/Core/config_core.h.in @@ -17,3 +17,4 @@ #cmakedefine01 USE_KRB5 #cmakedefine01 USE_FILELOG #cmakedefine01 USE_ODBC +#cmakedefine01 USE_REPLXX diff --git a/src/configure_config.cmake b/src/configure_config.cmake index af19c24b594..4c922ed686e 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -88,3 +88,6 @@ endif() if (TARGET ch_contrib::unixodbc) set(USE_ODBC 1) endif() +if (TARGET ch_contrib::replxx) + set(USE_REPLXX 1) +endif() From b23053d1f67e20f8ceb2e6ea5db9742203cbe931 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:51:13 +0300 Subject: [PATCH 217/403] Remove USE_XXHASH (reduntant) --- contrib/lz4-cmake/CMakeLists.txt | 2 +- src/Functions/FunctionsHashing.cpp | 2 -- src/Functions/FunctionsHashing.h | 18 +++--------------- .../StorageSystemBuildOptions.generated.cpp.in | 1 - 4 files changed, 4 insertions(+), 19 deletions(-) diff --git a/contrib/lz4-cmake/CMakeLists.txt b/contrib/lz4-cmake/CMakeLists.txt index 68662b12c85..94def029410 100644 --- a/contrib/lz4-cmake/CMakeLists.txt +++ b/contrib/lz4-cmake/CMakeLists.txt @@ -11,7 +11,7 @@ set (SRCS add_library (_lz4 ${SRCS}) add_library (ch_contrib::lz4 ALIAS _lz4) -target_compile_definitions (_lz4 PUBLIC LZ4_DISABLE_DEPRECATE_WARNINGS=1 USE_XXHASH=1) +target_compile_definitions (_lz4 PUBLIC LZ4_DISABLE_DEPRECATE_WARNINGS=1) if (SANITIZE STREQUAL "undefined") target_compile_options (_lz4 PRIVATE -fno-sanitize=undefined) endif () diff --git a/src/Functions/FunctionsHashing.cpp b/src/Functions/FunctionsHashing.cpp index 3f334e9c302..cbafd4bcec2 100644 --- a/src/Functions/FunctionsHashing.cpp +++ b/src/Functions/FunctionsHashing.cpp @@ -37,9 +37,7 @@ void registerFunctionsHashing(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); -#if USE_XXHASH factory.registerFunction(); factory.registerFunction(); -#endif } } diff --git a/src/Functions/FunctionsHashing.h b/src/Functions/FunctionsHashing.h index a42e6b0bf65..88a0e9524b3 100644 --- a/src/Functions/FunctionsHashing.h +++ b/src/Functions/FunctionsHashing.h @@ -12,10 +12,7 @@ #include #include #include - -#if USE_XXHASH -# include -#endif +#include #if USE_SSL # include @@ -551,9 +548,6 @@ struct ImplMetroHash64 static constexpr bool use_int_hash_for_pods = true; }; - -#if USE_XXHASH - struct ImplXxHash32 { static constexpr auto name = "xxHash32"; @@ -574,7 +568,6 @@ struct ImplXxHash32 static constexpr bool use_int_hash_for_pods = false; }; - struct ImplXxHash64 { static constexpr auto name = "xxHash64"; @@ -592,9 +585,6 @@ struct ImplXxHash64 static constexpr bool use_int_hash_for_pods = false; }; -#endif - - template class FunctionStringHashFixedString : public IFunction { @@ -1413,9 +1403,7 @@ using FunctionJavaHash = FunctionAnyHash; using FunctionJavaHashUTF16LE = FunctionAnyHash; using FunctionHiveHash = FunctionAnyHash; -#if USE_XXHASH - using FunctionXxHash32 = FunctionAnyHash; - using FunctionXxHash64 = FunctionAnyHash; -#endif +using FunctionXxHash32 = FunctionAnyHash; +using FunctionXxHash64 = FunctionAnyHash; } diff --git a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in index a14f4996d52..5c25322b4f0 100644 --- a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in +++ b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in @@ -33,7 +33,6 @@ const char * auto_config_build[] "USE_RDKAFKA", "@USE_RDKAFKA@", "USE_CAPNP", "@USE_CAPNP@", "USE_BASE64", "@USE_BASE64@", - "USE_XXHASH", "@USE_XXHASH@", "USE_HDFS", "@USE_HDFS@", "USE_SNAPPY", "@USE_SNAPPY@", "USE_PARQUET", "@USE_PARQUET@", From 9926f336e3c0d0e26320fec509d41cdb106dc280 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:51:13 +0300 Subject: [PATCH 218/403] Cleanup cpuid contrib --- contrib/libcpuid-cmake/CMakeLists.txt | 8 ++------ src/CMakeLists.txt | 5 ++++- src/Common/config.h.in | 1 + src/Common/getNumberOfPhysicalCPUCores.cpp | 1 + src/configure_config.cmake | 3 +++ 5 files changed, 11 insertions(+), 7 deletions(-) diff --git a/contrib/libcpuid-cmake/CMakeLists.txt b/contrib/libcpuid-cmake/CMakeLists.txt index 9baebb3ba1b..2b890579fbb 100644 --- a/contrib/libcpuid-cmake/CMakeLists.txt +++ b/contrib/libcpuid-cmake/CMakeLists.txt @@ -6,10 +6,7 @@ elseif(ENABLE_CPUID) endif() if (NOT ENABLE_CPUID) - add_library (cpuid INTERFACE) - - target_compile_definitions (cpuid INTERFACE USE_CPUID=0) - + message("Not using cpuid") return() endif() @@ -29,10 +26,9 @@ set (SRCS add_library (cpuid ${SRCS}) target_include_directories (cpuid SYSTEM PUBLIC "${LIBRARY_DIR}") -target_compile_definitions (cpuid PUBLIC USE_CPUID=1) target_compile_definitions (cpuid PRIVATE VERSION="v0.4.1") if (COMPILER_CLANG) target_compile_options (cpuid PRIVATE -Wno-reserved-id-macro) endif () -message (STATUS "Using cpuid") +add_library(ch_contrib::cpuid ALIAS cpuid) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a59ea743093..0318805156a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -322,7 +322,6 @@ target_link_libraries(clickhouse_common_io PUBLIC ch_contrib::re2) target_link_libraries(clickhouse_common_io PRIVATE ${EXECINFO_LIBRARIES} - cpuid PUBLIC boost::program_options boost::system @@ -332,6 +331,10 @@ target_link_libraries(clickhouse_common_io Poco::Foundation ) +if (TARGET ch_contrib::cpuid) + target_link_libraries(clickhouse_common_io PRIVATE ch_contrib::cpuid) +endif() + # Make dbms depend on roaring instead of clickhouse_common_io so that roaring itself can depend on clickhouse_common_io # That way we we can redirect malloc/free functions avoiding circular dependencies dbms_target_link_libraries(PUBLIC roaring) diff --git a/src/Common/config.h.in b/src/Common/config.h.in index 629f5d3566c..67a1ee4842e 100644 --- a/src/Common/config.h.in +++ b/src/Common/config.h.in @@ -2,6 +2,7 @@ // .h autogenerated by cmake! +#cmakedefine01 USE_CPUID #cmakedefine01 USE_BASE64 #cmakedefine01 USE_SSL #cmakedefine01 USE_HDFS diff --git a/src/Common/getNumberOfPhysicalCPUCores.cpp b/src/Common/getNumberOfPhysicalCPUCores.cpp index 13485c634e8..4c09f3d1ea0 100644 --- a/src/Common/getNumberOfPhysicalCPUCores.cpp +++ b/src/Common/getNumberOfPhysicalCPUCores.cpp @@ -1,5 +1,6 @@ #include "getNumberOfPhysicalCPUCores.h" +#include #if USE_CPUID # include #endif diff --git a/src/configure_config.cmake b/src/configure_config.cmake index 4c922ed686e..894bb65d01d 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -91,3 +91,6 @@ endif() if (TARGET ch_contrib::replxx) set(USE_REPLXX 1) endif() +if (TARGET ch_contrib::cpuid) + set(USE_CPUID 1) +endif() From a6d482d3e1309db02b0084ffaa90cbfccc3967df Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 09:51:13 +0300 Subject: [PATCH 219/403] Cleanup jemalloc contrib --- contrib/jemalloc-cmake/CMakeLists.txt | 7 +------ contrib/rocksdb-cmake/CMakeLists.txt | 4 ++-- programs/server/CMakeLists.txt | 4 +++- src/CMakeLists.txt | 19 +++++++++++++++---- src/Common/config.h.in | 1 + src/Common/memory.h | 9 +++++---- src/Common/new_delete.cpp | 1 + src/Core/config_core.h.in | 1 + src/configure_config.cmake | 3 +++ 9 files changed, 32 insertions(+), 17 deletions(-) diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index fb11879fb21..afb6ec06b55 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -12,9 +12,6 @@ else () endif () if (NOT ENABLE_JEMALLOC) - add_library(jemalloc INTERFACE) - target_compile_definitions(jemalloc INTERFACE USE_JEMALLOC=0) - message (STATUS "Not using jemalloc") return() endif () @@ -140,6 +137,4 @@ target_compile_options(jemalloc PRIVATE -Wno-redundant-decls) # for RTLD_NEXT target_compile_options(jemalloc PRIVATE -D_GNU_SOURCE) -set_property(TARGET jemalloc APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS USE_JEMALLOC=1) - -message (STATUS "Using jemalloc") +add_library(ch_contrib::jemalloc ALIAS jemalloc) diff --git a/contrib/rocksdb-cmake/CMakeLists.txt b/contrib/rocksdb-cmake/CMakeLists.txt index f91e7199727..902d29a9630 100644 --- a/contrib/rocksdb-cmake/CMakeLists.txt +++ b/contrib/rocksdb-cmake/CMakeLists.txt @@ -49,9 +49,9 @@ else() # but it does not have all the jemalloc files in include/... set(WITH_JEMALLOC ON) else() - if(WITH_JEMALLOC) + if(WITH_JEMALLOC AND TARGET ch_contrib::jemalloc) add_definitions(-DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE) - list(APPEND THIRDPARTY_LIBS jemalloc) + list(APPEND THIRDPARTY_LIBS ch_contrib::jemalloc) endif() endif() diff --git a/programs/server/CMakeLists.txt b/programs/server/CMakeLists.txt index 281c25d50eb..643fd2f0ec4 100644 --- a/programs/server/CMakeLists.txt +++ b/programs/server/CMakeLists.txt @@ -18,13 +18,15 @@ set (CLICKHOUSE_SERVER_LINK clickhouse_storages_system clickhouse_table_functions string_utils - jemalloc ${LINK_RESOURCE_LIB} PUBLIC daemon ) +if (TARGET ch_contrib::jemalloc) + list(APPEND CLICKHOUSE_SERVER_LINK PRIVATE ch_contrib::jemalloc) +endif() clickhouse_program_add(server) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0318805156a..4faa8c86141 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -177,10 +177,15 @@ if (((SANITIZE STREQUAL "thread") OR (SANITIZE STREQUAL "address")) AND COMPILER message(WARNING "Memory tracking is disabled, due to gcc sanitizers") else() add_library (clickhouse_new_delete STATIC Common/new_delete.cpp) - target_link_libraries (clickhouse_new_delete PRIVATE clickhouse_common_io jemalloc) + target_link_libraries (clickhouse_new_delete PRIVATE clickhouse_common_io) + if (TARGET ch_contrib::jemalloc) + target_link_libraries (clickhouse_new_delete PRIVATE ch_contrib::jemalloc) + endif() endif() -target_link_libraries (clickhouse_common_io PRIVATE jemalloc) +if (TARGET ch_contrib::jemalloc) + target_link_libraries (clickhouse_common_io PRIVATE ch_contrib::jemalloc) +endif() add_subdirectory(Access/Common) add_subdirectory(Common/ZooKeeper) @@ -256,12 +261,18 @@ endif() if (MAKE_STATIC_LIBRARIES OR NOT SPLIT_SHARED_LIBRARIES) add_library (dbms STATIC ${dbms_headers} ${dbms_sources}) - target_link_libraries (dbms PRIVATE jemalloc libdivide ${DBMS_COMMON_LIBRARIES}) + target_link_libraries (dbms PRIVATE libdivide ${DBMS_COMMON_LIBRARIES}) + if (TARGET ch_contrib::jemalloc) + target_link_libraries (dbms PRIVATE ch_contrib::jemalloc) + endif() set (all_modules dbms) else() add_library (dbms SHARED ${dbms_headers} ${dbms_sources}) target_link_libraries (dbms PUBLIC ${all_modules} ${DBMS_COMMON_LIBRARIES}) - target_link_libraries (clickhouse_interpreters PRIVATE jemalloc libdivide) + target_link_libraries (clickhouse_interpreters PRIVATE libdivide) + if (TARGET ch_contrib::jemalloc) + target_link_libraries (clickhouse_interpreters PRIVATE ch_contrib::jemalloc) + endif() list (APPEND all_modules dbms) # force all split libs to be linked if (OS_DARWIN) diff --git a/src/Common/config.h.in b/src/Common/config.h.in index 67a1ee4842e..3d785e0d0fb 100644 --- a/src/Common/config.h.in +++ b/src/Common/config.h.in @@ -21,3 +21,4 @@ #cmakedefine01 USE_HIVE #cmakedefine01 USE_ODBC #cmakedefine01 USE_REPLXX +#cmakedefine01 USE_JEMALLOC diff --git a/src/Common/memory.h b/src/Common/memory.h index 0dc163a54a1..41b10a57db4 100644 --- a/src/Common/memory.h +++ b/src/Common/memory.h @@ -4,12 +4,13 @@ #include #include +#include #if USE_JEMALLOC # include #endif -#if !USE_JEMALLOC || JEMALLOC_VERSION_MAJOR < 4 +#if !USE_JEMALLOC # include #endif @@ -37,7 +38,7 @@ inline ALWAYS_INLINE void deleteImpl(void * ptr) noexcept free(ptr); } -#if USE_JEMALLOC && JEMALLOC_VERSION_MAJOR >= 4 +#if USE_JEMALLOC inline ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size) noexcept { @@ -67,7 +68,7 @@ inline ALWAYS_INLINE size_t getActualAllocationSize(size_t size) { size_t actual_size = size; -#if USE_JEMALLOC && JEMALLOC_VERSION_MAJOR >= 5 +#if USE_JEMALLOC /// The nallocx() function allocates no memory, but it performs the same size computation as the mallocx() function /// @note je_mallocx() != je_malloc(). It's expected they don't differ much in allocation logic. if (likely(size != 0)) @@ -87,7 +88,7 @@ inline ALWAYS_INLINE void untrackMemory(void * ptr [[maybe_unused]], std::size_t { try { -#if USE_JEMALLOC && JEMALLOC_VERSION_MAJOR >= 5 +#if USE_JEMALLOC /// @note It's also possible to use je_malloc_usable_size() here. if (likely(ptr != nullptr)) CurrentMemoryTracker::free(sallocx(ptr, 0)); diff --git a/src/Common/new_delete.cpp b/src/Common/new_delete.cpp index 27db87809d3..8908d140b90 100644 --- a/src/Common/new_delete.cpp +++ b/src/Common/new_delete.cpp @@ -1,4 +1,5 @@ #include +#include #include #if defined(OS_DARWIN) && (USE_JEMALLOC) diff --git a/src/Core/config_core.h.in b/src/Core/config_core.h.in index 43f58396f47..5d37f8cf361 100644 --- a/src/Core/config_core.h.in +++ b/src/Core/config_core.h.in @@ -18,3 +18,4 @@ #cmakedefine01 USE_FILELOG #cmakedefine01 USE_ODBC #cmakedefine01 USE_REPLXX +#cmakedefine01 USE_JEMALLOC diff --git a/src/configure_config.cmake b/src/configure_config.cmake index 894bb65d01d..ce50ab87afc 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -94,3 +94,6 @@ endif() if (TARGET ch_contrib::cpuid) set(USE_CPUID 1) endif() +if (TARGET ch_contrib::jemalloc) + set(USE_JEMALLOC 1) +endif() From 97f9cf939efe0a255f2fabaad28ab9ac35eb751c Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 18 Jan 2022 22:34:22 +0300 Subject: [PATCH 220/403] Fix build hive w/o hdfs --- contrib/CMakeLists.txt | 6 +++--- contrib/hive-metastore-cmake/CMakeLists.txt | 6 +++++- src/CMakeLists.txt | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 8fd1a11f558..d0ab426a8dc 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -84,13 +84,13 @@ add_contrib (thrift-cmake thrift) # parquet/arrow/orc add_contrib (arrow-cmake arrow) # requires: snappy, thrift, double-conversion add_contrib (avro-cmake avro) # requires: snappy -add_contrib (hive-metastore-cmake hive-metastore) # requires: thrift/avro/arrow +add_contrib (protobuf-cmake protobuf) +add_contrib (libhdfs3-cmake libhdfs3) # requires: protobuf +add_contrib (hive-metastore-cmake hive-metastore) # requires: thrift/avro/arrow/libhdfs3 add_contrib (librdkafka-cmake librdkafka) add_contrib (cppkafka-cmake cppkafka) add_contrib (openldap-cmake openldap) add_contrib (grpc-cmake grpc) -add_contrib (protobuf-cmake protobuf) -add_contrib (libhdfs3-cmake libhdfs3) # requires: protobuf add_contrib (msgpack-c-cmake msgpack-c) if (ENABLE_FUZZING) diff --git a/contrib/hive-metastore-cmake/CMakeLists.txt b/contrib/hive-metastore-cmake/CMakeLists.txt index 4a1d90b7cef..9069d46cea7 100644 --- a/contrib/hive-metastore-cmake/CMakeLists.txt +++ b/contrib/hive-metastore-cmake/CMakeLists.txt @@ -1,4 +1,8 @@ -option(ENABLE_HIVE "Enable Hive" ${ENABLE_LIBRARIES}) +if (TARGET ch_contrib::hdfs) + option(ENABLE_HIVE "Enable Hive" ${ENABLE_LIBRARIES}) +elseif(ENABLE_HIVE) + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use Hive without HDFS") +endif() if (NOT ENABLE_HIVE) message("Hive disabled") diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4faa8c86141..7b508505ea2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -120,7 +120,7 @@ if (TARGET ch_contrib::hdfs) endif() add_headers_and_sources(dbms Storages/Cache) -if (TARGET ch_contrib::hivemetastore AND TARGET ch_contrib::hdfs) +if (TARGET ch_contrib::hivemetastore) add_headers_and_sources(dbms Storages/Hive) endif() From 78858f9ba99143e886990c5f1b298b7dbb02cf67 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 19 Jan 2022 09:16:10 +0300 Subject: [PATCH 221/403] Fix librdkafka/hdfs that depends on sasl2/krb5 --- contrib/CMakeLists.txt | 10 +++++----- contrib/libhdfs3-cmake/CMakeLists.txt | 3 ++- contrib/librdkafka-cmake/CMakeLists.txt | 6 +++--- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index d0ab426a8dc..b748818b66d 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -85,10 +85,6 @@ add_contrib (thrift-cmake thrift) add_contrib (arrow-cmake arrow) # requires: snappy, thrift, double-conversion add_contrib (avro-cmake avro) # requires: snappy add_contrib (protobuf-cmake protobuf) -add_contrib (libhdfs3-cmake libhdfs3) # requires: protobuf -add_contrib (hive-metastore-cmake hive-metastore) # requires: thrift/avro/arrow/libhdfs3 -add_contrib (librdkafka-cmake librdkafka) -add_contrib (cppkafka-cmake cppkafka) add_contrib (openldap-cmake openldap) add_contrib (grpc-cmake grpc) add_contrib (msgpack-c-cmake msgpack-c) @@ -108,7 +104,6 @@ if (ENABLE_TESTS) endif() add_contrib (llvm-cmake llvm) -add_contrib (libgsasl-cmake libgsasl) add_contrib (libxml2-cmake libxml2) add_contrib (aws-s3-cmake aws @@ -129,6 +124,11 @@ add_contrib (sentry-native-cmake sentry-native) # requires: curl add_contrib (fmtlib-cmake fmtlib) add_contrib (krb5-cmake krb5) add_contrib (cyrus-sasl-cmake cyrus-sasl) # for krb5 +add_contrib (libgsasl-cmake libgsasl) # requires krb5 +add_contrib (librdkafka-cmake librdkafka) # requires: libgsasl +add_contrib (libhdfs3-cmake libhdfs3) # requires: protobuf, krb5 +add_contrib (hive-metastore-cmake hive-metastore) # requires: thrift/avro/arrow/libhdfs3 +add_contrib (cppkafka-cmake cppkafka) add_contrib (libpqxx-cmake libpqxx) add_contrib (libpq-cmake libpq) add_contrib (nuraft-cmake NuRaft) diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index 180751f21d8..b2f785fa06f 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -9,7 +9,8 @@ if(NOT ENABLE_HDFS) return() endif() -if (${ENABLE_KRB5}) +if (TARGET ch_contrib::krb5) + message(STATUS "Enable kerberos for HDFS") SET(WITH_KERBEROS 1) else() SET(WITH_KERBEROS 0) diff --git a/contrib/librdkafka-cmake/CMakeLists.txt b/contrib/librdkafka-cmake/CMakeLists.txt index 45ff79b1377..d84abd06dec 100644 --- a/contrib/librdkafka-cmake/CMakeLists.txt +++ b/contrib/librdkafka-cmake/CMakeLists.txt @@ -87,14 +87,14 @@ set(SRCS "${RDKAFKA_SOURCE_DIR}/tinycthread_extra.c" ) -if(${ENABLE_CYRUS_SASL}) +if(TARGET ch_contrib::sasl2) message (STATUS "librdkafka with SASL support") set(WITH_SASL_CYRUS 1) endif() message (STATUS "librdkafka with SSL support") set(WITH_SSL 1) -if(${ENABLE_CYRUS_SASL}) +if(WITH_SASL_CYRUS) set(WITH_SASL_SCRAM 1) set(WITH_SASL_OAUTHBEARER 1) endif() @@ -127,7 +127,7 @@ target_link_libraries(_rdkafka ch_contrib::zstd OpenSSL::Crypto OpenSSL::SSL ) -if(${ENABLE_CYRUS_SASL}) +if(WITH_SASL_CYRUS) target_link_libraries(_rdkafka PRIVATE ch_contrib::sasl2) endif() From eabef7b1ee127b0beb8d22a52fe8c6a0dd8ac387 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 20 Jan 2022 10:50:57 +0300 Subject: [PATCH 222/403] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87860deea9d..166492e293a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,6 @@ * Added function `arrayLast`. Closes [#33390](https://github.com/ClickHouse/ClickHouse/issues/33390). [#33415](https://github.com/ClickHouse/ClickHouse/pull/33415) Added function `arrayLastIndex`. [#33465](https://github.com/ClickHouse/ClickHouse/pull/33465) ([Maksim Kita](https://github.com/kitaisreal)). * Add function `decodeURLFormComponent` slightly different to `decodeURLComponent`. Close [#10298](https://github.com/ClickHouse/ClickHouse/issues/10298). [#33451](https://github.com/ClickHouse/ClickHouse/pull/33451) ([SuperDJY](https://github.com/cmsxbc)). * Allow to split `GraphiteMergeTree` rollup rules for plain/tagged metrics (optional rule_type field). [#33494](https://github.com/ClickHouse/ClickHouse/pull/33494) ([Michail Safronov](https://github.com/msaf1980)). -* Potential issue, cannot be exploited: integer overflow may happen in array resize. [#33024](https://github.com/ClickHouse/ClickHouse/pull/33024) ([varadarajkumar](https://github.com/varadarajkumar)). #### Performance Improvement @@ -100,6 +99,7 @@ * Use `--database` option for clickhouse-local. [#32797](https://github.com/ClickHouse/ClickHouse/pull/32797) ([Kseniia Sumarokova](https://github.com/kssenii)). * Fix surprisingly bad code in SQL ordinary function `file`. Now it supports symlinks. [#32640](https://github.com/ClickHouse/ClickHouse/pull/32640) ([alexey-milovidov](https://github.com/alexey-milovidov)). * Updating `modification_time` for data part in `system.parts` after part movement [#32964](https://github.com/ClickHouse/ClickHouse/issues/32964). [#32965](https://github.com/ClickHouse/ClickHouse/pull/32965) ([save-my-heart](https://github.com/save-my-heart)). +* Potential issue, cannot be exploited: integer overflow may happen in array resize. [#33024](https://github.com/ClickHouse/ClickHouse/pull/33024) ([varadarajkumar](https://github.com/varadarajkumar)). #### Build/Testing/Packaging Improvement From e4f5444073c48c994c2fc46a2ea26ab2e21e310c Mon Sep 17 00:00:00 2001 From: taiyang-li <654010905@qq.com> Date: Thu, 20 Jan 2022 16:23:52 +0800 Subject: [PATCH 223/403] support explain create function query --- .../InterpreterCreateFunctionQuery.cpp | 2 +- src/Interpreters/InterpreterExplainQuery.cpp | 1 + .../UserDefinedSQLFunctionVisitor.cpp | 2 +- src/Parsers/ASTCreateFunctionQuery.cpp | 19 +++++++++++++++++-- src/Parsers/ASTCreateFunctionQuery.h | 6 ++++-- src/Parsers/ParserCreateFunctionQuery.cpp | 6 +++++- 6 files changed, 29 insertions(+), 7 deletions(-) diff --git a/src/Interpreters/InterpreterCreateFunctionQuery.cpp b/src/Interpreters/InterpreterCreateFunctionQuery.cpp index 2f345f8b237..20114fa0d75 100644 --- a/src/Interpreters/InterpreterCreateFunctionQuery.cpp +++ b/src/Interpreters/InterpreterCreateFunctionQuery.cpp @@ -41,7 +41,7 @@ BlockIO InterpreterCreateFunctionQuery::execute() auto & user_defined_function_factory = UserDefinedSQLFunctionFactory::instance(); - auto & function_name = create_function_query.function_name; + auto function_name = create_function_query.getFunctionName(); bool if_not_exists = create_function_query.if_not_exists; bool replace = create_function_query.or_replace; diff --git a/src/Interpreters/InterpreterExplainQuery.cpp b/src/Interpreters/InterpreterExplainQuery.cpp index 37b944d72d6..79f2233efa5 100644 --- a/src/Interpreters/InterpreterExplainQuery.cpp +++ b/src/Interpreters/InterpreterExplainQuery.cpp @@ -25,6 +25,7 @@ #include #include +#include "Parsers/formatAST.h" namespace DB { diff --git a/src/Interpreters/UserDefinedSQLFunctionVisitor.cpp b/src/Interpreters/UserDefinedSQLFunctionVisitor.cpp index 3e82930af9d..1adb3d5819a 100644 --- a/src/Interpreters/UserDefinedSQLFunctionVisitor.cpp +++ b/src/Interpreters/UserDefinedSQLFunctionVisitor.cpp @@ -59,7 +59,7 @@ ASTPtr UserDefinedSQLFunctionMatcher::tryToReplaceFunction(const ASTFunction & f if (function_arguments.size() != identifiers_raw.size()) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Function {} expects {} arguments actual arguments {}", - create_function_query->function_name, + create_function_query->getFunctionName(), identifiers_raw.size(), function_arguments.size()); diff --git a/src/Parsers/ASTCreateFunctionQuery.cpp b/src/Parsers/ASTCreateFunctionQuery.cpp index 9209b1f1869..a12eca3d334 100644 --- a/src/Parsers/ASTCreateFunctionQuery.cpp +++ b/src/Parsers/ASTCreateFunctionQuery.cpp @@ -10,7 +10,15 @@ namespace DB ASTPtr ASTCreateFunctionQuery::clone() const { - return std::make_shared(*this); + auto res = std::make_shared(*this); + res->children.clear(); + + res->function_name = function_name->clone(); + res->children.push_back(res->function_name); + + res->function_core = function_core->clone(); + res->children.push_back(res->function_core); + return res; } void ASTCreateFunctionQuery::formatImpl(const IAST::FormatSettings & settings, IAST::FormatState & state, IAST::FormatStateStacked frame) const @@ -27,7 +35,7 @@ void ASTCreateFunctionQuery::formatImpl(const IAST::FormatSettings & settings, I settings.ostr << (settings.hilite ? hilite_none : ""); - settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(function_name) << (settings.hilite ? hilite_none : ""); + settings.ostr << (settings.hilite ? hilite_identifier : "") << backQuoteIfNeed(getFunctionName()) << (settings.hilite ? hilite_none : ""); formatOnCluster(settings); @@ -35,4 +43,11 @@ void ASTCreateFunctionQuery::formatImpl(const IAST::FormatSettings & settings, I function_core->formatImpl(settings, state, frame); } +String ASTCreateFunctionQuery::getFunctionName() const +{ + String name; + tryGetIdentifierNameInto(function_name, name); + return name; +} + } diff --git a/src/Parsers/ASTCreateFunctionQuery.h b/src/Parsers/ASTCreateFunctionQuery.h index fccc1837135..b61441da6df 100644 --- a/src/Parsers/ASTCreateFunctionQuery.h +++ b/src/Parsers/ASTCreateFunctionQuery.h @@ -10,19 +10,21 @@ namespace DB class ASTCreateFunctionQuery : public IAST, public ASTQueryWithOnCluster { public: - String function_name; + ASTPtr function_name; ASTPtr function_core; bool or_replace = false; bool if_not_exists = false; - String getID(char) const override { return "CreateFunctionQuery"; } + String getID(char delim) const override { return "CreateFunctionQuery" + (delim + getFunctionName()); } ASTPtr clone() const override; void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override; ASTPtr getRewrittenASTWithoutOnCluster(const std::string &) const override { return removeOnCluster(clone()); } + + String getFunctionName() const; }; } diff --git a/src/Parsers/ParserCreateFunctionQuery.cpp b/src/Parsers/ParserCreateFunctionQuery.cpp index 55e6e2df1d2..08df6d8da7a 100644 --- a/src/Parsers/ParserCreateFunctionQuery.cpp +++ b/src/Parsers/ParserCreateFunctionQuery.cpp @@ -59,8 +59,12 @@ bool ParserCreateFunctionQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Exp auto create_function_query = std::make_shared(); node = create_function_query; - create_function_query->function_name = function_name->as().name(); + create_function_query->function_name = function_name; + create_function_query->children.push_back(function_name); + create_function_query->function_core = function_core; + create_function_query->children.push_back(function_core); + create_function_query->or_replace = or_replace; create_function_query->if_not_exists = if_not_exists; create_function_query->cluster = std::move(cluster_str); From 6f407e219af9eb7881a4df097239dec2d5cce37c Mon Sep 17 00:00:00 2001 From: cnmade Date: Thu, 20 Jan 2022 17:04:06 +0800 Subject: [PATCH 224/403] Translate zh/faq/index.md --- docs/zh/faq/index.md | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/docs/zh/faq/index.md b/docs/zh/faq/index.md index a44dbb31e89..dd29d73a013 100644 --- a/docs/zh/faq/index.md +++ b/docs/zh/faq/index.md @@ -1,8 +1,46 @@ --- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd toc_folder_title: F.A.Q. +toc_hidden: true toc_priority: 76 --- +# ClickHouse 问答 F.A.Q {#clickhouse-f-a-q} +本节文档是一个收集经常出现的ClickHouse相关问题的答案的地方。 + +类别: + +- **[常见问题](../faq/general/index.md)** + - [什么是 ClickHouse?](../index.md#what-is-clickhouse) + - [为何 ClickHouse 如此迅捷?](../faq/general/why-clickhouse-is-so-fast.md) + - [谁在使用 ClickHouse?](../faq/general/who-is-using-clickhouse.md) + - [“ClickHouse” 有什么含义?](../faq/general/dbms-naming.md) + - [ “Не тормозит” 有什么含义?](../faq/general/ne-tormozit.md) + - [什么是 OLAP?](../faq/general/olap.md) + - [什么是列存储数据库?](../faq/general/columnar-database.md) + - [为何不使用 MapReduce等技术?](../faq/general/mapreduce.md) +- **[应用案例](../faq/use-cases/index.md)** + - [我能把 ClickHouse 作为时序数据库来使用吗?](../faq/use-cases/time-series.md) + - [我能把 ClickHouse 作为 key-value 键值存储吗？](../faq/use-cases/key-value.md) +- **[运维操作](../faq/operations/index.md)** + - [如果想在生产环境部署，需要用哪个版本的 ClickHouse 呢?](../faq/operations/production.md) + - [是否可能从 ClickHouse 数据表中删除所有旧的数据记录?](../faq/operations/delete-old-data.md) +- **[集成开发](../faq/integration/index.md)** + - [如何从 ClickHouse 导出数据到一个文件?](../faq/integration/file-export.md) + - [如果我用ODBC链接Oracle数据库出现编码问题该怎么办?](../faq/integration/oracle-odbc.md) + +{## TODO +Question candidates: +- How to choose a primary key? +- How to add a column in ClickHouse? +- Too many parts +- How to filter ClickHouse table by an array column contents? +- How to insert all rows from one table to another of identical structure? +- How to kill a process (query) in ClickHouse? +- How to implement pivot (like in pandas)? +- How to remove the default ClickHouse user through users.d? +- Importing MySQL dump to ClickHouse +- Window function workarounds (row_number, lag/lead, running diff/sum/average) +##} + +{## [原始文档](https://clickhouse.com/docs/en/faq) ##} From 567b4f5b014eaec1091df89681090652d199b27a Mon Sep 17 00:00:00 2001 From: cnmade <278153+cnmade@users.noreply.github.com> Date: Thu, 20 Jan 2022 17:43:10 +0800 Subject: [PATCH 225/403] Update Readme.md, add Code Browser@github.dev github.dev provide free code browser, base on VSCode web version. Easy to read and navigate. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e12238577a7..dbadfbef682 100644 --- a/README.md +++ b/README.md @@ -11,4 +11,5 @@ ClickHouse® is an open-source column-oriented database management system that a * [Slack](https://join.slack.com/t/clickhousedb/shared_invite/zt-rxm3rdrk-lIUmhLC3V8WTaL0TGxsOmg) and [Telegram](https://telegram.me/clickhouse_en) allow chatting with ClickHouse users in real-time. * [Blog](https://clickhouse.com/blog/en/) contains various ClickHouse-related articles, as well as announcements and reports about events. * [Code Browser](https://clickhouse.com/codebrowser/html_report/ClickHouse/index.html) with syntax highlight and navigation. +* [Code Browser@github.dev](https://github.dev/ClickHouse/ClickHouse) with syntax highlight and navigation, powered by github.dev free of charge. * [Contacts](https://clickhouse.com/company/#contact) can help to get your questions answered if there are any. From 76a64bda6ad5155c514cbc30cf45ffb0da2abb85 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 20 Jan 2022 13:20:52 +0300 Subject: [PATCH 226/403] Update CHANGELOG.md --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 166492e293a..6cde6b9e401 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -77,7 +77,6 @@ * Validate config keys for external dictionaries. [#33095](https://github.com/ClickHouse/ClickHouse/issues/33095#issuecomment-1000577517). [#33130](https://github.com/ClickHouse/ClickHouse/pull/33130) ([Kseniia Sumarokova](https://github.com/kssenii)). * Send profile info inside `clickhouse-local`. Closes [#33093](https://github.com/ClickHouse/ClickHouse/issues/33093). [#33097](https://github.com/ClickHouse/ClickHouse/pull/33097) ([Kseniia Sumarokova](https://github.com/kssenii)). * Short circuit evaluation: support for function `throwIf`. Closes [#32969](https://github.com/ClickHouse/ClickHouse/issues/32969). [#32973](https://github.com/ClickHouse/ClickHouse/pull/32973) ([Maksim Kita](https://github.com/kitaisreal)). -* Added `Date32` date type support in dictionaries. Closes [#32913](https://github.com/ClickHouse/ClickHouse/issues/32913). [#32971](https://github.com/ClickHouse/ClickHouse/pull/32971) ([Maksim Kita](https://github.com/kitaisreal)). * (This only happens in unofficial builds). Fixed segfault when inserting data into compressed Decimal, String, FixedString and Array columns. This closes [#32939](https://github.com/ClickHouse/ClickHouse/issues/32939). [#32940](https://github.com/ClickHouse/ClickHouse/pull/32940) ([N. Kolotov](https://github.com/nkolotov)). * Added support for specifying subquery as SQL user defined function. Example: `CREATE FUNCTION test AS () -> (SELECT 1)`. Closes [#30755](https://github.com/ClickHouse/ClickHouse/issues/30755). [#32758](https://github.com/ClickHouse/ClickHouse/pull/32758) ([Maksim Kita](https://github.com/kitaisreal)). * Improve gRPC compression support for [#28671](https://github.com/ClickHouse/ClickHouse/issues/28671). [#32747](https://github.com/ClickHouse/ClickHouse/pull/32747) ([Vitaly Baranov](https://github.com/vitlibar)). From 51ca90b0309930c7cce53a73eeb8813cefc6e572 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 20 Jan 2022 13:22:01 +0300 Subject: [PATCH 227/403] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cde6b9e401..1e4ea95c08c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ * A tool for collecting diagnostics data if you need support. [#33175](https://github.com/ClickHouse/ClickHouse/pull/33175) ([Alexander Burmak](https://github.com/Alex-Burmak)). * Automatic cluster discovery via Zoo/Keeper. It allows to add replicas to the cluster without changing configuration on every server. [#31442](https://github.com/ClickHouse/ClickHouse/pull/31442) ([vdimir](https://github.com/vdimir)). * Implement hive table engine to access apache hive from clickhouse. This implements: [#29245](https://github.com/ClickHouse/ClickHouse/issues/29245). [#31104](https://github.com/ClickHouse/ClickHouse/pull/31104) ([taiyang-li](https://github.com/taiyang-li)). -* Add aggregate functions `cramersV`, `cramersVBiasCorrected`, `theilsU` and `contingency`. These functions calculate dependency (measure of association) between categorical values. All these functions are using cross-tab (histogram on pairs) for implementation. You can imagine it like a correlation coefficient but for any discrete values (not necessary numbers). [#33366](https://github.com/ClickHouse/ClickHouse/pull/33366) ([alexey-milovidov](https://github.com/alexey-milovidov)). Initial implementation by TODO +* Add aggregate functions `cramersV`, `cramersVBiasCorrected`, `theilsU` and `contingency`. These functions calculate dependency (measure of association) between categorical values. All these functions are using cross-tab (histogram on pairs) for implementation. You can imagine it like a correlation coefficient but for any discrete values (not necessary numbers). [#33366](https://github.com/ClickHouse/ClickHouse/pull/33366) ([alexey-milovidov](https://github.com/alexey-milovidov)). Initial implementation by [Vanyok-All-is-OK](https://github.com/Vanyok-All-is-OK) and [antikvist](https://github.com/antikvist). * Added table function `hdfsCluster` which allows processing files from HDFS in parallel from many nodes in a specified cluster, similarly to `s3Cluster`. [#32400](https://github.com/ClickHouse/ClickHouse/pull/32400) ([Zhichang Yu](https://github.com/yuzhichang)). * Adding support for disks backed by Azure Blob Storage, in a similar way it has been done for disks backed by AWS S3. [#31505](https://github.com/ClickHouse/ClickHouse/pull/31505) ([Jakub Kuklis](https://github.com/jkuklis)). * Allow `COMMENT` in `CREATE VIEW` (for all VIEW kinds). [#31062](https://github.com/ClickHouse/ClickHouse/pull/31062) ([Vasily Nemkov](https://github.com/Enmk)). From ba54a5ff80e17fb88b741ad1562c037002ec0116 Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 18 Jan 2022 14:52:27 +0300 Subject: [PATCH 228/403] Some improvements and fixes for Bool data type --- src/Common/FieldVisitorConvertToNumber.h | 2 + src/Common/FieldVisitorDump.cpp | 1 + src/Common/FieldVisitorDump.h | 1 + src/Common/FieldVisitorHash.cpp | 7 +++ src/Common/FieldVisitorHash.h | 1 + src/Common/FieldVisitorSum.cpp | 2 + src/Common/FieldVisitorSum.h | 1 + src/Common/FieldVisitorToString.cpp | 2 +- src/Common/FieldVisitorToString.h | 1 + src/Common/FieldVisitorWriteBinary.cpp | 5 ++ src/Common/FieldVisitorWriteBinary.h | 1 + src/Common/FieldVisitorsAccurateComparison.h | 16 +++++ src/Core/Field.cpp | 13 ++++ src/Core/Field.h | 32 +++++++++- src/DataTypes/FieldToDataType.cpp | 5 ++ src/DataTypes/FieldToDataType.h | 1 + src/Functions/FunctionsConversion.h | 15 +++-- src/Functions/FunctionsLogical.cpp | 5 +- src/Functions/registerFunctions.cpp | 2 + src/Functions/toBool.cpp | 62 +++++++++++++++++++ src/Interpreters/convertFieldToType.cpp | 2 +- src/Storages/MergeTree/MergeTreePartition.cpp | 6 ++ .../0_stateless/02179_bool_type.reference | 55 ++++++++++++++++ tests/queries/0_stateless/02179_bool_type.sql | 62 +++++++++++++++++++ 24 files changed, 287 insertions(+), 13 deletions(-) create mode 100644 src/Functions/toBool.cpp create mode 100644 tests/queries/0_stateless/02179_bool_type.reference create mode 100644 tests/queries/0_stateless/02179_bool_type.sql diff --git a/src/Common/FieldVisitorConvertToNumber.h b/src/Common/FieldVisitorConvertToNumber.h index 025fd667609..7bbb7f0708a 100644 --- a/src/Common/FieldVisitorConvertToNumber.h +++ b/src/Common/FieldVisitorConvertToNumber.h @@ -123,6 +123,8 @@ public: else return static_cast(x); } + + T operator() (const bool & x) const { return T(x); } }; } diff --git a/src/Common/FieldVisitorDump.cpp b/src/Common/FieldVisitorDump.cpp index d0203407900..6c869e05fd4 100644 --- a/src/Common/FieldVisitorDump.cpp +++ b/src/Common/FieldVisitorDump.cpp @@ -37,6 +37,7 @@ String FieldVisitorDump::operator() (const UInt256 & x) const { return formatQuo String FieldVisitorDump::operator() (const Int128 & x) const { return formatQuotedWithPrefix(x, "Int128_"); } String FieldVisitorDump::operator() (const Int256 & x) const { return formatQuotedWithPrefix(x, "Int256_"); } String FieldVisitorDump::operator() (const UUID & x) const { return formatQuotedWithPrefix(x, "UUID_"); } +String FieldVisitorDump::operator() (const bool & x) const { return formatQuotedWithPrefix(x, "Bool_"); } String FieldVisitorDump::operator() (const String & x) const diff --git a/src/Common/FieldVisitorDump.h b/src/Common/FieldVisitorDump.h index 22e34d66ff7..0b1b311999e 100644 --- a/src/Common/FieldVisitorDump.h +++ b/src/Common/FieldVisitorDump.h @@ -27,6 +27,7 @@ public: String operator() (const DecimalField & x) const; String operator() (const DecimalField & x) const; String operator() (const AggregateFunctionStateData & x) const; + String operator() (const bool & x) const; }; } diff --git a/src/Common/FieldVisitorHash.cpp b/src/Common/FieldVisitorHash.cpp index 80d5f2daf65..09b8b7908f3 100644 --- a/src/Common/FieldVisitorHash.cpp +++ b/src/Common/FieldVisitorHash.cpp @@ -146,4 +146,11 @@ void FieldVisitorHash::operator() (const Int256 & x) const hash.update(x); } +void FieldVisitorHash::operator() (const bool & x) const +{ + UInt8 type = Field::Types::Bool; + hash.update(type); + hash.update(x); +} + } diff --git a/src/Common/FieldVisitorHash.h b/src/Common/FieldVisitorHash.h index 6c786fda4ad..7527e13ca20 100644 --- a/src/Common/FieldVisitorHash.h +++ b/src/Common/FieldVisitorHash.h @@ -33,6 +33,7 @@ public: void operator() (const DecimalField & x) const; void operator() (const DecimalField & x) const; void operator() (const AggregateFunctionStateData & x) const; + void operator() (const bool & x) const; }; } diff --git a/src/Common/FieldVisitorSum.cpp b/src/Common/FieldVisitorSum.cpp index 0064830c08a..c3d7f4f8462 100644 --- a/src/Common/FieldVisitorSum.cpp +++ b/src/Common/FieldVisitorSum.cpp @@ -33,5 +33,7 @@ bool FieldVisitorSum::operator() (AggregateFunctionStateData &) const throw Exception("Cannot sum AggregateFunctionStates", ErrorCodes::LOGICAL_ERROR); } +bool FieldVisitorSum::operator() (bool &) const { throw Exception("Cannot sum Bools", ErrorCodes::LOGICAL_ERROR); } + } diff --git a/src/Common/FieldVisitorSum.h b/src/Common/FieldVisitorSum.h index e208933043b..3e868e46f71 100644 --- a/src/Common/FieldVisitorSum.h +++ b/src/Common/FieldVisitorSum.h @@ -27,6 +27,7 @@ public: bool operator() (Map &) const; bool operator() (UUID &) const; bool operator() (AggregateFunctionStateData &) const; + bool operator() (bool &) const; template bool operator() (DecimalField & x) const diff --git a/src/Common/FieldVisitorToString.cpp b/src/Common/FieldVisitorToString.cpp index 01b2db54735..6cc83f32a52 100644 --- a/src/Common/FieldVisitorToString.cpp +++ b/src/Common/FieldVisitorToString.cpp @@ -51,7 +51,6 @@ static String formatFloat(const Float64 x) return { buffer, buffer + builder.position() }; } - String FieldVisitorToString::operator() (const Null & x) const { return x.isNegativeInfinity() ? "-Inf" : (x.isPositiveInfinity() ? "+Inf" : "NULL"); } String FieldVisitorToString::operator() (const UInt64 & x) const { return formatQuoted(x); } String FieldVisitorToString::operator() (const Int64 & x) const { return formatQuoted(x); } @@ -67,6 +66,7 @@ String FieldVisitorToString::operator() (const UInt256 & x) const { return forma String FieldVisitorToString::operator() (const Int256 & x) const { return formatQuoted(x); } String FieldVisitorToString::operator() (const UUID & x) const { return formatQuoted(x); } String FieldVisitorToString::operator() (const AggregateFunctionStateData & x) const { return formatQuoted(x.data); } +String FieldVisitorToString::operator() (const bool & x) const { return x ? "true" : "false"; } String FieldVisitorToString::operator() (const Array & x) const { diff --git a/src/Common/FieldVisitorToString.h b/src/Common/FieldVisitorToString.h index 39709f1c272..991f7b4b2d7 100644 --- a/src/Common/FieldVisitorToString.h +++ b/src/Common/FieldVisitorToString.h @@ -27,6 +27,7 @@ public: String operator() (const DecimalField & x) const; String operator() (const DecimalField & x) const; String operator() (const AggregateFunctionStateData & x) const; + String operator() (const bool & x) const; }; } diff --git a/src/Common/FieldVisitorWriteBinary.cpp b/src/Common/FieldVisitorWriteBinary.cpp index d01188bef40..fc17b58b334 100644 --- a/src/Common/FieldVisitorWriteBinary.cpp +++ b/src/Common/FieldVisitorWriteBinary.cpp @@ -66,5 +66,10 @@ void FieldVisitorWriteBinary::operator() (const Map & x, WriteBuffer & buf) cons } } +void FieldVisitorWriteBinary::operator()(const bool & x, WriteBuffer & buf) const +{ + writeBinary(UInt8(x), buf); +} + } diff --git a/src/Common/FieldVisitorWriteBinary.h b/src/Common/FieldVisitorWriteBinary.h index ae864ca74f3..155cf0e1050 100644 --- a/src/Common/FieldVisitorWriteBinary.h +++ b/src/Common/FieldVisitorWriteBinary.h @@ -26,6 +26,7 @@ public: void operator() (const DecimalField & x, WriteBuffer & buf) const; void operator() (const DecimalField & x, WriteBuffer & buf) const; void operator() (const AggregateFunctionStateData & x, WriteBuffer & buf) const; + void operator() (const bool & x, WriteBuffer & buf) const; }; } diff --git a/src/Common/FieldVisitorsAccurateComparison.h b/src/Common/FieldVisitorsAccurateComparison.h index 795620da0cb..487f4f78a00 100644 --- a/src/Common/FieldVisitorsAccurateComparison.h +++ b/src/Common/FieldVisitorsAccurateComparison.h @@ -32,6 +32,14 @@ public: return l == r; return false; } + else if constexpr (std::is_same_v) + { + return operator()(UInt8(l), r); + } + else if constexpr (std::is_same_v) + { + return operator()(l, UInt8(r)); + } else { if constexpr (std::is_same_v) @@ -91,6 +99,14 @@ public: { return r.isPositiveInfinity(); } + else if constexpr (std::is_same_v) + { + return operator()(UInt8(l), r); + } + else if constexpr (std::is_same_v) + { + return operator()(l, UInt8(r)); + } else { if constexpr (std::is_same_v) diff --git a/src/Core/Field.cpp b/src/Core/Field.cpp index a85b7cff46e..70a1458c9f0 100644 --- a/src/Core/Field.cpp +++ b/src/Core/Field.cpp @@ -106,6 +106,12 @@ inline Field getBinaryValue(UInt8 type, ReadBuffer & buf) readStringBinary(value.data, buf); return value; } + case Field::Types::Bool: + { + UInt8 value; + readBinary(value, buf); + return bool(value); + } } return Field(); } @@ -346,6 +352,13 @@ Field Field::restoreFromDump(const std::string_view & dump_) return str; } + prefix = std::string_view{"Bool_"}; + if (dump.starts_with(prefix)) + { + bool value = parseFromString(dump.substr(prefix.length())); + return value; + } + prefix = std::string_view{"Array_["}; if (dump.starts_with(prefix)) { diff --git a/src/Core/Field.h b/src/Core/Field.h index 19573ed9831..b525e3a83ab 100644 --- a/src/Core/Field.h +++ b/src/Core/Field.h @@ -282,6 +282,7 @@ public: Int256 = 25, Map = 26, UUID = 27, + Bool = 28, }; }; @@ -323,7 +324,10 @@ public: template Field(T && rhs, enable_if_not_field_or_bool_or_stringlike_t = nullptr); - Field(bool rhs) : Field(castToNearestFieldType(rhs)) {} + Field(bool rhs) : Field(castToNearestFieldType(rhs)) + { + which = Types::Bool; + } /// Create a string inplace. Field(const std::string_view & str) { create(str.data(), str.size()); } @@ -376,7 +380,12 @@ public: enable_if_not_field_or_bool_or_stringlike_t & operator=(T && rhs); - Field & operator= (bool rhs) { return *this = castToNearestFieldType(rhs); } + Field & operator= (bool rhs) + { + *this = castToNearestFieldType(rhs); + which = Types::Bool; + return *this; + } Field & operator= (const std::string_view & str); Field & operator= (const String & str) { return *this = std::string_view{str}; } @@ -450,6 +459,7 @@ public: switch (which) { case Types::Null: return false; + case Types::Bool: [[fallthrough]]; case Types::UInt64: return get() < rhs.get(); case Types::UInt128: return get() < rhs.get(); case Types::UInt256: return get() < rhs.get(); @@ -487,6 +497,7 @@ public: switch (which) { case Types::Null: return true; + case Types::Bool: [[fallthrough]]; case Types::UInt64: return get() <= rhs.get(); case Types::UInt128: return get() <= rhs.get(); case Types::UInt256: return get() <= rhs.get(); @@ -524,6 +535,7 @@ public: switch (which) { case Types::Null: return true; + case Types::Bool: [[fallthrough]]; case Types::UInt64: return get() == rhs.get(); case Types::Int64: return get() == rhs.get(); case Types::Float64: @@ -580,6 +592,11 @@ public: case Types::Array: return f(field.template get()); case Types::Tuple: return f(field.template get()); case Types::Map: return f(field.template get()); + case Types::Bool: + { + bool value = bool(field.template get()); + return f(value); + } case Types::Decimal32: return f(field.template get>()); case Types::Decimal64: return f(field.template get>()); case Types::Decimal128: return f(field.template get>()); @@ -739,6 +756,7 @@ template <> struct Field::TypeToEnum>{ static const Typ template <> struct Field::TypeToEnum>{ static const Types::Which value = Types::Decimal256; }; template <> struct Field::TypeToEnum>{ static const Types::Which value = Types::Decimal64; }; template <> struct Field::TypeToEnum{ static const Types::Which value = Types::AggregateFunctionState; }; +template <> struct Field::TypeToEnum{ static const Types::Which value = Types::Bool; }; template <> struct Field::EnumToType { using Type = Null; }; template <> struct Field::EnumToType { using Type = UInt64; }; @@ -758,6 +776,7 @@ template <> struct Field::EnumToType { using Type = Dec template <> struct Field::EnumToType { using Type = DecimalField; }; template <> struct Field::EnumToType { using Type = DecimalField; }; template <> struct Field::EnumToType { using Type = DecimalField; }; +template <> struct Field::EnumToType { using Type = UInt64; }; inline constexpr bool isInt64OrUInt64FieldType(Field::Types::Which t) { @@ -765,6 +784,13 @@ inline constexpr bool isInt64OrUInt64FieldType(Field::Types::Which t) || t == Field::Types::UInt64; } +inline constexpr bool isInt64OrUInt64orBoolFieldType(Field::Types::Which t) +{ + return t == Field::Types::Int64 + || t == Field::Types::UInt64 + || t == Field::Types::Bool; +} + // Field value getter with type checking in debug builds. template NearestFieldType> & Field::get() @@ -781,7 +807,7 @@ NearestFieldType> & Field::get() // Disregard signedness when converting between int64 types. constexpr Field::Types::Which target = TypeToEnum::value; if (target != which - && (!isInt64OrUInt64FieldType(target) || !isInt64OrUInt64FieldType(which))) + && (!isInt64OrUInt64orBoolFieldType(target) || !isInt64OrUInt64orBoolFieldType(which))) throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid Field get from type {} to type {}", which, target); #endif diff --git a/src/DataTypes/FieldToDataType.cpp b/src/DataTypes/FieldToDataType.cpp index c1a8cacd5c2..8ca5ffac7c5 100644 --- a/src/DataTypes/FieldToDataType.cpp +++ b/src/DataTypes/FieldToDataType.cpp @@ -152,4 +152,9 @@ DataTypePtr FieldToDataType::operator() (const AggregateFunctionStateData & x) c return DataTypeFactory::instance().get(name); } +DataTypePtr FieldToDataType::operator()(const bool &) const +{ + return DataTypeFactory::instance().get("Bool"); +} + } diff --git a/src/DataTypes/FieldToDataType.h b/src/DataTypes/FieldToDataType.h index ca83ce868fc..72575c070f5 100644 --- a/src/DataTypes/FieldToDataType.h +++ b/src/DataTypes/FieldToDataType.h @@ -38,6 +38,7 @@ public: DataTypePtr operator() (const DecimalField & x) const; DataTypePtr operator() (const DecimalField & x) const; DataTypePtr operator() (const AggregateFunctionStateData & x) const; + DataTypePtr operator() (const bool & x) const; }; } diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index 62e62b5f5dc..f8d3a7a4e02 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -185,6 +185,15 @@ struct ConvertImpl bool result_is_bool = isBool(result_type); for (size_t i = 0; i < input_rows_count; ++i) { + if constexpr (std::is_same_v) + { + if (result_is_bool) + { + vec_to[i] = vec_from[i] != FromFieldType(0); + continue; + } + } + if constexpr (std::is_same_v != std::is_same_v) { throw Exception("Conversion between numeric types and UUID is not supported", ErrorCodes::NOT_IMPLEMENTED); @@ -269,12 +278,6 @@ struct ConvertImpl vec_to[i] = static_cast(vec_from[i]); } } - - if constexpr (std::is_same_v) - { - if (result_is_bool) - vec_to[i] = static_cast(vec_to[i]); - } } } diff --git a/src/Functions/FunctionsLogical.cpp b/src/Functions/FunctionsLogical.cpp index 87a2ecd4c57..1fdba05f29d 100644 --- a/src/Functions/FunctionsLogical.cpp +++ b/src/Functions/FunctionsLogical.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -503,7 +504,7 @@ DataTypePtr FunctionAnyArityLogical::getReturnTypeImpl(const DataTyp ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } - auto result_type = std::make_shared(); + auto result_type = DataTypeFactory::instance().get("Bool"); return has_nullable_arguments ? makeNullable(result_type) : result_type; @@ -711,7 +712,7 @@ DataTypePtr FunctionUnaryLogical::getReturnTypeImpl(const DataTypes + ") of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - return std::make_shared(); + return DataTypeFactory::instance().get("Bool"); } template