From 322db80cd9f609e2de4e119eda6245d1c60e8fca Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Thu, 17 Aug 2023 12:30:06 +0200 Subject: [PATCH 01/42] Fixed SELECTing from ReplacingMergeTree with do_not_merge_across_partitions_select_final --- .../QueryPlan/ReadFromMergeTree.cpp | 6 ++-- ...select_final_on_single_partition.reference | 29 ++++++++++++++++++ ...e_fix_select_final_on_single_partition.sql | 30 +++++++++++++++++++ 3 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/02814_ReplacingMergeTree_fix_select_final_on_single_partition.reference create mode 100644 tests/queries/0_stateless/02814_ReplacingMergeTree_fix_select_final_on_single_partition.sql diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 13de5d1d140..dc9ce5f0391 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -996,7 +996,8 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( /// MergeTreeReadPool and MergeTreeThreadSelectProcessor for parallel select. if (num_streams > 1 && settings.do_not_merge_across_partitions_select_final && std::distance(parts_to_merge_ranges[range_index], parts_to_merge_ranges[range_index + 1]) == 1 && - parts_to_merge_ranges[range_index]->data_part->info.level > 0) + parts_to_merge_ranges[range_index]->data_part->info.level > 0 + && data.merging_params.is_deleted_column.empty()) { sum_marks_in_lonely_parts += parts_to_merge_ranges[range_index]->getMarksCount(); lonely_parts.push_back(std::move(*parts_to_merge_ranges[range_index])); @@ -1052,7 +1053,8 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( /// with level > 0 then we won't postprocess this part if (settings.do_not_merge_across_partitions_select_final && std::distance(parts_to_merge_ranges[range_index], parts_to_merge_ranges[range_index + 1]) == 1 && - parts_to_merge_ranges[range_index]->data_part->info.level > 0) + parts_to_merge_ranges[range_index]->data_part->info.level > 0 && + data.merging_params.is_deleted_column.empty()) { partition_pipes.emplace_back(Pipe::unitePipes(std::move(pipes))); continue; diff --git a/tests/queries/0_stateless/02814_ReplacingMergeTree_fix_select_final_on_single_partition.reference b/tests/queries/0_stateless/02814_ReplacingMergeTree_fix_select_final_on_single_partition.reference new file mode 100644 index 00000000000..0510348cb11 --- /dev/null +++ b/tests/queries/0_stateless/02814_ReplacingMergeTree_fix_select_final_on_single_partition.reference @@ -0,0 +1,29 @@ +--- Based on https://github.com/ClickHouse/ClickHouse/issues/49685 +--- Verify that ReplacingMergeTree properly handles _is_deleted: +--- SELECT FINAL should take _is_deleted in consideration when there is only one partition. +-- { echoOn } + +CREATE TABLE t +( + `account_id` UInt64, + `_is_deleted` UInt8, + `_version` UInt64 +) +ENGINE = ReplacingMergeTree(_version, _is_deleted) +ORDER BY (account_id); +INSERT INTO t SELECT number, 0, 1 FROM numbers(1e3); +-- Mark first 100 rows as deleted. +INSERT INTO t SELECT number, 1, 1 FROM numbers(1e2); +-- Put everything is in one partition +OPTIMIZE TABLE t FINAL; +SELECT count() FROM t; +1000 +SELECT count() FROM t FINAL; +900 +-- Both should produce same number of rows. +-- Previously `do_not_merge_across_partitions_select_final = 1` showed more rows, +-- as if no rows were deleted. +SELECT count() FROM t FINAL SETTINGS do_not_merge_across_partitions_select_final = 1; +900 +SELECT count() FROM t FINAL SETTINGS do_not_merge_across_partitions_select_final = 0; +900 diff --git a/tests/queries/0_stateless/02814_ReplacingMergeTree_fix_select_final_on_single_partition.sql b/tests/queries/0_stateless/02814_ReplacingMergeTree_fix_select_final_on_single_partition.sql new file mode 100644 index 00000000000..bb9da732090 --- /dev/null +++ b/tests/queries/0_stateless/02814_ReplacingMergeTree_fix_select_final_on_single_partition.sql @@ -0,0 +1,30 @@ +--- Based on https://github.com/ClickHouse/ClickHouse/issues/49685 +--- Verify that ReplacingMergeTree properly handles _is_deleted: +--- SELECT FINAL should take _is_deleted in consideration when there is only one partition. +-- { echoOn } + +CREATE TABLE t +( + `account_id` UInt64, + `_is_deleted` UInt8, + `_version` UInt64 +) +ENGINE = ReplacingMergeTree(_version, _is_deleted) +ORDER BY (account_id); + +INSERT INTO t SELECT number, 0, 1 FROM numbers(1e3); +-- Mark first 100 rows as deleted. +INSERT INTO t SELECT number, 1, 1 FROM numbers(1e2); + +-- Put everything is in one partition +OPTIMIZE TABLE t FINAL; + +SELECT count() FROM t; +SELECT count() FROM t FINAL; + +-- Both should produce same number of rows. +-- Previously `do_not_merge_across_partitions_select_final = 1` showed more rows, +-- as if no rows were deleted. +SELECT count() FROM t FINAL SETTINGS do_not_merge_across_partitions_select_final = 1; +SELECT count() FROM t FINAL SETTINGS do_not_merge_across_partitions_select_final = 0; + From 258483504d6112fc728ed4e79d0db6e2c6de0389 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 19 Aug 2023 02:15:41 +0300 Subject: [PATCH 02/42] Update 02814_ReplacingMergeTree_fix_select_final_on_single_partition.sql --- ...ergeTree_fix_select_final_on_single_partition.sql | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/02814_ReplacingMergeTree_fix_select_final_on_single_partition.sql b/tests/queries/0_stateless/02814_ReplacingMergeTree_fix_select_final_on_single_partition.sql index bb9da732090..a89a1ff590a 100644 --- a/tests/queries/0_stateless/02814_ReplacingMergeTree_fix_select_final_on_single_partition.sql +++ b/tests/queries/0_stateless/02814_ReplacingMergeTree_fix_select_final_on_single_partition.sql @@ -1,8 +1,9 @@ --- Based on https://github.com/ClickHouse/ClickHouse/issues/49685 --- Verify that ReplacingMergeTree properly handles _is_deleted: ---- SELECT FINAL should take _is_deleted in consideration when there is only one partition. +--- SELECT FINAL should take `_is_deleted` into consideration when there is only one partition. -- { echoOn } +DROP TABLE IF EXISTS t; CREATE TABLE t ( `account_id` UInt64, @@ -13,18 +14,19 @@ ENGINE = ReplacingMergeTree(_version, _is_deleted) ORDER BY (account_id); INSERT INTO t SELECT number, 0, 1 FROM numbers(1e3); --- Mark first 100 rows as deleted. +-- Mark the first 100 rows as deleted. INSERT INTO t SELECT number, 1, 1 FROM numbers(1e2); --- Put everything is in one partition +-- Put everything in one partition OPTIMIZE TABLE t FINAL; SELECT count() FROM t; SELECT count() FROM t FINAL; --- Both should produce same number of rows. --- Previously `do_not_merge_across_partitions_select_final = 1` showed more rows, +-- Both should produce the same number of rows. +-- Previously, `do_not_merge_across_partitions_select_final = 1` showed more rows, -- as if no rows were deleted. SELECT count() FROM t FINAL SETTINGS do_not_merge_across_partitions_select_final = 1; SELECT count() FROM t FINAL SETTINGS do_not_merge_across_partitions_select_final = 0; +DROP TABLE t; From 7cd8237a42452cb57e09241e4a0556a68b129c34 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Tue, 22 Aug 2023 10:26:52 +0000 Subject: [PATCH 03/42] exceptions_kafka_consumers: initial --- src/Storages/Kafka/StorageKafka.cpp | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 54db0f29cb8..9ef3cbe8140 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -771,7 +771,27 @@ void StorageKafka::threadFunc(size_t idx) } catch (...) { - tryLogCurrentException(__PRETTY_FUNCTION__); + // !!! + // std::string getCurrentExceptionMessage(bool with_stacktrace, bool check_embedded_stacktrace /*= false*/, bool with_extra_info /*= true*/); + + auto last_exception = std::current_exception(); + tryLogException(last_exception, log, __PRETTY_FUNCTION__); + + auto exception_str = getExceptionMessage(last_exception, true /* with_stacktrace */); + + for (auto consumer_ptr_weak : all_consumers) + { + if (auto consumer_ptr = consumer_ptr_weak.lock()) + { + consumer_ptr->setExceptionInfo(exception_str); + } + } + + + + // tryLogCurrentException(__PRETTY_FUNCTION__); + + } mv_attached.store(false); From 741d98c964bd58793ff4817c1babb79761b2b533 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Wed, 23 Aug 2023 15:18:35 +0200 Subject: [PATCH 04/42] Updated reference to match the test after fixing typos --- ...ee_fix_select_final_on_single_partition.reference | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/02814_ReplacingMergeTree_fix_select_final_on_single_partition.reference b/tests/queries/0_stateless/02814_ReplacingMergeTree_fix_select_final_on_single_partition.reference index 0510348cb11..d19222b55ec 100644 --- a/tests/queries/0_stateless/02814_ReplacingMergeTree_fix_select_final_on_single_partition.reference +++ b/tests/queries/0_stateless/02814_ReplacingMergeTree_fix_select_final_on_single_partition.reference @@ -1,8 +1,9 @@ --- Based on https://github.com/ClickHouse/ClickHouse/issues/49685 --- Verify that ReplacingMergeTree properly handles _is_deleted: ---- SELECT FINAL should take _is_deleted in consideration when there is only one partition. +--- SELECT FINAL should take `_is_deleted` into consideration when there is only one partition. -- { echoOn } +DROP TABLE IF EXISTS t; CREATE TABLE t ( `account_id` UInt64, @@ -12,18 +13,19 @@ CREATE TABLE t ENGINE = ReplacingMergeTree(_version, _is_deleted) ORDER BY (account_id); INSERT INTO t SELECT number, 0, 1 FROM numbers(1e3); --- Mark first 100 rows as deleted. +-- Mark the first 100 rows as deleted. INSERT INTO t SELECT number, 1, 1 FROM numbers(1e2); --- Put everything is in one partition +-- Put everything in one partition OPTIMIZE TABLE t FINAL; SELECT count() FROM t; 1000 SELECT count() FROM t FINAL; 900 --- Both should produce same number of rows. --- Previously `do_not_merge_across_partitions_select_final = 1` showed more rows, +-- Both should produce the same number of rows. +-- Previously, `do_not_merge_across_partitions_select_final = 1` showed more rows, -- as if no rows were deleted. SELECT count() FROM t FINAL SETTINGS do_not_merge_across_partitions_select_final = 1; 900 SELECT count() FROM t FINAL SETTINGS do_not_merge_across_partitions_select_final = 0; 900 +DROP TABLE t; From aee8329a61210a3e65773bedcc8dc29fb9ac2f97 Mon Sep 17 00:00:00 2001 From: Bhavna Jindal Date: Wed, 16 Aug 2023 11:25:21 -0700 Subject: [PATCH 05/42] Updated openSSL to 3.0.10 to address CVEs --- contrib/openssl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/openssl b/contrib/openssl index 19cc035b6c6..245cb0291e0 160000 --- a/contrib/openssl +++ b/contrib/openssl @@ -1 +1 @@ -Subproject commit 19cc035b6c6f2283573d29c7ea7f7d675cf750ce +Subproject commit 245cb0291e0db99d9ccf3692fa76f440b2b054c2 From 6406043294bdf4763d93988db922a5a2c1ce61e8 Mon Sep 17 00:00:00 2001 From: Bhavna Jindal Date: Wed, 23 Aug 2023 08:33:01 -0700 Subject: [PATCH 06/42] Added openSSL header files for x86_64, ppc64le, s309x, aarch64 platforms --- .../linux_aarch64/include/openssl/cmp.h | 25 +++++++++++-------- .../linux_aarch64/include/openssl/opensslv.h | 12 ++++----- .../linux_aarch64/include/openssl/x509v3.h | 6 ++--- .../linux_ppc64le/include/openssl/cmp.h | 25 +++++++++++-------- .../linux_ppc64le/include/openssl/opensslv.h | 12 ++++----- .../linux_ppc64le/include/openssl/x509v3.h | 6 ++--- .../linux_s390x/include/openssl/cmp.h | 25 +++++++++++-------- .../linux_s390x/include/openssl/opensslv.h | 12 ++++----- .../linux_s390x/include/openssl/x509v3.h | 6 ++--- .../linux_x86_64/include/openssl/cmp.h | 25 +++++++++++-------- .../linux_x86_64/include/openssl/opensslv.h | 12 ++++----- .../linux_x86_64/include/openssl/x509v3.h | 6 ++--- 12 files changed, 96 insertions(+), 76 deletions(-) diff --git a/contrib/openssl-cmake/linux_aarch64/include/openssl/cmp.h b/contrib/openssl-cmake/linux_aarch64/include/openssl/cmp.h index 2476042c531..49825570d8c 100644 --- a/contrib/openssl-cmake/linux_aarch64/include/openssl/cmp.h +++ b/contrib/openssl-cmake/linux_aarch64/include/openssl/cmp.h @@ -1,8 +1,8 @@ /* * WARNING: do not edit! - * Generated by Makefile from ../include/openssl/cmp.h.in + * Generated by Makefile from include/openssl/cmp.h.in * - * Copyright 2007-2021 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2007-2023 The OpenSSL Project Authors. All Rights Reserved. * Copyright Nokia 2007-2019 * Copyright Siemens AG 2015-2019 * @@ -193,13 +193,16 @@ typedef ASN1_BIT_STRING OSSL_CMP_PKIFAILUREINFO; * -- CertReqMsg * } */ -# define OSSL_CMP_PKISTATUS_accepted 0 -# define OSSL_CMP_PKISTATUS_grantedWithMods 1 -# define OSSL_CMP_PKISTATUS_rejection 2 -# define OSSL_CMP_PKISTATUS_waiting 3 -# define OSSL_CMP_PKISTATUS_revocationWarning 4 +# define OSSL_CMP_PKISTATUS_request -3 +# define OSSL_CMP_PKISTATUS_trans -2 +# define OSSL_CMP_PKISTATUS_unspecified -1 +# define OSSL_CMP_PKISTATUS_accepted 0 +# define OSSL_CMP_PKISTATUS_grantedWithMods 1 +# define OSSL_CMP_PKISTATUS_rejection 2 +# define OSSL_CMP_PKISTATUS_waiting 3 +# define OSSL_CMP_PKISTATUS_revocationWarning 4 # define OSSL_CMP_PKISTATUS_revocationNotification 5 -# define OSSL_CMP_PKISTATUS_keyUpdateWarning 6 +# define OSSL_CMP_PKISTATUS_keyUpdateWarning 6 typedef ASN1_INTEGER OSSL_CMP_PKISTATUS; DECLARE_ASN1_ITEM(OSSL_CMP_PKISTATUS) @@ -439,11 +442,12 @@ int OSSL_CMP_CTX_build_cert_chain(OSSL_CMP_CTX *ctx, X509_STORE *own_trusted, int OSSL_CMP_CTX_set1_pkey(OSSL_CMP_CTX *ctx, EVP_PKEY *pkey); int OSSL_CMP_CTX_set1_referenceValue(OSSL_CMP_CTX *ctx, const unsigned char *ref, int len); -int OSSL_CMP_CTX_set1_secretValue(OSSL_CMP_CTX *ctx, const unsigned char *sec, - const int len); +int OSSL_CMP_CTX_set1_secretValue(OSSL_CMP_CTX *ctx, + const unsigned char *sec, int len); /* CMP message header and extra certificates: */ int OSSL_CMP_CTX_set1_recipient(OSSL_CMP_CTX *ctx, const X509_NAME *name); int OSSL_CMP_CTX_push0_geninfo_ITAV(OSSL_CMP_CTX *ctx, OSSL_CMP_ITAV *itav); +int OSSL_CMP_CTX_reset_geninfo_ITAVs(OSSL_CMP_CTX *ctx); int OSSL_CMP_CTX_set1_extraCertsOut(OSSL_CMP_CTX *ctx, STACK_OF(X509) *extraCertsOut); /* certificate template: */ @@ -499,6 +503,7 @@ ASN1_OCTET_STRING *OSSL_CMP_HDR_get0_recipNonce(const OSSL_CMP_PKIHEADER *hdr); OSSL_CMP_PKIHEADER *OSSL_CMP_MSG_get0_header(const OSSL_CMP_MSG *msg); int OSSL_CMP_MSG_get_bodytype(const OSSL_CMP_MSG *msg); int OSSL_CMP_MSG_update_transactionID(OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg); +int OSSL_CMP_MSG_update_recipNonce(OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg); OSSL_CRMF_MSG *OSSL_CMP_CTX_setup_CRM(OSSL_CMP_CTX *ctx, int for_KUR, int rid); OSSL_CMP_MSG *OSSL_CMP_MSG_read(const char *file, OSSL_LIB_CTX *libctx, const char *propq); diff --git a/contrib/openssl-cmake/linux_aarch64/include/openssl/opensslv.h b/contrib/openssl-cmake/linux_aarch64/include/openssl/opensslv.h index 81c1b93afaa..3c221e1ac23 100644 --- a/contrib/openssl-cmake/linux_aarch64/include/openssl/opensslv.h +++ b/contrib/openssl-cmake/linux_aarch64/include/openssl/opensslv.h @@ -1,6 +1,6 @@ /* * WARNING: do not edit! - * Generated by Makefile from ../include/openssl/opensslv.h.in + * Generated by Makefile from include/openssl/opensslv.h.in * * Copyright 1999-2020 The OpenSSL Project Authors. All Rights Reserved. * @@ -29,7 +29,7 @@ extern "C" { */ # define OPENSSL_VERSION_MAJOR 3 # define OPENSSL_VERSION_MINOR 0 -# define OPENSSL_VERSION_PATCH 7 +# define OPENSSL_VERSION_PATCH 10 /* * Additional version information @@ -74,21 +74,21 @@ extern "C" { * longer variant with OPENSSL_VERSION_PRE_RELEASE_STR and * OPENSSL_VERSION_BUILD_METADATA_STR appended. */ -# define OPENSSL_VERSION_STR "3.0.7" -# define OPENSSL_FULL_VERSION_STR "3.0.7" +# define OPENSSL_VERSION_STR "3.0.10" +# define OPENSSL_FULL_VERSION_STR "3.0.10" /* * SECTION 3: ADDITIONAL METADATA * * These strings are defined separately to allow them to be parsable. */ -# define OPENSSL_RELEASE_DATE "1 Nov 2022" +# define OPENSSL_RELEASE_DATE "1 Aug 2023" /* * SECTION 4: BACKWARD COMPATIBILITY */ -# define OPENSSL_VERSION_TEXT "OpenSSL 3.0.7 1 Nov 2022" +# define OPENSSL_VERSION_TEXT "OpenSSL 3.0.10 1 Aug 2023" /* Synthesize OPENSSL_VERSION_NUMBER with the layout 0xMNN00PPSL */ # ifdef OPENSSL_VERSION_PRE_RELEASE diff --git a/contrib/openssl-cmake/linux_aarch64/include/openssl/x509v3.h b/contrib/openssl-cmake/linux_aarch64/include/openssl/x509v3.h index fb4b49ca349..20b67455f20 100644 --- a/contrib/openssl-cmake/linux_aarch64/include/openssl/x509v3.h +++ b/contrib/openssl-cmake/linux_aarch64/include/openssl/x509v3.h @@ -1,8 +1,8 @@ /* * WARNING: do not edit! - * Generated by Makefile from ../include/openssl/x509v3.h.in + * Generated by Makefile from include/openssl/x509v3.h.in * - * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -177,7 +177,7 @@ typedef struct GENERAL_NAME_st { OTHERNAME *otherName; /* otherName */ ASN1_IA5STRING *rfc822Name; ASN1_IA5STRING *dNSName; - ASN1_TYPE *x400Address; + ASN1_STRING *x400Address; X509_NAME *directoryName; EDIPARTYNAME *ediPartyName; ASN1_IA5STRING *uniformResourceIdentifier; diff --git a/contrib/openssl-cmake/linux_ppc64le/include/openssl/cmp.h b/contrib/openssl-cmake/linux_ppc64le/include/openssl/cmp.h index 2476042c531..49825570d8c 100644 --- a/contrib/openssl-cmake/linux_ppc64le/include/openssl/cmp.h +++ b/contrib/openssl-cmake/linux_ppc64le/include/openssl/cmp.h @@ -1,8 +1,8 @@ /* * WARNING: do not edit! - * Generated by Makefile from ../include/openssl/cmp.h.in + * Generated by Makefile from include/openssl/cmp.h.in * - * Copyright 2007-2021 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2007-2023 The OpenSSL Project Authors. All Rights Reserved. * Copyright Nokia 2007-2019 * Copyright Siemens AG 2015-2019 * @@ -193,13 +193,16 @@ typedef ASN1_BIT_STRING OSSL_CMP_PKIFAILUREINFO; * -- CertReqMsg * } */ -# define OSSL_CMP_PKISTATUS_accepted 0 -# define OSSL_CMP_PKISTATUS_grantedWithMods 1 -# define OSSL_CMP_PKISTATUS_rejection 2 -# define OSSL_CMP_PKISTATUS_waiting 3 -# define OSSL_CMP_PKISTATUS_revocationWarning 4 +# define OSSL_CMP_PKISTATUS_request -3 +# define OSSL_CMP_PKISTATUS_trans -2 +# define OSSL_CMP_PKISTATUS_unspecified -1 +# define OSSL_CMP_PKISTATUS_accepted 0 +# define OSSL_CMP_PKISTATUS_grantedWithMods 1 +# define OSSL_CMP_PKISTATUS_rejection 2 +# define OSSL_CMP_PKISTATUS_waiting 3 +# define OSSL_CMP_PKISTATUS_revocationWarning 4 # define OSSL_CMP_PKISTATUS_revocationNotification 5 -# define OSSL_CMP_PKISTATUS_keyUpdateWarning 6 +# define OSSL_CMP_PKISTATUS_keyUpdateWarning 6 typedef ASN1_INTEGER OSSL_CMP_PKISTATUS; DECLARE_ASN1_ITEM(OSSL_CMP_PKISTATUS) @@ -439,11 +442,12 @@ int OSSL_CMP_CTX_build_cert_chain(OSSL_CMP_CTX *ctx, X509_STORE *own_trusted, int OSSL_CMP_CTX_set1_pkey(OSSL_CMP_CTX *ctx, EVP_PKEY *pkey); int OSSL_CMP_CTX_set1_referenceValue(OSSL_CMP_CTX *ctx, const unsigned char *ref, int len); -int OSSL_CMP_CTX_set1_secretValue(OSSL_CMP_CTX *ctx, const unsigned char *sec, - const int len); +int OSSL_CMP_CTX_set1_secretValue(OSSL_CMP_CTX *ctx, + const unsigned char *sec, int len); /* CMP message header and extra certificates: */ int OSSL_CMP_CTX_set1_recipient(OSSL_CMP_CTX *ctx, const X509_NAME *name); int OSSL_CMP_CTX_push0_geninfo_ITAV(OSSL_CMP_CTX *ctx, OSSL_CMP_ITAV *itav); +int OSSL_CMP_CTX_reset_geninfo_ITAVs(OSSL_CMP_CTX *ctx); int OSSL_CMP_CTX_set1_extraCertsOut(OSSL_CMP_CTX *ctx, STACK_OF(X509) *extraCertsOut); /* certificate template: */ @@ -499,6 +503,7 @@ ASN1_OCTET_STRING *OSSL_CMP_HDR_get0_recipNonce(const OSSL_CMP_PKIHEADER *hdr); OSSL_CMP_PKIHEADER *OSSL_CMP_MSG_get0_header(const OSSL_CMP_MSG *msg); int OSSL_CMP_MSG_get_bodytype(const OSSL_CMP_MSG *msg); int OSSL_CMP_MSG_update_transactionID(OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg); +int OSSL_CMP_MSG_update_recipNonce(OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg); OSSL_CRMF_MSG *OSSL_CMP_CTX_setup_CRM(OSSL_CMP_CTX *ctx, int for_KUR, int rid); OSSL_CMP_MSG *OSSL_CMP_MSG_read(const char *file, OSSL_LIB_CTX *libctx, const char *propq); diff --git a/contrib/openssl-cmake/linux_ppc64le/include/openssl/opensslv.h b/contrib/openssl-cmake/linux_ppc64le/include/openssl/opensslv.h index 81c1b93afaa..3c221e1ac23 100644 --- a/contrib/openssl-cmake/linux_ppc64le/include/openssl/opensslv.h +++ b/contrib/openssl-cmake/linux_ppc64le/include/openssl/opensslv.h @@ -1,6 +1,6 @@ /* * WARNING: do not edit! - * Generated by Makefile from ../include/openssl/opensslv.h.in + * Generated by Makefile from include/openssl/opensslv.h.in * * Copyright 1999-2020 The OpenSSL Project Authors. All Rights Reserved. * @@ -29,7 +29,7 @@ extern "C" { */ # define OPENSSL_VERSION_MAJOR 3 # define OPENSSL_VERSION_MINOR 0 -# define OPENSSL_VERSION_PATCH 7 +# define OPENSSL_VERSION_PATCH 10 /* * Additional version information @@ -74,21 +74,21 @@ extern "C" { * longer variant with OPENSSL_VERSION_PRE_RELEASE_STR and * OPENSSL_VERSION_BUILD_METADATA_STR appended. */ -# define OPENSSL_VERSION_STR "3.0.7" -# define OPENSSL_FULL_VERSION_STR "3.0.7" +# define OPENSSL_VERSION_STR "3.0.10" +# define OPENSSL_FULL_VERSION_STR "3.0.10" /* * SECTION 3: ADDITIONAL METADATA * * These strings are defined separately to allow them to be parsable. */ -# define OPENSSL_RELEASE_DATE "1 Nov 2022" +# define OPENSSL_RELEASE_DATE "1 Aug 2023" /* * SECTION 4: BACKWARD COMPATIBILITY */ -# define OPENSSL_VERSION_TEXT "OpenSSL 3.0.7 1 Nov 2022" +# define OPENSSL_VERSION_TEXT "OpenSSL 3.0.10 1 Aug 2023" /* Synthesize OPENSSL_VERSION_NUMBER with the layout 0xMNN00PPSL */ # ifdef OPENSSL_VERSION_PRE_RELEASE diff --git a/contrib/openssl-cmake/linux_ppc64le/include/openssl/x509v3.h b/contrib/openssl-cmake/linux_ppc64le/include/openssl/x509v3.h index fb4b49ca349..20b67455f20 100644 --- a/contrib/openssl-cmake/linux_ppc64le/include/openssl/x509v3.h +++ b/contrib/openssl-cmake/linux_ppc64le/include/openssl/x509v3.h @@ -1,8 +1,8 @@ /* * WARNING: do not edit! - * Generated by Makefile from ../include/openssl/x509v3.h.in + * Generated by Makefile from include/openssl/x509v3.h.in * - * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -177,7 +177,7 @@ typedef struct GENERAL_NAME_st { OTHERNAME *otherName; /* otherName */ ASN1_IA5STRING *rfc822Name; ASN1_IA5STRING *dNSName; - ASN1_TYPE *x400Address; + ASN1_STRING *x400Address; X509_NAME *directoryName; EDIPARTYNAME *ediPartyName; ASN1_IA5STRING *uniformResourceIdentifier; diff --git a/contrib/openssl-cmake/linux_s390x/include/openssl/cmp.h b/contrib/openssl-cmake/linux_s390x/include/openssl/cmp.h index 2476042c531..49825570d8c 100644 --- a/contrib/openssl-cmake/linux_s390x/include/openssl/cmp.h +++ b/contrib/openssl-cmake/linux_s390x/include/openssl/cmp.h @@ -1,8 +1,8 @@ /* * WARNING: do not edit! - * Generated by Makefile from ../include/openssl/cmp.h.in + * Generated by Makefile from include/openssl/cmp.h.in * - * Copyright 2007-2021 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2007-2023 The OpenSSL Project Authors. All Rights Reserved. * Copyright Nokia 2007-2019 * Copyright Siemens AG 2015-2019 * @@ -193,13 +193,16 @@ typedef ASN1_BIT_STRING OSSL_CMP_PKIFAILUREINFO; * -- CertReqMsg * } */ -# define OSSL_CMP_PKISTATUS_accepted 0 -# define OSSL_CMP_PKISTATUS_grantedWithMods 1 -# define OSSL_CMP_PKISTATUS_rejection 2 -# define OSSL_CMP_PKISTATUS_waiting 3 -# define OSSL_CMP_PKISTATUS_revocationWarning 4 +# define OSSL_CMP_PKISTATUS_request -3 +# define OSSL_CMP_PKISTATUS_trans -2 +# define OSSL_CMP_PKISTATUS_unspecified -1 +# define OSSL_CMP_PKISTATUS_accepted 0 +# define OSSL_CMP_PKISTATUS_grantedWithMods 1 +# define OSSL_CMP_PKISTATUS_rejection 2 +# define OSSL_CMP_PKISTATUS_waiting 3 +# define OSSL_CMP_PKISTATUS_revocationWarning 4 # define OSSL_CMP_PKISTATUS_revocationNotification 5 -# define OSSL_CMP_PKISTATUS_keyUpdateWarning 6 +# define OSSL_CMP_PKISTATUS_keyUpdateWarning 6 typedef ASN1_INTEGER OSSL_CMP_PKISTATUS; DECLARE_ASN1_ITEM(OSSL_CMP_PKISTATUS) @@ -439,11 +442,12 @@ int OSSL_CMP_CTX_build_cert_chain(OSSL_CMP_CTX *ctx, X509_STORE *own_trusted, int OSSL_CMP_CTX_set1_pkey(OSSL_CMP_CTX *ctx, EVP_PKEY *pkey); int OSSL_CMP_CTX_set1_referenceValue(OSSL_CMP_CTX *ctx, const unsigned char *ref, int len); -int OSSL_CMP_CTX_set1_secretValue(OSSL_CMP_CTX *ctx, const unsigned char *sec, - const int len); +int OSSL_CMP_CTX_set1_secretValue(OSSL_CMP_CTX *ctx, + const unsigned char *sec, int len); /* CMP message header and extra certificates: */ int OSSL_CMP_CTX_set1_recipient(OSSL_CMP_CTX *ctx, const X509_NAME *name); int OSSL_CMP_CTX_push0_geninfo_ITAV(OSSL_CMP_CTX *ctx, OSSL_CMP_ITAV *itav); +int OSSL_CMP_CTX_reset_geninfo_ITAVs(OSSL_CMP_CTX *ctx); int OSSL_CMP_CTX_set1_extraCertsOut(OSSL_CMP_CTX *ctx, STACK_OF(X509) *extraCertsOut); /* certificate template: */ @@ -499,6 +503,7 @@ ASN1_OCTET_STRING *OSSL_CMP_HDR_get0_recipNonce(const OSSL_CMP_PKIHEADER *hdr); OSSL_CMP_PKIHEADER *OSSL_CMP_MSG_get0_header(const OSSL_CMP_MSG *msg); int OSSL_CMP_MSG_get_bodytype(const OSSL_CMP_MSG *msg); int OSSL_CMP_MSG_update_transactionID(OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg); +int OSSL_CMP_MSG_update_recipNonce(OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg); OSSL_CRMF_MSG *OSSL_CMP_CTX_setup_CRM(OSSL_CMP_CTX *ctx, int for_KUR, int rid); OSSL_CMP_MSG *OSSL_CMP_MSG_read(const char *file, OSSL_LIB_CTX *libctx, const char *propq); diff --git a/contrib/openssl-cmake/linux_s390x/include/openssl/opensslv.h b/contrib/openssl-cmake/linux_s390x/include/openssl/opensslv.h index 81c1b93afaa..3c221e1ac23 100644 --- a/contrib/openssl-cmake/linux_s390x/include/openssl/opensslv.h +++ b/contrib/openssl-cmake/linux_s390x/include/openssl/opensslv.h @@ -1,6 +1,6 @@ /* * WARNING: do not edit! - * Generated by Makefile from ../include/openssl/opensslv.h.in + * Generated by Makefile from include/openssl/opensslv.h.in * * Copyright 1999-2020 The OpenSSL Project Authors. All Rights Reserved. * @@ -29,7 +29,7 @@ extern "C" { */ # define OPENSSL_VERSION_MAJOR 3 # define OPENSSL_VERSION_MINOR 0 -# define OPENSSL_VERSION_PATCH 7 +# define OPENSSL_VERSION_PATCH 10 /* * Additional version information @@ -74,21 +74,21 @@ extern "C" { * longer variant with OPENSSL_VERSION_PRE_RELEASE_STR and * OPENSSL_VERSION_BUILD_METADATA_STR appended. */ -# define OPENSSL_VERSION_STR "3.0.7" -# define OPENSSL_FULL_VERSION_STR "3.0.7" +# define OPENSSL_VERSION_STR "3.0.10" +# define OPENSSL_FULL_VERSION_STR "3.0.10" /* * SECTION 3: ADDITIONAL METADATA * * These strings are defined separately to allow them to be parsable. */ -# define OPENSSL_RELEASE_DATE "1 Nov 2022" +# define OPENSSL_RELEASE_DATE "1 Aug 2023" /* * SECTION 4: BACKWARD COMPATIBILITY */ -# define OPENSSL_VERSION_TEXT "OpenSSL 3.0.7 1 Nov 2022" +# define OPENSSL_VERSION_TEXT "OpenSSL 3.0.10 1 Aug 2023" /* Synthesize OPENSSL_VERSION_NUMBER with the layout 0xMNN00PPSL */ # ifdef OPENSSL_VERSION_PRE_RELEASE diff --git a/contrib/openssl-cmake/linux_s390x/include/openssl/x509v3.h b/contrib/openssl-cmake/linux_s390x/include/openssl/x509v3.h index fb4b49ca349..20b67455f20 100644 --- a/contrib/openssl-cmake/linux_s390x/include/openssl/x509v3.h +++ b/contrib/openssl-cmake/linux_s390x/include/openssl/x509v3.h @@ -1,8 +1,8 @@ /* * WARNING: do not edit! - * Generated by Makefile from ../include/openssl/x509v3.h.in + * Generated by Makefile from include/openssl/x509v3.h.in * - * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -177,7 +177,7 @@ typedef struct GENERAL_NAME_st { OTHERNAME *otherName; /* otherName */ ASN1_IA5STRING *rfc822Name; ASN1_IA5STRING *dNSName; - ASN1_TYPE *x400Address; + ASN1_STRING *x400Address; X509_NAME *directoryName; EDIPARTYNAME *ediPartyName; ASN1_IA5STRING *uniformResourceIdentifier; diff --git a/contrib/openssl-cmake/linux_x86_64/include/openssl/cmp.h b/contrib/openssl-cmake/linux_x86_64/include/openssl/cmp.h index 2476042c531..49825570d8c 100644 --- a/contrib/openssl-cmake/linux_x86_64/include/openssl/cmp.h +++ b/contrib/openssl-cmake/linux_x86_64/include/openssl/cmp.h @@ -1,8 +1,8 @@ /* * WARNING: do not edit! - * Generated by Makefile from ../include/openssl/cmp.h.in + * Generated by Makefile from include/openssl/cmp.h.in * - * Copyright 2007-2021 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2007-2023 The OpenSSL Project Authors. All Rights Reserved. * Copyright Nokia 2007-2019 * Copyright Siemens AG 2015-2019 * @@ -193,13 +193,16 @@ typedef ASN1_BIT_STRING OSSL_CMP_PKIFAILUREINFO; * -- CertReqMsg * } */ -# define OSSL_CMP_PKISTATUS_accepted 0 -# define OSSL_CMP_PKISTATUS_grantedWithMods 1 -# define OSSL_CMP_PKISTATUS_rejection 2 -# define OSSL_CMP_PKISTATUS_waiting 3 -# define OSSL_CMP_PKISTATUS_revocationWarning 4 +# define OSSL_CMP_PKISTATUS_request -3 +# define OSSL_CMP_PKISTATUS_trans -2 +# define OSSL_CMP_PKISTATUS_unspecified -1 +# define OSSL_CMP_PKISTATUS_accepted 0 +# define OSSL_CMP_PKISTATUS_grantedWithMods 1 +# define OSSL_CMP_PKISTATUS_rejection 2 +# define OSSL_CMP_PKISTATUS_waiting 3 +# define OSSL_CMP_PKISTATUS_revocationWarning 4 # define OSSL_CMP_PKISTATUS_revocationNotification 5 -# define OSSL_CMP_PKISTATUS_keyUpdateWarning 6 +# define OSSL_CMP_PKISTATUS_keyUpdateWarning 6 typedef ASN1_INTEGER OSSL_CMP_PKISTATUS; DECLARE_ASN1_ITEM(OSSL_CMP_PKISTATUS) @@ -439,11 +442,12 @@ int OSSL_CMP_CTX_build_cert_chain(OSSL_CMP_CTX *ctx, X509_STORE *own_trusted, int OSSL_CMP_CTX_set1_pkey(OSSL_CMP_CTX *ctx, EVP_PKEY *pkey); int OSSL_CMP_CTX_set1_referenceValue(OSSL_CMP_CTX *ctx, const unsigned char *ref, int len); -int OSSL_CMP_CTX_set1_secretValue(OSSL_CMP_CTX *ctx, const unsigned char *sec, - const int len); +int OSSL_CMP_CTX_set1_secretValue(OSSL_CMP_CTX *ctx, + const unsigned char *sec, int len); /* CMP message header and extra certificates: */ int OSSL_CMP_CTX_set1_recipient(OSSL_CMP_CTX *ctx, const X509_NAME *name); int OSSL_CMP_CTX_push0_geninfo_ITAV(OSSL_CMP_CTX *ctx, OSSL_CMP_ITAV *itav); +int OSSL_CMP_CTX_reset_geninfo_ITAVs(OSSL_CMP_CTX *ctx); int OSSL_CMP_CTX_set1_extraCertsOut(OSSL_CMP_CTX *ctx, STACK_OF(X509) *extraCertsOut); /* certificate template: */ @@ -499,6 +503,7 @@ ASN1_OCTET_STRING *OSSL_CMP_HDR_get0_recipNonce(const OSSL_CMP_PKIHEADER *hdr); OSSL_CMP_PKIHEADER *OSSL_CMP_MSG_get0_header(const OSSL_CMP_MSG *msg); int OSSL_CMP_MSG_get_bodytype(const OSSL_CMP_MSG *msg); int OSSL_CMP_MSG_update_transactionID(OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg); +int OSSL_CMP_MSG_update_recipNonce(OSSL_CMP_CTX *ctx, OSSL_CMP_MSG *msg); OSSL_CRMF_MSG *OSSL_CMP_CTX_setup_CRM(OSSL_CMP_CTX *ctx, int for_KUR, int rid); OSSL_CMP_MSG *OSSL_CMP_MSG_read(const char *file, OSSL_LIB_CTX *libctx, const char *propq); diff --git a/contrib/openssl-cmake/linux_x86_64/include/openssl/opensslv.h b/contrib/openssl-cmake/linux_x86_64/include/openssl/opensslv.h index 81c1b93afaa..3c221e1ac23 100644 --- a/contrib/openssl-cmake/linux_x86_64/include/openssl/opensslv.h +++ b/contrib/openssl-cmake/linux_x86_64/include/openssl/opensslv.h @@ -1,6 +1,6 @@ /* * WARNING: do not edit! - * Generated by Makefile from ../include/openssl/opensslv.h.in + * Generated by Makefile from include/openssl/opensslv.h.in * * Copyright 1999-2020 The OpenSSL Project Authors. All Rights Reserved. * @@ -29,7 +29,7 @@ extern "C" { */ # define OPENSSL_VERSION_MAJOR 3 # define OPENSSL_VERSION_MINOR 0 -# define OPENSSL_VERSION_PATCH 7 +# define OPENSSL_VERSION_PATCH 10 /* * Additional version information @@ -74,21 +74,21 @@ extern "C" { * longer variant with OPENSSL_VERSION_PRE_RELEASE_STR and * OPENSSL_VERSION_BUILD_METADATA_STR appended. */ -# define OPENSSL_VERSION_STR "3.0.7" -# define OPENSSL_FULL_VERSION_STR "3.0.7" +# define OPENSSL_VERSION_STR "3.0.10" +# define OPENSSL_FULL_VERSION_STR "3.0.10" /* * SECTION 3: ADDITIONAL METADATA * * These strings are defined separately to allow them to be parsable. */ -# define OPENSSL_RELEASE_DATE "1 Nov 2022" +# define OPENSSL_RELEASE_DATE "1 Aug 2023" /* * SECTION 4: BACKWARD COMPATIBILITY */ -# define OPENSSL_VERSION_TEXT "OpenSSL 3.0.7 1 Nov 2022" +# define OPENSSL_VERSION_TEXT "OpenSSL 3.0.10 1 Aug 2023" /* Synthesize OPENSSL_VERSION_NUMBER with the layout 0xMNN00PPSL */ # ifdef OPENSSL_VERSION_PRE_RELEASE diff --git a/contrib/openssl-cmake/linux_x86_64/include/openssl/x509v3.h b/contrib/openssl-cmake/linux_x86_64/include/openssl/x509v3.h index fb4b49ca349..20b67455f20 100644 --- a/contrib/openssl-cmake/linux_x86_64/include/openssl/x509v3.h +++ b/contrib/openssl-cmake/linux_x86_64/include/openssl/x509v3.h @@ -1,8 +1,8 @@ /* * WARNING: do not edit! - * Generated by Makefile from ../include/openssl/x509v3.h.in + * Generated by Makefile from include/openssl/x509v3.h.in * - * Copyright 1999-2021 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 1999-2023 The OpenSSL Project Authors. All Rights Reserved. * * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy @@ -177,7 +177,7 @@ typedef struct GENERAL_NAME_st { OTHERNAME *otherName; /* otherName */ ASN1_IA5STRING *rfc822Name; ASN1_IA5STRING *dNSName; - ASN1_TYPE *x400Address; + ASN1_STRING *x400Address; X509_NAME *directoryName; EDIPARTYNAME *ediPartyName; ASN1_IA5STRING *uniformResourceIdentifier; From 7f31258956a85bb0494cff526eb06804bf50effb Mon Sep 17 00:00:00 2001 From: ltrk2 <107155950+ltrk2@users.noreply.github.com> Date: Wed, 23 Aug 2023 11:14:06 -0700 Subject: [PATCH 07/42] Enable using simdjson on s390x --- contrib/CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 2557ebf78ae..7b98cf43b09 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -136,9 +136,7 @@ add_contrib (aws-cmake ) add_contrib (base64-cmake base64) -if (NOT ARCH_S390X) add_contrib (simdjson-cmake simdjson) -endif() add_contrib (rapidjson-cmake rapidjson) add_contrib (fastops-cmake fastops) add_contrib (libuv-cmake libuv) From 8d8490ac5805c01b51603eec4333d2326febbece Mon Sep 17 00:00:00 2001 From: ltrk2 <107155950+ltrk2@users.noreply.github.com> Date: Wed, 23 Aug 2023 11:14:34 -0700 Subject: [PATCH 08/42] Implement endianness-independent serialization for UUIDs --- .../Serializations/SerializationUUID.cpp | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/DataTypes/Serializations/SerializationUUID.cpp b/src/DataTypes/Serializations/SerializationUUID.cpp index 93658fd05a3..947394a2bd8 100644 --- a/src/DataTypes/Serializations/SerializationUUID.cpp +++ b/src/DataTypes/Serializations/SerializationUUID.cpp @@ -7,6 +7,7 @@ #include #include +#include namespace DB { @@ -136,23 +137,31 @@ void SerializationUUID::deserializeBinary(IColumn & column, ReadBuffer & istr, c void SerializationUUID::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const { const typename ColumnVector::Container & x = typeid_cast &>(column).getData(); - - size_t size = x.size(); - - if (limit == 0 || offset + limit > size) + if (const size_t size = x.size(); limit == 0 || offset + limit > size) limit = size - offset; - if (limit) + if (limit == 0) + return; + + if constexpr (std::endian::native == std::endian::big) + { + std::ranges::for_each( + x | std::views::drop(offset) | std::views::take(limit), [&ostr](const auto & uuid) { writeBinaryLittleEndian(uuid, ostr); }); + } + else ostr.write(reinterpret_cast(&x[offset]), sizeof(UUID) * limit); } void SerializationUUID::deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double /*avg_value_size_hint*/) const { typename ColumnVector::Container & x = typeid_cast &>(column).getData(); - size_t initial_size = x.size(); + const size_t initial_size = x.size(); x.resize(initial_size + limit); - size_t size = istr.readBig(reinterpret_cast(&x[initial_size]), sizeof(UUID) * limit); + const size_t size = istr.readBig(reinterpret_cast(&x[initial_size]), sizeof(UUID) * limit); x.resize(initial_size + size / sizeof(UUID)); -} + if constexpr (std::endian::native == std::endian::big) + std::ranges::for_each( + x | std::views::drop(initial_size), [](auto & uuid) { transformEndianness(uuid); }); +} } From 8cd56d8c0889352e410675376534f53961b499b0 Mon Sep 17 00:00:00 2001 From: ltrk2 <107155950+ltrk2@users.noreply.github.com> Date: Wed, 23 Aug 2023 11:14:49 -0700 Subject: [PATCH 09/42] Improve existing serialization implementations --- .../SerializationDecimalBase.cpp | 37 ++++++------------- .../Serializations/SerializationNumber.cpp | 9 +---- 2 files changed, 13 insertions(+), 33 deletions(-) diff --git a/src/DataTypes/Serializations/SerializationDecimalBase.cpp b/src/DataTypes/Serializations/SerializationDecimalBase.cpp index 494d0aa9168..37edfffc0d4 100644 --- a/src/DataTypes/Serializations/SerializationDecimalBase.cpp +++ b/src/DataTypes/Serializations/SerializationDecimalBase.cpp @@ -7,6 +7,7 @@ #include #include +#include namespace DB { @@ -29,21 +30,13 @@ template void SerializationDecimalBase::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const { const typename ColumnType::Container & x = typeid_cast(column).getData(); - - size_t size = x.size(); - - if (limit == 0 || offset + limit > size) + if (const size_t size = x.size(); limit == 0 || offset + limit > size) limit = size - offset; - if constexpr (std::endian::native == std::endian::big && sizeof(T) >= 2) + + if constexpr (std::endian::native == std::endian::big) { - for (size_t i = 0; i < limit; i++) - { - auto tmp(x[offset+i]); - char *start = reinterpret_cast(&tmp); - char *end = start + sizeof(FieldType); - std::reverse(start, end); - ostr.write(reinterpret_cast(&tmp), sizeof(FieldType)); - } + std::ranges::for_each( + x | std::views::drop(offset) | std::views::take(limit), [&ostr](const auto & d) { writeBinaryLittleEndian(d, ostr); }); } else ostr.write(reinterpret_cast(&x[offset]), sizeof(FieldType) * limit); @@ -69,20 +62,14 @@ template void SerializationDecimalBase::deserializeBinaryBulk(IColumn & column, ReadBuffer & istr, size_t limit, double) const { typename ColumnType::Container & x = typeid_cast(column).getData(); - size_t initial_size = x.size(); + const size_t initial_size = x.size(); x.resize(initial_size + limit); - size_t size = istr.readBig(reinterpret_cast(&x[initial_size]), sizeof(FieldType) * limit); - if constexpr (std::endian::native == std::endian::big && sizeof(T) >= 2) - { - for (size_t i = 0; i < limit; i++) - { - char *start = reinterpret_cast(&x[initial_size + i]); - char *end = start + sizeof(FieldType); - std::reverse(start, end); - } - } - + const size_t size = istr.readBig(reinterpret_cast(&x[initial_size]), sizeof(FieldType) * limit); x.resize(initial_size + size / sizeof(FieldType)); + + if constexpr (std::endian::native == std::endian::big) + std::ranges::for_each( + x | std::views::drop(initial_size), [](auto & d) { transformEndianness(d); }); } template class SerializationDecimalBase; diff --git a/src/DataTypes/Serializations/SerializationNumber.cpp b/src/DataTypes/Serializations/SerializationNumber.cpp index df6c0848bbe..fc3a5f0db24 100644 --- a/src/DataTypes/Serializations/SerializationNumber.cpp +++ b/src/DataTypes/Serializations/SerializationNumber.cpp @@ -145,15 +145,8 @@ void SerializationNumber::serializeBinaryBulk(const IColumn & column, WriteBu if constexpr (std::endian::native == std::endian::big && sizeof(T) >= 2) { - static constexpr auto to_little_endian = [](auto i) - { - transformEndianness(i); - return i; - }; - std::ranges::for_each( - x | std::views::drop(offset) | std::views::take(limit) | std::views::transform(to_little_endian), - [&ostr](const auto & i) { ostr.write(reinterpret_cast(&i), sizeof(typename ColumnVector::ValueType)); }); + x | std::views::drop(offset) | std::views::take(limit), [&ostr](const auto & i) { writeBinaryLittleEndian(i, ostr); }); } else ostr.write(reinterpret_cast(&x[offset]), sizeof(typename ColumnVector::ValueType) * limit); From e7213ef5b96941200a22fac39434e195fd727e7d Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 23 Aug 2023 22:46:04 +0000 Subject: [PATCH 10/42] exceptions_kafka_consumers: new test, MV exceptions propagated --- src/Storages/Kafka/KafkaConsumer.cpp | 26 ++++++-- src/Storages/Kafka/KafkaConsumer.h | 4 +- src/Storages/Kafka/StorageKafka.cpp | 27 ++++----- .../test_kafka_bad_messages/test.py | 59 ++++++++++++++++++- 4 files changed, 94 insertions(+), 22 deletions(-) diff --git a/src/Storages/Kafka/KafkaConsumer.cpp b/src/Storages/Kafka/KafkaConsumer.cpp index 9e558940012..31ec322fda8 100644 --- a/src/Storages/Kafka/KafkaConsumer.cpp +++ b/src/Storages/Kafka/KafkaConsumer.cpp @@ -542,15 +542,33 @@ void KafkaConsumer::storeLastReadMessageOffset() } } -void KafkaConsumer::setExceptionInfo(const cppkafka::Error & err) +void KafkaConsumer::setExceptionInfo(const cppkafka::Error & err, bool with_stacktrace) { - setExceptionInfo(err.to_string()); + setExceptionInfo(err.to_string(), with_stacktrace); } -void KafkaConsumer::setExceptionInfo(const String & text) +void KafkaConsumer::setExceptionInfo(const std::string & text, bool with_stacktrace) { + std::string exceptionWithTrace; + + if (with_stacktrace) + { + try + { + throw Exception(); + } + catch(std::exception & ex) + { + exceptionWithTrace = text + getExceptionStackTraceString(ex); + } + } + else + { + exceptionWithTrace = text; + } + std::lock_guard lock(exception_mutex); - exceptions_buffer.push_back({text, static_cast(Poco::Timestamp().epochTime())}); + exceptions_buffer.push_back({exceptionWithTrace, static_cast(Poco::Timestamp().epochTime())}); } /* diff --git a/src/Storages/Kafka/KafkaConsumer.h b/src/Storages/Kafka/KafkaConsumer.h index 91bb2ae8d77..1c3ddd85873 100644 --- a/src/Storages/Kafka/KafkaConsumer.h +++ b/src/Storages/Kafka/KafkaConsumer.h @@ -105,8 +105,8 @@ public: auto currentTimestamp() const { return current[-1].get_timestamp(); } const auto & currentHeaderList() const { return current[-1].get_header_list(); } String currentPayload() const { return current[-1].get_payload(); } - void setExceptionInfo(const cppkafka::Error & err); - void setExceptionInfo(const String & text); + void setExceptionInfo(const cppkafka::Error & err, bool with_stacktrace = true); + void setExceptionInfo(const std::string & text, bool with_stacktrace = true); void setRDKafkaStat(const std::string & stat_json_string) { std::lock_guard lock(rdkafka_stat_mutex); diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 9ef3cbe8140..4aca84d1f26 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -732,6 +732,8 @@ void StorageKafka::threadFunc(size_t idx) { assert(idx < tasks.size()); auto task = tasks[idx]; + std::string exception_str; + try { auto table_id = getStorageID(); @@ -771,27 +773,24 @@ void StorageKafka::threadFunc(size_t idx) } catch (...) { - // !!! - // std::string getCurrentExceptionMessage(bool with_stacktrace, bool check_embedded_stacktrace /*= false*/, bool with_extra_info /*= true*/); + /// do bare minimum in catch block + LockMemoryExceptionInThread lock_memory_tracker(VariableContext::Global); + exception_str = getCurrentExceptionMessage(true /* with_stacktrace */); + } - auto last_exception = std::current_exception(); - tryLogException(last_exception, log, __PRETTY_FUNCTION__); + if (!exception_str.empty()) + { + LOG_ERROR(log, "{} {}", __PRETTY_FUNCTION__, exception_str); - auto exception_str = getExceptionMessage(last_exception, true /* with_stacktrace */); - - for (auto consumer_ptr_weak : all_consumers) + auto safe_consumers = getSafeConsumers(); + for (auto consumer_ptr_weak : safe_consumers.consumers) { + /// propagate materialized view exception to all consumers if (auto consumer_ptr = consumer_ptr_weak.lock()) { - consumer_ptr->setExceptionInfo(exception_str); + consumer_ptr->setExceptionInfo(exception_str, false /* no stacktrace, reuse passed one */); } } - - - - // tryLogCurrentException(__PRETTY_FUNCTION__); - - } mv_attached.store(false); diff --git a/tests/integration/test_kafka_bad_messages/test.py b/tests/integration/test_kafka_bad_messages/test.py index a634ce36631..6a351e6563a 100644 --- a/tests/integration/test_kafka_bad_messages/test.py +++ b/tests/integration/test_kafka_bad_messages/test.py @@ -320,14 +320,14 @@ def test_bad_messages_parsing_exception(kafka_cluster, max_retries=20): ) expected_result = """avro::Exception: Invalid data file. Magic does not match: : while parsing Kafka message (topic: Avro_err, partition: 0, offset: 0)\\'|1|1|1|default|kafka_Avro -Cannot parse input: expected \\'{\\' before: \\'qwertyuiop\\': while parsing Kafka message (topic: JSONEachRow_err, partition: 0, offset: 0)\\'|1|1|1|default|kafka_JSONEachRow +Cannot parse input: expected \\'{\\' before: \\'qwertyuiop\\': while parsing Kafka message (topic: JSONEachRow_err, partition: 0, offset: 0|1|1|1|default|kafka_JSONEachRow """ retries = 0 result_system_kafka_consumers = "" while True: result_system_kafka_consumers = instance.query( """ - SELECT exceptions.text[1], length(exceptions.text) > 1 AND length(exceptions.text) < 15, length(exceptions.time) > 1 AND length(exceptions.time) < 15, abs(dateDiff('second', exceptions.time[1], now())) < 40, database, table FROM system.kafka_consumers ORDER BY table, assignments.partition_id[1] + SELECT substr(exceptions.text[1], 1, 131), length(exceptions.text) > 1 AND length(exceptions.text) < 15, length(exceptions.time) > 1 AND length(exceptions.time) < 15, abs(dateDiff('second', exceptions.time[1], now())) < 40, database, table FROM system.kafka_consumers ORDER BY table, assignments.partition_id[1] """ ) result_system_kafka_consumers = result_system_kafka_consumers.replace("\t", "|") @@ -344,6 +344,61 @@ Cannot parse input: expected \\'{\\' before: \\'qwertyuiop\\': while parsing Kaf ]: kafka_delete_topic(admin_client, f"{format_name}_err") +def test_bad_messages_to_mv(kafka_cluster, max_retries=20): + admin_client = KafkaAdminClient( + bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) + ) + + + kafka_create_topic(admin_client, "tomv") + + instance.query( + f""" + DROP TABLE IF EXISTS kafka_materialized; + DROP TABLE IF EXISTS kafka_consumer; + DROP TABLE IF EXISTS kafka; + + CREATE TABLE kafka (key UInt64, value String) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = 'tomv', + kafka_group_name = 'tomv', + kafka_format = 'JSONEachRow', + kafka_num_consumers = 1; + + CREATE TABLE kafka_materialized(`key` UInt64, `value` UInt64) ENGINE = Log; + + CREATE MATERIALIZED VIEW kafka_consumer TO kafka_materialized + (`key` UInt64, `value` UInt64) AS + SELECT key, CAST(value, 'UInt64') AS value + FROM kafka; + """ + ) + + kafka_produce( + kafka_cluster, "tomv", ['{"key":10, "value":"aaa"}'] + ) + + expected_result = """Code: 6. DB::Exception: Cannot parse string \\'aaa\\' as UInt64: syntax error at begin of string. Note: there are toUInt64OrZero and to|1|1|1|default|kafka +""" + retries = 0 + result_system_kafka_consumers = "" + while True: + result_system_kafka_consumers = instance.query( + """ + SELECT substr(exceptions.text[1], 1, 131), length(exceptions.text) > 1 AND length(exceptions.text) < 15, length(exceptions.time) > 1 AND length(exceptions.time) < 15, abs(dateDiff('second', exceptions.time[1], now())) < 40, database, table FROM system.kafka_consumers ORDER BY table, assignments.partition_id[1] + """ + ) + result_system_kafka_consumers = result_system_kafka_consumers.replace("\t", "|") + if result_system_kafka_consumers == expected_result or retries > max_retries: + break + retries += 1 + time.sleep(1) + + assert result_system_kafka_consumers == expected_result + + kafka_delete_topic(admin_client, "tomv") + if __name__ == "__main__": cluster.start() From ec0c12960add84dba714807cad0019c876e5649b Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Wed, 23 Aug 2023 23:10:25 +0000 Subject: [PATCH 11/42] exceptions_kafka_consumers: stylecheck --- src/Storages/Kafka/KafkaConsumer.cpp | 2 +- tests/integration/test_kafka_bad_messages/test.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/Storages/Kafka/KafkaConsumer.cpp b/src/Storages/Kafka/KafkaConsumer.cpp index 31ec322fda8..076b2c50559 100644 --- a/src/Storages/Kafka/KafkaConsumer.cpp +++ b/src/Storages/Kafka/KafkaConsumer.cpp @@ -557,7 +557,7 @@ void KafkaConsumer::setExceptionInfo(const std::string & text, bool with_stacktr { throw Exception(); } - catch(std::exception & ex) + catch (const std::exception & ex) { exceptionWithTrace = text + getExceptionStackTraceString(ex); } diff --git a/tests/integration/test_kafka_bad_messages/test.py b/tests/integration/test_kafka_bad_messages/test.py index 6a351e6563a..aeceadcaf25 100644 --- a/tests/integration/test_kafka_bad_messages/test.py +++ b/tests/integration/test_kafka_bad_messages/test.py @@ -344,12 +344,12 @@ Cannot parse input: expected \\'{\\' before: \\'qwertyuiop\\': while parsing Kaf ]: kafka_delete_topic(admin_client, f"{format_name}_err") + def test_bad_messages_to_mv(kafka_cluster, max_retries=20): admin_client = KafkaAdminClient( bootstrap_servers="localhost:{}".format(kafka_cluster.kafka_port) ) - kafka_create_topic(admin_client, "tomv") instance.query( @@ -375,9 +375,7 @@ def test_bad_messages_to_mv(kafka_cluster, max_retries=20): """ ) - kafka_produce( - kafka_cluster, "tomv", ['{"key":10, "value":"aaa"}'] - ) + kafka_produce(kafka_cluster, "tomv", ['{"key":10, "value":"aaa"}']) expected_result = """Code: 6. DB::Exception: Cannot parse string \\'aaa\\' as UInt64: syntax error at begin of string. Note: there are toUInt64OrZero and to|1|1|1|default|kafka """ From 8c1f0251e6fb4925dc2ad6b3fd1b28bad229bd3b Mon Sep 17 00:00:00 2001 From: pufit Date: Wed, 23 Aug 2023 22:18:24 -0400 Subject: [PATCH 12/42] Get keeper node from clickhouse configs --- programs/keeper-client/KeeperClient.cpp | 35 ++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/programs/keeper-client/KeeperClient.cpp b/programs/keeper-client/KeeperClient.cpp index 893be4d25d1..cf8dddc3e2b 100644 --- a/programs/keeper-client/KeeperClient.cpp +++ b/programs/keeper-client/KeeperClient.cpp @@ -2,6 +2,7 @@ #include "Commands.h" #include #include +#include #include #include #include @@ -155,6 +156,11 @@ void KeeperClient::defineOptions(Poco::Util::OptionSet & options) .argument("") .binding("operation-timeout")); + options.addOption( + Poco::Util::Option("config-file", "c", "if set, will try to get a connection string from clickhouse config. default `config.xml`") + .argument("") + .binding("config-file")); + options.addOption( Poco::Util::Option("history-file", "", "set path of history file. default `~/.keeper-client-history`") .argument("") @@ -211,7 +217,7 @@ void KeeperClient::initialize(Poco::Util::Application & /* self */) } } - Poco::Logger::root().setLevel(config().getString("log-level", "error")); + Poco::Logger::root().setLevel(config().getString("log-level", "information")); EventNotifier::init(); } @@ -311,8 +317,31 @@ int KeeperClient::main(const std::vector & /* args */) return 0; } - auto host = config().getString("host", "localhost"); - auto port = config().getString("port", "9181"); + DB::ConfigProcessor config_processor(config().getString("config-file", "config.xml")); + + /// This will handle a situation when clickhouse is running on the embedded config, but config.d folder is also present. + config_processor.registerEmbeddedConfig("config.xml", ""); + auto clickhouse_config = config_processor.loadConfig(); + + String host; + String port; + String prefix = "zookeeper.node[0]."; + + if (!config().has("host") && !config().has("port") && clickhouse_config.configuration->has(prefix + "host")) + { + LOG_INFO(&Poco::Logger::get("KeeperClient"), "Found keeper node in the config.xml, will use it for connection"); + if (clickhouse_config.configuration->has(prefix + "secure")) + host = "secure://"; + + host += clickhouse_config.configuration->getString(prefix + "host"); + port = clickhouse_config.configuration->getString(prefix + "port"); + } + else + { + host = config().getString("host", "localhost"); + port = config().getString("port", "9181"); + } + zk_args.hosts = {host + ":" + port}; zk_args.connection_timeout_ms = config().getInt("connection-timeout", 10) * 1000; zk_args.session_timeout_ms = config().getInt("session-timeout", 10) * 1000; From 94ae9f640f7df4960236b9885ac5dd7d8f6eb555 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Thu, 24 Aug 2023 07:28:26 +0000 Subject: [PATCH 13/42] exceptions_kafka_consumers: test fix --- tests/integration/test_kafka_bad_messages/test.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_kafka_bad_messages/test.py b/tests/integration/test_kafka_bad_messages/test.py index aeceadcaf25..777d976933d 100644 --- a/tests/integration/test_kafka_bad_messages/test.py +++ b/tests/integration/test_kafka_bad_messages/test.py @@ -325,9 +325,10 @@ Cannot parse input: expected \\'{\\' before: \\'qwertyuiop\\': while parsing Kaf retries = 0 result_system_kafka_consumers = "" while True: + # filter out stacktrace in exceptions.text[1] because it is hardly stable enough result_system_kafka_consumers = instance.query( """ - SELECT substr(exceptions.text[1], 1, 131), length(exceptions.text) > 1 AND length(exceptions.text) < 15, length(exceptions.time) > 1 AND length(exceptions.time) < 15, abs(dateDiff('second', exceptions.time[1], now())) < 40, database, table FROM system.kafka_consumers ORDER BY table, assignments.partition_id[1] + SELECT substr(exceptions.text[1], 1, 131), length(exceptions.text) > 1 AND length(exceptions.text) < 15, length(exceptions.time) > 1 AND length(exceptions.time) < 15, abs(dateDiff('second', exceptions.time[1], now())) < 40, database, table FROM system.kafka_consumers WHERE table in('kafka_Avro', 'kafka_JSONEachRow') ORDER BY table, assignments.partition_id[1] """ ) result_system_kafka_consumers = result_system_kafka_consumers.replace("\t", "|") @@ -356,9 +357,9 @@ def test_bad_messages_to_mv(kafka_cluster, max_retries=20): f""" DROP TABLE IF EXISTS kafka_materialized; DROP TABLE IF EXISTS kafka_consumer; - DROP TABLE IF EXISTS kafka; + DROP TABLE IF EXISTS kafka1; - CREATE TABLE kafka (key UInt64, value String) + CREATE TABLE kafka1 (key UInt64, value String) ENGINE = Kafka SETTINGS kafka_broker_list = 'kafka1:19092', kafka_topic_list = 'tomv', @@ -371,20 +372,20 @@ def test_bad_messages_to_mv(kafka_cluster, max_retries=20): CREATE MATERIALIZED VIEW kafka_consumer TO kafka_materialized (`key` UInt64, `value` UInt64) AS SELECT key, CAST(value, 'UInt64') AS value - FROM kafka; + FROM kafka1; """ ) kafka_produce(kafka_cluster, "tomv", ['{"key":10, "value":"aaa"}']) - expected_result = """Code: 6. DB::Exception: Cannot parse string \\'aaa\\' as UInt64: syntax error at begin of string. Note: there are toUInt64OrZero and to|1|1|1|default|kafka + expected_result = """Code: 6. DB::Exception: Cannot parse string \\'aaa\\' as UInt64: syntax error at begin of string. Note: there are toUInt64OrZero and to|1|1|1|default|kafka1 """ retries = 0 result_system_kafka_consumers = "" while True: result_system_kafka_consumers = instance.query( """ - SELECT substr(exceptions.text[1], 1, 131), length(exceptions.text) > 1 AND length(exceptions.text) < 15, length(exceptions.time) > 1 AND length(exceptions.time) < 15, abs(dateDiff('second', exceptions.time[1], now())) < 40, database, table FROM system.kafka_consumers ORDER BY table, assignments.partition_id[1] + SELECT substr(exceptions.text[1], 1, 131), length(exceptions.text) > 1 AND length(exceptions.text) < 15, length(exceptions.time) > 1 AND length(exceptions.time) < 15, abs(dateDiff('second', exceptions.time[1], now())) < 40, database, table FROM system.kafka_consumers WHERE table='kafka1' ORDER BY table, assignments.partition_id[1] """ ) result_system_kafka_consumers = result_system_kafka_consumers.replace("\t", "|") From 901240eeded2e6f113ac7f84fec09dd93373aa4e Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Thu, 24 Aug 2023 08:44:08 +0000 Subject: [PATCH 14/42] exceptions_kafka_consumers: clang-tidy improvement --- src/Storages/Kafka/StorageKafka.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 4aca84d1f26..aaf716219c2 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -783,7 +783,7 @@ void StorageKafka::threadFunc(size_t idx) LOG_ERROR(log, "{} {}", __PRETTY_FUNCTION__, exception_str); auto safe_consumers = getSafeConsumers(); - for (auto consumer_ptr_weak : safe_consumers.consumers) + for (auto const & consumer_ptr_weak : safe_consumers.consumers) { /// propagate materialized view exception to all consumers if (auto consumer_ptr = consumer_ptr_weak.lock()) From f0f2d416dde4e0ccb1d4b3d3db17145458aeb42a Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 24 Aug 2023 15:34:32 +0300 Subject: [PATCH 15/42] Aggregator merge and destroy states in batch --- src/AggregateFunctions/IAggregateFunction.h | 13 +++++ src/Interpreters/Aggregator.cpp | 61 ++++++++++----------- src/Interpreters/JIT/compileFunction.cpp | 45 +++++++++++++-- src/Interpreters/JIT/compileFunction.h | 2 +- 4 files changed, 81 insertions(+), 40 deletions(-) diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index b460a66ea22..b89e179ee90 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -169,6 +169,10 @@ public: throw Exception(ErrorCodes::NOT_IMPLEMENTED, "merge() with thread pool parameter isn't implemented for {} ", getName()); } + /// Merges states (on which src places points to) with other states (on which dst places points to) of current aggregation function + /// then destroy states (on which src places points to). + virtual void mergeAndDestroyBatch(AggregateDataPtr * dst_places, AggregateDataPtr * src_places, size_t size, size_t offset, Arena * arena) const = 0; + /// Serializes state (to transmit it over the network, for example). virtual void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional version = std::nullopt) const = 0; /// NOLINT @@ -506,6 +510,15 @@ public: static_cast(this)->merge(places[i] + place_offset, rhs[i], arena); } + void mergeAndDestroyBatch(AggregateDataPtr * dst_places, AggregateDataPtr * rhs_places, size_t size, size_t offset, Arena * arena) const override + { + for (size_t i = 0; i < size; ++i) + { + static_cast(this)->merge(dst_places[i] + offset, rhs_places[i] + offset, arena); + static_cast(this)->destroy(rhs_places[i] + offset); + } + } + void addBatchSinglePlace( /// NOLINT size_t row_begin, size_t row_end, diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 088349f3485..9a269b9f56a 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -2479,48 +2479,21 @@ void NO_INLINE Aggregator::mergeDataNullKey( } } - template void NO_INLINE Aggregator::mergeDataImpl(Table & table_dst, Table & table_src, Arena * arena) const { if constexpr (Method::low_cardinality_optimization || Method::one_key_nullable_optimization) mergeDataNullKey(table_dst, table_src, arena); + PaddedPODArray dst_places; + PaddedPODArray src_places; + auto merge = [&](AggregateDataPtr & __restrict dst, AggregateDataPtr & __restrict src, bool inserted) { if (!inserted) { -#if USE_EMBEDDED_COMPILER - if constexpr (use_compiled_functions) - { - const auto & compiled_functions = compiled_aggregate_functions_holder->compiled_aggregate_functions; - compiled_functions.merge_aggregate_states_function(dst, src); - - if (compiled_aggregate_functions_holder->compiled_aggregate_functions.functions_count != params.aggregates_size) - { - for (size_t i = 0; i < params.aggregates_size; ++i) - { - if (!is_aggregate_function_compiled[i]) - aggregate_functions[i]->merge( - dst + offsets_of_aggregate_states[i], src + offsets_of_aggregate_states[i], arena); - } - - for (size_t i = 0; i < params.aggregates_size; ++i) - { - if (!is_aggregate_function_compiled[i]) - aggregate_functions[i]->destroy(src + offsets_of_aggregate_states[i]); - } - } - } - else -#endif - { - for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_functions[i]->merge(dst + offsets_of_aggregate_states[i], src + offsets_of_aggregate_states[i], arena); - - for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_functions[i]->destroy(src + offsets_of_aggregate_states[i]); - } + dst_places.push_back(dst); + src_places.push_back(src); } else { @@ -2531,8 +2504,30 @@ void NO_INLINE Aggregator::mergeDataImpl(Table & table_dst, Table & table_src, A }; table_src.template mergeToViaEmplace(table_dst, std::move(merge)); - table_src.clearAndShrink(); + +#if USE_EMBEDDED_COMPILER + if constexpr (use_compiled_functions) + { + const auto & compiled_functions = compiled_aggregate_functions_holder->compiled_aggregate_functions; + compiled_functions.merge_aggregate_states_function(dst_places.data(), src_places.data(), dst_places.size()); + + for (size_t i = 0; i < params.aggregates_size; ++i) + { + if (!is_aggregate_function_compiled[i]) + aggregate_functions[i]->mergeAndDestroyBatch( + dst_places.data(), src_places.data(), dst_places.size(), offsets_of_aggregate_states[i], arena); + } + + return; + } +#endif + + for (size_t i = 0; i < params.aggregates_size; ++i) + { + aggregate_functions[i]->mergeAndDestroyBatch( + dst_places.data(), src_places.data(), dst_places.size(), offsets_of_aggregate_states[i], arena); + } } diff --git a/src/Interpreters/JIT/compileFunction.cpp b/src/Interpreters/JIT/compileFunction.cpp index fb8dec665b4..f50a122f9a2 100644 --- a/src/Interpreters/JIT/compileFunction.cpp +++ b/src/Interpreters/JIT/compileFunction.cpp @@ -357,27 +357,60 @@ static void compileMergeAggregatesStates(llvm::Module & module, const std::vecto llvm::IRBuilder<> b(module.getContext()); auto * aggregate_data_place_type = b.getInt8Ty()->getPointerTo(); - auto * merge_aggregates_states_func_declaration = llvm::FunctionType::get(b.getVoidTy(), { aggregate_data_place_type, aggregate_data_place_type }, false); - auto * merge_aggregates_states_func = llvm::Function::Create(merge_aggregates_states_func_declaration, llvm::Function::ExternalLinkage, name, module); + auto * aggregate_data_places_type = aggregate_data_place_type->getPointerTo(); + auto * size_type = b.getInt64Ty(); + + auto * merge_aggregates_states_func_declaration + = llvm::FunctionType::get(b.getVoidTy(), {aggregate_data_places_type, aggregate_data_places_type, size_type}, false); + auto * merge_aggregates_states_func + = llvm::Function::Create(merge_aggregates_states_func_declaration, llvm::Function::ExternalLinkage, name, module); auto * arguments = merge_aggregates_states_func->args().begin(); - llvm::Value * aggregate_data_place_dst_arg = arguments++; - llvm::Value * aggregate_data_place_src_arg = arguments++; + llvm::Value * aggregate_data_places_dst_arg = arguments++; + llvm::Value * aggregate_data_places_src_arg = arguments++; + llvm::Value * aggregate_places_size_arg = arguments++; auto * entry = llvm::BasicBlock::Create(b.getContext(), "entry", merge_aggregates_states_func); b.SetInsertPoint(entry); + /// Initialize loop + + auto * end = llvm::BasicBlock::Create(b.getContext(), "end", merge_aggregates_states_func); + auto * loop = llvm::BasicBlock::Create(b.getContext(), "loop", merge_aggregates_states_func); + b.CreateCondBr(b.CreateICmpEQ(aggregate_places_size_arg, llvm::ConstantInt::get(size_type, 0)), end, loop); + + b.SetInsertPoint(loop); + + /// Loop + + auto * counter_phi = b.CreatePHI(size_type, 2); + counter_phi->addIncoming(llvm::ConstantInt::get(size_type, 0), entry); + for (const auto & function_to_compile : functions) { + auto * aggregate_data_place_dst = b.CreateLoad(aggregate_data_place_type, + b.CreateInBoundsGEP(aggregate_data_place_type->getPointerTo(), aggregate_data_places_dst_arg, counter_phi)); + auto * aggregate_data_place_src = b.CreateLoad(aggregate_data_place_type, + b.CreateInBoundsGEP(aggregate_data_place_type->getPointerTo(), aggregate_data_places_src_arg, counter_phi)); + size_t aggregate_function_offset = function_to_compile.aggregate_data_offset; - auto * aggregate_data_place_merge_dst_with_offset = b.CreateConstInBoundsGEP1_64(b.getInt8Ty(), aggregate_data_place_dst_arg, aggregate_function_offset); - auto * aggregate_data_place_merge_src_with_offset = b.CreateConstInBoundsGEP1_64(b.getInt8Ty(), aggregate_data_place_src_arg, aggregate_function_offset); + auto * aggregate_data_place_merge_dst_with_offset = b.CreateConstInBoundsGEP1_64(b.getInt8Ty(), aggregate_data_place_dst, aggregate_function_offset); + auto * aggregate_data_place_merge_src_with_offset = b.CreateConstInBoundsGEP1_64(b.getInt8Ty(), aggregate_data_place_src, aggregate_function_offset); const auto * aggregate_function_ptr = function_to_compile.function; aggregate_function_ptr->compileMerge(b, aggregate_data_place_merge_dst_with_offset, aggregate_data_place_merge_src_with_offset); } + /// End of loop + + auto * current_block = b.GetInsertBlock(); + auto * incremeted_counter = b.CreateAdd(counter_phi, llvm::ConstantInt::get(size_type, 1)); + counter_phi->addIncoming(incremeted_counter, current_block); + + b.CreateCondBr(b.CreateICmpEQ(incremeted_counter, aggregate_places_size_arg), end, loop); + + b.SetInsertPoint(end); b.CreateRetVoid(); } diff --git a/src/Interpreters/JIT/compileFunction.h b/src/Interpreters/JIT/compileFunction.h index fe5abe1988c..84abfa0925a 100644 --- a/src/Interpreters/JIT/compileFunction.h +++ b/src/Interpreters/JIT/compileFunction.h @@ -56,7 +56,7 @@ struct AggregateFunctionWithOffset using JITCreateAggregateStatesFunction = void (*)(AggregateDataPtr); using JITAddIntoAggregateStatesFunction = void (*)(ColumnDataRowsOffset, ColumnDataRowsOffset, ColumnData *, AggregateDataPtr *); using JITAddIntoAggregateStatesFunctionSinglePlace = void (*)(ColumnDataRowsOffset, ColumnDataRowsOffset, ColumnData *, AggregateDataPtr); -using JITMergeAggregateStatesFunction = void (*)(AggregateDataPtr, AggregateDataPtr); +using JITMergeAggregateStatesFunction = void (*)(AggregateDataPtr *, AggregateDataPtr *, size_t); using JITInsertAggregateStatesIntoColumnsFunction = void (*)(ColumnDataRowsOffset, ColumnDataRowsOffset, ColumnData *, AggregateDataPtr *); struct CompiledAggregateFunctions From f58b4a812d4dbaefdf104a3e751c01440c9858f6 Mon Sep 17 00:00:00 2001 From: Ilya Golshtein Date: Thu, 24 Aug 2023 13:10:21 +0000 Subject: [PATCH 16/42] exceptions_kafka_consumers: code review suggestions are addressed --- src/Storages/Kafka/KafkaConsumer.cpp | 17 ++----- .../test_kafka_bad_messages/test.py | 48 ++++++++----------- 2 files changed, 22 insertions(+), 43 deletions(-) diff --git a/src/Storages/Kafka/KafkaConsumer.cpp b/src/Storages/Kafka/KafkaConsumer.cpp index 076b2c50559..31d431e27fe 100644 --- a/src/Storages/Kafka/KafkaConsumer.cpp +++ b/src/Storages/Kafka/KafkaConsumer.cpp @@ -549,26 +549,15 @@ void KafkaConsumer::setExceptionInfo(const cppkafka::Error & err, bool with_stac void KafkaConsumer::setExceptionInfo(const std::string & text, bool with_stacktrace) { - std::string exceptionWithTrace; + std::string enriched_text = text; if (with_stacktrace) { - try - { - throw Exception(); - } - catch (const std::exception & ex) - { - exceptionWithTrace = text + getExceptionStackTraceString(ex); - } - } - else - { - exceptionWithTrace = text; + enriched_text.append(StackTrace().toString()); } std::lock_guard lock(exception_mutex); - exceptions_buffer.push_back({exceptionWithTrace, static_cast(Poco::Timestamp().epochTime())}); + exceptions_buffer.push_back({enriched_text, static_cast(Poco::Timestamp().epochTime())}); } /* diff --git a/tests/integration/test_kafka_bad_messages/test.py b/tests/integration/test_kafka_bad_messages/test.py index 777d976933d..1633f230f83 100644 --- a/tests/integration/test_kafka_bad_messages/test.py +++ b/tests/integration/test_kafka_bad_messages/test.py @@ -322,22 +322,17 @@ def test_bad_messages_parsing_exception(kafka_cluster, max_retries=20): expected_result = """avro::Exception: Invalid data file. Magic does not match: : while parsing Kafka message (topic: Avro_err, partition: 0, offset: 0)\\'|1|1|1|default|kafka_Avro Cannot parse input: expected \\'{\\' before: \\'qwertyuiop\\': while parsing Kafka message (topic: JSONEachRow_err, partition: 0, offset: 0|1|1|1|default|kafka_JSONEachRow """ - retries = 0 - result_system_kafka_consumers = "" - while True: - # filter out stacktrace in exceptions.text[1] because it is hardly stable enough - result_system_kafka_consumers = instance.query( - """ - SELECT substr(exceptions.text[1], 1, 131), length(exceptions.text) > 1 AND length(exceptions.text) < 15, length(exceptions.time) > 1 AND length(exceptions.time) < 15, abs(dateDiff('second', exceptions.time[1], now())) < 40, database, table FROM system.kafka_consumers WHERE table in('kafka_Avro', 'kafka_JSONEachRow') ORDER BY table, assignments.partition_id[1] - """ - ) - result_system_kafka_consumers = result_system_kafka_consumers.replace("\t", "|") - if result_system_kafka_consumers == expected_result or retries > max_retries: - break - retries += 1 - time.sleep(1) + # filter out stacktrace in exceptions.text[1] because it is hardly stable enough + result_system_kafka_consumers = instance.query_with_retry( + """ + SELECT substr(exceptions.text[1], 1, 131), length(exceptions.text) > 1 AND length(exceptions.text) < 15, length(exceptions.time) > 1 AND length(exceptions.time) < 15, abs(dateDiff('second', exceptions.time[1], now())) < 40, database, table FROM system.kafka_consumers WHERE table in('kafka_Avro', 'kafka_JSONEachRow') ORDER BY table, assignments.partition_id[1] + """, + retry_count=max_retries, + sleep_time=1, + check_callback=lambda res: res.replace("\t", "|") == expected_result, + ) - assert result_system_kafka_consumers == expected_result + assert result_system_kafka_consumers.replace("\t", "|") == expected_result for format_name in [ "Avro", @@ -380,21 +375,16 @@ def test_bad_messages_to_mv(kafka_cluster, max_retries=20): expected_result = """Code: 6. DB::Exception: Cannot parse string \\'aaa\\' as UInt64: syntax error at begin of string. Note: there are toUInt64OrZero and to|1|1|1|default|kafka1 """ - retries = 0 - result_system_kafka_consumers = "" - while True: - result_system_kafka_consumers = instance.query( - """ - SELECT substr(exceptions.text[1], 1, 131), length(exceptions.text) > 1 AND length(exceptions.text) < 15, length(exceptions.time) > 1 AND length(exceptions.time) < 15, abs(dateDiff('second', exceptions.time[1], now())) < 40, database, table FROM system.kafka_consumers WHERE table='kafka1' ORDER BY table, assignments.partition_id[1] - """ - ) - result_system_kafka_consumers = result_system_kafka_consumers.replace("\t", "|") - if result_system_kafka_consumers == expected_result or retries > max_retries: - break - retries += 1 - time.sleep(1) + result_system_kafka_consumers = instance.query_with_retry( + """ + SELECT substr(exceptions.text[1], 1, 131), length(exceptions.text) > 1 AND length(exceptions.text) < 15, length(exceptions.time) > 1 AND length(exceptions.time) < 15, abs(dateDiff('second', exceptions.time[1], now())) < 40, database, table FROM system.kafka_consumers WHERE table='kafka1' ORDER BY table, assignments.partition_id[1] + """, + retry_count=max_retries, + sleep_time=1, + check_callback=lambda res: res.replace("\t", "|") == expected_result, + ) - assert result_system_kafka_consumers == expected_result + assert result_system_kafka_consumers.replace("\t", "|") == expected_result kafka_delete_topic(admin_client, "tomv") From abd791bf2686152bea30255cc74c2a0890b56490 Mon Sep 17 00:00:00 2001 From: pufit Date: Thu, 24 Aug 2023 12:49:25 -0400 Subject: [PATCH 17/42] Fix tests --- tests/integration/test_keeper_client/test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_keeper_client/test.py b/tests/integration/test_keeper_client/test.py index 8f7056a5afd..6143860448f 100644 --- a/tests/integration/test_keeper_client/test.py +++ b/tests/integration/test_keeper_client/test.py @@ -33,6 +33,8 @@ def keeper_query(query: str): str(cluster.get_instance_ip("zoo1")), "--port", str(cluster.zookeeper_port), + "--log-level", + "error", "-q", query, ], From 091a27359a85a50db19da57638c2e626226fbc48 Mon Sep 17 00:00:00 2001 From: pufit Date: Thu, 24 Aug 2023 21:45:10 -0400 Subject: [PATCH 18/42] More careful logs handling --- programs/keeper-client/KeeperClient.cpp | 9 ++++++++- tests/integration/test_keeper_client/test.py | 2 -- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/programs/keeper-client/KeeperClient.cpp b/programs/keeper-client/KeeperClient.cpp index cf8dddc3e2b..eacc4be98ae 100644 --- a/programs/keeper-client/KeeperClient.cpp +++ b/programs/keeper-client/KeeperClient.cpp @@ -217,7 +217,14 @@ void KeeperClient::initialize(Poco::Util::Application & /* self */) } } - Poco::Logger::root().setLevel(config().getString("log-level", "information")); + String default_log_level; + if (config().has("query")) + /// We don't want to see any information log in query mode, unless it was set explicitly + default_log_level = "error"; + else + default_log_level = "information"; + + Poco::Logger::root().setLevel( config().getString("log-level", default_log_level)); EventNotifier::init(); } diff --git a/tests/integration/test_keeper_client/test.py b/tests/integration/test_keeper_client/test.py index 6143860448f..8f7056a5afd 100644 --- a/tests/integration/test_keeper_client/test.py +++ b/tests/integration/test_keeper_client/test.py @@ -33,8 +33,6 @@ def keeper_query(query: str): str(cluster.get_instance_ip("zoo1")), "--port", str(cluster.zookeeper_port), - "--log-level", - "error", "-q", query, ], From 8dd3448844e5673dc809ea076d088f8b7712111f Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 25 Aug 2023 19:29:47 +0300 Subject: [PATCH 19/42] Update DatabaseCatalog.h --- src/Interpreters/DatabaseCatalog.h | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h index 50a6ef437bf..da362225f97 100644 --- a/src/Interpreters/DatabaseCatalog.h +++ b/src/Interpreters/DatabaseCatalog.h @@ -44,10 +44,7 @@ public: if (database) { for (auto table_it = database->getTablesIterator(context); table_it->isValid(); table_it->next()) - { - const auto & storage_id = table_it->table()->getStorageID(); - result.emplace_back(storage_id.getTableName()); - } + result.emplace_back(table_it->name()); } return result; } From bf12415215e1d95ec03bf615a6ffdfbfde9e8b2f Mon Sep 17 00:00:00 2001 From: jsc0218 Date: Sun, 27 Aug 2023 21:18:13 -0400 Subject: [PATCH 20/42] Ignore foreign keys in tables definition --- src/Parsers/ASTForeignKeyDeclaration.h | 26 ++++++++ src/Parsers/ParserCreateQuery.cpp | 63 +++++++++++++++++++ src/Parsers/ParserCreateQuery.h | 7 +++ ...oreign_keys_in_tables_definition.reference | 3 + ...nore_foreign_keys_in_tables_definition.sql | 29 +++++++++ 5 files changed, 128 insertions(+) create mode 100644 src/Parsers/ASTForeignKeyDeclaration.h create mode 100644 tests/queries/0_stateless/02863_ignore_foreign_keys_in_tables_definition.reference create mode 100644 tests/queries/0_stateless/02863_ignore_foreign_keys_in_tables_definition.sql diff --git a/src/Parsers/ASTForeignKeyDeclaration.h b/src/Parsers/ASTForeignKeyDeclaration.h new file mode 100644 index 00000000000..d157ea81d5f --- /dev/null +++ b/src/Parsers/ASTForeignKeyDeclaration.h @@ -0,0 +1,26 @@ +#pragma once + +#include + +namespace DB +{ + +/* + * Currently ignore the foreign key node, flesh it out when needed + */ +class ASTForeignKeyDeclaration : public IAST +{ +public: + String name; + + String getID(char) const override { return "Foreign Key"; } + + ASTPtr clone() const override + { + auto res = std::make_shared(); + res->name = name; + return res; + } +}; + +} diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index fb6dae248c0..ad0728ff5f8 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -222,17 +223,69 @@ bool ParserProjectionDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & return true; } +bool ParserForeignKeyDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) +{ + ParserKeyword s_references("REFERENCES"); + ParserCompoundIdentifier table_name_p(true, true); + ParserExpression expression_p; + + ASTPtr name; + ASTPtr expr; + + if (!expression_p.parse(pos, expr, expected)) + return false; + + if (!s_references.ignore(pos, expected)) + return false; + + if (!table_name_p.parse(pos, name, expected)) + return false; + + if (!expression_p.parse(pos, expr, expected)) + return false; + + ParserKeyword s_on("ON"); + while (s_on.ignore(pos, expected)) + { + ParserKeyword s_delete("DELETE"); + ParserKeyword s_update("UPDATE"); + + if (!s_delete.ignore(pos, expected) && !s_update.ignore(pos, expected)) + return false; + + ParserKeyword s_restrict("RESTRICT"); + ParserKeyword s_cascade("CASCADE"); + ParserKeyword s_set_null("SET NULL"); + ParserKeyword s_no_action("NO ACTION"); + ParserKeyword s_set_default("SET DEFAULT"); + + if (!s_restrict.ignore(pos, expected) && !s_cascade.ignore(pos, expected) && + !s_set_null.ignore(pos, expected) && !s_no_action.ignore(pos, expected) && + !s_set_default.ignore(pos, expected)) + { + return false; + } + } + + auto foreign_key = std::make_shared(); + foreign_key->name = "Foreign Key"; + node = foreign_key; + + return true; +} bool ParserTablePropertyDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ParserKeyword s_index("INDEX"); ParserKeyword s_constraint("CONSTRAINT"); ParserKeyword s_projection("PROJECTION"); + ParserKeyword s_foreign_key("FOREIGN KEY"); ParserKeyword s_primary_key("PRIMARY KEY"); ParserIndexDeclaration index_p; ParserConstraintDeclaration constraint_p; ParserProjectionDeclaration projection_p; + ParserForeignKeyDeclaration foreign_key_p; ParserColumnDeclaration column_p{true, true}; ParserExpression primary_key_p; @@ -258,6 +311,11 @@ bool ParserTablePropertyDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expecte if (!primary_key_p.parse(pos, new_node, expected)) return false; } + else if (s_foreign_key.ignore(pos, expected)) + { + if (!foreign_key_p.parse(pos, new_node, expected)) + return false; + } else { if (!column_p.parse(pos, new_node, expected)) @@ -321,6 +379,11 @@ bool ParserTablePropertiesDeclarationList::parseImpl(Pos & pos, ASTPtr & node, E constraints->children.push_back(elem); else if (elem->as()) projections->children.push_back(elem); + else if (elem->as()) + { + /// Ignore the foreign key node + continue; + } else if (elem->as() || elem->as()) { if (primary_key) diff --git a/src/Parsers/ParserCreateQuery.h b/src/Parsers/ParserCreateQuery.h index 0a98923436c..4062ed25c6b 100644 --- a/src/Parsers/ParserCreateQuery.h +++ b/src/Parsers/ParserCreateQuery.h @@ -403,6 +403,13 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; +class ParserForeignKeyDeclaration : public IParserBase +{ +protected: + const char * getName() const override { return "foreign key declaration"; } + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; +}; + class ParserTablePropertyDeclaration : public IParserBase { protected: diff --git a/tests/queries/0_stateless/02863_ignore_foreign_keys_in_tables_definition.reference b/tests/queries/0_stateless/02863_ignore_foreign_keys_in_tables_definition.reference new file mode 100644 index 00000000000..6e82dd5d023 --- /dev/null +++ b/tests/queries/0_stateless/02863_ignore_foreign_keys_in_tables_definition.reference @@ -0,0 +1,3 @@ +CREATE TABLE default.child\n(\n `id` Int32,\n `pid` Int32\n)\nENGINE = MergeTree\nPRIMARY KEY id\nORDER BY id\nSETTINGS index_granularity = 8192 +CREATE TABLE default.child2\n(\n `id` Int32,\n `pid` Int32\n)\nENGINE = MergeTree\nPRIMARY KEY id\nORDER BY id\nSETTINGS index_granularity = 8192 +CREATE TABLE default.child3\n(\n `id` Int32,\n `pid` Int32\n)\nENGINE = MergeTree\nPRIMARY KEY id\nORDER BY id\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/02863_ignore_foreign_keys_in_tables_definition.sql b/tests/queries/0_stateless/02863_ignore_foreign_keys_in_tables_definition.sql new file mode 100644 index 00000000000..2d814643116 --- /dev/null +++ b/tests/queries/0_stateless/02863_ignore_foreign_keys_in_tables_definition.sql @@ -0,0 +1,29 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/53380 + + +drop table if exists parent; +drop table if exists child; + +create table parent (id int, primary key(id)) engine MergeTree; +create table child (id int, pid int, primary key(id), foreign key(pid)) engine MergeTree; -- { clientError SYNTAX_ERROR } +create table child (id int, pid int, primary key(id), foreign key(pid) references) engine MergeTree; -- { clientError SYNTAX_ERROR } +create table child (id int, pid int, primary key(id), foreign key(pid) references parent(pid)) engine MergeTree; + +show create table child; + +create table child2 (id int, pid int, primary key(id), + foreign key(pid) references parent(pid) on delete) engine MergeTree; -- { clientError SYNTAX_ERROR } +create table child2 (id int, pid int, primary key(id), + foreign key(pid) references parent(pid) on delete cascade) engine MergeTree; + +show create table child2; + +create table child3 (id int, pid int, primary key(id), + foreign key(pid) references parent(pid) on delete cascade on update restrict) engine MergeTree; + +show create table child3; + +drop table child3; +drop table child2; +drop table child; +drop table parent; \ No newline at end of file From bb12f7deccd90791e39dfee9cd02ba1ff14459ea Mon Sep 17 00:00:00 2001 From: xuelei Date: Sun, 27 Aug 2023 23:41:23 -0300 Subject: [PATCH 21/42] deal with conflict --- src/Disks/ObjectStorages/S3/registerDiskS3.cpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/Disks/ObjectStorages/S3/registerDiskS3.cpp b/src/Disks/ObjectStorages/S3/registerDiskS3.cpp index d1264affaea..3f33b5c533a 100644 --- a/src/Disks/ObjectStorages/S3/registerDiskS3.cpp +++ b/src/Disks/ObjectStorages/S3/registerDiskS3.cpp @@ -104,13 +104,9 @@ void registerDiskS3(DiskFactory & factory, bool global_skip_access_check) { String endpoint = context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); S3::URI uri(endpoint); - - if (uri.key.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "No key in S3 uri: {}", uri.uri.toString()); - - if (uri.key.back() != '/') - throw Exception(ErrorCodes::BAD_ARGUMENTS, "S3 path must ends with '/', but '{}' doesn't.", uri.key); - + if(!uri.key.ends_with('/')) + uri.key.push_back('/'); + S3Capabilities s3_capabilities = getCapabilitiesFromConfig(config, config_prefix); std::shared_ptr s3_storage; From 643dd499b8295e6e9a6ba21cd5d0dc8fef1add31 Mon Sep 17 00:00:00 2001 From: xuelei Date: Sun, 27 Aug 2023 23:46:53 -0300 Subject: [PATCH 22/42] deal with conflict --- src/Disks/ObjectStorages/S3/diskSettings.cpp | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index ea40c49ff4b..3ab16f93c0c 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -24,12 +24,7 @@ namespace DB { - -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; -} - + std::unique_ptr getSettings(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context) { const Settings & settings = context->getSettingsRef(); @@ -50,6 +45,8 @@ std::unique_ptr getClient( { String endpoint = context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); S3::URI uri(endpoint); + if(!uri.key.ends_with('/')) + uri.key.push_back('/'); S3::PocoHTTPClientConfiguration client_configuration = S3::ClientFactory::instance().createClientConfiguration( config.getString(config_prefix + ".region", ""), @@ -61,9 +58,6 @@ std::unique_ptr getClient( settings.request_settings.put_request_throttler, uri.uri.getScheme()); - if (uri.key.back() != '/') - throw Exception(ErrorCodes::BAD_ARGUMENTS, "S3 path must ends with '/', but '{}' doesn't.", uri.key); - client_configuration.connectTimeoutMs = config.getUInt(config_prefix + ".connect_timeout_ms", 1000); client_configuration.requestTimeoutMs = config.getUInt(config_prefix + ".request_timeout_ms", 3000); client_configuration.maxConnections = config.getUInt(config_prefix + ".max_connections", 100); From 3c457c47c20f6969c2d25664988df907f33980a0 Mon Sep 17 00:00:00 2001 From: xiaolei565 Date: Mon, 28 Aug 2023 11:05:30 +0800 Subject: [PATCH 23/42] Update diskSettings.cpp --- src/Disks/ObjectStorages/S3/diskSettings.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index 3ab16f93c0c..433d5bc4268 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -24,7 +24,7 @@ namespace DB { - + std::unique_ptr getSettings(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context) { const Settings & settings = context->getSettingsRef(); From b9e872ddf7162122b888e362065ffabb571f8821 Mon Sep 17 00:00:00 2001 From: xiaolei565 Date: Mon, 28 Aug 2023 11:28:31 +0800 Subject: [PATCH 24/42] Update diskSettings.cpp --- src/Disks/ObjectStorages/S3/diskSettings.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index 433d5bc4268..29e2bb44fc4 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -24,7 +24,7 @@ namespace DB { - + std::unique_ptr getSettings(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context) { const Settings & settings = context->getSettingsRef(); From 8d2c03ea26fb035fcee8aff96dae3da9521a52c1 Mon Sep 17 00:00:00 2001 From: xiaolei565 Date: Mon, 28 Aug 2023 11:41:53 +0800 Subject: [PATCH 25/42] Update registerDiskS3.cpp --- src/Disks/ObjectStorages/S3/registerDiskS3.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Disks/ObjectStorages/S3/registerDiskS3.cpp b/src/Disks/ObjectStorages/S3/registerDiskS3.cpp index 3f33b5c533a..7592a30917e 100644 --- a/src/Disks/ObjectStorages/S3/registerDiskS3.cpp +++ b/src/Disks/ObjectStorages/S3/registerDiskS3.cpp @@ -106,7 +106,7 @@ void registerDiskS3(DiskFactory & factory, bool global_skip_access_check) S3::URI uri(endpoint); if(!uri.key.ends_with('/')) uri.key.push_back('/'); - + S3Capabilities s3_capabilities = getCapabilitiesFromConfig(config, config_prefix); std::shared_ptr s3_storage; From 58a11600959a3638ba48e2949b96e4cb11f7086b Mon Sep 17 00:00:00 2001 From: xiaolei565 Date: Mon, 28 Aug 2023 12:37:54 +0800 Subject: [PATCH 26/42] Update diskSettings.cpp --- src/Disks/ObjectStorages/S3/diskSettings.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index 29e2bb44fc4..f371be2c41b 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -45,6 +45,7 @@ std::unique_ptr getClient( { String endpoint = context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); S3::URI uri(endpoint); + if(!uri.key.ends_with('/')) uri.key.push_back('/'); From ef6e1fd2d677a4cb8d9dd8704101d75a364e06c3 Mon Sep 17 00:00:00 2001 From: xiaolei565 Date: Mon, 28 Aug 2023 12:38:08 +0800 Subject: [PATCH 27/42] Update diskSettings.cpp --- src/Disks/ObjectStorages/S3/diskSettings.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index f371be2c41b..c098925cf95 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -45,7 +45,7 @@ std::unique_ptr getClient( { String endpoint = context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); S3::URI uri(endpoint); - + if(!uri.key.ends_with('/')) uri.key.push_back('/'); From 853551bd3eba1d8ecba052a829468348a0f63d96 Mon Sep 17 00:00:00 2001 From: xiaolei565 Date: Mon, 28 Aug 2023 12:38:33 +0800 Subject: [PATCH 28/42] Update registerDiskS3.cpp --- src/Disks/ObjectStorages/S3/registerDiskS3.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Disks/ObjectStorages/S3/registerDiskS3.cpp b/src/Disks/ObjectStorages/S3/registerDiskS3.cpp index 7592a30917e..98328077ab7 100644 --- a/src/Disks/ObjectStorages/S3/registerDiskS3.cpp +++ b/src/Disks/ObjectStorages/S3/registerDiskS3.cpp @@ -104,6 +104,7 @@ void registerDiskS3(DiskFactory & factory, bool global_skip_access_check) { String endpoint = context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); S3::URI uri(endpoint); + if(!uri.key.ends_with('/')) uri.key.push_back('/'); From 7b4902133de43ad242d05fcdb9e571f0c1460e65 Mon Sep 17 00:00:00 2001 From: xiaolei565 Date: Mon, 28 Aug 2023 16:54:13 +0800 Subject: [PATCH 29/42] Update registerDiskS3.cpp From 636e7ddae7252de53cf8483b67e9f2cf48bba969 Mon Sep 17 00:00:00 2001 From: xuelei Date: Mon, 28 Aug 2023 06:05:21 -0300 Subject: [PATCH 30/42] deal with conflict --- src/Disks/ObjectStorages/S3/diskSettings.cpp | 1 - src/Disks/ObjectStorages/S3/registerDiskS3.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index c098925cf95..29e2bb44fc4 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -45,7 +45,6 @@ std::unique_ptr getClient( { String endpoint = context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); S3::URI uri(endpoint); - if(!uri.key.ends_with('/')) uri.key.push_back('/'); diff --git a/src/Disks/ObjectStorages/S3/registerDiskS3.cpp b/src/Disks/ObjectStorages/S3/registerDiskS3.cpp index 98328077ab7..7592a30917e 100644 --- a/src/Disks/ObjectStorages/S3/registerDiskS3.cpp +++ b/src/Disks/ObjectStorages/S3/registerDiskS3.cpp @@ -104,7 +104,6 @@ void registerDiskS3(DiskFactory & factory, bool global_skip_access_check) { String endpoint = context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); S3::URI uri(endpoint); - if(!uri.key.ends_with('/')) uri.key.push_back('/'); From 1747b4190127a07338b84c4d950c500e8c014429 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 28 Aug 2023 11:48:16 +0000 Subject: [PATCH 31/42] Update version_date.tsv and changelogs after v23.6.3.87-stable --- docker/keeper/Dockerfile | 2 +- docker/server/Dockerfile.alpine | 2 +- docker/server/Dockerfile.ubuntu | 2 +- docs/changelogs/v23.6.3.87-stable.md | 58 ++++++++++++++++++++++++++++ utils/list-versions/version_date.tsv | 2 + 5 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 docs/changelogs/v23.6.3.87-stable.md diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index c7206550bd8..ab0d89c3184 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \ esac ARG REPOSITORY="https://s3.amazonaws.com/clickhouse-builds/22.4/31c367d3cd3aefd316778601ff6565119fe36682/package_release" -ARG VERSION="23.7.4.5" +ARG VERSION="23.7.5.30" ARG PACKAGES="clickhouse-keeper" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index a3a1f912e90..ac7d6a40518 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -33,7 +33,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="23.7.4.5" +ARG VERSION="23.7.5.30" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 4936cfeccb0..10697142a51 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -23,7 +23,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="23.7.4.5" +ARG VERSION="23.7.5.30" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # set non-empty deb_location_url url to create a docker image diff --git a/docs/changelogs/v23.6.3.87-stable.md b/docs/changelogs/v23.6.3.87-stable.md new file mode 100644 index 00000000000..8db499f308a --- /dev/null +++ b/docs/changelogs/v23.6.3.87-stable.md @@ -0,0 +1,58 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v23.6.3.87-stable (36911c17d0f) FIXME as compared to v23.6.2.18-stable (89f39a7ccfe) + +#### Performance Improvement +* Backported in [#52751](https://github.com/ClickHouse/ClickHouse/issues/52751): Fix incorrect projection analysis which invalidates primary keys. This issue only exists when `query_plan_optimize_primary_key = 1, query_plan_optimize_projection = 1` . This fixes [#48823](https://github.com/ClickHouse/ClickHouse/issues/48823) . This fixes [#51173](https://github.com/ClickHouse/ClickHouse/issues/51173) . [#52308](https://github.com/ClickHouse/ClickHouse/pull/52308) ([Amos Bird](https://github.com/amosbird)). + +#### Build/Testing/Packaging Improvement +* Backported in [#52911](https://github.com/ClickHouse/ClickHouse/issues/52911): Add `clickhouse-keeper-client` symlink to the clickhouse-server package. [#51882](https://github.com/ClickHouse/ClickHouse/pull/51882) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backported in [#53023](https://github.com/ClickHouse/ClickHouse/issues/53023): Packing inline cache into docker images sometimes causes strange special effects. Since we don't use it at all, it's good to go. [#53008](https://github.com/ClickHouse/ClickHouse/pull/53008) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backported in [#53290](https://github.com/ClickHouse/ClickHouse/issues/53290): The compiler's profile data (`-ftime-trace`) is uploaded to ClickHouse Cloud., the second attempt after [#53100](https://github.com/ClickHouse/ClickHouse/issues/53100). [#53213](https://github.com/ClickHouse/ClickHouse/pull/53213) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#53465](https://github.com/ClickHouse/ClickHouse/issues/53465): Preserve environment parameters in `clickhouse start` command. Fixes [#51962](https://github.com/ClickHouse/ClickHouse/issues/51962). [#53418](https://github.com/ClickHouse/ClickHouse/pull/53418) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Fix for moving 'IN' conditions to PREWHERE [#51610](https://github.com/ClickHouse/ClickHouse/pull/51610) ([Alexander Gololobov](https://github.com/davenger)). +* Fix binary arithmetic for Nullable(IPv4) [#51642](https://github.com/ClickHouse/ClickHouse/pull/51642) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Support IPv4 and IPv6 as dictionary attributes [#51756](https://github.com/ClickHouse/ClickHouse/pull/51756) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Check refcount in `RemoveManyObjectStorageOperation::finalize` instead of `execute` [#51954](https://github.com/ClickHouse/ClickHouse/pull/51954) ([vdimir](https://github.com/vdimir)). +* Fix ORDER BY tuple of WINDOW functions [#52145](https://github.com/ClickHouse/ClickHouse/pull/52145) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix error in `groupArrayMoving` functions [#52161](https://github.com/ClickHouse/ClickHouse/pull/52161) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Disable expression templates for time intervals [#52335](https://github.com/ClickHouse/ClickHouse/pull/52335) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix `countSubstrings()` hang with empty needle and a column haystack [#52409](https://github.com/ClickHouse/ClickHouse/pull/52409) ([Sergei Trifonov](https://github.com/serxa)). +* Fixed inserting into Buffer engine [#52440](https://github.com/ClickHouse/ClickHouse/pull/52440) ([Vasily Nemkov](https://github.com/Enmk)). +* The implementation of AnyHash was non-conformant. [#52448](https://github.com/ClickHouse/ClickHouse/pull/52448) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix abort in function `transform` [#52513](https://github.com/ClickHouse/ClickHouse/pull/52513) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix possible error "Cannot drain connections: cancel first" [#52585](https://github.com/ClickHouse/ClickHouse/pull/52585) ([Kruglov Pavel](https://github.com/Avogar)). +* init and destroy ares channel on demand.. [#52634](https://github.com/ClickHouse/ClickHouse/pull/52634) ([Arthur Passos](https://github.com/arthurpassos)). +* Fix crash in function `tuple` with one sparse column argument [#52659](https://github.com/ClickHouse/ClickHouse/pull/52659) ([Anton Popov](https://github.com/CurtizJ)). +* clickhouse-keeper: fix implementation of server with poll() [#52833](https://github.com/ClickHouse/ClickHouse/pull/52833) ([Andy Fiddaman](https://github.com/citrus-it)). +* Fix password leak in show create mysql table [#52962](https://github.com/ClickHouse/ClickHouse/pull/52962) ([Duc Canh Le](https://github.com/canhld94)). +* Not-ready Set [#53162](https://github.com/ClickHouse/ClickHouse/pull/53162) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix incorrect normal projection AST format [#53347](https://github.com/ClickHouse/ClickHouse/pull/53347) ([Amos Bird](https://github.com/amosbird)). +* Fix loading lazy database during system.table select query [#53372](https://github.com/ClickHouse/ClickHouse/pull/53372) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* Fix: interpolate expression takes source column instead of same name aliased from select expression. [#53572](https://github.com/ClickHouse/ClickHouse/pull/53572) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Correctly handle totals and extremes with `DelayedSource` [#53644](https://github.com/ClickHouse/ClickHouse/pull/53644) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix: sorted distinct with sparse columns [#53711](https://github.com/ClickHouse/ClickHouse/pull/53711) ([Igor Nikonov](https://github.com/devcrafter)). +* Fix fuzzer crash in parseDateTime() [#53764](https://github.com/ClickHouse/ClickHouse/pull/53764) ([Robert Schulze](https://github.com/rschu1ze)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix: logical error in grace hash join [#51737](https://github.com/ClickHouse/ClickHouse/pull/51737) ([Igor Nikonov](https://github.com/devcrafter)). +* Pin rust nightly (to make it stable) [#51903](https://github.com/ClickHouse/ClickHouse/pull/51903) ([Azat Khuzhin](https://github.com/azat)). +* Fix crash in comparison functions due to incorrect query analysis [#52172](https://github.com/ClickHouse/ClickHouse/pull/52172) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Less replication errors [#52382](https://github.com/ClickHouse/ClickHouse/pull/52382) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Improve logging macros [#52519](https://github.com/ClickHouse/ClickHouse/pull/52519) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix deadlocks in StorageTableFunctionProxy [#52626](https://github.com/ClickHouse/ClickHouse/pull/52626) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Increase min protocol version for sparse serialization [#52835](https://github.com/ClickHouse/ClickHouse/pull/52835) ([Anton Popov](https://github.com/CurtizJ)). +* Docker improvements [#52869](https://github.com/ClickHouse/ClickHouse/pull/52869) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Disable test_reverse_dns_query/test.py [#53195](https://github.com/ClickHouse/ClickHouse/pull/53195) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Disable test_host_regexp_multiple_ptr_records/test.py [#53211](https://github.com/ClickHouse/ClickHouse/pull/53211) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Get rid of describe_parameters for the best robot token [#53833](https://github.com/ClickHouse/ClickHouse/pull/53833) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 0f15c761d2a..07e5899fe13 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,7 +1,9 @@ +v23.7.5.30-stable 2023-08-28 v23.7.4.5-stable 2023-08-08 v23.7.3.14-stable 2023-08-05 v23.7.2.25-stable 2023-08-03 v23.7.1.2470-stable 2023-07-27 +v23.6.3.87-stable 2023-08-28 v23.6.2.18-stable 2023-07-09 v23.6.1.1524-stable 2023-06-30 v23.5.4.25-stable 2023-06-29 From 504af09ec9bf60ab29a37b7a8ba14ba805f7e858 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 28 Aug 2023 11:50:34 +0000 Subject: [PATCH 32/42] Update version_date.tsv and changelogs after v23.3.11.5-lts --- docker/keeper/Dockerfile | 2 +- docker/server/Dockerfile.alpine | 2 +- docker/server/Dockerfile.ubuntu | 2 +- docs/changelogs/v23.3.11.5-lts.md | 17 +++++++++++++++++ utils/list-versions/version_date.tsv | 4 ++++ 5 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 docs/changelogs/v23.3.11.5-lts.md diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index c7206550bd8..ab0d89c3184 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \ esac ARG REPOSITORY="https://s3.amazonaws.com/clickhouse-builds/22.4/31c367d3cd3aefd316778601ff6565119fe36682/package_release" -ARG VERSION="23.7.4.5" +ARG VERSION="23.7.5.30" ARG PACKAGES="clickhouse-keeper" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index a3a1f912e90..ac7d6a40518 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -33,7 +33,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="23.7.4.5" +ARG VERSION="23.7.5.30" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 4936cfeccb0..10697142a51 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -23,7 +23,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="23.7.4.5" +ARG VERSION="23.7.5.30" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # set non-empty deb_location_url url to create a docker image diff --git a/docs/changelogs/v23.3.11.5-lts.md b/docs/changelogs/v23.3.11.5-lts.md new file mode 100644 index 00000000000..b671c7e5bb6 --- /dev/null +++ b/docs/changelogs/v23.3.11.5-lts.md @@ -0,0 +1,17 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v23.3.11.5-lts (5762a23a76d) FIXME as compared to v23.3.10.5-lts (d8737007f9e) + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Fix: sorted distinct with sparse columns [#53711](https://github.com/ClickHouse/ClickHouse/pull/53711) ([Igor Nikonov](https://github.com/devcrafter)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Get rid of describe_parameters for the best robot token [#53833](https://github.com/ClickHouse/ClickHouse/pull/53833) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 0f15c761d2a..78b8b07a0c6 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,9 +1,12 @@ +v23.7.5.30-stable 2023-08-28 v23.7.4.5-stable 2023-08-08 v23.7.3.14-stable 2023-08-05 v23.7.2.25-stable 2023-08-03 v23.7.1.2470-stable 2023-07-27 +v23.6.3.87-stable 2023-08-28 v23.6.2.18-stable 2023-07-09 v23.6.1.1524-stable 2023-06-30 +v23.5.5.92-stable 2023-08-28 v23.5.4.25-stable 2023-06-29 v23.5.3.24-stable 2023-06-17 v23.5.2.7-stable 2023-06-10 @@ -14,6 +17,7 @@ v23.4.4.16-stable 2023-06-17 v23.4.3.48-stable 2023-06-12 v23.4.2.11-stable 2023-05-02 v23.4.1.1943-stable 2023-04-27 +v23.3.11.5-lts 2023-08-28 v23.3.10.5-lts 2023-08-23 v23.3.9.55-lts 2023-08-21 v23.3.8.21-lts 2023-07-13 From e89488f37a006eb1d160c18d984a07fbbef8c375 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 28 Aug 2023 11:56:30 +0000 Subject: [PATCH 33/42] Update version_date.tsv and changelogs after v23.5.5.92-stable --- docker/keeper/Dockerfile | 2 +- docker/server/Dockerfile.alpine | 2 +- docker/server/Dockerfile.ubuntu | 2 +- docs/changelogs/v23.5.5.92-stable.md | 62 ++++++++++++++++++++++++++++ utils/list-versions/version_date.tsv | 4 ++ 5 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 docs/changelogs/v23.5.5.92-stable.md diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index c7206550bd8..ab0d89c3184 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \ esac ARG REPOSITORY="https://s3.amazonaws.com/clickhouse-builds/22.4/31c367d3cd3aefd316778601ff6565119fe36682/package_release" -ARG VERSION="23.7.4.5" +ARG VERSION="23.7.5.30" ARG PACKAGES="clickhouse-keeper" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index a3a1f912e90..ac7d6a40518 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -33,7 +33,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="23.7.4.5" +ARG VERSION="23.7.5.30" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 4936cfeccb0..10697142a51 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -23,7 +23,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="23.7.4.5" +ARG VERSION="23.7.5.30" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # set non-empty deb_location_url url to create a docker image diff --git a/docs/changelogs/v23.5.5.92-stable.md b/docs/changelogs/v23.5.5.92-stable.md new file mode 100644 index 00000000000..ade39b7545d --- /dev/null +++ b/docs/changelogs/v23.5.5.92-stable.md @@ -0,0 +1,62 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v23.5.5.92-stable (557edaddace) FIXME as compared to v23.5.4.25-stable (190f962abcf) + +#### Performance Improvement +* Backported in [#52749](https://github.com/ClickHouse/ClickHouse/issues/52749): Fix incorrect projection analysis which invalidates primary keys. This issue only exists when `query_plan_optimize_primary_key = 1, query_plan_optimize_projection = 1` . This fixes [#48823](https://github.com/ClickHouse/ClickHouse/issues/48823) . This fixes [#51173](https://github.com/ClickHouse/ClickHouse/issues/51173) . [#52308](https://github.com/ClickHouse/ClickHouse/pull/52308) ([Amos Bird](https://github.com/amosbird)). + +#### Build/Testing/Packaging Improvement +* Backported in [#51886](https://github.com/ClickHouse/ClickHouse/issues/51886): Update cargo dependencies. [#51721](https://github.com/ClickHouse/ClickHouse/pull/51721) ([Raúl Marín](https://github.com/Algunenano)). +* Backported in [#52909](https://github.com/ClickHouse/ClickHouse/issues/52909): Add `clickhouse-keeper-client` symlink to the clickhouse-server package. [#51882](https://github.com/ClickHouse/ClickHouse/pull/51882) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backported in [#53021](https://github.com/ClickHouse/ClickHouse/issues/53021): Packing inline cache into docker images sometimes causes strange special effects. Since we don't use it at all, it's good to go. [#53008](https://github.com/ClickHouse/ClickHouse/pull/53008) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backported in [#53289](https://github.com/ClickHouse/ClickHouse/issues/53289): The compiler's profile data (`-ftime-trace`) is uploaded to ClickHouse Cloud., the second attempt after [#53100](https://github.com/ClickHouse/ClickHouse/issues/53100). [#53213](https://github.com/ClickHouse/ClickHouse/pull/53213) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Backported in [#53463](https://github.com/ClickHouse/ClickHouse/issues/53463): Preserve environment parameters in `clickhouse start` command. Fixes [#51962](https://github.com/ClickHouse/ClickHouse/issues/51962). [#53418](https://github.com/ClickHouse/ClickHouse/pull/53418) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Fix backward compatibility for IP types hashing in aggregate functions [#50551](https://github.com/ClickHouse/ClickHouse/pull/50551) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Fix segfault in MathUnary [#51499](https://github.com/ClickHouse/ClickHouse/pull/51499) ([Ilya Yatsishin](https://github.com/qoega)). +* Fix for moving 'IN' conditions to PREWHERE [#51610](https://github.com/ClickHouse/ClickHouse/pull/51610) ([Alexander Gololobov](https://github.com/davenger)). +* Fix binary arithmetic for Nullable(IPv4) [#51642](https://github.com/ClickHouse/ClickHouse/pull/51642) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Support IPv4 and IPv6 as dictionary attributes [#51756](https://github.com/ClickHouse/ClickHouse/pull/51756) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Fix reading from empty column in `parseSipHashKey` [#51804](https://github.com/ClickHouse/ClickHouse/pull/51804) ([Nikita Taranov](https://github.com/nickitat)). +* Fix async connect to hosts with multiple ips [#51934](https://github.com/ClickHouse/ClickHouse/pull/51934) ([Kruglov Pavel](https://github.com/Avogar)). +* Allow parametric UDFs [#51964](https://github.com/ClickHouse/ClickHouse/pull/51964) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix ORDER BY tuple of WINDOW functions [#52145](https://github.com/ClickHouse/ClickHouse/pull/52145) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix error in `groupArrayMoving` functions [#52161](https://github.com/ClickHouse/ClickHouse/pull/52161) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Disable expression templates for time intervals [#52335](https://github.com/ClickHouse/ClickHouse/pull/52335) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix `countSubstrings()` hang with empty needle and a column haystack [#52409](https://github.com/ClickHouse/ClickHouse/pull/52409) ([Sergei Trifonov](https://github.com/serxa)). +* Fixed inserting into Buffer engine [#52440](https://github.com/ClickHouse/ClickHouse/pull/52440) ([Vasily Nemkov](https://github.com/Enmk)). +* The implementation of AnyHash was non-conformant. [#52448](https://github.com/ClickHouse/ClickHouse/pull/52448) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix possible error "Cannot drain connections: cancel first" [#52585](https://github.com/ClickHouse/ClickHouse/pull/52585) ([Kruglov Pavel](https://github.com/Avogar)). +* init and destroy ares channel on demand.. [#52634](https://github.com/ClickHouse/ClickHouse/pull/52634) ([Arthur Passos](https://github.com/arthurpassos)). +* Fix crash in function `tuple` with one sparse column argument [#52659](https://github.com/ClickHouse/ClickHouse/pull/52659) ([Anton Popov](https://github.com/CurtizJ)). +* clickhouse-keeper: fix implementation of server with poll() [#52833](https://github.com/ClickHouse/ClickHouse/pull/52833) ([Andy Fiddaman](https://github.com/citrus-it)). +* Fix password leak in show create mysql table [#52962](https://github.com/ClickHouse/ClickHouse/pull/52962) ([Duc Canh Le](https://github.com/canhld94)). +* Not-ready Set [#53162](https://github.com/ClickHouse/ClickHouse/pull/53162) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix incorrect normal projection AST format [#53347](https://github.com/ClickHouse/ClickHouse/pull/53347) ([Amos Bird](https://github.com/amosbird)). +* Fix loading lazy database during system.table select query [#53372](https://github.com/ClickHouse/ClickHouse/pull/53372) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)). +* Fix: interpolate expression takes source column instead of same name aliased from select expression. [#53572](https://github.com/ClickHouse/ClickHouse/pull/53572) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Correctly handle totals and extremes with `DelayedSource` [#53644](https://github.com/ClickHouse/ClickHouse/pull/53644) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix: sorted distinct with sparse columns [#53711](https://github.com/ClickHouse/ClickHouse/pull/53711) ([Igor Nikonov](https://github.com/devcrafter)). +* Fix fuzzer crash in parseDateTime() [#53764](https://github.com/ClickHouse/ClickHouse/pull/53764) ([Robert Schulze](https://github.com/rschu1ze)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Decoupled commits from [#51180](https://github.com/ClickHouse/ClickHouse/issues/51180) for backports [#51561](https://github.com/ClickHouse/ClickHouse/pull/51561) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Fix MergeTreeMarksLoader segfaulting if marks file is longer than expected [#51636](https://github.com/ClickHouse/ClickHouse/pull/51636) ([Michael Kolupaev](https://github.com/al13n321)). +* Fix source image for sqllogic [#51728](https://github.com/ClickHouse/ClickHouse/pull/51728) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Pin rust nightly (to make it stable) [#51903](https://github.com/ClickHouse/ClickHouse/pull/51903) ([Azat Khuzhin](https://github.com/azat)). +* Fix crash in comparison functions due to incorrect query analysis [#52172](https://github.com/ClickHouse/ClickHouse/pull/52172) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Less replication errors [#52382](https://github.com/ClickHouse/ClickHouse/pull/52382) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Improve logging macros [#52519](https://github.com/ClickHouse/ClickHouse/pull/52519) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix deadlocks in StorageTableFunctionProxy [#52626](https://github.com/ClickHouse/ClickHouse/pull/52626) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Disable test_reverse_dns_query/test.py [#53195](https://github.com/ClickHouse/ClickHouse/pull/53195) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Disable test_host_regexp_multiple_ptr_records/test.py [#53211](https://github.com/ClickHouse/ClickHouse/pull/53211) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Get rid of describe_parameters for the best robot token [#53833](https://github.com/ClickHouse/ClickHouse/pull/53833) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 0f15c761d2a..78b8b07a0c6 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,9 +1,12 @@ +v23.7.5.30-stable 2023-08-28 v23.7.4.5-stable 2023-08-08 v23.7.3.14-stable 2023-08-05 v23.7.2.25-stable 2023-08-03 v23.7.1.2470-stable 2023-07-27 +v23.6.3.87-stable 2023-08-28 v23.6.2.18-stable 2023-07-09 v23.6.1.1524-stable 2023-06-30 +v23.5.5.92-stable 2023-08-28 v23.5.4.25-stable 2023-06-29 v23.5.3.24-stable 2023-06-17 v23.5.2.7-stable 2023-06-10 @@ -14,6 +17,7 @@ v23.4.4.16-stable 2023-06-17 v23.4.3.48-stable 2023-06-12 v23.4.2.11-stable 2023-05-02 v23.4.1.1943-stable 2023-04-27 +v23.3.11.5-lts 2023-08-28 v23.3.10.5-lts 2023-08-23 v23.3.9.55-lts 2023-08-21 v23.3.8.21-lts 2023-07-13 From fa1b5fe2a817edcaae15e554c1e32e55d1ae65dc Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 28 Aug 2023 12:00:47 +0000 Subject: [PATCH 34/42] Update version_date.tsv and changelogs after v22.8.21.38-lts --- docker/keeper/Dockerfile | 2 +- docker/server/Dockerfile.alpine | 2 +- docker/server/Dockerfile.ubuntu | 2 +- docs/changelogs/v22.8.21.38-lts.md | 36 ++++++++++++++++++++++++++++ utils/list-versions/version_date.tsv | 5 ++++ 5 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 docs/changelogs/v22.8.21.38-lts.md diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile index c7206550bd8..ab0d89c3184 100644 --- a/docker/keeper/Dockerfile +++ b/docker/keeper/Dockerfile @@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \ esac ARG REPOSITORY="https://s3.amazonaws.com/clickhouse-builds/22.4/31c367d3cd3aefd316778601ff6565119fe36682/package_release" -ARG VERSION="23.7.4.5" +ARG VERSION="23.7.5.30" ARG PACKAGES="clickhouse-keeper" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine index a3a1f912e90..ac7d6a40518 100644 --- a/docker/server/Dockerfile.alpine +++ b/docker/server/Dockerfile.alpine @@ -33,7 +33,7 @@ RUN arch=${TARGETARCH:-amd64} \ # lts / testing / prestable / etc ARG REPO_CHANNEL="stable" ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}" -ARG VERSION="23.7.4.5" +ARG VERSION="23.7.5.30" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # user/group precreated explicitly with fixed uid/gid on purpose. diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu index 4936cfeccb0..10697142a51 100644 --- a/docker/server/Dockerfile.ubuntu +++ b/docker/server/Dockerfile.ubuntu @@ -23,7 +23,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list ARG REPO_CHANNEL="stable" ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main" -ARG VERSION="23.7.4.5" +ARG VERSION="23.7.5.30" ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static" # set non-empty deb_location_url url to create a docker image diff --git a/docs/changelogs/v22.8.21.38-lts.md b/docs/changelogs/v22.8.21.38-lts.md new file mode 100644 index 00000000000..fc919b25735 --- /dev/null +++ b/docs/changelogs/v22.8.21.38-lts.md @@ -0,0 +1,36 @@ +--- +sidebar_position: 1 +sidebar_label: 2023 +--- + +# 2023 Changelog + +### ClickHouse release v22.8.21.38-lts (70872e9859e) FIXME as compared to v22.8.20.11-lts (c9ca79e24e8) + +#### Build/Testing/Packaging Improvement +* Backported in [#53017](https://github.com/ClickHouse/ClickHouse/issues/53017): Packing inline cache into docker images sometimes causes strange special effects. Since we don't use it at all, it's good to go. [#53008](https://github.com/ClickHouse/ClickHouse/pull/53008) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). +* Backported in [#53459](https://github.com/ClickHouse/ClickHouse/issues/53459): Preserve environment parameters in `clickhouse start` command. Fixes [#51962](https://github.com/ClickHouse/ClickHouse/issues/51962). [#53418](https://github.com/ClickHouse/ClickHouse/pull/53418) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + +#### Bug Fix (user-visible misbehavior in an official stable release) + +* Fix Block structure mismatch in Pipe::unitePipes for FINAL [#51492](https://github.com/ClickHouse/ClickHouse/pull/51492) ([Nikita Taranov](https://github.com/nickitat)). +* Fix ORDER BY tuple of WINDOW functions [#52145](https://github.com/ClickHouse/ClickHouse/pull/52145) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix `countSubstrings()` hang with empty needle and a column haystack [#52409](https://github.com/ClickHouse/ClickHouse/pull/52409) ([Sergei Trifonov](https://github.com/serxa)). +* The implementation of AnyHash was non-conformant. [#52448](https://github.com/ClickHouse/ClickHouse/pull/52448) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* init and destroy ares channel on demand.. [#52634](https://github.com/ClickHouse/ClickHouse/pull/52634) ([Arthur Passos](https://github.com/arthurpassos)). +* clickhouse-keeper: fix implementation of server with poll() [#52833](https://github.com/ClickHouse/ClickHouse/pull/52833) ([Andy Fiddaman](https://github.com/citrus-it)). +* Not-ready Set [#53162](https://github.com/ClickHouse/ClickHouse/pull/53162) ([Nikolai Kochetov](https://github.com/KochetovNicolai)). +* Fix incorrect normal projection AST format [#53347](https://github.com/ClickHouse/ClickHouse/pull/53347) ([Amos Bird](https://github.com/amosbird)). +* Fix: interpolate expression takes source column instead of same name aliased from select expression. [#53572](https://github.com/ClickHouse/ClickHouse/pull/53572) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)). +* Correctly handle totals and extremes with `DelayedSource` [#53644](https://github.com/ClickHouse/ClickHouse/pull/53644) ([Antonio Andelic](https://github.com/antonio2368)). +* Fix: sorted distinct with sparse columns [#53711](https://github.com/ClickHouse/ClickHouse/pull/53711) ([Igor Nikonov](https://github.com/devcrafter)). + +#### NOT FOR CHANGELOG / INSIGNIFICANT + +* Fix crash in comparison functions due to incorrect query analysis [#52172](https://github.com/ClickHouse/ClickHouse/pull/52172) ([Alexey Milovidov](https://github.com/alexey-milovidov)). +* Fix deadlocks in StorageTableFunctionProxy [#52626](https://github.com/ClickHouse/ClickHouse/pull/52626) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Disable test_reverse_dns_query/test.py [#53195](https://github.com/ClickHouse/ClickHouse/pull/53195) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Disable test_host_regexp_multiple_ptr_records/test.py [#53211](https://github.com/ClickHouse/ClickHouse/pull/53211) ([Alexander Tokmakov](https://github.com/tavplubix)). +* Fix broken `02862_sorted_distinct_sparse_fix` [#53738](https://github.com/ClickHouse/ClickHouse/pull/53738) ([Antonio Andelic](https://github.com/antonio2368)). +* Get rid of describe_parameters for the best robot token [#53833](https://github.com/ClickHouse/ClickHouse/pull/53833) ([Mikhail f. Shiryaev](https://github.com/Felixoid)). + diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 0f15c761d2a..6d09f8501b3 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,9 +1,12 @@ +v23.7.5.30-stable 2023-08-28 v23.7.4.5-stable 2023-08-08 v23.7.3.14-stable 2023-08-05 v23.7.2.25-stable 2023-08-03 v23.7.1.2470-stable 2023-07-27 +v23.6.3.87-stable 2023-08-28 v23.6.2.18-stable 2023-07-09 v23.6.1.1524-stable 2023-06-30 +v23.5.5.92-stable 2023-08-28 v23.5.4.25-stable 2023-06-29 v23.5.3.24-stable 2023-06-17 v23.5.2.7-stable 2023-06-10 @@ -14,6 +17,7 @@ v23.4.4.16-stable 2023-06-17 v23.4.3.48-stable 2023-06-12 v23.4.2.11-stable 2023-05-02 v23.4.1.1943-stable 2023-04-27 +v23.3.11.5-lts 2023-08-28 v23.3.10.5-lts 2023-08-23 v23.3.9.55-lts 2023-08-21 v23.3.8.21-lts 2023-07-13 @@ -64,6 +68,7 @@ v22.9.4.32-stable 2022-10-26 v22.9.3.18-stable 2022-09-30 v22.9.2.7-stable 2022-09-23 v22.9.1.2603-stable 2022-09-22 +v22.8.21.38-lts 2023-08-28 v22.8.20.11-lts 2023-07-09 v22.8.19.10-lts 2023-06-17 v22.8.18.31-lts 2023-06-12 From 9190fb2ba337ccd802dfa80562912fe3992e68b3 Mon Sep 17 00:00:00 2001 From: Davit Vardanyan <78792753+davvard@users.noreply.github.com> Date: Mon, 28 Aug 2023 16:18:12 +0400 Subject: [PATCH 35/42] Fix: Deserialize --- contrib/usearch | 2 +- .../MergeTree/MergeTreeIndexUSearch.cpp | 21 +++++++++---------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/contrib/usearch b/contrib/usearch index 387b78b28b1..c6329e9bfb8 160000 --- a/contrib/usearch +++ b/contrib/usearch @@ -1 +1 @@ -Subproject commit 387b78b28b17b8954024ffc81e97cbcfa10d1f30 +Subproject commit c6329e9bfb893a92c28d825cc7982580f65bbfae diff --git a/src/Storages/MergeTree/MergeTreeIndexUSearch.cpp b/src/Storages/MergeTree/MergeTreeIndexUSearch.cpp index 731aa2309e3..5f07b3be914 100644 --- a/src/Storages/MergeTree/MergeTreeIndexUSearch.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexUSearch.cpp @@ -35,7 +35,7 @@ USearchIndexWithSerialization::USearchIndexWithSerialization(size_t dime } template -void USearchIndexWithSerialization::serialize([[maybe_unused]] WriteBuffer & ostr) const +void USearchIndexWithSerialization::serialize(WriteBuffer & ostr) const { auto callback = [&ostr](void * from, size_t n) { @@ -43,21 +43,20 @@ void USearchIndexWithSerialization::serialize([[maybe_unused]] WriteBuff return true; }; - Base::stream(callback); + Base::save_to_stream(callback); } template -void USearchIndexWithSerialization::deserialize([[maybe_unused]] ReadBuffer & istr) +void USearchIndexWithSerialization::deserialize(ReadBuffer & istr) { - BufferBase::Position & pos = istr.position(); - unum::usearch::memory_mapped_file_t memory_map(pos, istr.buffer().size() - istr.count()); - Base::view(std::move(memory_map)); - pos += Base::stream_length(); + auto callback = [&istr](void * from, size_t n) + { + istr.readStrict(reinterpret_cast(from), n); + return true; + }; + + Base::load_from_stream(callback); - auto copy = Base::copy(); - if (!copy) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Could not copy usearch index"); - Base::swap(copy.index); } template From ebc7dbb62838a2f7cbce6df1ef8736ed119d3b4b Mon Sep 17 00:00:00 2001 From: xuelei Date: Mon, 28 Aug 2023 09:39:22 -0300 Subject: [PATCH 36/42] The configuration of S3 append '/' automatically if needed --- src/Disks/ObjectStorages/S3/diskSettings.cpp | 2 +- src/Disks/ObjectStorages/S3/registerDiskS3.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Disks/ObjectStorages/S3/diskSettings.cpp b/src/Disks/ObjectStorages/S3/diskSettings.cpp index 29e2bb44fc4..ae2e3b40fa2 100644 --- a/src/Disks/ObjectStorages/S3/diskSettings.cpp +++ b/src/Disks/ObjectStorages/S3/diskSettings.cpp @@ -45,7 +45,7 @@ std::unique_ptr getClient( { String endpoint = context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); S3::URI uri(endpoint); - if(!uri.key.ends_with('/')) + if (!uri.key.ends_with('/')) uri.key.push_back('/'); S3::PocoHTTPClientConfiguration client_configuration = S3::ClientFactory::instance().createClientConfiguration( diff --git a/src/Disks/ObjectStorages/S3/registerDiskS3.cpp b/src/Disks/ObjectStorages/S3/registerDiskS3.cpp index 7592a30917e..69bdfe01a36 100644 --- a/src/Disks/ObjectStorages/S3/registerDiskS3.cpp +++ b/src/Disks/ObjectStorages/S3/registerDiskS3.cpp @@ -104,7 +104,7 @@ void registerDiskS3(DiskFactory & factory, bool global_skip_access_check) { String endpoint = context->getMacros()->expand(config.getString(config_prefix + ".endpoint")); S3::URI uri(endpoint); - if(!uri.key.ends_with('/')) + if (!uri.key.ends_with('/')) uri.key.push_back('/'); S3Capabilities s3_capabilities = getCapabilitiesFromConfig(config, config_prefix); From 4c78c088472b0033bf6bf380ebf52b797bf820f9 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 28 Aug 2023 13:02:24 +0000 Subject: [PATCH 37/42] Add test --- src/Storages/MergeTree/MergeTreeIndexAnnoy.h | 3 +- .../MergeTree/MergeTreeIndexUSearch.cpp | 10 +++--- .../MergeTree/MergeTreeIndexUSearch.h | 9 ++++-- .../0_stateless/02354_usearch_index.reference | 5 +++ .../0_stateless/02354_usearch_index.sql | 32 +++++++++++++++++++ 5 files changed, 50 insertions(+), 9 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeIndexAnnoy.h b/src/Storages/MergeTree/MergeTreeIndexAnnoy.h index cfc3b7519b8..d2765f10a22 100644 --- a/src/Storages/MergeTree/MergeTreeIndexAnnoy.h +++ b/src/Storages/MergeTree/MergeTreeIndexAnnoy.h @@ -25,6 +25,7 @@ public: template using AnnoyIndexWithSerializationPtr = std::shared_ptr>; + template struct MergeTreeIndexGranuleAnnoy final : public IMergeTreeIndexGranule { @@ -43,6 +44,7 @@ struct MergeTreeIndexGranuleAnnoy final : public IMergeTreeIndexGranule AnnoyIndexWithSerializationPtr index; }; + template struct MergeTreeIndexAggregatorAnnoy final : IMergeTreeIndexAggregator { @@ -104,7 +106,6 @@ private: const String distance_function; }; - } #endif diff --git a/src/Storages/MergeTree/MergeTreeIndexUSearch.cpp b/src/Storages/MergeTree/MergeTreeIndexUSearch.cpp index 5f07b3be914..70e2b8f76df 100644 --- a/src/Storages/MergeTree/MergeTreeIndexUSearch.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexUSearch.cpp @@ -56,7 +56,6 @@ void USearchIndexWithSerialization::deserialize(ReadBuffer & istr) }; Base::load_from_stream(callback); - } template @@ -245,18 +244,17 @@ std::vector MergeTreeIndexConditionUSearch::getUsefulRangesImpl(MergeTre throw Exception(ErrorCodes::LOGICAL_ERROR, "Attempt to optimize query with where without distance"); const std::vector reference_vector = ann_condition.getReferenceVector(); + const auto granule = std::dynamic_pointer_cast>(idx_granule); if (granule == nullptr) throw Exception(ErrorCodes::LOGICAL_ERROR, "Granule has the wrong type"); const USearchIndexWithSerializationPtr index = granule->index; + if (ann_condition.getDimensions() != index->dimensions()) - throw Exception( - ErrorCodes::INCORRECT_QUERY, - "The dimension of the space in the request ({}) " + throw Exception(ErrorCodes::INCORRECT_QUERY, "The dimension of the space in the request ({}) " "does not match the dimension in the index ({})", - ann_condition.getDimensions(), - index->dimensions()); + ann_condition.getDimensions(), index->dimensions()); auto result = index->search(reference_vector.data(), limit); std::vector neighbors(result.size()); /// indexes of dots which were closest to the reference vector diff --git a/src/Storages/MergeTree/MergeTreeIndexUSearch.h b/src/Storages/MergeTree/MergeTreeIndexUSearch.h index f1fde934fd5..98fb05b6f1a 100644 --- a/src/Storages/MergeTree/MergeTreeIndexUSearch.h +++ b/src/Storages/MergeTree/MergeTreeIndexUSearch.h @@ -27,6 +27,7 @@ public: template using USearchIndexWithSerializationPtr = std::shared_ptr>; + template struct MergeTreeIndexGranuleUSearch final : public IMergeTreeIndexGranule { @@ -45,6 +46,7 @@ struct MergeTreeIndexGranuleUSearch final : public IMergeTreeIndexGranule USearchIndexWithSerializationPtr index; }; + template struct MergeTreeIndexAggregatorUSearch final : IMergeTreeIndexAggregator { @@ -64,7 +66,11 @@ struct MergeTreeIndexAggregatorUSearch final : IMergeTreeIndexAggregator class MergeTreeIndexConditionUSearch final : public IMergeTreeIndexConditionApproximateNearestNeighbor { public: - MergeTreeIndexConditionUSearch(const IndexDescription & index_description, const SelectQueryInfo & query, const String & distance_function, ContextPtr context); + MergeTreeIndexConditionUSearch( + const IndexDescription & index_description, + const SelectQueryInfo & query, + const String & distance_function, + ContextPtr context); ~MergeTreeIndexConditionUSearch() override = default; @@ -98,7 +104,6 @@ private: const String distance_function; }; - } diff --git a/tests/queries/0_stateless/02354_usearch_index.reference b/tests/queries/0_stateless/02354_usearch_index.reference index a93209f6ba8..f966b5ce33f 100644 --- a/tests/queries/0_stateless/02354_usearch_index.reference +++ b/tests/queries/0_stateless/02354_usearch_index.reference @@ -141,3 +141,8 @@ Expression (Projection) Description: usearch GRANULARITY 4 Parts: 1/1 Granules: 1/4 +--- Test correctness of Usearch index with > 1 mark +1 [1,0,0,0] +9000 [9000,0,0,0] +1 (1,0,0,0) +9000 (9000,0,0,0) diff --git a/tests/queries/0_stateless/02354_usearch_index.sql b/tests/queries/0_stateless/02354_usearch_index.sql index f21767ea6de..f771e2835fa 100644 --- a/tests/queries/0_stateless/02354_usearch_index.sql +++ b/tests/queries/0_stateless/02354_usearch_index.sql @@ -228,3 +228,35 @@ ORDER BY L2Distance(vector, [10.0, 0.0, 10.0, 0.0]) LIMIT 3; DROP TABLE tab; + +SELECT '--- Test correctness of Usearch index with > 1 mark'; + +CREATE TABLE tab(id Int32, vector Array(Float32), INDEX usearch_index vector TYPE usearch()) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity_bytes=0, min_rows_for_wide_part = 0, min_bytes_for_wide_part = 0; -- disable adaptive granularity due to bug +INSERT INTO tab SELECT number, [toFloat32(number), 0., 0., 0.] from numbers(10000); + +SELECT * +FROM tab +ORDER BY L2Distance(vector, [1.0, 0.0, 0.0, 0.0]) +LIMIT 1; + +SELECT * +FROM tab +ORDER BY L2Distance(vector, [9000.0, 0.0, 0.0, 0.0]) +LIMIT 1; + +DROP TABLE tab; + +CREATE TABLE tab(id Int32, vector Tuple(Float32, Float32, Float32, Float32), INDEX usearch_index vector TYPE usearch()) ENGINE = MergeTree ORDER BY id SETTINGS index_granularity_bytes=0, min_rows_for_wide_part = 0, min_bytes_for_wide_part = 0; -- disable adaptive granularity due to bug +INSERT INTO tab SELECT number, (toFloat32(number), 0., 0., 0.) from numbers(10000); + +SELECT * +FROM tab +ORDER BY L2Distance(vector, (1.0, 0.0, 0.0, 0.0)) +LIMIT 1; + +SELECT * +FROM tab +ORDER BY L2Distance(vector, (9000.0, 0.0, 0.0, 0.0)) +LIMIT 1; + +DROP TABLE tab; From c91b409bcf3b997dd1318a873f996235bac5ac99 Mon Sep 17 00:00:00 2001 From: jsc0218 Date: Mon, 28 Aug 2023 09:19:46 -0400 Subject: [PATCH 38/42] remove trailing white spaces to pass format check --- src/Parsers/ASTForeignKeyDeclaration.h | 2 +- src/Parsers/ParserCreateQuery.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Parsers/ASTForeignKeyDeclaration.h b/src/Parsers/ASTForeignKeyDeclaration.h index d157ea81d5f..43c5995055d 100644 --- a/src/Parsers/ASTForeignKeyDeclaration.h +++ b/src/Parsers/ASTForeignKeyDeclaration.h @@ -15,7 +15,7 @@ public: String getID(char) const override { return "Foreign Key"; } - ASTPtr clone() const override + ASTPtr clone() const override { auto res = std::make_shared(); res->name = name; diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index ad0728ff5f8..c0d684ff6c6 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -379,7 +379,7 @@ bool ParserTablePropertiesDeclarationList::parseImpl(Pos & pos, ASTPtr & node, E constraints->children.push_back(elem); else if (elem->as()) projections->children.push_back(elem); - else if (elem->as()) + else if (elem->as()) { /// Ignore the foreign key node continue; From e52934ce615177b5a1fb2564e9ca5a4fcca8231e Mon Sep 17 00:00:00 2001 From: ltrk2 <107155950+ltrk2@users.noreply.github.com> Date: Mon, 28 Aug 2023 08:38:48 -0700 Subject: [PATCH 39/42] Make UInt128TrivialHash endianness-independent --- src/Common/HashTable/Hash.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/HashTable/Hash.h b/src/Common/HashTable/Hash.h index 49ab875297c..6bc5884e5bd 100644 --- a/src/Common/HashTable/Hash.h +++ b/src/Common/HashTable/Hash.h @@ -402,7 +402,7 @@ struct UInt128HashCRC32 : public UInt128Hash {}; struct UInt128TrivialHash { - size_t operator()(UInt128 x) const { return x.items[0]; } + size_t operator()(UInt128 x) const { return x.items[UInt128::_impl::little(0)]; } }; struct UUIDTrivialHash From dcecf52a680e58bbcc1898eaf7295121876e88bb Mon Sep 17 00:00:00 2001 From: HarryLeeIBM Date: Mon, 28 Aug 2023 11:35:04 -0700 Subject: [PATCH 40/42] Fix StripeLog storage endian issue for s390x --- src/Formats/IndexForNativeFormat.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Formats/IndexForNativeFormat.cpp b/src/Formats/IndexForNativeFormat.cpp index 91ae1e39280..bb410125378 100644 --- a/src/Formats/IndexForNativeFormat.cpp +++ b/src/Formats/IndexForNativeFormat.cpp @@ -20,8 +20,8 @@ void IndexOfBlockForNativeFormat::read(ReadBuffer & istr) auto & column = columns.emplace_back(); readBinary(column.name, istr); readBinary(column.type, istr); - readBinary(column.location.offset_in_compressed_file, istr); - readBinary(column.location.offset_in_decompressed_block, istr); + readBinaryLittleEndian(column.location.offset_in_compressed_file, istr); + readBinaryLittleEndian(column.location.offset_in_decompressed_block, istr); } } @@ -34,8 +34,8 @@ void IndexOfBlockForNativeFormat::write(WriteBuffer & ostr) const const auto & column = columns[i]; writeBinary(column.name, ostr); writeBinary(column.type, ostr); - writeBinary(column.location.offset_in_compressed_file, ostr); - writeBinary(column.location.offset_in_decompressed_block, ostr); + writeBinaryLittleEndian(column.location.offset_in_compressed_file, ostr); + writeBinaryLittleEndian(column.location.offset_in_decompressed_block, ostr); } } From 9f9c4293fac73302a13381b6d207986678ec5160 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 28 Aug 2023 20:10:49 +0000 Subject: [PATCH 41/42] Update usearch to v2.0.2 --- contrib/usearch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/usearch b/contrib/usearch index c6329e9bfb8..f942b6f334b 160000 --- a/contrib/usearch +++ b/contrib/usearch @@ -1 +1 @@ -Subproject commit c6329e9bfb893a92c28d825cc7982580f65bbfae +Subproject commit f942b6f334b31716f9bdb02eb6a25fa6b222f5ba From 7f4fc3744d04f216beb810fd3d69f8a63f9c0e40 Mon Sep 17 00:00:00 2001 From: pufit Date: Mon, 28 Aug 2023 22:23:01 -0400 Subject: [PATCH 42/42] resolve issues --- programs/keeper-client/KeeperClient.cpp | 31 +++++++++++++++---------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/programs/keeper-client/KeeperClient.cpp b/programs/keeper-client/KeeperClient.cpp index eacc4be98ae..6e151bf5dc3 100644 --- a/programs/keeper-client/KeeperClient.cpp +++ b/programs/keeper-client/KeeperClient.cpp @@ -224,7 +224,7 @@ void KeeperClient::initialize(Poco::Util::Application & /* self */) else default_log_level = "information"; - Poco::Logger::root().setLevel( config().getString("log-level", default_log_level)); + Poco::Logger::root().setLevel(config().getString("log-level", default_log_level)); EventNotifier::init(); } @@ -330,26 +330,33 @@ int KeeperClient::main(const std::vector & /* args */) config_processor.registerEmbeddedConfig("config.xml", ""); auto clickhouse_config = config_processor.loadConfig(); - String host; - String port; - String prefix = "zookeeper.node[0]."; + Poco::Util::AbstractConfiguration::Keys keys; + clickhouse_config.configuration->keys("zookeeper", keys); - if (!config().has("host") && !config().has("port") && clickhouse_config.configuration->has(prefix + "host")) + if (!config().has("host") && !config().has("port") && !keys.empty()) { LOG_INFO(&Poco::Logger::get("KeeperClient"), "Found keeper node in the config.xml, will use it for connection"); - if (clickhouse_config.configuration->has(prefix + "secure")) - host = "secure://"; - host += clickhouse_config.configuration->getString(prefix + "host"); - port = clickhouse_config.configuration->getString(prefix + "port"); + for (const auto & key : keys) + { + String prefix = "zookeeper." + key; + String host = clickhouse_config.configuration->getString(prefix + ".host"); + String port = clickhouse_config.configuration->getString(prefix + ".port"); + + if (clickhouse_config.configuration->has(prefix + ".secure")) + host = "secure://" + host; + + zk_args.hosts.push_back(host + ":" + port); + } } else { - host = config().getString("host", "localhost"); - port = config().getString("port", "9181"); + String host = config().getString("host", "localhost"); + String port = config().getString("port", "9181"); + + zk_args.hosts.push_back(host + ":" + port); } - zk_args.hosts = {host + ":" + port}; zk_args.connection_timeout_ms = config().getInt("connection-timeout", 10) * 1000; zk_args.session_timeout_ms = config().getInt("session-timeout", 10) * 1000; zk_args.operation_timeout_ms = config().getInt("operation-timeout", 10) * 1000;