From bb995c74ccfaf9ca28edebbd55fba5d32a4739bb Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 13 Jun 2024 16:30:38 +0200 Subject: [PATCH 01/13] Revert "Revert "S3: reduce retires time for queries, increase retries count for backups"" This reverts commit 7683f06188d8dc901bd912c4ace935a4b3f498e2. --- src/Backups/BackupIO_S3.cpp | 6 +++--- src/Core/Settings.h | 1 + src/Core/SettingsChangesHistory.h | 1 + src/IO/S3/Client.h | 2 +- .../integration/test_mask_sensitive_info/configs/users.xml | 1 + 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index 92f086295a0..1ea59c1d38b 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -54,9 +54,9 @@ namespace S3::PocoHTTPClientConfiguration client_configuration = S3::ClientFactory::instance().createClientConfiguration( settings.auth_settings.region, context->getRemoteHostFilter(), - static_cast(global_settings.s3_max_redirects), - static_cast(global_settings.s3_retry_attempts), - global_settings.enable_s3_requests_logging, + static_cast(local_settings.s3_max_redirects), + static_cast(local_settings.backup_restore_s3_retry_attempts), + local_settings.enable_s3_requests_logging, /* for_disk_s3 = */ false, request_settings.get_request_throttler, request_settings.put_request_throttler, diff --git a/src/Core/Settings.h b/src/Core/Settings.h index b3e83092a77..7f99243e285 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -517,6 +517,7 @@ class IColumn; M(UInt64, backup_restore_keeper_value_max_size, 1048576, "Maximum size of data of a [Zoo]Keeper's node during backup", 0) \ M(UInt64, backup_restore_batch_size_for_keeper_multiread, 10000, "Maximum size of batch for multiread request to [Zoo]Keeper during backup or restore", 0) \ M(UInt64, backup_restore_batch_size_for_keeper_multi, 1000, "Maximum size of batch for multi request to [Zoo]Keeper during backup or restore", 0) \ + M(UInt64, backup_restore_s3_retry_attempts, 1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore.", 0) \ M(UInt64, max_backup_bandwidth, 0, "The maximum read speed in bytes per second for particular backup on server. Zero means unlimited.", 0) \ \ M(Bool, log_profile_events, true, "Log query performance statistics into the query_log, query_thread_log and query_views_log.", 0) \ diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 69bc8c5d207..b447421671e 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -115,6 +115,7 @@ static const std::map& error, long attemptedRetries) const override; diff --git a/tests/integration/test_mask_sensitive_info/configs/users.xml b/tests/integration/test_mask_sensitive_info/configs/users.xml index f129a5bb3e3..f767216e907 100644 --- a/tests/integration/test_mask_sensitive_info/configs/users.xml +++ b/tests/integration/test_mask_sensitive_info/configs/users.xml @@ -2,6 +2,7 @@ 5 + 5 From 13e00d4f8b621c800b286e383b20765e669d5163 Mon Sep 17 00:00:00 2001 From: Sema Checherinda Date: Thu, 13 Jun 2024 17:40:42 +0200 Subject: [PATCH 02/13] put backup_restore_s3_retry_attempts to 24.6 --- src/Core/SettingsChangesHistory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index b447421671e..496c7a46798 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -105,6 +105,7 @@ static const std::map Date: Thu, 20 Jun 2024 19:09:15 +0200 Subject: [PATCH 03/13] backup_restore_s3_retry_attempts moved to 24.7 --- src/Core/SettingsChangesHistory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h index 0370368a8f8..5f332f6301e 100644 --- a/src/Core/SettingsChangesHistory.h +++ b/src/Core/SettingsChangesHistory.h @@ -85,6 +85,7 @@ namespace SettingsChangesHistory /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972) static const std::map settings_changes_history = { + {"24.7", {{"backup_restore_s3_retry_attempts", 0, 1000, "A new setting."}}}, {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"}, {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"}, {"input_format_parquet_use_native_reader", false, false, "When reading Parquet files, to use native reader instead of arrow reader."}, @@ -114,7 +115,6 @@ static const std::map Date: Wed, 10 Jul 2024 17:57:09 +0800 Subject: [PATCH 04/13] fix the bug that LogicalExpressionOptimizerPass lost logical type of constant --- src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp index 698602ca5bc..bd8b6f9faa1 100644 --- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp +++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp @@ -11,6 +11,7 @@ #include #include +#include namespace DB { @@ -615,6 +616,7 @@ private: bool is_any_nullable = false; Tuple args; args.reserve(equals_functions.size()); + DataTypes tuple_element_types; /// first we create tuple from RHS of equals functions for (const auto & equals : equals_functions) { @@ -627,16 +629,18 @@ private: if (const auto * rhs_literal = equals_arguments[1]->as()) { args.push_back(rhs_literal->getValue()); + tuple_element_types.push_back(rhs_literal->getResultType()); } else { const auto * lhs_literal = equals_arguments[0]->as(); assert(lhs_literal); args.push_back(lhs_literal->getValue()); + tuple_element_types.push_back(lhs_literal->getResultType()); } } - auto rhs_node = std::make_shared(std::move(args)); + auto rhs_node = std::make_shared(std::move(args), std::make_shared(std::move(tuple_element_types))); auto in_function = std::make_shared("in"); From 1ac2933d7d079ea31767da6304fa22712d38d6f4 Mon Sep 17 00:00:00 2001 From: chloro <13125187405@163.com> Date: Wed, 10 Jul 2024 22:12:15 +0800 Subject: [PATCH 05/13] add test --- ...03203_optimize_disjunctions_chain_to_in.reference | 2 ++ .../03203_optimize_disjunctions_chain_to_in.sql | 12 ++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 tests/queries/0_stateless/03203_optimize_disjunctions_chain_to_in.reference create mode 100644 tests/queries/0_stateless/03203_optimize_disjunctions_chain_to_in.sql diff --git a/tests/queries/0_stateless/03203_optimize_disjunctions_chain_to_in.reference b/tests/queries/0_stateless/03203_optimize_disjunctions_chain_to_in.reference new file mode 100644 index 00000000000..353c8f98b03 --- /dev/null +++ b/tests/queries/0_stateless/03203_optimize_disjunctions_chain_to_in.reference @@ -0,0 +1,2 @@ +2020-01-01 +2020-01-02 diff --git a/tests/queries/0_stateless/03203_optimize_disjunctions_chain_to_in.sql b/tests/queries/0_stateless/03203_optimize_disjunctions_chain_to_in.sql new file mode 100644 index 00000000000..e48138ab990 --- /dev/null +++ b/tests/queries/0_stateless/03203_optimize_disjunctions_chain_to_in.sql @@ -0,0 +1,12 @@ +SET allow_experimental_analyzer=1; +CREATE OR REPLACE TABLE foo (i Date) ENGINE MergeTree ORDER BY i; +INSERT INTO foo VALUES ('2020-01-01'); +INSERT INTO foo VALUES ('2020-01-02'); + +SET optimize_min_equality_disjunction_chain_length = 3; +SELECT * +FROM foo +WHERE (foo.i = parseDateTimeBestEffort('2020-01-01')) + OR (foo.i = parseDateTimeBestEffort('2020-01-02')) + OR (foo.i = parseDateTimeBestEffort('2020-01-03')) +ORDER BY foo.i ASC From faa0cb93fb524c14155e3ab409a099654cec8b91 Mon Sep 17 00:00:00 2001 From: chloro <13125187405@163.com> Date: Wed, 10 Jul 2024 23:37:27 +0800 Subject: [PATCH 06/13] Empty commit From 1fb0bdf51093d3482950f1cfd9d1ad0bb727a69e Mon Sep 17 00:00:00 2001 From: chloro <13125187405@163.com> Date: Thu, 11 Jul 2024 01:00:49 +0800 Subject: [PATCH 07/13] fix test failed --- .../0_stateless/03203_optimize_disjunctions_chain_to_in.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/03203_optimize_disjunctions_chain_to_in.sql b/tests/queries/0_stateless/03203_optimize_disjunctions_chain_to_in.sql index e48138ab990..f9ba28bcd60 100644 --- a/tests/queries/0_stateless/03203_optimize_disjunctions_chain_to_in.sql +++ b/tests/queries/0_stateless/03203_optimize_disjunctions_chain_to_in.sql @@ -1,5 +1,5 @@ SET allow_experimental_analyzer=1; -CREATE OR REPLACE TABLE foo (i Date) ENGINE MergeTree ORDER BY i; +CREATE TABLE foo (i Date) ENGINE MergeTree ORDER BY i; INSERT INTO foo VALUES ('2020-01-01'); INSERT INTO foo VALUES ('2020-01-02'); From 2f391cfa6c18bea786fbe21c692b7c7e33b6d34b Mon Sep 17 00:00:00 2001 From: chloro <13125187405@163.com> Date: Thu, 11 Jul 2024 17:50:46 +0800 Subject: [PATCH 08/13] restart ci From 82e13d785a460592e68662f0d94d4c0ce6d79057 Mon Sep 17 00:00:00 2001 From: chloro <13125187405@163.com> Date: Thu, 11 Jul 2024 22:23:09 +0800 Subject: [PATCH 09/13] restart ci From bc1b19591d1c78ba9b782722a27c1d29ce1d8d7d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 15 Jul 2024 05:13:38 +0200 Subject: [PATCH 10/13] Disable suspend on Ctrl+Z --- src/Client/ClientBase.cpp | 5 +++-- src/Client/LineReader.h | 3 +-- src/Client/ReplxxLineReader.cpp | 3 +++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 5d472ba99b9..c6a070219a3 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -80,6 +80,7 @@ #include #include "config.h" + namespace fs = std::filesystem; using namespace std::literals; @@ -2565,12 +2566,12 @@ void ClientBase::runInteractive() word_break_characters, highlight_callback); #else + (void)word_break_characters; LineReader lr( history_file, getClientConfiguration().has("multiline"), query_extenders, - query_delimiters, - word_break_characters); + query_delimiters); #endif static const std::initializer_list> backslash_aliases = diff --git a/src/Client/LineReader.h b/src/Client/LineReader.h index 0172bd7ec22..8c101401190 100644 --- a/src/Client/LineReader.h +++ b/src/Client/LineReader.h @@ -46,8 +46,7 @@ public: Patterns delimiters, std::istream & input_stream_ = std::cin, std::ostream & output_stream_ = std::cout, - int in_fd_ = STDIN_FILENO - ); + int in_fd_ = STDIN_FILENO); virtual ~LineReader() = default; diff --git a/src/Client/ReplxxLineReader.cpp b/src/Client/ReplxxLineReader.cpp index 46600168695..3b3508d1a58 100644 --- a/src/Client/ReplxxLineReader.cpp +++ b/src/Client/ReplxxLineReader.cpp @@ -362,6 +362,9 @@ ReplxxLineReader::ReplxxLineReader( rx.bind_key(Replxx::KEY::control('N'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_NEXT, code); }); rx.bind_key(Replxx::KEY::control('P'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_PREVIOUS, code); }); + /// We don't want the default, "suspend" behavior, it confuses people. + rx.bind_key_internal(replxx::Replxx::KEY::control('Z'), "insert_character"); + auto commit_action = [this](char32_t code) { /// If we allow multiline and there is already something in the input, start a newline. From de2e789c06e255b7a31a6caf530043fad8b248a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 15 Jul 2024 19:44:30 +0200 Subject: [PATCH 11/13] Change env variable to run tests with SMT --- tests/clickhouse-test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 90fb9611151..8df1dd2a8eb 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -3386,13 +3386,13 @@ def parse_args(): parser.add_argument( "--replace-replicated-with-shared", action="store_true", - default=os.environ.get("USE_META_IN_KEEPER_FOR_MERGE_TREE", False), + default=os.environ.get("REPLACE_RMT_WITH_SMT", False), help="Replace ReplicatedMergeTree engine with SharedMergeTree", ) parser.add_argument( "--replace-non-replicated-with-shared", action="store_true", - default=False, + default=os.environ.get("REPLACE_MT_WITH_SMT", False), help="Replace ordinary MergeTree engine with SharedMergeTree", ) From 1f752eb313804fcdb1e360a69b195a2481fd970a Mon Sep 17 00:00:00 2001 From: Yinzuo Jiang Date: Tue, 16 Jul 2024 15:59:10 +0800 Subject: [PATCH 12/13] Update build-cross-riscv.md --- docs/en/development/build-cross-riscv.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/build-cross-riscv.md b/docs/en/development/build-cross-riscv.md index 759d97823e2..dd97b6081e8 100644 --- a/docs/en/development/build-cross-riscv.md +++ b/docs/en/development/build-cross-riscv.md @@ -11,7 +11,7 @@ This is for the case when you have Linux machine and want to use it to build `cl The cross-build for RISC-V 64 is based on the [Build instructions](../development/build.md), follow them first. -## Install Clang-16 +## Install Clang-18 Follow the instructions from https://apt.llvm.org/ for your Ubuntu or Debian setup or do ``` From 630a29f537ecae672dcc295f3f29cea5478253a2 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 16 Jul 2024 15:11:17 +0200 Subject: [PATCH 13/13] Fix bsdtar for @nikitamikhaylov --- tests/ci/compress_files.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ci/compress_files.py b/tests/ci/compress_files.py index 70c0aca6a36..f49c872b5a0 100644 --- a/tests/ci/compress_files.py +++ b/tests/ci/compress_files.py @@ -58,14 +58,14 @@ def decompress_fast(archive_path: Path, result_path: Optional[Path] = None) -> N archive_path, result_path, ) - program_part = "--use-compress-program='zstd --threads=0'" + program_part = "--use-compress-program='zstd --threads=0 -d'" elif PIGZ.exists(): logging.info( "pigz found, will compress and decompress faster ('%s' -> '%s')", archive_path, result_path, ) - program_part = "--use-compress-program='pigz'" + program_part = "--use-compress-program='pigz -d'" else: program_part = "-z" logging.info(