From 8401cd8c41fed0bc26b754bf51ff0f12979fe205 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Wed, 21 Feb 2024 00:47:54 +0100 Subject: [PATCH 1/7] impl --- contrib/aws | 2 +- src/IO/S3/Client.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/aws b/contrib/aws index 9eb5097a0ab..5f0542b3ad7 160000 --- a/contrib/aws +++ b/contrib/aws @@ -1 +1 @@ -Subproject commit 9eb5097a0abfa837722cca7a5114a25837817bf2 +Subproject commit 5f0542b3ad7eef25b0540d37d778207e0345ea8f diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index 7f0ede72740..108dfdb7706 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -731,7 +731,7 @@ std::string Client::getRegionForBucket(const std::string & bucket, bool force_de if (outcome.IsSuccess()) { const auto & result = outcome.GetResult(); - region = result.GetRegion(); + region = result.GetBucketRegion(); } else { From 1d335ca645b2a7a2455b3e161bb87fe3e6d887f9 Mon Sep 17 00:00:00 2001 From: Pablo Musa Date: Fri, 23 Feb 2024 18:34:44 +0100 Subject: [PATCH 2/7] Fix typos on postgresql.md --- docs/en/engines/database-engines/postgresql.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/database-engines/postgresql.md b/docs/en/engines/database-engines/postgresql.md index 294d1202bdd..ae323680688 100644 --- a/docs/en/engines/database-engines/postgresql.md +++ b/docs/en/engines/database-engines/postgresql.md @@ -10,7 +10,7 @@ Allows to connect to databases on a remote [PostgreSQL](https://www.postgresql.o Gives the real-time access to table list and table structure from remote PostgreSQL with the help of `SHOW TABLES` and `DESCRIBE TABLE` queries. -Supports table structure modifications (`ALTER TABLE ... ADD|DROP COLUMN`). If `use_table_cache` parameter (see the Engine Parameters below) it set to `1`, the table structure is cached and not checked for being modified, but can be updated with `DETACH` and `ATTACH` queries. +Supports table structure modifications (`ALTER TABLE ... ADD|DROP COLUMN`). If `use_table_cache` parameter (see the Engine Parameters below) is set to `1`, the table structure is cached and not checked for being modified, but can be updated with `DETACH` and `ATTACH` queries. ## Creating a Database {#creating-a-database} From dd690dd05d7e04a2a86a11d9bcb0ebf3542f3482 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 24 Feb 2024 19:23:28 +0100 Subject: [PATCH 3/7] Expose fatal.log separately for fuzzer Otherwise you have to download the full server.log Signed-off-by: Azat Khuzhin --- docker/test/fuzzer/run-fuzzer.sh | 6 ++++++ tests/ci/ast_fuzzer_check.py | 1 + 2 files changed, 7 insertions(+) diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index ca6bff9c6be..69422992cfb 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -387,6 +387,11 @@ if [ -f core.zst ]; then fi rg --text -F '' server.log > fatal.log ||: +FATAL_LINK='' +if [ -s fatal.log ]; then + FATAL_LINK='fatal.log' +fi + dmesg -T > dmesg.log ||: zstd --threads=0 --rm server.log @@ -419,6 +424,7 @@ p.links a { padding: 5px; margin: 3px; background: #FFF; line-height: 2; white-s main.log dmesg.log ${CORE_LINK} + ${FATAL_LINK}

diff --git a/tests/ci/ast_fuzzer_check.py b/tests/ci/ast_fuzzer_check.py index 0a69d8aab49..6e3da7fa816 100644 --- a/tests/ci/ast_fuzzer_check.py +++ b/tests/ci/ast_fuzzer_check.py @@ -114,6 +114,7 @@ def main(): "report.html": workspace_path / "report.html", "core.zst": workspace_path / "core.zst", "dmesg.log": workspace_path / "dmesg.log", + "fatal.log": workspace_path / "fatal.log", } compressed_server_log_path = workspace_path / "server.log.zst" From a70a100db05b57b85e4ca6f295242fe6642cb936 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 24 Feb 2024 07:52:53 +0100 Subject: [PATCH 4/7] Support io_uring for StorageFile Signed-off-by: Azat Khuzhin --- src/Storages/StorageFile.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index 595573b566d..7d674fea9ca 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -26,6 +26,8 @@ #include #include #include +#include +#include #include #include @@ -92,6 +94,7 @@ namespace ErrorCodes extern const int CANNOT_EXTRACT_TABLE_STRUCTURE; extern const int CANNOT_DETECT_FORMAT; extern const int CANNOT_COMPILE_REGEXP; + extern const int UNSUPPORTED_METHOD; } namespace @@ -276,6 +279,22 @@ std::unique_ptr selectReadBuffer( ProfileEvents::increment(ProfileEvents::CreatedReadBufferOrdinary); } + else if (read_method == LocalFSReadMethod::io_uring && !use_table_fd) + { +#if USE_LIBURING + auto & reader = context->getIOURingReader(); + if (!reader.isSupported()) + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "io_uring is not supported by this system"); + + res = std::make_unique( + reader, + Priority{}, + current_path, + context->getSettingsRef().max_read_buffer_size); +#else + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Read method io_uring is only supported in Linux"); +#endif + } else { if (use_table_fd) From 35e948a20a5dc4ec6685facb8aa7be2196b486fe Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 24 Feb 2024 07:55:22 +0100 Subject: [PATCH 5/7] Detect io_uring in tests Signed-off-by: Azat Khuzhin --- tests/clickhouse-test | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 9c21f1fd2a2..275f89548d4 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -574,6 +574,27 @@ def get_localzone(): return os.getenv("TZ", "/".join(os.readlink("/etc/localtime").split("/")[-2:])) +def supports_io_uring(): + return not subprocess.call( + [ + args.binary, + "-q", + "select * from file('/dev/null', 'LineAsString')", + "--storage_file_read_method", + "io_uring", + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + +def get_local_filesystem_methods(): + methods = ["read", "pread", "mmap", "pread_threadpool"] + if supports_io_uring(): + methods.append("io_uring") + return methods + + class SettingsRandomizer: settings = { "max_insert_threads": lambda: 0 @@ -614,10 +635,7 @@ class SettingsRandomizer: 0.2, 0.5, 1, 10 * 1024 * 1024 * 1024 ), "local_filesystem_read_method": lambda: random.choice( - # Allow to use uring only when running on Linux - ["read", "pread", "mmap", "pread_threadpool", "io_uring"] - if platform.system().lower() == "linux" - else ["read", "pread", "mmap", "pread_threadpool"] + get_local_filesystem_methods() ), "remote_filesystem_read_method": lambda: random.choice(["read", "threadpool"]), "local_filesystem_read_prefetch": lambda: random.randint(0, 1), From 554d993d8ba4268d112385250e060b1500e53bc8 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sat, 24 Feb 2024 19:37:13 +0100 Subject: [PATCH 6/7] clickhouse-test: fix style check (unused import platform) Signed-off-by: Azat Khuzhin --- tests/clickhouse-test | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 275f89548d4..f438c6f4f31 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -13,7 +13,6 @@ import sys import os import os.path import glob -import platform import signal import re import copy From 65474216e07abe990ee8fb525a10cf1b45044f60 Mon Sep 17 00:00:00 2001 From: rogeryk Date: Sun, 25 Feb 2024 16:50:41 +0800 Subject: [PATCH 7/7] Update build-osx.md The NASM is required, The corresponding installation is missing from the doc. --- docs/en/development/build-osx.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/build-osx.md b/docs/en/development/build-osx.md index 39ccc9a78c3..21b9446aa66 100644 --- a/docs/en/development/build-osx.md +++ b/docs/en/development/build-osx.md @@ -37,7 +37,7 @@ sudo xcode-select --install ``` bash brew update -brew install ccache cmake ninja libtool gettext llvm gcc binutils grep findutils +brew install ccache cmake ninja libtool gettext llvm gcc binutils grep findutils nasm ``` ## Checkout ClickHouse Sources {#checkout-clickhouse-sources}