From 8401cd8c41fed0bc26b754bf51ff0f12979fe205 Mon Sep 17 00:00:00 2001
From: Nikita Taranov
Date: Wed, 21 Feb 2024 00:47:54 +0100
Subject: [PATCH 1/7] impl
---
contrib/aws | 2 +-
src/IO/S3/Client.cpp | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/contrib/aws b/contrib/aws
index 9eb5097a0ab..5f0542b3ad7 160000
--- a/contrib/aws
+++ b/contrib/aws
@@ -1 +1 @@
-Subproject commit 9eb5097a0abfa837722cca7a5114a25837817bf2
+Subproject commit 5f0542b3ad7eef25b0540d37d778207e0345ea8f
diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp
index 7f0ede72740..108dfdb7706 100644
--- a/src/IO/S3/Client.cpp
+++ b/src/IO/S3/Client.cpp
@@ -731,7 +731,7 @@ std::string Client::getRegionForBucket(const std::string & bucket, bool force_de
if (outcome.IsSuccess())
{
const auto & result = outcome.GetResult();
- region = result.GetRegion();
+ region = result.GetBucketRegion();
}
else
{
From 1d335ca645b2a7a2455b3e161bb87fe3e6d887f9 Mon Sep 17 00:00:00 2001
From: Pablo Musa
Date: Fri, 23 Feb 2024 18:34:44 +0100
Subject: [PATCH 2/7] Fix typos on postgresql.md
---
docs/en/engines/database-engines/postgresql.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/en/engines/database-engines/postgresql.md b/docs/en/engines/database-engines/postgresql.md
index 294d1202bdd..ae323680688 100644
--- a/docs/en/engines/database-engines/postgresql.md
+++ b/docs/en/engines/database-engines/postgresql.md
@@ -10,7 +10,7 @@ Allows to connect to databases on a remote [PostgreSQL](https://www.postgresql.o
Gives the real-time access to table list and table structure from remote PostgreSQL with the help of `SHOW TABLES` and `DESCRIBE TABLE` queries.
-Supports table structure modifications (`ALTER TABLE ... ADD|DROP COLUMN`). If `use_table_cache` parameter (see the Engine Parameters below) it set to `1`, the table structure is cached and not checked for being modified, but can be updated with `DETACH` and `ATTACH` queries.
+Supports table structure modifications (`ALTER TABLE ... ADD|DROP COLUMN`). If `use_table_cache` parameter (see the Engine Parameters below) is set to `1`, the table structure is cached and not checked for being modified, but can be updated with `DETACH` and `ATTACH` queries.
## Creating a Database {#creating-a-database}
From dd690dd05d7e04a2a86a11d9bcb0ebf3542f3482 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin
Date: Sat, 24 Feb 2024 19:23:28 +0100
Subject: [PATCH 3/7] Expose fatal.log separately for fuzzer
Otherwise you have to download the full server.log
Signed-off-by: Azat Khuzhin
---
docker/test/fuzzer/run-fuzzer.sh | 6 ++++++
tests/ci/ast_fuzzer_check.py | 1 +
2 files changed, 7 insertions(+)
diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh
index ca6bff9c6be..69422992cfb 100755
--- a/docker/test/fuzzer/run-fuzzer.sh
+++ b/docker/test/fuzzer/run-fuzzer.sh
@@ -387,6 +387,11 @@ if [ -f core.zst ]; then
fi
rg --text -F '' server.log > fatal.log ||:
+FATAL_LINK=''
+if [ -s fatal.log ]; then
+ FATAL_LINK='fatal.log'
+fi
+
dmesg -T > dmesg.log ||:
zstd --threads=0 --rm server.log
@@ -419,6 +424,7 @@ p.links a { padding: 5px; margin: 3px; background: #FFF; line-height: 2; white-s
main.log
dmesg.log
${CORE_LINK}
+ ${FATAL_LINK}
diff --git a/tests/ci/ast_fuzzer_check.py b/tests/ci/ast_fuzzer_check.py
index 0a69d8aab49..6e3da7fa816 100644
--- a/tests/ci/ast_fuzzer_check.py
+++ b/tests/ci/ast_fuzzer_check.py
@@ -114,6 +114,7 @@ def main():
"report.html": workspace_path / "report.html",
"core.zst": workspace_path / "core.zst",
"dmesg.log": workspace_path / "dmesg.log",
+ "fatal.log": workspace_path / "fatal.log",
}
compressed_server_log_path = workspace_path / "server.log.zst"
From a70a100db05b57b85e4ca6f295242fe6642cb936 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin
Date: Sat, 24 Feb 2024 07:52:53 +0100
Subject: [PATCH 4/7] Support io_uring for StorageFile
Signed-off-by: Azat Khuzhin
---
src/Storages/StorageFile.cpp | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp
index 595573b566d..7d674fea9ca 100644
--- a/src/Storages/StorageFile.cpp
+++ b/src/Storages/StorageFile.cpp
@@ -26,6 +26,8 @@
#include
#include
#include
+#include
+#include
#include
#include
@@ -92,6 +94,7 @@ namespace ErrorCodes
extern const int CANNOT_EXTRACT_TABLE_STRUCTURE;
extern const int CANNOT_DETECT_FORMAT;
extern const int CANNOT_COMPILE_REGEXP;
+ extern const int UNSUPPORTED_METHOD;
}
namespace
@@ -276,6 +279,22 @@ std::unique_ptr selectReadBuffer(
ProfileEvents::increment(ProfileEvents::CreatedReadBufferOrdinary);
}
+ else if (read_method == LocalFSReadMethod::io_uring && !use_table_fd)
+ {
+#if USE_LIBURING
+ auto & reader = context->getIOURingReader();
+ if (!reader.isSupported())
+ throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "io_uring is not supported by this system");
+
+ res = std::make_unique(
+ reader,
+ Priority{},
+ current_path,
+ context->getSettingsRef().max_read_buffer_size);
+#else
+ throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Read method io_uring is only supported in Linux");
+#endif
+ }
else
{
if (use_table_fd)
From 35e948a20a5dc4ec6685facb8aa7be2196b486fe Mon Sep 17 00:00:00 2001
From: Azat Khuzhin
Date: Sat, 24 Feb 2024 07:55:22 +0100
Subject: [PATCH 5/7] Detect io_uring in tests
Signed-off-by: Azat Khuzhin
---
tests/clickhouse-test | 26 ++++++++++++++++++++++----
1 file changed, 22 insertions(+), 4 deletions(-)
diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 9c21f1fd2a2..275f89548d4 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -574,6 +574,27 @@ def get_localzone():
return os.getenv("TZ", "/".join(os.readlink("/etc/localtime").split("/")[-2:]))
+def supports_io_uring():
+ return not subprocess.call(
+ [
+ args.binary,
+ "-q",
+ "select * from file('/dev/null', 'LineAsString')",
+ "--storage_file_read_method",
+ "io_uring",
+ ],
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ )
+
+
+def get_local_filesystem_methods():
+ methods = ["read", "pread", "mmap", "pread_threadpool"]
+ if supports_io_uring():
+ methods.append("io_uring")
+ return methods
+
+
class SettingsRandomizer:
settings = {
"max_insert_threads": lambda: 0
@@ -614,10 +635,7 @@ class SettingsRandomizer:
0.2, 0.5, 1, 10 * 1024 * 1024 * 1024
),
"local_filesystem_read_method": lambda: random.choice(
- # Allow to use uring only when running on Linux
- ["read", "pread", "mmap", "pread_threadpool", "io_uring"]
- if platform.system().lower() == "linux"
- else ["read", "pread", "mmap", "pread_threadpool"]
+ get_local_filesystem_methods()
),
"remote_filesystem_read_method": lambda: random.choice(["read", "threadpool"]),
"local_filesystem_read_prefetch": lambda: random.randint(0, 1),
From 554d993d8ba4268d112385250e060b1500e53bc8 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin
Date: Sat, 24 Feb 2024 19:37:13 +0100
Subject: [PATCH 6/7] clickhouse-test: fix style check (unused import platform)
Signed-off-by: Azat Khuzhin
---
tests/clickhouse-test | 1 -
1 file changed, 1 deletion(-)
diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 275f89548d4..f438c6f4f31 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -13,7 +13,6 @@ import sys
import os
import os.path
import glob
-import platform
import signal
import re
import copy
From 65474216e07abe990ee8fb525a10cf1b45044f60 Mon Sep 17 00:00:00 2001
From: rogeryk
Date: Sun, 25 Feb 2024 16:50:41 +0800
Subject: [PATCH 7/7] Update build-osx.md
The NASM is required, The corresponding installation is missing from the doc.
---
docs/en/development/build-osx.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/docs/en/development/build-osx.md b/docs/en/development/build-osx.md
index 39ccc9a78c3..21b9446aa66 100644
--- a/docs/en/development/build-osx.md
+++ b/docs/en/development/build-osx.md
@@ -37,7 +37,7 @@ sudo xcode-select --install
``` bash
brew update
-brew install ccache cmake ninja libtool gettext llvm gcc binutils grep findutils
+brew install ccache cmake ninja libtool gettext llvm gcc binutils grep findutils nasm
```
## Checkout ClickHouse Sources {#checkout-clickhouse-sources}