simplify build, add read method randomized tests, fix typos

contrib/liburing-cmake/CMakeLists.txt

@@ -24,71 +24,11 @@ set (SRCS
 set (LIBURING_COMPAT_INCLUDE_DIR "${ClickHouse_BINARY_DIR}/contrib/liburing/src/include-compat")
 set (LIBURING_COMPAT_HEADER "${LIBURING_COMPAT_INCLUDE_DIR}/liburing/compat.h")
 
-include(CheckCSourceCompiles)
-
-# The below checks should run against the bundled contrib/libc-headers
-set (CMAKE_REQUIRED_INCLUDES ${CMAKE_C_STANDARD_INCLUDE_DIRECTORIES})
-
-check_c_source_compiles ("
-#include <linux/fs.h>
-int main(int argc, char **argv)
-{
-  __kernel_rwf_t x;
-  x = 0;
-  return x;
-}" LIBURING_CONFIG_HAS_KERNEL_RWF_T)
-
-check_c_source_compiles ("
-#include <linux/time.h>
-#include <linux/time_types.h>
-int main(int argc, char **argv)
-{
-  struct __kernel_timespec ts;
-  ts.tv_sec = 0;
-  ts.tv_nsec = 1;
-  return 0;
-}" LIBURING_CONFIG_HAS_KERNEL_TIMESPEC)
-
-check_c_source_compiles ("
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <string.h>
-int main(int argc, char **argv)
-{
-  struct open_how how;
-  how.flags = 0;
-  how.mode = 0;
-  how.resolve = 0;
-  return 0;
-}" LIBURING_CONFIG_HAS_OPEN_HOW)
-
-check_c_source_compiles ("
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <string.h>
-#include <linux/stat.h>
-int main(int argc, char **argv)
-{
-  struct statx x;
-  return memset(&x, 0, sizeof(x)) != NULL;
-}" LIBURING_CONFIG_HAS_STATX)
-
-check_c_source_compiles ("
-#include <sys/types.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <string.h>
-#include <linux/stat.h>
-int main(int argc, char **argv)
-{
-  struct statx x;
-  return memset(&x, 0, sizeof(x)) != NULL;
-}" LIBURING_CONFIG_HAS_GLIBC_STATX)
-
-unset (CMAKE_REQUIRED_INCLUDES)
+set (LIBURING_CONFIG_HAS_KERNEL_RWF_T,    "yes")
+set (LIBURING_CONFIG_HAS_KERNEL_TIMESPEC, "no")
+set (LIBURING_CONFIG_HAS_OPEN_HOW,        "no")
+set (LIBURING_CONFIG_HAS_STATX,           "no")
+set (LIBURING_CONFIG_HAS_GLIBC_STATX,     "no")
 
 add_compile_definitions(LIBURING_INTERNAL)
 configure_file (compat.h.in ${LIBURING_COMPAT_HEADER})
@@ -96,4 +36,4 @@ configure_file (compat.h.in ${LIBURING_COMPAT_HEADER})
 add_library (_liburing ${SRCS})
 add_library (ch_contrib::liburing ALIAS _liburing)
 
-target_include_directories (_liburing PUBLIC ${LIBURING_COMPAT_INCLUDE_DIR} "${LIBURING_SOURCE_DIR}/include")
+target_include_directories (_liburing SYSTEM PUBLIC ${LIBURING_COMPAT_INCLUDE_DIR} "${LIBURING_SOURCE_DIR}/include")

docker/test/fasttest/run.sh

@@ -135,6 +135,7 @@ function clone_submodules
             contrib/replxx
             contrib/wyhash
             contrib/hashidsxx
+            contrib/liburing
         )
 
         git submodule sync
@@ -157,6 +158,7 @@ function run_cmake
         "-DENABLE_NURAFT=1"
         "-DENABLE_JEMALLOC=1"
         "-DENABLE_REPLXX=1"
+        "-DENABLE_LIBURING=1"
     )
 
     # TODO remove this? we don't use ccache anyway. An option would be to download it

src/Disks/IO/IOUringReader.cpp

@@ -81,7 +81,7 @@ std::future<IAsynchronousReader::Result> IOUringReader::submit(Request request)
     });
 
     if (!is_newly_inserted)
-        return makeFailedResult(ErrorCodes::LOGICAL_ERROR, "Tried enqueuing read request for %lu that is already submitted", request_id);
+        return makeFailedResult(ErrorCodes::LOGICAL_ERROR, "Tried enqueuing read request for {} that is already submitted", request_id);
 
     EnqueuedRequest & enqueued = kv->second;
     trySubmitRequest(request_id, enqueued, false);
@@ -147,7 +147,7 @@ void IOUringReader::monitorRing()
         int ret = io_uring_wait_cqe(&ring, &cqe);
 
         if (ret < 0)
-            throwFromErrno(fmt::format("Failed waiting for io_uring CQEs: ", ret), ErrorCodes::IO_URING_WAIT_ERROR);
+            throwFromErrno(fmt::format("Failed waiting for io_uring CQEs: {}", ret), ErrorCodes::IO_URING_WAIT_ERROR);
 
         // user_data zero means a noop event sent from the destructor meant to interrupt the thread
         if (cancelled.load(std::memory_order_relaxed) || cqe->user_data == 0)
@@ -160,7 +160,7 @@ void IOUringReader::monitorRing()
         const auto it = enqueued_requests.find(request_id);
         if (it == enqueued_requests.end())
             throwFromErrno(
-                fmt::format("Got a completion event for a request %lu that was not submitted", request_id),
+                fmt::format("Got a completion event for a request {} that was not submitted", request_id),
                 ErrorCodes::LOGICAL_ERROR);
 
         auto & enqueued = it->second;

src/Disks/IO/IOUringReader.h

@@ -4,25 +4,8 @@
 #include <Common/ThreadPool.h>
 #include <IO/AsynchronousReader.h>
 #include <unordered_map>
-
-#if defined(__clang__)
-#pragma clang diagnostic push
-
-#pragma clang diagnostic ignored "-Wold-style-cast"
-#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
-
-#ifdef HAS_RESERVED_IDENTIFIER
-#pragma clang diagnostic ignored "-Wreserved-identifier"
-#endif
-
-#endif
-
 #include <liburing.h>
 
-#if defined(__clang__)
-#pragma clang diagnostic pop
-#endif
-
 namespace DB
 {
 

tests/clickhouse-test

@@ -425,6 +425,7 @@ class SettingsRandomizer:
         "read_in_order_two_level_merge_threshold": lambda: random.randint(0, 100),
         "optimize_aggregation_in_order": lambda: random.randint(0, 1),
         "aggregation_in_order_max_block_bytes": lambda: random.randint(0, 50000000),
+        "local_filesystem_read_method": lambda: random.choice(['pread', 'pread_threadpool', 'io_uring']),
     }
 
     @staticmethod