Corrosion fixes + review fixes

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.15)
+cmake_minimum_required(VERSION 3.20)
 
 project(ClickHouse LANGUAGES C CXX ASM)
 
@@ -65,14 +65,6 @@ if (ENABLE_CHECK_HEAVY_BUILDS)
     set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --as=${RLIMIT_AS} --data=${RLIMIT_DATA} --cpu=${RLIMIT_CPU} ${CMAKE_CXX_COMPILER_LAUNCHER})
 endif ()
 
-if (CMAKE_BUILD_TYPE STREQUAL "Debug")
-    set(CARGO_RELEASE_FLAG "" CACHE INTERNAL "")
-    set(TARGET_DIR "debug" CACHE INTERNAL "")
-else ()
-    set(CARGO_RELEASE_FLAG "--release" CACHE INTERNAL "")
-    set(TARGET_DIR "release" CACHE INTERNAL "")
-endif ()
-
 if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None")
     set (CMAKE_BUILD_TYPE "RelWithDebInfo")
     message (STATUS "CMAKE_BUILD_TYPE is not set, set to default = ${CMAKE_BUILD_TYPE}")
@@ -98,8 +90,6 @@ if (USE_STATIC_LIBRARIES)
     list(REVERSE CMAKE_FIND_LIBRARY_SUFFIXES)
 endif ()
 
-option (ENABLE_RUST "Build libraries written in Rust" ON)
-
 option (ENABLE_FUZZING "Fuzzy testing using libfuzzer" OFF)
 
 if (ENABLE_FUZZING)
@@ -586,9 +576,9 @@ include_directories(${ConfigIncludePath})
 include (cmake/warnings.cmake)
 include (cmake/print_flags.cmake)
 
-#if (ENABLE_RUST)
-add_subdirectory (rust)
-#endif()
+if (ENABLE_RUST)
+    add_subdirectory (rust)
+endif()
 
 add_subdirectory (base)
 add_subdirectory (src)

base/glibc-compatibility/spawn.h

@@ -7,21 +7,6 @@ extern "C" {
 
 #include <features.h>
 
-#define __NEED_mode_t
-#define __NEED_pid_t
-#define __NEED_sigset_t
-
-struct sched_param;
-
-#define POSIX_SPAWN_RESETIDS 1
-#define POSIX_SPAWN_SETPGROUP 2
-#define POSIX_SPAWN_SETSIGDEF 4
-#define POSIX_SPAWN_SETSIGMASK 8
-#define POSIX_SPAWN_SETSCHEDPARAM 16
-#define POSIX_SPAWN_SETSCHEDULER 32
-#define POSIX_SPAWN_USEVFORK 64
-#define POSIX_SPAWN_SETSID 128
-
 typedef struct {
 	int __flags;
 	pid_t __pgrp;
@@ -39,40 +24,6 @@ typedef struct {
 
 int posix_spawn(pid_t *__restrict, const char *__restrict, const posix_spawn_file_actions_t *,
 	const posix_spawnattr_t *__restrict, char *const *__restrict, char *const *__restrict);
-int posix_spawnp(pid_t *__restrict, const char *__restrict, const posix_spawn_file_actions_t *,
-	const posix_spawnattr_t *__restrict, char *const *__restrict, char *const *__restrict);
-
-int posix_spawnattr_init(posix_spawnattr_t *);
-int posix_spawnattr_destroy(posix_spawnattr_t *);
-
-int posix_spawnattr_setflags(posix_spawnattr_t *, short);
-int posix_spawnattr_getflags(const posix_spawnattr_t *__restrict, short *__restrict);
-
-int posix_spawnattr_setpgroup(posix_spawnattr_t *, pid_t);
-int posix_spawnattr_getpgroup(const posix_spawnattr_t *__restrict, pid_t *__restrict);
-
-int posix_spawnattr_setsigmask(posix_spawnattr_t *__restrict, const sigset_t *__restrict);
-int posix_spawnattr_getsigmask(const posix_spawnattr_t *__restrict, sigset_t *__restrict);
-
-int posix_spawnattr_setsigdefault(posix_spawnattr_t *__restrict, const sigset_t *__restrict);
-int posix_spawnattr_getsigdefault(const posix_spawnattr_t *__restrict, sigset_t *__restrict);
-
-int posix_spawnattr_setschedparam(posix_spawnattr_t *__restrict, const struct sched_param *__restrict);
-int posix_spawnattr_getschedparam(const posix_spawnattr_t *__restrict, struct sched_param *__restrict);
-int posix_spawnattr_setschedpolicy(posix_spawnattr_t *, int);
-int posix_spawnattr_getschedpolicy(const posix_spawnattr_t *__restrict, int *__restrict);
-
-int posix_spawn_file_actions_init(posix_spawn_file_actions_t *);
-int posix_spawn_file_actions_destroy(posix_spawn_file_actions_t *);
-
-int posix_spawn_file_actions_addopen(posix_spawn_file_actions_t *__restrict, int, const char *__restrict, int, mode_t);
-int posix_spawn_file_actions_addclose(posix_spawn_file_actions_t *, int);
-int posix_spawn_file_actions_adddup2(posix_spawn_file_actions_t *, int, int);
-
-#if defined(_BSD_SOURCE) || defined(_GNU_SOURCE)
-int posix_spawn_file_actions_addchdir_np(posix_spawn_file_actions_t *__restrict, const char *__restrict);
-int posix_spawn_file_actions_addfchdir_np(posix_spawn_file_actions_t *, int);
-#endif
 
 #ifdef __cplusplus
 }

contrib/CMakeLists.txt

@@ -92,9 +92,9 @@ add_contrib (openldap-cmake openldap)
 add_contrib (grpc-cmake grpc)
 add_contrib (msgpack-c-cmake msgpack-c)
 
-#if (ENABLE_RUST)
-add_contrib (corrosion-cmake corrosion)
-#endif()
+if (ENABLE_RUST)
+    add_contrib (corrosion-cmake corrosion)
+endif()
 
 if (ENABLE_FUZZING)
     add_contrib (libprotobuf-mutator-cmake libprotobuf-mutator)

contrib/corrosion-cmake/CMakeLists.txt

@@ -1,2 +1,52 @@
-list (APPEND CMAKE_MODULE_PATH "${ClickHouse_SOURCE_DIR}/contrib/corrosion/cmake")
+if (NOT ENABLE_LIBRARIES)
+  option(ENABLE_RUST "Enable rust" FALSE)
+else()
+  list (APPEND CMAKE_MODULE_PATH "${ClickHouse_SOURCE_DIR}/contrib/corrosion/cmake")
+  find_package(Rust)
+  option(ENABLE_RUST "Enable rust" $Rust_FOUND)
+endif()
+
+if(NOT ENABLE_RUST)
+  message(STATUS "Not using rust")
+  return()
+endif()
+
+# Define function corrosion_import_crate()
 include ("${ClickHouse_SOURCE_DIR}/contrib/corrosion/cmake/Corrosion.cmake")
+
+message(STATUS "Checking Rust toolchain for current target")
+
+if(CMAKE_TOOLCHAIN_FILE MATCHES "linux/toolchain-x86_64")
+    set(Rust_CARGO_TARGET "x86_64-unknown-linux-gnu")
+    execute_process(COMMAND rustup target add x86_64-unknown-linux-gnu)
+endif()
+
+if(CMAKE_TOOLCHAIN_FILE MATCHES "linux/toolchain-aarch64")
+    set(Rust_CARGO_TARGET "aarch64-unknown-linux-gnu")
+    execute_process(COMMAND rustup target add aarch64-unknown-linux-gnu)
+endif()
+
+if((CMAKE_TOOLCHAIN_FILE MATCHES "darwin") AND (CMAKE_TOOLCHAIN_FILE MATCHES "toolchain-aarch64"))
+    set(Rust_CARGO_TARGET "aarch64-apple-darwin")
+    execute_process(COMMAND rustup target add aarch64-apple-darwin)
+endif()
+
+if((CMAKE_TOOLCHAIN_FILE MATCHES "darwin") AND (CMAKE_TOOLCHAIN_FILE MATCHES "x86_64"))
+    set(Rust_CARGO_TARGET "x86_64-apple-darwin")
+    execute_process(COMMAND rustup target add x86_64-apple-darwin)
+endif()
+
+if((CMAKE_TOOLCHAIN_FILE MATCHES "freebsd") AND (CMAKE_TOOLCHAIN_FILE MATCHES "x86_64"))
+    set(Rust_CARGO_TARGET "x86_64-unknown-freebsd")
+    execute_process(COMMAND rustup target add x86_64-unknown-freebsd)
+
+endif()
+
+if(CMAKE_TOOLCHAIN_FILE MATCHES "ppc64le")
+    set(Rust_CARGO_TARGET "powerpc64le-unknown-linux-gnu")
+    execute_process(COMMAND rustup target add powerpc64le-unknown-linux-gnu)
+endif()
+
+execute_process(rustup target default ${Rust_CARGO_TARGET})
+
+message(STATUS "Switched Rust target to ${Rust_CARGO_TARGET}")

docker/packager/binary/Dockerfile

@@ -23,6 +23,12 @@ RUN apt-get install \
         libc6-dev-arm64-cross \
         --yes
 
+# Install CMake 3.20+ for Rust compilation
+RUN apt purge cmake --yes
+RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null
+RUN apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main'
+RUN apt update && apt install cmake --yes
+
 ENV CC=clang-${LLVM_VERSION}
 ENV CXX=clang++-${LLVM_VERSION}
 

docker/packager/binary/build.sh

@@ -55,7 +55,7 @@ ccache --zero-stats ||:
 if [ "$BUILD_MUSL_KEEPER" == "1" ]
 then
     # build keeper with musl separately
-    cmake --debug-trycompile -DBUILD_STANDALONE_KEEPER=1 -DENABLE_CLICKHOUSE_KEEPER=1 -DCMAKE_VERBOSE_MAKEFILE=1 -DUSE_MUSL=1 -LA -DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
+    cmake --debug-trycompile -DENABLE_RUST=1 -DBUILD_STANDALONE_KEEPER=1 -DENABLE_CLICKHOUSE_KEEPER=1 -DCMAKE_VERBOSE_MAKEFILE=1 -DUSE_MUSL=1 -LA -DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
     # shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty.
     ninja $NINJA_FLAGS clickhouse-keeper
 
@@ -70,10 +70,10 @@ then
     rm -f CMakeCache.txt
 
     # Build the rest of binaries
-    cmake --debug-trycompile -DBUILD_STANDALONE_KEEPER=0 -DCREATE_KEEPER_SYMLINK=0 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
+    cmake --debug-trycompile -DENABLE_RUST=1 -DBUILD_STANDALONE_KEEPER=0 -DCREATE_KEEPER_SYMLINK=0 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
 else
     # Build everything
-    cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
+    cmake --debug-trycompile -DENABLE_RUST=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
 fi
 
 if [ "coverity" == "$COMBINED_OUTPUT" ]

docker/test/fasttest/Dockerfile

@@ -19,6 +19,12 @@ RUN apt-get update \
         pv \
     --yes --no-install-recommends
 
+# Install CMake 3.20+ for Rust compilation
+RUN apt purge cmake --yes
+RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null
+RUN apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main'
+RUN apt update && apt install cmake --yes
+
 RUN pip3 install numpy scipy pandas Jinja2
 
 ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz"

rust/BLAKE3/Cargo.lock

@@ -65,9 +65,9 @@ dependencies = [
 
 [[package]]
 name = "generic-array"
-version = "0.14.5"
+version = "0.14.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd48d33ec7f05fbfa152300fdad764757cbded343c1aa1cff2fbaf4134851803"
+checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9"
 dependencies = [
  "typenum",
  "version_check",
@@ -75,9 +75,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.126"
+version = "0.2.132"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836"
+checksum = "8371e4e5341c3a96db127eb2465ac681ced4c433e01dd0e938adbef26ba93ba5"
 
 [[package]]
 name = "typenum"

rust/BLAKE3/Cargo.toml

@@ -5,8 +5,8 @@ version = "0.1.0"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-blake3 = "1.2.*"
-libc = "0.2.*"
+blake3 = "1.2.0"
+libc = "0.2.132"
 
 [lib]
 crate-type = ["staticlib"]

rust/BLAKE3/include/blake3.h

@@ -1,10 +1,7 @@
-#include <cstdarg>
-#include <cstddef>
-#include <cstdint>
-#include <cstdlib>
-#include <ostream>
-#include <new>
+#ifndef BLAKE3_H
+#define BLAKE3_H
 
+#include <cstdint>
 
 
 extern "C" {
@@ -16,3 +13,5 @@ char *blake3_apply_shim_msan_compat(const char *begin, uint32_t size, uint8_t *o
 void blake3_free_char_pointer(char *ptr_to_free);
 
 } // extern "C"
+
+#endif /* BLAKE3_H */