Better compile with low memory; Fix macos build, add utils/build/build_debian_unbundled_split.sh test script (#4030)

2024-09-19 16:20:50 +00:00 · 2019-01-11 15:40:19 +03:00 · 2019-01-11 15:40:19 +03:00 · 497b6ec2ad
commit 497b6ec2ad
parent baafb84018
8 changed files with 39 additions and 26 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -90,8 +90,6 @@ if (GLIBC_COMPATIBILITY)
    set (USE_INTERNAL_MEMCPY ON)
 endif ()

-set (COMPILER_FLAGS "${COMPILER_FLAGS}")
-
 string(REGEX MATCH "-?[0-9]+(.[0-9]+)?$" COMPILER_POSTFIX ${CMAKE_CXX_COMPILER})

 find_program (LLD_PATH NAMES "lld${COMPILER_POSTFIX}" "lld")
@ -108,10 +106,15 @@ if (LINKER_NAME)
    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}")
 endif ()

-option (PIPE "-pipe compiler option [less /tmp usage, more ram usage]" ON)
-if (PIPE)
-    set (COMPILER_FLAGS                  "${COMPILER_FLAGS} -pipe")
-endif ()
+cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd
+if(NOT AVAILABLE_PHYSICAL_MEMORY OR AVAILABLE_PHYSICAL_MEMORY GREATER 8000)
+    option(COMPILER_PIPE "-pipe compiler option [less /tmp usage, more ram usage]" ON)
+endif()
+if(COMPILER_PIPE)
+    set(COMPILER_FLAGS "${COMPILER_FLAGS} -pipe")
+else()
+    message(STATUS "Disabling compiler -pipe option (have only ${AVAILABLE_PHYSICAL_MEMORY} mb of memory)")
+endif()

 include (cmake/test_cpu.cmake)

--- a/cmake/find_ssl.cmake
+++ b/cmake/find_ssl.cmake
@ -6,7 +6,7 @@ set (OPENSSL_USE_STATIC_LIBS ${USE_STATIC_LIBRARIES})

 if (NOT USE_INTERNAL_SSL_LIBRARY)
    if (APPLE)
-        set (OPENSSL_ROOT_DIR "/usr/local/opt/openssl")
+        set (OPENSSL_ROOT_DIR "/usr/local/opt/openssl" CACHE INTERNAL "")
        # https://rt.openssl.org/Ticket/Display.html?user=guest&pass=guest&id=2232
        if (USE_STATIC_LIBRARIES)
            message(WARNING "Disable USE_STATIC_LIBRARIES if you have linking problems with OpenSSL on MacOS")
--- a/cmake/limit_jobs.cmake
+++ b/cmake/limit_jobs.cmake
@ -4,6 +4,7 @@
 # include (cmake/limit_jobs.cmake)

 cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd
+cmake_host_system_information(RESULT NUMBER_OF_LOGICAL_CORES QUERY NUMBER_OF_LOGICAL_CORES)

 option(PARALLEL_COMPILE_JOBS "Define the maximum number of concurrent compilation jobs" "")
 if (NOT PARALLEL_COMPILE_JOBS AND AVAILABLE_PHYSICAL_MEMORY AND MAX_COMPILER_MEMORY)
@ -12,7 +13,7 @@ if (NOT PARALLEL_COMPILE_JOBS AND AVAILABLE_PHYSICAL_MEMORY AND MAX_COMPILER_MEM
        set (PARALLEL_COMPILE_JOBS 1)
    endif ()
 endif ()
-if (PARALLEL_COMPILE_JOBS)
+if (PARALLEL_COMPILE_JOBS AND (NOT NUMBER_OF_LOGICAL_CORES OR PARALLEL_COMPILE_JOBS LESS NUMBER_OF_LOGICAL_CORES))
    set(CMAKE_JOB_POOL_COMPILE compile_job_pool${CMAKE_CURRENT_SOURCE_DIR})
    string (REGEX REPLACE "[^a-zA-Z0-9]+" "_" CMAKE_JOB_POOL_COMPILE ${CMAKE_JOB_POOL_COMPILE})
    set_property(GLOBAL APPEND PROPERTY JOB_POOLS ${CMAKE_JOB_POOL_COMPILE}=${PARALLEL_COMPILE_JOBS})
@ -25,13 +26,12 @@ if (NOT PARALLEL_LINK_JOBS AND AVAILABLE_PHYSICAL_MEMORY AND MAX_LINKER_MEMORY)
        set (PARALLEL_LINK_JOBS 1)
    endif ()
 endif ()
-if (PARALLEL_COMPILE_JOBS OR PARALLEL_LINK_JOBS)
-    message(STATUS "${CMAKE_CURRENT_SOURCE_DIR}: Have ${AVAILABLE_PHYSICAL_MEMORY} megabytes of memory. Limiting concurrent linkers jobs to ${PARALLEL_LINK_JOBS} and compiler jobs to ${PARALLEL_COMPILE_JOBS}")
-endif ()
-
-if (LLVM_PARALLEL_LINK_JOBS)
+if (PARALLEL_LINK_JOBS AND (NOT NUMBER_OF_LOGICAL_CORES OR PARALLEL_COMPILE_JOBS LESS NUMBER_OF_LOGICAL_CORES))
    set(CMAKE_JOB_POOL_LINK link_job_pool${CMAKE_CURRENT_SOURCE_DIR})
    string (REGEX REPLACE "[^a-zA-Z0-9]+" "_" CMAKE_JOB_POOL_LINK ${CMAKE_JOB_POOL_LINK})
    set_property(GLOBAL APPEND PROPERTY JOB_POOLS ${CMAKE_JOB_POOL_LINK}=${PARALLEL_LINK_JOBS})
 endif ()

+if (PARALLEL_COMPILE_JOBS OR PARALLEL_LINK_JOBS)
+    message(STATUS "${CMAKE_CURRENT_SOURCE_DIR}: Have ${AVAILABLE_PHYSICAL_MEMORY} megabytes of memory. Limiting concurrent linkers jobs to ${PARALLEL_LINK_JOBS} and compiler jobs to ${PARALLEL_COMPILE_JOBS}")
+endif ()
--- a/dbms/CMakeLists.txt
+++ b/dbms/CMakeLists.txt
@ -2,13 +2,17 @@ if (USE_INCLUDE_WHAT_YOU_USE)
    set (CMAKE_CXX_INCLUDE_WHAT_YOU_USE ${IWYU_PATH})
 endif ()

-set (MAX_COMPILER_MEMORY 2500 CACHE INTERNAL "")
-if (MAKE_STATIC_LIBRARIES)
-    set (MAX_LINKER_MEMORY 3500 CACHE INTERNAL "")
+if(COMPILER_PIPE)
+    set(MAX_COMPILER_MEMORY 2500)
 else()
-    set (MAX_LINKER_MEMORY 2500 CACHE INTERNAL "")
-endif ()
-include (../cmake/limit_jobs.cmake)
+    set(MAX_COMPILER_MEMORY 1500)
+endif()
+if(MAKE_STATIC_LIBRARIES)
+    set(MAX_LINKER_MEMORY 3500)
+else()
+    set(MAX_LINKER_MEMORY 2500)
+endif()
+include(../cmake/limit_jobs.cmake)

 include(cmake/find_vectorclass.cmake)

--- a/dbms/programs/copier/ClusterCopier.cpp
+++ b/dbms/programs/copier/ClusterCopier.cpp
@ -1997,7 +1997,7 @@ protected:
        };

        {
-            ThreadPool thread_pool(std::min(num_shards, UInt64(getNumberOfPhysicalCPUCores())));
+            ThreadPool thread_pool(std::min<UInt64>(num_shards, getNumberOfPhysicalCPUCores()));

            for (UInt64 shard_index = 0; shard_index < num_shards; ++shard_index)
                thread_pool.schedule([=] { do_for_shard(shard_index); });
--- a/utils/CMakeLists.txt
+++ b/utils/CMakeLists.txt
@ -3,12 +3,12 @@ if (NOT NO_WERROR)
    set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror")
 endif ()

-if (MAKE_STATIC_LIBRARIES)
-    set (MAX_LINKER_MEMORY 3500 CACHE INTERNAL "")
+if(MAKE_STATIC_LIBRARIES)
+    set(MAX_LINKER_MEMORY 3500)
 else()
-    set (MAX_LINKER_MEMORY 2500 CACHE INTERNAL "")
-endif ()
-include (../cmake/limit_jobs.cmake)
+    set(MAX_LINKER_MEMORY 2500)
+endif()
+include(../cmake/limit_jobs.cmake)

 # Utils used in package
 add_subdirectory (config-processor)
--- a/utils/build/build_debian_unbundled.sh
+++ b/utils/build/build_debian_unbundled.sh
@ -22,5 +22,5 @@ env TEST_RUN=1 \
    `# Use all possible contrib libs from system` \
    `# psmisc - killall` \
    `# gdb - symbol test in pbuilder` \
-    EXTRAPACKAGES="psmisc libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libboost-thread-dev zlib1g-dev liblz4-dev libdouble-conversion-dev libsparsehash-dev librdkafka-dev libpoco-dev unixodbc-dev libsparsehash-dev libgoogle-perftools-dev libzstd-dev libre2-dev libunwind-dev googletest libcctz-dev libcapnp-dev libjemalloc-dev libssl-dev $EXTRAPACKAGES" \
+    EXTRAPACKAGES="psmisc libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libboost-thread-dev zlib1g-dev liblz4-dev libdouble-conversion-dev libsparsehash-dev librdkafka-dev libpoco-dev unixodbc-dev libsparsehash-dev libgoogle-perftools-dev libzstd-dev libre2-dev libunwind-dev googletest libcctz-dev libcapnp-dev libjemalloc-dev libssl-dev libunwind-dev libxml2-dev libgsasl7-dev $EXTRAPACKAGES" \
    pdebuild --configfile $ROOT_DIR/debian/.pbuilderrc $PDEBUILD_OPT
--- a/utils/build/build_debian_unbundled_split.sh
+++ b/utils/build/build_debian_unbundled_split.sh
@ -0,0 +1,6 @@
+#!/bin/bash
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
+CMAKE_FLAGS+=" -DCLICKHOUSE_SPLIT_BINARY=1 "
+. $CUR_DIR/build_debian_unbundled.sh