From d6a90b6079506e26b67577e35114bee2b8d70bd7 Mon Sep 17 00:00:00 2001 From: proller Date: Wed, 22 May 2019 13:16:16 +0300 Subject: [PATCH] Build fixes (#5368) --- cmake/dbms_glob_sources.cmake | 2 +- dbms/programs/clang/CMakeLists.txt | 4 +-- dbms/programs/clang/copy_headers.sh | 38 ++++++++++++---------- debian/.pbuilderrc | 9 +++-- debian/control | 1 + debian/pbuilder-hooks/B90test-server | 10 +++--- libs/libglibc-compatibility/CMakeLists.txt | 32 ++++-------------- release | 2 ++ 8 files changed, 45 insertions(+), 53 deletions(-) diff --git a/cmake/dbms_glob_sources.cmake b/cmake/dbms_glob_sources.cmake index 2d2200c2c55..5ea9fe88ca3 100644 --- a/cmake/dbms_glob_sources.cmake +++ b/cmake/dbms_glob_sources.cmake @@ -5,7 +5,7 @@ endmacro() macro(add_headers_and_sources prefix common_path) add_glob(${prefix}_headers RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${common_path}/*.h) - add_glob(${prefix}_sources ${common_path}/*.cpp ${common_path}/*.h) + add_glob(${prefix}_sources ${common_path}/*.cpp ${common_path}/*.c ${common_path}/*.h) endmacro() macro(add_headers_only prefix common_path) diff --git a/dbms/programs/clang/CMakeLists.txt b/dbms/programs/clang/CMakeLists.txt index 8bf9e3760e2..82f520614f4 100644 --- a/dbms/programs/clang/CMakeLists.txt +++ b/dbms/programs/clang/CMakeLists.txt @@ -26,8 +26,8 @@ elseif (EXISTS ${INTERNAL_COMPILER_BIN_ROOT}${INTERNAL_COMPILER_EXECUTABLE}) set (COPY_HEADERS_COMPILER "${INTERNAL_COMPILER_BIN_ROOT}${INTERNAL_COMPILER_EXECUTABLE}") endif () -if (COPY_HEADERS_COMPILER AND OS_LINUX) - add_custom_target (copy-headers [ -f ${TMP_HEADERS_DIR}/dbms/src/Interpreters/SpecializedAggregator.h ] || env CLANG=${COPY_HEADERS_COMPILER} BUILD_PATH=${ClickHouse_BINARY_DIR} DESTDIR=${ClickHouse_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/copy_headers.sh ${ClickHouse_SOURCE_DIR} ${TMP_HEADERS_DIR} DEPENDS ${COPY_HEADERS_DEPENDS} WORKING_DIRECTORY ${ClickHouse_SOURCE_DIR} SOURCES copy_headers.sh) +if (COPY_HEADERS_COMPILER) + add_custom_target (copy-headers [ -f ${TMP_HEADERS_DIR}/dbms/src/Interpreters/SpecializedAggregator.h ] || env CLANG=${COPY_HEADERS_COMPILER} BUILD_PATH=${ClickHouse_BINARY_DIR} DESTDIR=${ClickHouse_SOURCE_DIR} CMAKE_CXX_COMPILER_VERSION=${CMAKE_CXX_COMPILER_VERSION} ${CMAKE_CURRENT_SOURCE_DIR}/copy_headers.sh ${ClickHouse_SOURCE_DIR} ${TMP_HEADERS_DIR} DEPENDS ${COPY_HEADERS_DEPENDS} WORKING_DIRECTORY ${ClickHouse_SOURCE_DIR} SOURCES copy_headers.sh) if (USE_INTERNAL_LLVM_LIBRARY) set (CLANG_HEADERS_DIR "${ClickHouse_SOURCE_DIR}/contrib/llvm/clang/lib/Headers") diff --git a/dbms/programs/clang/copy_headers.sh b/dbms/programs/clang/copy_headers.sh index 2f7b2f9f5cf..45a58855c91 100755 --- a/dbms/programs/clang/copy_headers.sh +++ b/dbms/programs/clang/copy_headers.sh @@ -38,26 +38,28 @@ for header in $START_HEADERS; do START_HEADERS_INCLUDE+="-include $header " done -# Опция -mcx16 для того, чтобы выбиралось больше заголовочных файлов (с запасом). -# The latter options are the same that are added while building packages. -# TODO: Does not work on macos: GCC_ROOT=`$CLANG -v 2>&1 | grep "Selected GCC installation"| sed -n -e 's/^.*: //p'` -for src_file in $(echo | $CLANG -M -xc++ -std=c++1z -Wall -Werror -msse4 -mcx16 -mpopcnt -O3 -g -fPIC -fstack-protector -D_FORTIFY_SOURCE=2 \ +# TODO: Does not work on macos? +GCC_ROOT=${GCC_ROOT:=/usr/lib/clang/${CMAKE_CXX_COMPILER_VERSION}} + +# Опция -mcx16 для того, чтобы выбиралось больше заголовочных файлов (с запасом). +# The latter options are the same that are added while building packages. +for src_file in $(echo | $CLANG -M -xc++ -std=c++1z -Wall -Werror -msse2 -msse4 -mcx16 -mpopcnt -O3 -g -fPIC -fstack-protector -D_FORTIFY_SOURCE=2 \ -I $GCC_ROOT/include \ -I $GCC_ROOT/include-fixed \ $(cat "$BUILD_PATH/include_directories.txt") \ $START_HEADERS_INCLUDE \ - | tr -d '\\' | - sed --posix -E -e 's/^-\.o://'); + sed -E -e 's/^-\.o://'); do dst_file=$src_file; - [ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed --posix -E -e "s!^$BUILD_PATH!!") - [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed --posix -E -e "s!^$DESTDIR!!") - dst_file=$(echo $dst_file | sed --posix -E -e 's/build\///') # for simplicity reasons, will put generated headers near the rest. - mkdir -p "$DST/$(echo $dst_file | sed --posix -E -e 's/\/[^/]*$/\//')"; + [ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed -E -e "s!^$BUILD_PATH!!") + [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!") + dst_file=$(echo $dst_file | sed -E -e 's/build\///') # for simplicity reasons, will put generated headers near the rest. + mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')"; cp "$src_file" "$DST/$dst_file"; done @@ -68,9 +70,9 @@ done for src_file in $(ls -1 $($CLANG -v -xc++ - <<<'' 2>&1 | grep '^ /' | grep 'include' | grep -E '/lib/clang/|/include/clang/')/*.h | grep -vE 'arm|altivec|Intrin'); do dst_file=$src_file; - [ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed --posix -E -e "s!^$BUILD_PATH!!") - [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed --posix -E -e "s!^$DESTDIR!!") - mkdir -p "$DST/$(echo $dst_file | sed --posix -E -e 's/\/[^/]*$/\//')"; + [ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed -E -e "s!^$BUILD_PATH!!") + [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!") + mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')"; cp "$src_file" "$DST/$dst_file"; done @@ -79,9 +81,9 @@ if [ -d "$SOURCE_PATH/contrib/boost/libs/smart_ptr/include/boost/smart_ptr/detai for src_file in $(ls -1 $SOURCE_PATH/contrib/boost/libs/smart_ptr/include/boost/smart_ptr/detail/*); do dst_file=$src_file; - [ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed --posix -E -e "s!^$BUILD_PATH!!") - [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed --posix -E -e "s!^$DESTDIR!!") - mkdir -p "$DST/$(echo $dst_file | sed --posix -E -e 's/\/[^/]*$/\//')"; + [ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed -E -e "s!^$BUILD_PATH!!") + [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!") + mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')"; cp "$src_file" "$DST/$dst_file"; done fi @@ -90,9 +92,9 @@ if [ -d "$SOURCE_PATH/contrib/boost/boost/smart_ptr/detail" ]; then for src_file in $(ls -1 $SOURCE_PATH/contrib/boost/boost/smart_ptr/detail/*); do dst_file=$src_file; - [ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed --posix -E -e "s!^$BUILD_PATH!!") - [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed --posix -E -e "s!^$DESTDIR!!") - mkdir -p "$DST/$(echo $dst_file | sed --posix -E -e 's/\/[^/]*$/\//')"; + [ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed -E -e "s!^$BUILD_PATH!!") + [ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!") + mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')"; cp "$src_file" "$DST/$dst_file"; done fi diff --git a/debian/.pbuilderrc b/debian/.pbuilderrc index e4277a73152..4eb4c6b7306 100644 --- a/debian/.pbuilderrc +++ b/debian/.pbuilderrc @@ -3,10 +3,13 @@ # # ubuntu: # prepare old (trusty or earlier) host system: + +# sudo ln -s gutsy /usr/share/debootstrap/scripts/eoan # sudo ln -s gutsy /usr/share/debootstrap/scripts/disco # sudo ln -s gutsy /usr/share/debootstrap/scripts/cosmic # sudo ln -s gutsy /usr/share/debootstrap/scripts/artful # sudo ln -s gutsy /usr/share/debootstrap/scripts/bionic +# sudo ln -s sid /usr/share/debootstrap/scripts/buster # build ubuntu: # sudo DIST=trusty pbuilder create --configfile debian/.pbuilderrc && DIST=trusty pdebuild --configfile debian/.pbuilderrc # sudo DIST=xenial pbuilder create --configfile debian/.pbuilderrc && DIST=xenial pdebuild --configfile debian/.pbuilderrc @@ -56,7 +59,7 @@ DEBIAN_SUITES=($UNSTABLE_CODENAME $TESTING_CODENAME $STABLE_CODENAME $STABLE_BAC "experimental" "unstable" "testing" "stable") # List of Ubuntu suites. Update these when needed. -UBUNTU_SUITES=("disco" "cosmic" "bionic" "artful" "zesty" "xenial" "trusty" "devel") +UBUNTU_SUITES=("eoan" "disco" "cosmic" "bionic" "artful" "zesty" "xenial" "trusty" "devel") # Set a default distribution if none is used. Note that you can set your own default (i.e. ${DIST:="unstable"}). HOST_DIST=`lsb_release --short --codename` @@ -167,12 +170,12 @@ esac if [ "$ARCH" != arm64 ]; then case "$DIST" in -# TODO: fix llvm-8 and use for "disco" +# TODO: fix llvm-8 and use for "disco" and "eoan" "experimental") EXTRAPACKAGES+=" liblld-8-dev libclang-8-dev llvm-8-dev liblld-8 " export CMAKE_FLAGS="-DLLVM_VERSION=8 $CMAKE_FLAGS" ;; - "disco" | "cosmic" | "testing" | "unstable") + "eoan" | "disco" | "cosmic" | "testing" | "unstable") EXTRAPACKAGES+=" liblld-7-dev libclang-7-dev llvm-7-dev liblld-7 " export CMAKE_FLAGS="-DLLVM_VERSION=7 $CMAKE_FLAGS" ;; diff --git a/debian/control b/debian/control index 5720230e8c1..ae1a8d498a6 100644 --- a/debian/control +++ b/debian/control @@ -11,6 +11,7 @@ Build-Depends: debhelper (>= 9), libicu-dev, libreadline-dev, gperf, + python, tzdata Standards-Version: 3.9.8 diff --git a/debian/pbuilder-hooks/B90test-server b/debian/pbuilder-hooks/B90test-server index 65cd0713001..35f482e9739 100755 --- a/debian/pbuilder-hooks/B90test-server +++ b/debian/pbuilder-hooks/B90test-server @@ -8,10 +8,10 @@ PACKAGE_INSTALL=${PACKAGE_INSTALL=1} TEST_PORT_RANDOM=${TEST_PORT_RANDOM=1} if [ "${PACKAGE_INSTALL}" ]; then - for PKG in $(ls /tmp/buildd/*.deb | sed -e's,.*/,,;s,_.*,,' ); do - apt-get install -y --force-yes "$PKG" ||: - apt-get remove -y "$PKG" ||: - done + #for PKG in $(ls /tmp/buildd/*.deb | sed -e's,.*/,,;s,_.*,,' ); do + # apt-get install -y --force-yes "$PKG" ||: + # apt-get remove -y "$PKG" ||: + #done dpkg --auto-deconfigure -i /tmp/buildd/*.deb ||: apt install -y -f --allow-downgrades ||: @@ -45,6 +45,8 @@ export CLICKHOUSE_PORT_TCP_SECURE=${CLICKHOUSE_PORT_TCP_SECURE:=9440} export CLICKHOUSE_PORT_HTTPS=${CLICKHOUSE_PORT_HTTPS:=8443} if [ "${TEST_CONNECT}" ]; then + sed -i 's/ssl_conf = ssl_sect/#ssl_conf = ssl_sect/g' /etc/ssl/openssl.cnf + cat /etc/ssl/openssl.cnf [ "${TEST_PORT_RANDOM}" ] && echo "${CLICKHOUSE_PORT_HTTP}${CLICKHOUSE_PORT_TCP}${CLICKHOUSE_PORT_INTERSERVER}" > /etc/clickhouse-server/config.d/port.xml if [ "${TEST_SSL}" ]; then diff --git a/libs/libglibc-compatibility/CMakeLists.txt b/libs/libglibc-compatibility/CMakeLists.txt index 4747953afde..c5824553118 100644 --- a/libs/libglibc-compatibility/CMakeLists.txt +++ b/libs/libglibc-compatibility/CMakeLists.txt @@ -7,37 +7,19 @@ if(COMPILER_CLANG) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-builtin-requires-header") endif() +add_headers_and_sources(glibc_compatibility .) +add_headers_and_sources(glibc_compatibility musl) +list(APPEND glibc_compatibility_sources musl/syscall.s musl/longjmp.s) -set(GLIBC_COMPATIBILITY_SOURCES -glibc-compatibility.c -musl/pipe2.c -musl/fallocate.c -musl/longjmp.s -musl/vasprintf.c -musl/lgamma.c -musl/posix_spawn.c -musl/futimens.c -musl/syscall.s -musl/syscall_ret.c -musl/sched_cpucount.c -musl/glob.c -musl/exp2f.c -musl/pwritev.c -musl/getrandom.c -musl/fcntl.c -musl/timespec_get.c -musl/sched_getcpu.c -) - +list(REMOVE_ITEM glibc_compatibility_sources musl/getentropy.c) if(HAVE_SYS_RANDOM_H) - list(APPEND GLIBC_COMPATIBILITY_SOURCES musl/getentropy.c) + list(APPEND glibc_compatibility_sources musl/getentropy.c) endif() if(MAKE_STATIC_LIBRARIES) - set(GLIBC_COMPATIBILITY_SOURCES ${GLIBC_COMPATIBILITY_SOURCES} - libcxxabi/cxa_thread_atexit.cpp) + list(APPEND glibc_compatibility_sources libcxxabi/cxa_thread_atexit.cpp) endif() -add_library(glibc-compatibility ${GLIBC_COMPATIBILITY_SOURCES}) +add_library(glibc-compatibility ${glibc_compatibility_sources}) target_include_directories(glibc-compatibility PRIVATE libcxxabi) diff --git a/release b/release index 8be67e99640..d9a18e87d8b 100755 --- a/release +++ b/release @@ -3,6 +3,8 @@ # If you have "no space left" error, you can change the location of temporary files with BUILDPLACE environment variable. # Advanced usage: +# Test gcc-9: +# env DIST=disco EXTRAPACKAGES="gcc-9 g++-9" DEB_CC=gcc-9 DEB_CXX=g++-9 CMAKE_FLAGS=" -DNO_WERROR=1 " ./release # Test gcc-8: # env DIST=bionic EXTRAPACKAGES="gcc-8 g++-8" DEB_CC=gcc-8 DEB_CXX=g++-8 CMAKE_FLAGS=" -DNO_WERROR=1 " ./release # Clang6 build: