From 3a97fbd0e7363b318a45ee6f215074c93c5aca1d Mon Sep 17 00:00:00 2001 From: Roman Tsisyk Date: Sat, 6 Jan 2018 15:46:18 +0300 Subject: [PATCH] Remove hard-coded paths in Interpreters/Compiler.cpp Interpreters/Compiler.cpp contained hard-coded paths for system's includes needed for query compiler. These paths were not portable between different Linux distros and gcc/clang versions. For example, Debian/Ubuntu use /usr/lib/gcc/x86_64-linux-gnu/*/include, RHEL/Fedora use /usr/lib/gcc/x86_64-redhat-linux/*/include, others use /usr/lib/gcc/*/include (without x86_64-XXX triplet). Patch 68850012b "Embedded compiler fixes" attempted to fix this problem by adding CMAKE_LIBRARY_ARCHITECTURE after /usr/lib. Unfortunally, CMAKE_LIBRARY_ARCHITECTURE is not defined on RHEL/Fedora because someone decided to omit "-gnu" from x86_64-redhat-linux (see RHBZ#1531678). Patch 70e35d0bc "Build fixes (#1718)" added a workaround for undefined CMAKE_LIBRARY_ARCHITECTURE on RHEL/Fedora, but hasn't fixed problem with missing /usr/lib/gcc/x86_64-redhat-linux/*/include/ in the list of hardcoded paths. Remove hard-coded paths and get the list of `-isystem` includes directly from bundled clickhouse-clang. Other changes: - Enable RPATH for the build directory to get working binaries without installing them by `make install`. --- .gitignore | 1 + CMakeLists.txt | 7 +++++++ dbms/CMakeLists.txt | 3 +++ dbms/scripts/gen-compiler-includes.sh | 16 ++++++++++++++++ dbms/src/Interpreters/CMakeLists.txt | 11 ++++++++++- dbms/src/Interpreters/Compiler.cpp | 21 ++++----------------- dbms/src/Interpreters/config_compile.h.in | 6 +----- dbms/src/Server/CMakeLists.txt | 7 +++++-- 8 files changed, 47 insertions(+), 25 deletions(-) create mode 100755 dbms/scripts/gen-compiler-includes.sh diff --git a/.gitignore b/.gitignore index bb339eac140..03498eeff73 100644 --- a/.gitignore +++ b/.gitignore @@ -187,6 +187,7 @@ debian/files dbms/src/Server/data/* dbms/src/Server/metadata/* dbms/src/Server/status +dbms/src/Interpreters/CompilerIncludes.h config-9001.xml *-preprocessed.xml diff --git a/CMakeLists.txt b/CMakeLists.txt index 164a90e06a7..de0ad6af676 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,6 +79,13 @@ if (USE_STATIC_LIBRARIES) list(REVERSE CMAKE_FIND_LIBRARY_SUFFIXES) endif () +# Don't skip the full RPATH for the build tree to get working binaries +# without running `make install`. +set (CMAKE_SKIP_BUILD_RPATH OFF) + +# The RPATH to be used when installing +set (CMAKE_INSTALL_RPATH "") + if (CMAKE_LIBRARY_ARCHITECTURE MATCHES "amd64.*|x86_64.*|AMD64.*") option (USE_INTERNAL_MEMCPY "Use internal implementation of 'memcpy' function instead of provided by libc. Only for x86_64." ON) diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 02d6c46bcab..e5cbd0b241a 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -88,6 +88,9 @@ list (APPEND dbms_headers list (APPEND dbms_sources src/TableFunctions/TableFunctionFactory.cpp) list (APPEND dbms_headers src/TableFunctions/ITableFunction.h src/TableFunctions/TableFunctionFactory.h) +if (USE_EMBEDDED_COMPILER) + list (APPEND dbms_headers ${CMAKE_CURRENT_SOURCE_DIR}/src/Interpreters/CompilerIncludes.h) +endif () list(REMOVE_ITEM clickhouse_common_io_sources src/Common/StringUtils.cpp) diff --git a/dbms/scripts/gen-compiler-includes.sh b/dbms/scripts/gen-compiler-includes.sh new file mode 100755 index 00000000000..5101456b42f --- /dev/null +++ b/dbms/scripts/gen-compiler-includes.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# +# Generates the list of -isystem includes from gcc or clang compiler +# Usage: ./gen-includes.sh [COMPILER] +# + +# Set local to C to capture GCC's "search starts here" text in English +export LANG=C +export LC_ALL=C +export LC_MESSAGES=C +CC=$1 || cc +if test -z "$CC"; then CC=cc; fi + +echo "" | $CC -x c++ -E -Wp,-v - 2>&1| + sed -n '/#include <...> search starts here:/,/End of search list./p' | + sed -e '1d;$d' -e 's/^[ \t ]*/" -isystem /;s/$/"/' diff --git a/dbms/src/Interpreters/CMakeLists.txt b/dbms/src/Interpreters/CMakeLists.txt index 8b9d118b120..e4a59084391 100644 --- a/dbms/src/Interpreters/CMakeLists.txt +++ b/dbms/src/Interpreters/CMakeLists.txt @@ -5,12 +5,21 @@ else () set (PATH_SHARE "/usr/share" CACHE STRING "") endif () +if (USE_EMBEDDED_COMPILER) + # Generate CompilerIncludes.h which contains all + # `-isystem` includes for bundled clickhouse-clang + add_custom_command(OUTPUT CompilerIncludes.h + COMMAND ${PROJECT_SOURCE_DIR}/dbms/scripts/gen-compiler-includes.sh + ${PROJECT_BINARY_DIR}/dbms/src/Server/clickhouse-clang + > ${CMAKE_CURRENT_BINARY_DIR}/CompilerIncludes.h + DEPENDS clickhouse-clang) +endif () + set (INTERNAL_COMPILER_EXECUTABLE "clickhouse-clang" CACHE STRING "") set (INTERNAL_LINKER_EXECUTABLE "clickhouse-lld" CACHE STRING "") set (INTERNAL_COMPILER_NO_WARNING OFF CACHE INTERNAL "") set (INTERNAL_COMPILER_HEADERS "${PATH_SHARE}/clickhouse/headers" CACHE STRING "") set (INTERNAL_COMPILER_HEADERS_ROOT "${INTERNAL_COMPILER_HEADERS}" CACHE STRING "") -set (INTERNAL_COMPILER_CUSTOM_ROOT ON CACHE INTERNAL "") set (INTERNAL_COMPILER_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_RELEASE} ${CXX_FLAGS_INTERNAL_COMPILER} -x c++ -march=native -fPIC -fvisibility=hidden -fno-implement-inlines -nostdinc -nostdinc++ -Wno-unused-command-line-argument -Bprefix=${PATH_SHARE}/clickhouse -isysroot=${INTERNAL_COMPILER_HEADERS_ROOT}" CACHE STRING "") string(REPLACE "-no-pie" "" INTERNAL_COMPILER_FLAGS ${INTERNAL_COMPILER_FLAGS}) diff --git a/dbms/src/Interpreters/Compiler.cpp b/dbms/src/Interpreters/Compiler.cpp index b63dd438a40..d104d47c48c 100644 --- a/dbms/src/Interpreters/Compiler.cpp +++ b/dbms/src/Interpreters/Compiler.cpp @@ -222,23 +222,10 @@ void Compiler::compile( INTERNAL_COMPILER_EXECUTABLE " " INTERNAL_COMPILER_FLAGS - - #if INTERNAL_COMPILER_CUSTOM_ROOT - /// To get correct order merge this results carefully: - /// echo | clang -x c++ -E -Wp,-v - - /// echo | g++ -x c++ -E -Wp,-v - - - " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/c++/*" - " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/" CMAKE_LIBRARY_ARCHITECTURE "/c++/*" - " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/c++/*/backward" - " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/clang/*/include" /// if compiler is clang (from package) - " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/local/lib/clang/*/include" /// if clang installed manually - " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/lib/gcc/" CMAKE_LIBRARY_ARCHITECTURE "/*/include-fixed" - " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/lib/gcc/" CMAKE_LIBRARY_ARCHITECTURE "/*/include" - " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/local/include" /// if something installed manually - " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/" CMAKE_LIBRARY_ARCHITECTURE - " -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include" - #endif + #if USE_EMBEDDED_COMPILER + /// Contains `echo "" | clickhouse-clang -x c++ -E -Wp,-v -` + #include "CompilerIncludes.h" + #endif /* USE_EMBEDDED_COMPILER */ " -I " INTERNAL_COMPILER_HEADERS "/dbms/src/" " -I " INTERNAL_COMPILER_HEADERS "/contrib/libcityhash/include/" " -I " INTERNAL_COMPILER_HEADERS "/contrib/libpcg-random/include/" diff --git a/dbms/src/Interpreters/config_compile.h.in b/dbms/src/Interpreters/config_compile.h.in index 19f91fd3449..ccd34b8f444 100644 --- a/dbms/src/Interpreters/config_compile.h.in +++ b/dbms/src/Interpreters/config_compile.h.in @@ -1,16 +1,12 @@ #pragma once -#cmakedefine CMAKE_LIBRARY_ARCHITECTURE "@CMAKE_LIBRARY_ARCHITECTURE@" -#if !defined(CMAKE_LIBRARY_ARCHITECTURE) -#define CMAKE_LIBRARY_ARCHITECTURE "" -#endif #cmakedefine PATH_SHARE "@PATH_SHARE@" #cmakedefine INTERNAL_COMPILER_FLAGS "@INTERNAL_COMPILER_FLAGS@" #cmakedefine INTERNAL_COMPILER_EXECUTABLE "@INTERNAL_COMPILER_EXECUTABLE@" #cmakedefine INTERNAL_LINKER_EXECUTABLE "@INTERNAL_LINKER_EXECUTABLE@" #cmakedefine INTERNAL_COMPILER_HEADERS "@INTERNAL_COMPILER_HEADERS@" #cmakedefine INTERNAL_COMPILER_HEADERS_ROOT "@INTERNAL_COMPILER_HEADERS_ROOT@" -#cmakedefine01 INTERNAL_COMPILER_CUSTOM_ROOT +#cmakedefine01 USE_EMBEDDED_COMPILER #cmakedefine INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR "@INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR@" #cmakedefine INTERNAL_Poco_Foundation_INCLUDE_DIR "@INTERNAL_Poco_Foundation_INCLUDE_DIR@" #cmakedefine INTERNAL_Poco_Util_INCLUDE_DIR "@INTERNAL_Poco_Util_INCLUDE_DIR@" diff --git a/dbms/src/Server/CMakeLists.txt b/dbms/src/Server/CMakeLists.txt index 8c24d377ed3..5f75b40d1a3 100644 --- a/dbms/src/Server/CMakeLists.txt +++ b/dbms/src/Server/CMakeLists.txt @@ -46,6 +46,10 @@ target_link_libraries (clickhouse-format-lib clickhouse_common_io ${Boost_PROGRA if (USE_EMBEDDED_COMPILER) link_directories (${LLVM_LIBRARY_DIRS}) add_subdirectory ("Compiler-${LLVM_VERSION}") + # Link clickhouse-clang binary even if CLICKHOUSE_SPLIT_BINARY is OFF + # in order to get system include paths for Interpreters/Compiler.cpp + add_executable (clickhouse-clang clickhouse-clang.cpp) + target_link_libraries (clickhouse-clang clickhouse-compiler-lib) endif () if (CLICKHOUSE_SPLIT_BINARY) @@ -70,8 +74,7 @@ if (CLICKHOUSE_SPLIT_BINARY) set (CLICKHOUSE_ALL_TARGETS clickhouse-server clickhouse-client clickhouse-local clickhouse-benchmark clickhouse-performance-test clickhouse-extract-from-config clickhouse-format) if (USE_EMBEDDED_COMPILER) - add_executable (clickhouse-clang clickhouse-clang.cpp) - target_link_libraries (clickhouse-clang clickhouse-compiler-lib) + # clickhouse-clang is linked above add_executable (clickhouse-lld clickhouse-lld.cpp) target_link_libraries (clickhouse-lld clickhouse-compiler-lib) install (TARGETS clickhouse-clang clickhouse-lld RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)