thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 01:25:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Use copy-headers from cmake. (working compiler test without package install) (#2456)

Browse Source 
* Try use copy-headers from cmake [WIP]

* Using internal compiler without install

* 32bit Compile fix

* Fixes

* fix

* clean

* fix

* clean
This commit is contained in:
proller 2018-06-04 23:26:17 +03:00 committed by alexey-milovidov
parent f69ecf7a9d
commit 4d12a823da
9 changed files with 59 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
copy_headers.sh
View File

@ -1,6 +1,7 @@
#!/bin/bash -e
# set -x
#echo "Args: $*"; env | sort
#set -x
# Этот скрипт собирает все заголовочные файлы, нужные для компиляции некоторого translation unit-а
# и копирует их с сохранением путей в директорию DST.
@ -52,6 +53,7 @@ for src_file in $(echo | $CLANG -M -xc++ -std=c++1z -Wall -Werror -msse4 -mcx16
sed -E -e 's/^-\.o://');
do
dst_file=$src_file;
[ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed -E -e "s!^$BUILD_PATH!!")
[ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!")
dst_file=$(echo $dst_file | sed -E -e 's/build\///') # for simplicity reasons, will put generated headers near the rest.
mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')";
@ -65,6 +67,7 @@ done
for src_file in $(ls -1 $($CLANG -v -xc++ - <<<'' 2>&1 | grep '^ /' | grep 'include' | grep -E '/lib/clang/|/include/clang/')/*.h | grep -vE 'arm|altivec|Intrin');
do
dst_file=$src_file;
[ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed -E -e "s!^$BUILD_PATH!!")
[ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!")
mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')";
cp "$src_file" "$DST/$dst_file";
@ -74,6 +77,7 @@ done
for src_file in $(ls -1 $SOURCE_PATH/contrib/boost/libs/smart_ptr/include/boost/smart_ptr/detail/*);
do
dst_file=$src_file;
[ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed -E -e "s!^$BUILD_PATH!!")
[ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!")
mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')";
cp "$src_file" "$DST/$dst_file";
@ -82,6 +86,7 @@ done
for src_file in $(ls -1 $SOURCE_PATH/contrib/boost/boost/smart_ptr/detail/*);
do
dst_file=$src_file;
[ -n $BUILD_PATH ] && dst_file=$(echo $dst_file | sed -E -e "s!^$BUILD_PATH!!")
[ -n $DESTDIR ] && dst_file=$(echo $dst_file | sed -E -e "s!^$DESTDIR!!")
mkdir -p "$DST/$(echo $dst_file | sed -E -e 's/\/[^/]*$/\//')";
cp "$src_file" "$DST/$dst_file";

4
dbms/CMakeLists.txt
View File

@ -242,7 +242,7 @@ target_include_directories (clickhouse_common_io BEFORE PUBLIC ${DOUBLE_CONVERSI
target_include_directories (clickhouse_common_io BEFORE PRIVATE ${COMMON_INCLUDE_DIR})
# TODO: fix and enable:
if (0 AND USE_EMBEDDED_COMPILER)
if (USE_EMBEDDED_COMPILER)
add_custom_target(copy-headers ALL env CLANG=${CMAKE_CURRENT_BINARY_DIR}/src/Server/clickhouse-clang BUILD_PATH=${ClickHouse_BINARY_DIR} DESTDIR=${ClickHouse_SOURCE_DIR} ${ClickHouse_SOURCE_DIR}/copy_headers.sh ${ClickHouse_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/headers DEPENDS clickhouse-clang WORKING_DIRECTORY ${ClickHouse_SOURCE_DIR} SOURCES ${ClickHouse_SOURCE_DIR}/copy_headers.sh)
install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/headers DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/clickhouse COMPONENT clickhouse)
@ -253,7 +253,7 @@ if (0 AND USE_EMBEDDED_COMPILER)
add_dependencies(copy-headers copy-headers-clang)
endif ()
add_dependencies(clickhouse-bundle copy-headers)
add_dependencies(clickhouse-server copy-headers)
endif ()
add_subdirectory (tests)

17
dbms/src/Interpreters/CMakeLists.txt
View File

@ -5,8 +5,9 @@ else ()
set (PATH_SHARE "/usr/share" CACHE STRING "")
endif ()
set (INTERNAL_COMPILER_EXECUTABLE "${CMAKE_INSTALL_FULL_BINDIR}/clickhouse-clang" CACHE STRING "")
set (INTERNAL_LINKER_EXECUTABLE "${CMAKE_INSTALL_FULL_BINDIR}/clickhouse-lld" CACHE STRING "")
set (INTERNAL_COMPILER_BIN_ROOT "${CMAKE_INSTALL_FULL_BINDIR}/" CACHE STRING "")
set (INTERNAL_COMPILER_EXECUTABLE "clickhouse-clang" CACHE STRING "")
set (INTERNAL_LINKER_EXECUTABLE "clickhouse-lld" CACHE STRING "")
# Disabling asan reporting for these tools
if (CMAKE_BUILD_TYPE_UC STREQUAL "ASAN")
@ -21,6 +22,8 @@ set (INTERNAL_COMPILER_HEADERS_ROOT "${INTERNAL_COMPILER_HEADERS}" CACHE STRING
set (INTERNAL_COMPILER_CUSTOM_ROOT ON CACHE INTERNAL "")
set (INTERNAL_COMPILER_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${CMAKE_BUILD_TYPE_UC}} ${CXX_FLAGS_INTERNAL_COMPILER} -x c++ -march=native -fPIC -fvisibility=hidden -fno-implement-inlines -nostdinc -nostdinc++ -Wno-unused-command-line-argument -Bprefix=${PATH_SHARE}/clickhouse -isysroot=${INTERNAL_COMPILER_HEADERS_ROOT}" CACHE STRING "")
# TODO: use libs from package: -nodefaultlibs -lm -lc -lgcc_s -lgcc -lc++ -lc++abi
string(REPLACE "-no-pie" "" INTERNAL_COMPILER_FLAGS ${INTERNAL_COMPILER_FLAGS})
if (INTERNAL_COMPILER_NO_WARNING)
string (REPLACE "-Wall" "" INTERNAL_COMPILER_FLAGS ${INTERNAL_COMPILER_FLAGS})
@ -35,12 +38,12 @@ if (NOT DOUBLE_CONVERSION_INCLUDE_DIR)
get_target_property(DOUBLE_CONVERSION_INCLUDE_DIR ${DOUBLE_CONVERSION_LIBRARIES} INTERFACE_INCLUDE_DIRECTORIES)
endif ()
string (REPLACE ${ClickHouse_SOURCE_DIR} ${INTERNAL_COMPILER_HEADERS} INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR ${DOUBLE_CONVERSION_INCLUDE_DIR})
string (REPLACE ${ClickHouse_SOURCE_DIR} ${INTERNAL_COMPILER_HEADERS} INTERNAL_Boost_INCLUDE_DIRS ${Boost_INCLUDE_DIRS})
string (REPLACE ${ClickHouse_SOURCE_DIR} ${INTERNAL_COMPILER_HEADERS} INTERNAL_Poco_Foundation_INCLUDE_DIR ${Poco_Foundation_INCLUDE_DIR})
string (REPLACE ${ClickHouse_SOURCE_DIR} ${INTERNAL_COMPILER_HEADERS} INTERNAL_Poco_Util_INCLUDE_DIR ${Poco_Util_INCLUDE_DIR})
string (REPLACE ${ClickHouse_SOURCE_DIR} "" INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR ${DOUBLE_CONVERSION_INCLUDE_DIR})
string (REPLACE ${ClickHouse_SOURCE_DIR} "" INTERNAL_Boost_INCLUDE_DIRS ${Boost_INCLUDE_DIRS})
string (REPLACE ${ClickHouse_SOURCE_DIR} "" INTERNAL_Poco_Foundation_INCLUDE_DIR ${Poco_Foundation_INCLUDE_DIR})
string (REPLACE ${ClickHouse_SOURCE_DIR} "" INTERNAL_Poco_Util_INCLUDE_DIR ${Poco_Util_INCLUDE_DIR})
message (STATUS "Using internal compiler: headers=${INTERNAL_COMPILER_HEADERS} : ${INTERNAL_COMPILER_ENV} ${INTERNAL_COMPILER_EXECUTABLE} ${INTERNAL_COMPILER_FLAGS}; ${INTERNAL_LINKER_EXECUTABLE}")
message (STATUS "Using internal=${USE_INTERNAL_LLVM_LIBRARY} compiler=${USE_EMBEDDED_COMPILER}: headers=${INTERNAL_COMPILER_HEADERS} : ${INTERNAL_COMPILER_ENV} ${INTERNAL_COMPILER_EXECUTABLE} ${INTERNAL_COMPILER_FLAGS}; ${INTERNAL_LINKER_EXECUTABLE}")
set (CONFIG_COMPILE ${ClickHouse_BINARY_DIR}/dbms/src/Interpreters/config_compile.h)
configure_file (${ClickHouse_SOURCE_DIR}/dbms/src/Interpreters/config_compile.h.in ${CONFIG_COMPILE})

55
dbms/src/Interpreters/Compiler.cpp
View File

@ -1,16 +1,14 @@
#include <Poco/DirectoryIterator.h>
#include <Common/ClickHouseRevision.h>
#include <Poco/Util/Application.h>
#include <ext/unlock_guard.h>
#include <Common/ClickHouseRevision.h>
#include <Common/SipHash.h>
#include <Common/ShellCommand.h>
#include <Common/StringUtils/StringUtils.h>
#include <IO/Operators.h>
#include <IO/ReadBufferFromString.h>
#include <IO/ReadBufferFromFileDescriptor.h>
#include <IO/WriteBufferFromFile.h>
#include <Interpreters/Compiler.h>
#include <Interpreters/config_compile.h>
@ -218,15 +216,20 @@ void Compiler::compile(
std::stringstream command;
auto compiler_executable_root = Poco::Util::Application::instance().config().getString("compiler_executable_root", INTERNAL_COMPILER_BIN_ROOT);
auto compiler_headers = Poco::Util::Application::instance().config().getString("compiler_headers", INTERNAL_COMPILER_HEADERS);
auto compiler_headers_root = Poco::Util::Application::instance().config().getString("compiler_headers_root", INTERNAL_COMPILER_HEADERS_ROOT);
LOG_DEBUG(log, "Using internal compiler: compiler_executable_root=" << compiler_executable_root << "; compiler_headers_root=" << compiler_headers_root << "; compiler_headers=" << compiler_headers);
/// Slightly unconvenient.
command <<
"("
INTERNAL_COMPILER_ENV
" " INTERNAL_COMPILER_EXECUTABLE
" " << compiler_executable_root << INTERNAL_COMPILER_EXECUTABLE
" " INTERNAL_COMPILER_FLAGS
/// It is hard to correctly call a ld program manually, because it is easy to skip critical flags, which might lead to
/// unhandled exceptions. Therefore pass path to llvm's lld directly to clang.
" -fuse-ld=" INTERNAL_LINKER_EXECUTABLE
" -fuse-ld=" << compiler_executable_root << INTERNAL_LINKER_EXECUTABLE
#if INTERNAL_COMPILER_CUSTOM_ROOT
@ -234,30 +237,34 @@ void Compiler::compile(
/// echo | clang -x c++ -E -Wp,-v -
/// echo | g++ -x c++ -E -Wp,-v -
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/c++/*"
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/" CMAKE_LIBRARY_ARCHITECTURE "/c++/*"
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/c++/*/backward"
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/clang/*/include" /// if compiler is clang (from package)
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/local/lib/clang/*/include" /// if clang installed manually
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/lib/clang/*/include" /// if clang build from submodules
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/lib/gcc/" CMAKE_LIBRARY_ARCHITECTURE "/*/include-fixed"
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/lib/gcc/" CMAKE_LIBRARY_ARCHITECTURE "/*/include"
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/local/include" /// if something installed manually
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include/" CMAKE_LIBRARY_ARCHITECTURE
" -isystem " INTERNAL_COMPILER_HEADERS_ROOT "/usr/include"
" -isystem " << compiler_headers_root << "/usr/include/c++/*"
" -isystem " << compiler_headers_root << "/usr/include/" CMAKE_LIBRARY_ARCHITECTURE "/c++/*"
" -isystem " << compiler_headers_root << "/usr/include/c++/*/backward"
" -isystem " << compiler_headers_root << "/usr/include/clang/*/include" /// if compiler is clang (from package)
" -isystem " << compiler_headers_root << "/usr/local/lib/clang/*/include" /// if clang installed manually
" -isystem " << compiler_headers_root << "/usr/lib/clang/*/include" /// if clang build from submodules
" -isystem " << compiler_headers_root << "/usr/lib/gcc/" CMAKE_LIBRARY_ARCHITECTURE "/*/include-fixed"
" -isystem " << compiler_headers_root << "/usr/lib/gcc/" CMAKE_LIBRARY_ARCHITECTURE "/*/include"
" -isystem " << compiler_headers_root << "/usr/local/include" /// if something installed manually
" -isystem " << compiler_headers_root << "/usr/include/" CMAKE_LIBRARY_ARCHITECTURE
" -isystem " << compiler_headers_root << "/usr/include"
#endif
" -I " INTERNAL_COMPILER_HEADERS "/dbms/src/"
" -I " INTERNAL_COMPILER_HEADERS "/contrib/libcityhash/include/"
" -I " INTERNAL_COMPILER_HEADERS "/contrib/libpcg-random/include/"
" -I " INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR
" -I " INTERNAL_Poco_Foundation_INCLUDE_DIR
" -I " INTERNAL_Boost_INCLUDE_DIRS
" -I " INTERNAL_COMPILER_HEADERS "/libs/libcommon/include/"
" -I " << compiler_headers << "/dbms/src/"
" -I " << compiler_headers << "/contrib/libcityhash/include/"
" -I " << compiler_headers << "/contrib/libpcg-random/include/"
" -I " << compiler_headers << INTERNAL_DOUBLE_CONVERSION_INCLUDE_DIR
" -I " << compiler_headers << INTERNAL_Poco_Foundation_INCLUDE_DIR
" -I " << compiler_headers << INTERNAL_Boost_INCLUDE_DIRS
" -I " << compiler_headers << "/libs/libcommon/include/"
" " << additional_compiler_flags <<
" -shared -o " << so_tmp_file_path << " " << cpp_file_path
<< " 2>&1"
") || echo Return code: $?";
#if !NDEBUG
LOG_TRACE(log, "Compile command: " << command.str());
#endif
std::string compile_result;
{

1
dbms/src/Interpreters/config_compile.h.in
View File

@ -6,6 +6,7 @@
#endif
#cmakedefine PATH_SHARE "@PATH_SHARE@"
#cmakedefine INTERNAL_COMPILER_FLAGS "@INTERNAL_COMPILER_FLAGS@"
#cmakedefine INTERNAL_COMPILER_BIN_ROOT "@INTERNAL_COMPILER_BIN_ROOT@"
#cmakedefine INTERNAL_LINKER_EXECUTABLE "@INTERNAL_LINKER_EXECUTABLE@"
#cmakedefine INTERNAL_COMPILER_EXECUTABLE "@INTERNAL_COMPILER_EXECUTABLE@"
#cmakedefine INTERNAL_COMPILER_ENV "@INTERNAL_COMPILER_ENV@"

3
dbms/tests/CMakeLists.txt
View File

@ -26,5 +26,6 @@ if (ENABLE_TESTS)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/CTestCustom.cmake ${CMAKE_BINARY_DIR})
# maybe add --no-long ?
add_test(NAME with_server COMMAND bash -c "env BUILD_DIR=${ClickHouse_BINARY_DIR} TEST_OPT0='--skip compile' ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse-test-server")
# if you want disable some tests: env TEST_OPT0='--skip compile'
add_test(NAME with_server COMMAND bash -c "env BUILD_DIR=${ClickHouse_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/clickhouse-test-server")
endif ()

5
dbms/tests/clickhouse-test-server
View File

@ -6,6 +6,7 @@ set -o pipefail
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd ../.. && pwd)
#TODO: DATA_DIR=${DATA_DIR:=`mktemp -d /tmp/clickhouse.test..XXXXX`}
DATA_DIR=${DATA_DIR:=/tmp/clickhouse}
LOG_DIR=${LOG_DIR:=$DATA_DIR/log}
BUILD_DIR=${BUILD_DIR:=$ROOT_DIR/build${BUILD_TYPE}}
@ -63,12 +64,16 @@ fi
# --remote_servers.test_shard_localhost_secure.shard.replica.port=$CLICKHOUSE_PORT_TCP_SECURE \
# --remote_servers.test_shard_localhost.shard.replica.port=$CLICKHOUSE_PORT_TCP \
# If run from compile dir - use in-place compile binary and headers
[ -n "$BIN_DIR" ] && INTERNAL_COMPILER_PARAMS="--compiler_executable_root=$BUILD_DIR/dbms/src/Server/ --compiler_headers=$BUILD_DIR/dbms/headers/ --compiler_headers_root=$BUILD_DIR/dbms/headers/"
$GDB ${BIN_DIR}clickhouse-server --config-file=$CLICKHOUSE_CONFIG -- \
--http_port=$CLICKHOUSE_PORT_HTTP \
--tcp_port=$CLICKHOUSE_PORT_TCP \
--https_port=$CLICKHOUSE_PORT_HTTPS \
--tcp_port_secure=$CLICKHOUSE_PORT_TCP_SECURE \
--interserver_http_port=$CLICKHOUSE_PORT_INTERSERVER \
$INTERNAL_COMPILER_PARAMS \
> $LOG_DIR/stdout 2>&1 &
CH_PID=$!
sleep 3

4
debian/rules vendored
View File

@ -101,8 +101,8 @@ override_dh_install:
#TODO: use from cmake:
# In case building clickhouse-server, adding to package binary of clang, ld and header files - for dynamic compilation.
mkdir -p $(DESTDIR)/usr/share/clickhouse/headers
CLANG=$(DESTDIR)/usr/bin/clickhouse-clang DESTDIR=$(DESTDIR) ./copy_headers.sh . $(DESTDIR)/usr/share/clickhouse/headers
#mkdir -p $(DESTDIR)/usr/share/clickhouse/headers
#CLANG=$(DESTDIR)/usr/bin/clickhouse-clang DESTDIR=$(DESTDIR) ./copy_headers.sh . $(DESTDIR)/usr/share/clickhouse/headers
# fake metrika files when private dir is empty
mkdir -p $(DESTDIR)/etc/clickhouse-server/metrika

1
utils/travis/pbuilder.sh
View File

@ -14,7 +14,6 @@ date
env TEST_RUN=${TEST_RUN=1} \
TEST_PORT_RANDOM= \
`# Skip tests:` \
`# 00281 requires internal compiler` \
`# 00416 requires patched poco from contrib/` \
TEST_OPT="--skip long pocopatch $TEST_OPT" \
TEST_SSL="" `# <Error> Application: SSL context exception: Error loading certificate from file /etc/clickhouse-server/server.crt: No error -- when using system poco on artful` \