From f749405bf0166afa52edb94a55daca8dc7e86b0b Mon Sep 17 00:00:00 2001 From: proller Date: Fri, 2 Mar 2018 03:17:25 +0300 Subject: [PATCH] libressl as submodule (#1983) * Allow use bundled *ssl library * fix * Add submodule * Fixes * fix * fixes * fixes * fix * fix * update poco * fix warnings * fix * fix --- .gitmodules | 3 +++ CMakeLists.txt | 14 +++++++--- cmake/arch.cmake | 10 +++++++ cmake/find_capnp.cmake | 2 +- cmake/find_openssl.cmake | 23 ---------------- cmake/find_poco.cmake | 4 +-- cmake/find_rdkafka.cmake | 2 +- cmake/find_ssl.cmake | 44 +++++++++++++++++++++++++++++++ cmake/find_zlib.cmake | 11 +++++++- contrib/CMakeLists.txt | 37 +++++++++++++++++++++++--- contrib/libboost/CMakeLists.txt | 5 ++-- contrib/poco | 2 +- contrib/ssl | 1 + dbms/CMakeLists.txt | 4 ++- dbms/src/Interpreters/Cluster.cpp | 1 - 15 files changed, 122 insertions(+), 41 deletions(-) delete mode 100644 cmake/find_openssl.cmake create mode 100644 cmake/find_ssl.cmake create mode 160000 contrib/ssl diff --git a/.gitmodules b/.gitmodules index e076a6a3d78..98e031e8b6f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -31,3 +31,6 @@ [submodule "contrib/re2"] path = contrib/re2 url = https://github.com/google/re2.git +[submodule "contrib/ssl"] + path = contrib/ssl + url = https://github.com/ClickHouse-Extras/ssl.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 2bffc353cef..cf546c72f83 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,8 +37,14 @@ set (CMAKE_CONFIGURATION_TYPES "RelWithDebInfo;Debug;Release;MinSizeRel;ASan;UBS include (cmake/arch.cmake) -set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wall") # -Werror is also added inside directories with our own code. -set (CXX_WARNING_FLAGS "${CXX_WARNING_FLAGS} -Wnon-virtual-dtor") +if (NOT MSVC) + set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wall") # -Werror is also added inside directories with our own code. +endif () + +if (COMPILER_GCC OR COMPILER_CLANG) + set (CXX_WARNING_FLAGS "${CXX_WARNING_FLAGS} -Wnon-virtual-dtor") +endif () + if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") # clang: warning: argument unused during compilation: '-stdlib=libc++' # clang: warning: argument unused during compilation: '-specs=/usr/share/dpkg/no-pie-compile.specs' [-Wunused-command-line-argument] @@ -50,7 +56,7 @@ if (ARCH_LINUX) endif () option (TEST_COVERAGE "Enables flags for test coverage" OFF) -option (ENABLE_TESTS "Enables tests" ON) +option (ENABLE_TESTS "Enables tests" ${NOT_MSVC}) option (USE_STATIC_LIBRARIES "Set to FALSE to use shared libraries" ON) option (MAKE_STATIC_LIBRARIES "Set to FALSE to make shared libraries" ${USE_STATIC_LIBRARIES}) @@ -222,7 +228,7 @@ message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE include(GNUInstallDirs) -include (cmake/find_openssl.cmake) +include (cmake/find_ssl.cmake) if (NOT OPENSSL_FOUND) message (FATAL_ERROR "Need openssl for build. debian tip: sudo apt install libssl-dev") endif () diff --git a/cmake/arch.cmake b/cmake/arch.cmake index 1191d70bd19..f61bac96ab0 100644 --- a/cmake/arch.cmake +++ b/cmake/arch.cmake @@ -17,3 +17,13 @@ endif () if (CMAKE_SYSTEM MATCHES "FreeBSD") set (ARCH_FREEBSD 1) endif () + +if (NOT MSVC) + set (NOT_MSVC 1) +endif () + +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set (COMPILER_GCC 1) +elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + set (COMPILER_CLANG 1) +endif () diff --git a/cmake/find_capnp.cmake b/cmake/find_capnp.cmake index 885b7ec3c94..22301acc3ee 100644 --- a/cmake/find_capnp.cmake +++ b/cmake/find_capnp.cmake @@ -1,4 +1,4 @@ -option (ENABLE_CAPNP "Enable Cap'n Proto" ON) +option (ENABLE_CAPNP "Enable Cap'n Proto" ${NOT_MSVC}) if (ENABLE_CAPNP) diff --git a/cmake/find_openssl.cmake b/cmake/find_openssl.cmake deleted file mode 100644 index 512ca24dfe5..00000000000 --- a/cmake/find_openssl.cmake +++ /dev/null @@ -1,23 +0,0 @@ -set (OPENSSL_USE_STATIC_LIBS ${USE_STATIC_LIBRARIES}) -if (APPLE) - set (OPENSSL_ROOT_DIR "/usr/local/opt/openssl") - # https://rt.openssl.org/Ticket/Display.html?user=guest&pass=guest&id=2232 - if (USE_STATIC_LIBRARIES) - message(WARNING "Disable USE_STATIC_LIBRARIES if you have linking problems with OpenSSL on MacOS") - endif () -endif () -find_package (OpenSSL) -if (NOT OPENSSL_FOUND) - # Try to find manually. - set (OPENSSL_INCLUDE_PATHS "/usr/local/opt/openssl/include") - set (OPENSSL_PATHS "/usr/local/opt/openssl/lib") - find_path (OPENSSL_INCLUDE_DIR NAMES openssl/ssl.h PATHS ${OPENSSL_INCLUDE_PATHS}) - find_library (OPENSSL_SSL_LIBRARY ssl PATHS ${OPENSSL_PATHS}) - find_library (OPENSSL_CRYPTO_LIBRARY crypto PATHS ${OPENSSL_PATHS}) - if (OPENSSL_SSL_LIBRARY AND OPENSSL_CRYPTO_LIBRARY AND OPENSSL_INCLUDE_DIR) - set (OPENSSL_LIBRARIES ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY}) - set (OPENSSL_FOUND 1) - endif () -endif () - -message (STATUS "Using openssl=${OPENSSL_FOUND}: ${OPENSSL_INCLUDE_DIR} : ${OPENSSL_LIBRARIES}") diff --git a/cmake/find_poco.cmake b/cmake/find_poco.cmake index 0750067429a..9051d63ae47 100644 --- a/cmake/find_poco.cmake +++ b/cmake/find_poco.cmake @@ -57,8 +57,8 @@ elseif (NOT MISSING_INTERNAL_POCO_LIBRARY) if (USE_STATIC_LIBRARIES AND USE_INTERNAL_ZLIB_LIBRARY) list (APPEND Poco_INCLUDE_DIRS - "${ClickHouse_SOURCE_DIR}/contrib/zlib-ng/" - "${ClickHouse_BINARY_DIR}/contrib/zlib-ng/" + "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}/" + "${ClickHouse_BINARY_DIR}/contrib/${INTERNAL_ZLIB_NAME}/" ) endif () diff --git a/cmake/find_rdkafka.cmake b/cmake/find_rdkafka.cmake index c662a73cb37..e5964b51dc7 100644 --- a/cmake/find_rdkafka.cmake +++ b/cmake/find_rdkafka.cmake @@ -1,4 +1,4 @@ -option (ENABLE_RDKAFKA "Enable kafka" ON) +option (ENABLE_RDKAFKA "Enable kafka" ${NOT_MSVC}) if (ENABLE_RDKAFKA) diff --git a/cmake/find_ssl.cmake b/cmake/find_ssl.cmake new file mode 100644 index 00000000000..ec40e498da1 --- /dev/null +++ b/cmake/find_ssl.cmake @@ -0,0 +1,44 @@ +option (USE_INTERNAL_SSL_LIBRARY "Set to FALSE to use system *ssl library instead of bundled" ${MSVC}) + +set (OPENSSL_USE_STATIC_LIBS ${USE_STATIC_LIBRARIES}) + +if (NOT USE_INTERNAL_SSL_LIBRARY) + if (APPLE) + set (OPENSSL_ROOT_DIR "/usr/local/opt/openssl") + # https://rt.openssl.org/Ticket/Display.html?user=guest&pass=guest&id=2232 + if (USE_STATIC_LIBRARIES) + message(WARNING "Disable USE_STATIC_LIBRARIES if you have linking problems with OpenSSL on MacOS") + endif () + endif () + find_package (OpenSSL) + + if (NOT OPENSSL_FOUND) + # Try to find manually. + set (OPENSSL_INCLUDE_PATHS "/usr/local/opt/openssl/include") + set (OPENSSL_PATHS "/usr/local/opt/openssl/lib") + find_path (OPENSSL_INCLUDE_DIR NAMES openssl/ssl.h PATHS ${OPENSSL_INCLUDE_PATHS}) + find_library (OPENSSL_SSL_LIBRARY ssl PATHS ${OPENSSL_PATHS}) + find_library (OPENSSL_CRYPTO_LIBRARY crypto PATHS ${OPENSSL_PATHS}) + if (OPENSSL_SSL_LIBRARY AND OPENSSL_CRYPTO_LIBRARY AND OPENSSL_INCLUDE_DIR) + set (OPENSSL_LIBRARIES ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY}) + set (OPENSSL_FOUND 1) + endif () + endif () +endif () + +if (NOT OPENSSL_FOUND) + set (USE_INTERNAL_SSL_LIBRARY 1) + set (OPENSSL_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/ssl") + set (OPENSSL_INCLUDE_DIR "${OPENSSL_ROOT_DIR}/include") + if (NOT USE_STATIC_LIBRARIES AND TARGET crypto-shared AND TARGET ssl-shared) + set (OPENSSL_CRYPTO_LIBRARY crypto-shared) + set (OPENSSL_SSL_LIBRARY ssl-shared) + else () + set (OPENSSL_CRYPTO_LIBRARY crypto) + set (OPENSSL_SSL_LIBRARY ssl) + endif () + set (OPENSSL_LIBRARIES ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY}) + set (OPENSSL_FOUND 1) +endif () + +message (STATUS "Using ssl=${OPENSSL_FOUND}: ${OPENSSL_INCLUDE_DIR} : ${OPENSSL_LIBRARIES}") diff --git a/cmake/find_zlib.cmake b/cmake/find_zlib.cmake index bafb1bb75b8..17350f9fd58 100644 --- a/cmake/find_zlib.cmake +++ b/cmake/find_zlib.cmake @@ -5,6 +5,15 @@ if (NOT USE_INTERNAL_ZLIB_LIBRARY) endif () if (NOT ZLIB_FOUND) + if (NOT MSVC) + set (INTERNAL_ZLIB_NAME "zlib-ng") + else () + set (INTERNAL_ZLIB_NAME "zlib") + if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}") + message (WARNING "Will use standard zlib, please clone manually:\n git clone https://github.com/madler/zlib.git ${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}") + endif () + endif () + set (USE_INTERNAL_ZLIB_LIBRARY 1) set (ZLIB_COMPAT 1) # for zlib-ng, also enables WITH_GZFILEOP set (WITH_NATIVE_INSTRUCTIONS ${ARCHNATIVE}) @@ -15,7 +24,7 @@ if (NOT ZLIB_FOUND) set(WITH_NEON 1 CACHE INTERNAL "") set(WITH_ACLE 1 CACHE INTERNAL "") endif () - set (ZLIB_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/zlib-ng" "${ClickHouse_BINARY_DIR}/contrib/zlib-ng") # generated zconf.h + set (ZLIB_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}" "${ClickHouse_BINARY_DIR}/contrib/${INTERNAL_ZLIB_NAME}") # generated zconf.h set (ZLIB_INCLUDE_DIRS ${ZLIB_INCLUDE_DIR}) # for poco set (ZLIB_FOUND 1) # for poco if (USE_STATIC_LIBRARIES) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 28445658fe9..d7631a8c9d6 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -1,4 +1,6 @@ -set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast") +if (NOT MSVC) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast") +endif () if (USE_INTERNAL_BOOST_LIBRARY) add_subdirectory (libboost) @@ -48,7 +50,7 @@ if (USE_INTERNAL_UNWIND_LIBRARY) endif () if (USE_INTERNAL_ZLIB_LIBRARY) - add_subdirectory (zlib-ng) + add_subdirectory (${INTERNAL_ZLIB_NAME}) # todo: make pull to Dead2/zlib-ng and remove: # We should use same defines when including zlib.h as used when zlib compiled target_compile_definitions (zlib PUBLIC ZLIB_COMPAT WITH_GZFILEOP) @@ -81,12 +83,33 @@ if (NOT ARCH_ARM) add_subdirectory (libcpuid) endif () +if (USE_INTERNAL_SSL_LIBRARY) + if (NOT MAKE_STATIC_LIBRARIES) + set (BUILD_SHARED 1) + endif () + set (USE_SHARED ${USE_STATIC_LIBRARIES}) + add_subdirectory (ssl) + target_include_directories(${OPENSSL_CRYPTO_LIBRARY} PUBLIC ${OPENSSL_INCLUDE_DIR}) + target_include_directories(${OPENSSL_SSL_LIBRARY} PUBLIC ${OPENSSL_INCLUDE_DIR}) +endif () + if (USE_INTERNAL_RDKAFKA_LIBRARY) set (RDKAFKA_BUILD_EXAMPLES OFF CACHE INTERNAL "") set (RDKAFKA_BUILD_TESTS OFF CACHE INTERNAL "") - set (RDKAFKA_BUILD_STATIC ON CACHE INTERNAL "") + set (RDKAFKA_BUILD_STATIC ${MAKE_STATIC_LIBRARIES} CACHE INTERNAL "") mark_as_advanced (ZLIB_INCLUDE_DIR) + + if (USE_INTERNAL_SSL_LIBRARY) + add_library(bundled-ssl ALIAS ${OPENSSL_SSL_LIBRARY}) + set (WITH_BUNDLED_SSL 1) + endif () + add_subdirectory (librdkafka) + + if (USE_INTERNAL_SSL_LIBRARY) + target_include_directories(rdkafka PRIVATE BEFORE ${OPENSSL_INCLUDE_DIR}) + endif () + target_include_directories(rdkafka PRIVATE BEFORE ${ZLIB_INCLUDE_DIR}) endif () if (USE_INTERNAL_CAPNP_LIBRARY) @@ -104,17 +127,23 @@ endif () if (USE_INTERNAL_POCO_LIBRARY) + set (ALLOW_DUPLICATE_CUSTOM_TARGETS 1) set (save_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) set (save_CMAKE_C_FLAGS ${CMAKE_C_FLAGS}) set (_save ${ENABLE_TESTS}) set (ENABLE_TESTS 0) set (CMAKE_DISABLE_FIND_PACKAGE_ZLIB 1) + if (USE_INTERNAL_SSL_LIBRARY) + set (DISABLE_INTERNAL_OPENSSL 1) + set (ENABLE_NETSSL 0) # TODO! + set (ENABLE_CRYPTO 0) # TODO! + endif () add_subdirectory (poco) unset (CMAKE_DISABLE_FIND_PACKAGE_ZLIB) set (ENABLE_TESTS ${_save}) set (CMAKE_CXX_FLAGS ${save_CMAKE_CXX_FLAGS}) set (CMAKE_C_FLAGS ${save_CMAKE_C_FLAGS}) - if (OPENSSL_FOUND) + if (OPENSSL_FOUND AND TARGET Crypto) # Bug in poco https://github.com/pocoproject/poco/pull/2100 found on macos target_include_directories(Crypto PUBLIC ${OPENSSL_INCLUDE_DIR}) endif () diff --git a/contrib/libboost/CMakeLists.txt b/contrib/libboost/CMakeLists.txt index 8cfe084636a..8533951a14b 100644 --- a/contrib/libboost/CMakeLists.txt +++ b/contrib/libboost/CMakeLists.txt @@ -1,5 +1,6 @@ -add_definitions(-Wno-unused-variable) -add_definitions(-Wno-deprecated-declarations) +if (NOT MSVC) + add_definitions(-Wno-unused-variable -Wno-deprecated-declarations) +endif () add_library(boost_program_options_internal boost_1_65_0/libs/program_options/src/cmdline.cpp diff --git a/contrib/poco b/contrib/poco index 3d885f5380f..cf1ad2e9a30 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit 3d885f5380f24b4b91d8d4cf18c8cbc083d3ef8d +Subproject commit cf1ad2e9a30ee9161772dc7bc9bf6e165cc51768 diff --git a/contrib/ssl b/contrib/ssl new file mode 160000 index 00000000000..6fbe1c6f404 --- /dev/null +++ b/contrib/ssl @@ -0,0 +1 @@ +Subproject commit 6fbe1c6f404193989c5f6a63115d80fbe34ce2a3 diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 7725f5f7f85..0a46bf0c665 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -18,7 +18,9 @@ get_property (BUILD_INCLUDE_DIRECTORIES DIRECTORY ${ClickHouse_SOURCE_DIR} PROPE string (TIMESTAMP BUILD_DATE "%Y-%m-%d" UTC) configure_file (${CMAKE_CURRENT_SOURCE_DIR}/src/Common/config_build.cpp.in ${CONFIG_BUILD}) -set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra") +if (NOT MSVC) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra") +endif () if (NOT NO_WERROR) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror") diff --git a/dbms/src/Interpreters/Cluster.cpp b/dbms/src/Interpreters/Cluster.cpp index 995a93e4324..3f8efbb9c64 100644 --- a/dbms/src/Interpreters/Cluster.cpp +++ b/dbms/src/Interpreters/Cluster.cpp @@ -10,7 +10,6 @@ #include #include #include -#include namespace DB {