Allow build with external double-conversion lz4 zstd re2 zookeeper (#531)

* Allow build with external double-conversion lz4 zstd re2 zookeeper

* fix

* fix

* freebsd fixes

* spaces

* fix

* fix

* fix

* wip

* move

* fix

CMakeLists.txt

@@ -3,7 +3,6 @@ cmake_minimum_required (VERSION 2.6)
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${ClickHouse_SOURCE_DIR}/cmake/Modules/")
 
-
 if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
 	# Require at least gcc 5
 	if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5 AND NOT CMAKE_VERSION VERSION_LESS 2.8.9)
@@ -36,7 +35,6 @@ message (STATUS "CMAKE_BUILD_TYPE: " ${CMAKE_BUILD_TYPE} )
 # TSan is not supported due to false positive errors in libstdc++ and necessity to rebuild libstdc++ with TSan
 set (CMAKE_CONFIGURATION_TYPES "RelWithDebInfo;Debug;Release;MinSizeRel;ASan;UBSan" CACHE STRING "" FORCE)
 
-
 if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*)")
 	set (AARCH64 1)
 endif ()
@@ -53,39 +51,6 @@ if (USE_STATIC_LIBRARIES)
 	list(REVERSE CMAKE_FIND_LIBRARY_SUFFIXES)
 endif ()
 
-
-option (UNBUNDLED "Try find all libraries in system (if fail - use bundled from contrib/)" OFF)
-if (UNBUNDLED)
-	set(NOT_UNBUNDLED 0)
-else ()
-	set(NOT_UNBUNDLED 1)
-endif ()
-
-message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ; USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES} UNBUNDLED=${UNBUNDLED}")
-
-option (USE_INTERNAL_BOOST_LIBRARY "Set to FALSE to use system boost library instead of bundled" ${NOT_UNBUNDLED})
-
-if (USE_INTERNAL_BOOST_LIBRARY)
-	add_definitions(-DBOOST_SYSTEM_NO_DEPRECATED)
-endif ()
-
-option (USE_INTERNAL_POCO_LIBRARY "Set to FALSE to use system poco library instead of bundled" ${NOT_UNBUNDLED})
-
-if (CMAKE_SYSTEM MATCHES "FreeBSD")
-	set(NOT_FREEBSD 0)
-	option (USE_INTERNAL_GPERFTOOLS_LIBRARY "Set to FALSE to use system gperftools (tcmalloc) library instead of bundled" ${NOT_FREEBSD})
-else ()
-	set(NOT_FREEBSD 1)
-	option (USE_INTERNAL_GPERFTOOLS_LIBRARY "Set to FALSE to use system gperftools (tcmalloc) library instead of bundled" ${NOT_UNBUNDLED})
-endif ()
-
-
-option (USE_INTERNAL_ZLIB_LIBRARY "Set to FALSE to use system zlib library instead of bundled" ${NOT_UNBUNDLED})
-
-option (ENABLE_LIBTCMALLOC "Set to TRUE to enable libtcmalloc." ON)
-
-option (DEBUG_LIBTCMALLOC "Set to TRUE to use debug version of libtcmalloc." OFF)
-
 option (GLIBC_COMPATIBILITY "Set to TRUE to enable compatibility with older glibc libraries. Note that it is not compatible with ASan." OFF)
 
 if (GLIBC_COMPATIBILITY)
@@ -115,7 +80,6 @@ if (ARCHNATIVE)
 	set (COMPILER_FLAGS                   "${COMPILER_FLAGS} -march=native")
 endif ()
 
-
 set (CMAKE_BUILD_COLOR_MAKEFILE          ON)
 set (CMAKE_CXX_FLAGS                     "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} -std=gnu++1y ${PLATFORM_EXTRA_CXX_FLAG} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS} ${GLIBC_COMPATIBILITY_COMPILE_FLAGS}")
 #set (CMAKE_CXX_FLAGS_RELEASE             "${CMAKE_CXX_FLAGS_RELEASE}")
@@ -184,14 +148,30 @@ else ()
 	set (CLICKHOUSE_ETC_DIR ${CMAKE_INSTALL_PREFIX}/etc)
 endif ()
 
+
+option (UNBUNDLED "Try find all libraries in system (if fail - use bundled from contrib/)" OFF)
+if (UNBUNDLED)
+	set(NOT_UNBUNDLED 0)
+else ()
+	set(NOT_UNBUNDLED 1)
+endif ()
+
+message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ; USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES} UNBUNDLED=${UNBUNDLED}")
+
 include (cmake/find_openssl.cmake)
 include (cmake/find_icu4c.cmake)
 include (cmake/find_boost.cmake)
 include (cmake/find_zlib.cmake)
+include (cmake/find_zstd.cmake)
 include (cmake/find_poco.cmake)
+include (cmake/find_lz4.cmake)
+include (cmake/find_sparsehash.cmake)
 include (cmake/find_libtool.cmake)
 include (cmake/find_rt.cmake)
 include (cmake/find_readline_edit.cmake)
+include (cmake/find_zookeeper.cmake)
+include (cmake/find_double-conversion.cmake)
+include (cmake/find_re2.cmake)
 include (cmake/find_gperftools.cmake)
 include (cmake/find_jemalloc.cmake)
 

cmake/dbms_include.cmake

@@ -9,6 +9,4 @@ include_directories (${ClickHouse_BINARY_DIR}/libs/libcommon/include)
 include_directories (${ClickHouse_SOURCE_DIR}/libs/libpocoext/include)
 include_directories (${ClickHouse_SOURCE_DIR}/libs/libzkutil/include)
 include_directories (${ClickHouse_SOURCE_DIR}/libs/libmysqlxx/include)
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libzookeeper/include)
 include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libcityhash/include)
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libdouble-conversion)

cmake/find_boost.cmake

@@ -1,3 +1,5 @@
+option (USE_INTERNAL_BOOST_LIBRARY "Set to FALSE to use system boost library instead of bundled" ${NOT_UNBUNDLED})
+
 if (NOT USE_INTERNAL_BOOST_LIBRARY)
 	set (Boost_USE_STATIC_LIBS ${USE_STATIC_LIBRARIES})
 	set (BOOST_ROOT "/usr/local")
@@ -16,6 +18,7 @@ if (NOT USE_INTERNAL_BOOST_LIBRARY)
 endif ()
 
 if (NOT Boost_SYSTEM_LIBRARY)
+	add_definitions(-DBOOST_SYSTEM_NO_DEPRECATED)
 	set (USE_INTERNAL_BOOST_LIBRARY 1)
 	set (Boost_PROGRAM_OPTIONS_LIBRARY boost_program_options_internal)
 	set (Boost_SYSTEM_LIBRARY boost_system_internal)
@@ -24,4 +27,4 @@ if (NOT Boost_SYSTEM_LIBRARY)
 	include_directories (BEFORE ${Boost_INCLUDE_DIRS})
 endif ()
 
-message(STATUS "Using Boost: ${Boost_INCLUDE_DIRS} : ${Boost_PROGRAM_OPTIONS_LIBRARY},${Boost_SYSTEM_LIBRARY},${Boost_FILESYSTEM_LIBRARY}")
+message (STATUS "Using Boost: ${Boost_INCLUDE_DIRS} : ${Boost_PROGRAM_OPTIONS_LIBRARY},${Boost_SYSTEM_LIBRARY},${Boost_FILESYSTEM_LIBRARY}")

cmake/find_double-conversion.cmake

@@ -0,0 +1,17 @@
+option (USE_INTERNAL_DOUBLE_CONVERSION_LIBRARY "Set to FALSE to use system double-conversion library instead of bundled" ${NOT_UNBUNDLED})
+
+if (NOT USE_INTERNAL_DOUBLE_CONVERSION_LIBRARY)
+	find_library (DOUBLE_CONVERSION_LIBRARY double-conversion)
+	find_path (DOUBLE_CONVERSION_INCLUDE_DIR NAMES double-conversion/double-conversion.h PATHS ${DOUBLE_CONVERSION_INCLUDE_PATHS})
+endif ()
+
+if (DOUBLE_CONVERSION_LIBRARY AND DOUBLE_CONVERSION_INCLUDE_DIR)
+	include_directories (${DOUBLE_CONVERSION_INCLUDE_DIR})
+else ()
+	set (USE_INTERNAL_DOUBLE_CONVERSION_LIBRARY 1)
+	set (DOUBLE_CONVERSION_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libdouble-conversion")
+	include_directories (BEFORE ${DOUBLE_CONVERSION_INCLUDE_DIR})
+	set (DOUBLE_CONVERSION_LIBRARY double-conversion)
+endif ()
+
+message (STATUS "Using double-conversion: ${DOUBLE_CONVERSION_INCLUDE_DIR} : ${DOUBLE_CONVERSION_LIBRARY}")

cmake/find_gperftools.cmake

@@ -1,3 +1,11 @@
+if (CMAKE_SYSTEM MATCHES "FreeBSD")
+	option (USE_INTERNAL_GPERFTOOLS_LIBRARY "Set to FALSE to use system gperftools (tcmalloc) library instead of bundled" OFF)
+else ()
+	option (USE_INTERNAL_GPERFTOOLS_LIBRARY "Set to FALSE to use system gperftools (tcmalloc) library instead of bundled" ${NOT_UNBUNDLED})
+endif ()
+option (ENABLE_LIBTCMALLOC "Set to TRUE to enable libtcmalloc" ON)
+option (DEBUG_LIBTCMALLOC "Set to TRUE to use debug version of libtcmalloc" OFF)
+
 if (ENABLE_LIBTCMALLOC)
 	#contrib/libtcmalloc doesnt build debug version, try find in system
 	if (DEBUG_LIBTCMALLOC OR NOT USE_INTERNAL_GPERFTOOLS_LIBRARY)

cmake/find_icu4c.cmake

@@ -6,5 +6,5 @@ find_library (ICUDATA icudata PATHS ${ICU_PATHS})
 set (ICU_LIBS ${ICUI18N} ${ICUUC} ${ICUDATA})
 
 find_path (ICU_INCLUDE_DIR NAMES unicode/unistr.h PATHS ${ICU_INCLUDE_PATHS})
-message(STATUS "Using icu: ${ICU_INCLUDE_DIR} : ${ICU_LIBS}")
+message (STATUS "Using icu: ${ICU_INCLUDE_DIR} : ${ICU_LIBS}")
 include_directories (${ICU_INCLUDE_DIR})

cmake/find_jemalloc.cmake

@@ -1,3 +1,5 @@
+option (ENABLE_JEMALLOC "Set to TRUE to use jemalloc instead of tcmalloc" OFF)
+
 if (ENABLE_JEMALLOC)
 	find_package(JeMalloc)
 

cmake/find_libtool.cmake

@@ -1,3 +1,3 @@
 set (LTDL_PATHS "/usr/local/opt/libtool/lib")
 find_library (LTDL_LIB ltdl PATHSS ${LTDL_PATHS})
-message(STATUS "Using ltdl: ${LTDL_LIB}")
+message (STATUS "Using ltdl: ${LTDL_LIB}")

cmake/find_lz4.cmake

@@ -0,0 +1,17 @@
+option (USE_INTERNAL_LZ4_LIBRARY "Set to FALSE to use system lz4 library instead of bundled" ${NOT_UNBUNDLED})
+
+if (NOT USE_INTERNAL_LZ4_LIBRARY)
+	find_library (LZ4_LIBRARY lz4)
+	find_path (LZ4_INCLUDE_DIR NAMES lz4.h PATHS ${LZ4_INCLUDE_PATHS})
+endif ()
+
+if (LZ4_LIBRARY AND LZ4_INCLUDE_DIR)
+	include_directories (${LZ4_INCLUDE_DIR})
+else ()
+	set (USE_INTERNAL_LZ4_LIBRARY 1)
+	set (LZ4_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/liblz4/include/lz4")
+	include_directories (BEFORE ${LZ4_INCLUDE_DIR})
+	set (LZ4_LIBRARY lz4)
+endif ()
+
+message (STATUS "Using lz4: ${LZ4_INCLUDE_DIR} : ${LZ4_LIBRARY}")

cmake/find_openssl.cmake

@@ -12,5 +12,5 @@ if (NOT OPENSSL_FOUND)
 	set (OPENSSL_LIBRARIES ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY})
 endif ()
 
-message(STATUS "Using openssl: ${OPENSSL_INCLUDE_DIR} : ${OPENSSL_LIBRARIES}")
+message (STATUS "Using openssl: ${OPENSSL_INCLUDE_DIR} : ${OPENSSL_LIBRARIES}")
 include_directories (${OPENSSL_INCLUDE_DIR})

cmake/find_poco.cmake

@@ -1,3 +1,4 @@
+option (USE_INTERNAL_POCO_LIBRARY "Set to FALSE to use system poco library instead of bundled" ${NOT_UNBUNDLED})
 
 if (NOT USE_INTERNAL_POCO_LIBRARY)
 	find_package (Poco COMPONENTS Net NetSSL XML Data Crypto DataODBC MongoDB)

cmake/find_re2.cmake

@@ -0,0 +1,22 @@
+#TODO: option (USE_INTERNAL_RE2_LIBRARY "Set to FALSE to use system re2 library instead of bundled" ${NOT_UNBUNDLED})
+set (USE_INTERNAL_RE2_LIBRARY ON)
+
+if (NOT USE_INTERNAL_RE2_LIBRARY)
+	# TODO! re2_st
+	find_library (RE2_LIBRARY re2)
+	find_path (RE2_INCLUDE_DIR NAMES re2/re2.h PATHS ${RE2_INCLUDE_PATHS})
+endif ()
+
+if (RE2_LIBRARY AND RE2_INCLUDE_DIR)
+	include_directories (${RE2_INCLUDE_DIR})
+else ()
+	set (USE_INTERNAL_RE2_LIBRARY 1)
+	set (RE2_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libre2")
+	set (RE2_ST_INCLUDE_DIR "${ClickHouse_BINARY_DIR}/contrib/libre2")
+	include_directories (BEFORE ${RE2_INCLUDE_DIR})
+	include_directories (BEFORE ${RE2_ST_INCLUDE_DIR})
+	set (RE2_LIBRARY re2)
+	set (RE2_ST_LIBRARY re2_st)
+endif ()
+
+message (STATUS "Using re2: ${RE2_INCLUDE_DIR} : ${RE2_LIBRARY}; ${RE2_ST_INCLUDE_DIR} : ${RE2_ST_LIBRARY}")

cmake/find_readline_edit.cmake

@@ -21,7 +21,7 @@ if (READLINE_LIB AND TERMCAP_LIB)
 	message (STATUS "Using line editing libraries (readline): ${READLINE_INCLUDE_DIR} : ${LINE_EDITING_LIBS}")
 elseif (EDIT_LIB)
 	find_library (CURSES_LIB NAMES curses)
-	set(USE_LIBEDIT 1)
+	set (USE_LIBEDIT 1)
 	find_path (READLINE_INCLUDE_DIR NAMES editline/readline.h PATHS ${READLINE_INCLUDE_PATHS})
 	set (LINE_EDITING_LIBS ${EDIT_LIB} ${CURSES_LIB} ${TERMCAP_LIB})
 	message (STATUS "Using line editing libraries (edit): ${READLINE_INCLUDE_DIR} : ${LINE_EDITING_LIBS}")
@@ -35,7 +35,7 @@ endif ()
 include (CheckCXXSourceRuns)
 
 set (CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES} ${LINE_EDITING_LIBS})
-check_cxx_source_runs("
+check_cxx_source_runs ("
 	#include <stdio.h>
 	#include <readline/readline.h>
 	#include <readline/history.h>

cmake/find_sparsehash.cmake

@@ -0,0 +1,15 @@
+option (USE_INTERNAL_SPARCEHASH_LIBRARY "Set to FALSE to use system sparsehash library instead of bundled" ${NOT_UNBUNDLED})
+
+if (NOT USE_INTERNAL_SPARCEHASH_LIBRARY)
+	find_path (SPARCEHASH_INCLUDE_DIR NAMES sparsehash/sparse_hash_map PATHS ${SPARCEHASH_INCLUDE_PATHS})
+endif ()
+
+if (SPARCEHASH_INCLUDE_DIR)
+	include_directories (${SPARCEHASH_INCLUDE_DIR})
+else ()
+	set (USE_INTERNAL_SPARCEHASH_LIBRARY 1)
+	set (SPARCEHASH_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libsparsehash")
+	include_directories (BEFORE ${SPARCEHASH_INCLUDE_DIR})
+endif ()
+
+message (STATUS "Using sparsehash: ${SPARCEHASH_INCLUDE_DIR}")

cmake/find_zlib.cmake

@@ -1,3 +1,5 @@
+option (USE_INTERNAL_ZLIB_LIBRARY "Set to FALSE to use system zlib library instead of bundled" ${NOT_UNBUNDLED})
+
 if (NOT USE_INTERNAL_ZLIB_LIBRARY)
 	find_package (ZLIB)
 	if (ZLIB_FOUND)

cmake/find_zookeeper.cmake

@@ -0,0 +1,17 @@
+option (USE_INTERNAL_ZOOKEEPER_LIBRARY "Set to FALSE to use system zookeeper library instead of bundled" ${NOT_UNBUNDLED})
+
+if (NOT USE_INTERNAL_ZOOKEEPER_LIBRARY)
+	find_library (ZOOKEEPER_LIBRARY zookeeper_mt)
+	find_path (ZOOKEEPER_INCLUDE_DIR NAMES zookeeper/zookeeper.h PATHS ${ZOOKEEPER_INCLUDE_PATHS})
+endif ()
+
+if (ZOOKEEPER_LIBRARY AND ZOOKEEPER_INCLUDE_DIR)
+	include_directories (${ZOOKEEPER_INCLUDE_DIR})
+else ()
+	set (USE_INTERNAL_ZOOKEEPER_LIBRARY 1)
+	set (ZOOKEEPER_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libzookeeper/include/zookeeper")
+	include_directories (BEFORE ${ZOOKEEPER_INCLUDE_DIR})
+	set (ZOOKEEPER_LIBRARY zookeeper_mt)
+endif ()
+
+message (STATUS "Using zookeeper: ${ZOOKEEPER_INCLUDE_DIR} : ${ZOOKEEPER_LIBRARY}")

cmake/find_zstd.cmake

@@ -0,0 +1,17 @@
+option (USE_INTERNAL_ZSTD_LIBRARY "Set to FALSE to use system zstd library instead of bundled" ${NOT_UNBUNDLED})
+
+if (NOT USE_INTERNAL_ZSTD_LIBRARY)
+	find_library (ZSTD_LIBRARY zstd)
+	find_path (ZSTD_INCLUDE_DIR NAMES zstd.h PATHS ${ZSTD_INCLUDE_PATHS})
+endif ()
+
+if (ZSTD_LIBRARY AND ZSTD_INCLUDE_DIR)
+	include_directories (${ZSTD_INCLUDE_DIR})
+else ()
+	set (USE_INTERNAL_ZSTD_LIBRARY 1)
+	set (ZSTD_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libzstd/include/zstd")
+	include_directories (BEFORE ${ZSTD_INCLUDE_DIR})
+	set (ZSTD_LIBRARY zstd)
+endif ()
+
+message (STATUS "Using zstd: ${ZSTD_INCLUDE_DIR} : ${ZSTD_LIBRARY}")

contrib/CMakeLists.txt

@@ -8,14 +8,30 @@ if (USE_INTERNAL_POCO_LIBRARY)
 	add_subdirectory (libpoco)
 endif ()
 
-add_subdirectory (liblz4)
+if (USE_INTERNAL_LZ4_LIBRARY)
-add_subdirectory (libzstd)
+	add_subdirectory (liblz4)
-add_subdirectory (libre2)
+endif ()
-add_subdirectory (libdouble-conversion)
+
-add_subdirectory (libzookeeper)
+if (USE_INTERNAL_ZSTD_LIBRARY)
+	add_subdirectory (libzstd)
+endif ()
+
+if (USE_INTERNAL_RE2_LIBRARY)
+	add_subdirectory (libre2)
+endif ()
+
+if (USE_INTERNAL_DOUBLE_CONVERSION_LIBRARY)
+	add_subdirectory (libdouble-conversion)
+endif ()
+
+if (USE_INTERNAL_ZOOKEEPER_LIBRARY)
+	add_subdirectory (libzookeeper)
+endif ()
+
 add_subdirectory (libcityhash)
 add_subdirectory (libfarmhash)
 add_subdirectory (libmetrohash)
+
 if (USE_INTERNAL_ZLIB_LIBRARY)
 	add_subdirectory (libzlib-ng)
 endif ()

contrib/libzookeeper/CMakeLists.txt

@@ -6,7 +6,7 @@ endif()
 
 include_directories (include/zookeeper src)
 
-add_library (zookeeper
+add_library (zookeeper_mt
 src/zookeeper.c
 src/zookeeper.jute.c
 src/zk_hashtable.c

contrib/libzstd/CMakeLists.txt

@@ -40,7 +40,7 @@ endfunction()
 
 # Define library directory, where sources and header files are located
 SET(LIBRARY_DIR include/zstd)
-INCLUDE_DIRECTORIES(${LIBRARY_DIR} ${LIBRARY_DIR}/common)
+INCLUDE_DIRECTORIES(BEFORE ${LIBRARY_DIR} ${LIBRARY_DIR}/common)
 
 # Read file content
 FILE(READ ${LIBRARY_DIR}/zstd.h HEADER_CONTENT)
@@ -80,7 +80,7 @@ SET(ZSTD_LEGACY_SUPPORT true)
 
 IF (ZSTD_LEGACY_SUPPORT)
     SET(LIBRARY_LEGACY_DIR ${LIBRARY_DIR}/legacy)
-    INCLUDE_DIRECTORIES(${LIBRARY_LEGACY_DIR})
+    INCLUDE_DIRECTORIES(BEFORE ${LIBRARY_LEGACY_DIR})
     ADD_DEFINITIONS(-D ZSTD_LEGACY_SUPPORT=1)
 
     SET(Sources ${Sources}

dbms/CMakeLists.txt

@@ -1,14 +1,9 @@
 include(${ClickHouse_SOURCE_DIR}/cmake/dbms_include.cmake)
 
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/liblz4/include/)
 include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libdivide)
 include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libcpuid/include/)
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libzstd/include/)
 include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libfarmhash)
 include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libmetrohash/src)
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libsparsehash)
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libre2/)
-include_directories (BEFORE ${ClickHouse_BINARY_DIR}/contrib/libre2/)
 include_directories (${ClickHouse_SOURCE_DIR}/libs/libdaemon/include/)
 include_directories (${ODBC_INCLUDE_DIRECTORIES})
 
@@ -17,6 +12,10 @@ if (NOT NO_WERROR)
 	set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror")
 endif ()
 
+include (${ClickHouse_SOURCE_DIR}/cmake/find_iconv.cmake)
+
+find_package (Threads)
+
 add_subdirectory (src)
 
 add_library(string_utils
@@ -118,7 +117,6 @@ if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
 		PROPERTIES COMPILE_FLAGS -g0)
 endif ()
 
-
 if (NOT AARCH64)
 	set (LINK_LIBRARIES_ONLY_ON_X86_64 cpuid)
 endif()
@@ -129,21 +127,19 @@ else()
     set(PLATFORM_LIBS "")
 endif()
 
-include (${ClickHouse_SOURCE_DIR}/cmake/find_iconv.cmake)
-
-find_package (Threads)
-
 target_link_libraries(dbms
 	common
 	zkutil
 	mysqlxx
 	cityhash farmhash metrohash
-	lz4 zstd
+	${LZ4_LIBRARY}
+	${ZSTD_LIBRARY}
 	string_utils
-	double-conversion
+	${DOUBLE_CONVERSION_LIBRARY}
 	${ZLIB_LIBRARIES}
 	${LINK_LIBRARIES_ONLY_ON_X86_64}
-	re2 re2_st
+	${RE2_LIBRARY}
+	${RE2_ST_LIBRARY}
 	${OPENSSL_CRYPTO_LIBRARY}
 	${Boost_SYSTEM_LIBRARY}
 	${Poco_Data_LIBRARY}

dbms/src/IO/CompressedReadBufferBase.cpp

@@ -8,8 +8,8 @@
 	#include <quicklz/quicklz_level1.h>
 #endif
 
-#include <lz4/lz4.h>
+#include <lz4.h>
-#include <zstd/zstd.h>
+#include <zstd.h>
 
 #include <DB/Common/PODArray.h>
 #include <DB/Common/ProfileEvents.h>

dbms/src/IO/CompressedWriteBuffer.cpp

@@ -5,9 +5,9 @@
 	#include <quicklz/quicklz_level1.h>
 #endif
 
-#include <lz4/lz4.h>
+#include <lz4.h>
-#include <lz4/lz4hc.h>
+#include <lz4hc.h>
-#include <zstd/zstd.h>
+#include <zstd.h>
 
 #include <DB/Common/unaligned.h>
 

dbms/tests/clickhouse-test

@@ -188,7 +188,7 @@ def main(args):
 								report_testcase.append(skipped)
 								print("{0} - no reference file".format(MSG_UNKNOWN))
 							else:
-								result_is_different = subprocess.call(['cmp', '--quiet', reference_file, stdout_file], stdout = PIPE)
+								result_is_different = subprocess.call(['cmp', '-s', reference_file, stdout_file], stdout = PIPE)
 
 								if result_is_different:
 									(diff, _) = Popen(['diff', '--side-by-side', reference_file, stdout_file], stdout = PIPE).communicate()

doc/build_debian.sh

@@ -7,6 +7,8 @@
 
 # install compiler and libs
 sudo apt install -y git bash cmake gcc-6 g++-6 libicu-dev libreadline-dev libmysqlclient-dev unixodbc-dev libltdl-dev libssl-dev
+# for -DUNBUNDLED=1 mode:
+#sudo apt install -y libboost-dev zlib1g-dev liblz4-dev libdouble-conversion-dev libzstd-dev libre2-dev libzookeeper-mt-dev libsparsehash-dev  #  libpoco-dev
 
 # install testing only stuff if you want:
 sudo apt install -y python python-lxml python-termcolor curl perl

doc/build_freebsd.sh

@@ -20,10 +20,10 @@ mkdir -p ClickHouse/build
 cd ClickHouse/build
 cmake .. -DUSE_INTERNAL_GPERFTOOLS_LIBRARY=0
 #  WIP: variant with libs from ports:
-# sudo pkg install boost-libs
+# sudo pkg install boost-libs libzookeeper libdouble-conversion zstd liblz4 sparsehash
 #  Check UNIXODBC option:
 # make -C /usr/ports/devel/poco config reinstall
-# cmake .. -DUSE_INTERNAL_BOOST_LIBRARY=0 -DUSE_INTERNAL_POCO_LIBRARY=0 -DUSE_INTERNAL_GPERFTOOLS_LIBRARY=0
+# cmake .. -DUNBUNDLED=1 -DUSE_STATIC_LIBRARIES=0 -DNO_WERROR=1
 
 make -C dbms/src/Server -j $(nproc || sysctl -n hw.ncpu || echo 2)
 cd ../..

libs/libzkutil/CMakeLists.txt

@@ -23,6 +23,6 @@ add_library(zkutil
 
 find_package (Threads)
 
-target_link_libraries(zkutil zookeeper ${Poco_Foundation_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} string_utils)
+target_link_libraries(zkutil ${ZOOKEEPER_LIBRARY} ${Poco_Foundation_LIBRARY} ${CMAKE_THREAD_LIBS_INIT} string_utils)
 
 add_subdirectory (src)

utils/compressor/CMakeLists.txt

@@ -1,10 +1,8 @@
 
-include_directories (BEFORE ${ClickHouse_SOURCE_DIR}/contrib/libzstd/include)
-
 add_executable (clickhouse-compressor main.cpp)
 target_link_libraries (clickhouse-compressor dbms ${Boost_PROGRAM_OPTIONS_LIBRARY})
 
 install (TARGETS clickhouse-compressor RUNTIME DESTINATION bin COMPONENT clickhouse-compressor)
 
 add_executable (zstd_test zstd_test.cpp)
-target_link_libraries (zstd_test zstd)
+target_link_libraries (zstd_test ${ZSTD_LIBRARY})

utils/compressor/zstd_test.cpp

@@ -1,5 +1,5 @@
 #include <unistd.h>
-#include <zstd/zstd.h>
+#include <zstd.h>
 #include <vector>
 #include <stdexcept>