diff --git a/.gitmodules b/.gitmodules index a9e50ab8f6d..20b5cb88ced 100644 --- a/.gitmodules +++ b/.gitmodules @@ -31,9 +31,6 @@ [submodule "contrib/ssl"] path = contrib/ssl url = https://github.com/ClickHouse-Extras/ssl.git -[submodule "contrib/boost"] - path = contrib/boost - url = https://github.com/ClickHouse-Extras/boost.git [submodule "contrib/llvm"] path = contrib/llvm url = https://github.com/ClickHouse-Extras/llvm @@ -46,6 +43,21 @@ [submodule "contrib/unixodbc"] path = contrib/unixodbc url = https://github.com/ClickHouse-Extras/UnixODBC.git +[submodule "contrib/protobuf"] + path = contrib/protobuf + url = https://github.com/ClickHouse-Extras/protobuf.git +[submodule "contrib/boost"] + path = contrib/boost + url = https://github.com/ClickHouse-Extras/boost-extra.git [submodule "contrib/base64"] path = contrib/base64 url = https://github.com/aklomp/base64.git +[submodule "contrib/libhdfs3"] + path = contrib/libhdfs3 + url = https://github.com/ClickHouse-Extras/libhdfs3.git +[submodule "contrib/libxml2"] + path = contrib/libxml2 + url = https://github.com/GNOME/libxml2.git +[submodule "contrib/libgsasl"] + path = contrib/libgsasl + url = git@github.com:ClickHouse-Extras/libgsasl.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 3972cd66ca0..b9262c5c554 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -274,6 +274,9 @@ include (cmake/find_rdkafka.cmake) include (cmake/find_capnp.cmake) include (cmake/find_llvm.cmake) include (cmake/find_cpuid.cmake) +include (cmake/find_libgsasl.cmake) +include (cmake/find_libxml2.cmake) +include (cmake/find_hdfs3.cmake) include (cmake/find_consistent-hashing.cmake) include (cmake/find_base64.cmake) if (ENABLE_TESTS) diff --git a/cmake/find_hdfs3.cmake b/cmake/find_hdfs3.cmake new file mode 100644 index 00000000000..7620ad1ed46 --- /dev/null +++ b/cmake/find_hdfs3.cmake @@ -0,0 +1,21 @@ +if (NOT ARCH_ARM) + option (ENABLE_HDFS "Enable HDFS" ${NOT_UNBUNDLED}) +endif () + +if (ENABLE_HDFS) +option (USE_INTERNAL_HDFS3_LIBRARY "Set to FALSE to use system HDFS3 instead of bundled" ON) + +if (NOT USE_INTERNAL_HDFS3_LIBRARY) + find_package(hdfs3) +endif () + +if (HDFS3_LIBRARY AND HDFS3_INCLUDE_DIR) +else () + set(HDFS3_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include") + set(HDFS3_LIBRARY hdfs3) +endif() +set (USE_HDFS 1) + +endif() + +message (STATUS "Using hdfs3: ${HDFS3_INCLUDE_DIR} : ${HDFS3_LIBRARY}") diff --git a/cmake/find_libxml2.cmake b/cmake/find_libxml2.cmake new file mode 100644 index 00000000000..cfababfbf63 --- /dev/null +++ b/cmake/find_libxml2.cmake @@ -0,0 +1,20 @@ +option (USE_INTERNAL_LIBXML2_LIBRARY "Set to FALSE to use system libxml2 library instead of bundled" ${NOT_UNBUNDLED}) + +if (USE_INTERNAL_LIBXML2_LIBRARY AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libxml2/libxml.h") + message (WARNING "submodule contrib/libxml2 is missing. to fix try run: \n git submodule update --init --recursive") + set (USE_INTERNAL_LIBXML2_LIBRARY 0) +endif () + +if (NOT USE_INTERNAL_LIBXML2_LIBRARY) + find_library (LIBXML2_LIBRARY libxml2) + find_path (LIBXML2_INCLUDE_DIR NAMES libxml.h PATHS ${LIBXML2_INCLUDE_PATHS}) +endif () + +if (LIBXML2_LIBRARY AND LIBXML2_INCLUDE_DIR) +else () + set (LIBXML2_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libxml2/include ${ClickHouse_SOURCE_DIR}/contrib/libxml2-cmake/linux_x86_64/include) + set (USE_INTERNAL_LIBXML2_LIBRARY 1) + set (LIBXML2_LIBRARY libxml2) +endif () + +message (STATUS "Using libxml2: ${LIBXML2_INCLUDE_DIR} : ${LIBXML2_LIBRARY}") diff --git a/cmake/find_protobuf.cmake b/cmake/find_protobuf.cmake new file mode 100644 index 00000000000..5daf5a0c186 --- /dev/null +++ b/cmake/find_protobuf.cmake @@ -0,0 +1,80 @@ +option (USE_INTERNAL_PROTOBUF_LIBRARY "Set to FALSE to use system protobuf instead of bundled" ON) + +if (NOT USE_INTERNAL_PROTOBUF_LIBRARY) + find_package(Protobuf) +endif () + +if (Protobuf_LIBRARY AND Protobuf_INCLUDE_DIR) +else () + set(Protobuf_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/contrib/protobuf/src) + + set(Protobuf_LIBRARY libprotobuf) + set(Protobuf_PROTOC_LIBRARY libprotoc) + set(Protobuf_LITE_LIBRARY libprotobuf-lite) + + set(Protobuf_PROTOC_EXECUTABLE ${CMAKE_BINARY_DIR}/contrib/protobuf/cmake/protoc) + + if(NOT DEFINED PROTOBUF_GENERATE_CPP_APPEND_PATH) + set(PROTOBUF_GENERATE_CPP_APPEND_PATH TRUE) + endif() + + function(PROTOBUF_GENERATE_CPP SRCS HDRS) + if(NOT ARGN) + message(SEND_ERROR "Error: PROTOBUF_GENERATE_CPP() called without any proto files") + return() + endif() + + if(PROTOBUF_GENERATE_CPP_APPEND_PATH) + # Create an include path for each file specified + foreach(FIL ${ARGN}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(ABS_PATH ${ABS_FIL} PATH) + list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _protobuf_include_path -I ${ABS_PATH}) + endif() + endforeach() + else() + set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR}) + endif() + + if(DEFINED PROTOBUF_IMPORT_DIRS AND NOT DEFINED Protobuf_IMPORT_DIRS) + set(Protobuf_IMPORT_DIRS "${PROTOBUF_IMPORT_DIRS}") + endif() + + if(DEFINED Protobuf_IMPORT_DIRS) + foreach(DIR ${Protobuf_IMPORT_DIRS}) + get_filename_component(ABS_PATH ${DIR} ABSOLUTE) + list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _protobuf_include_path -I ${ABS_PATH}) + endif() + endforeach() + endif() + + set(${SRCS}) + set(${HDRS}) + foreach(FIL ${ARGN}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(FIL_WE ${FIL} NAME_WE) + + list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc") + list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h") + + add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc" + "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h" + COMMAND ${Protobuf_PROTOC_EXECUTABLE} + ARGS --cpp_out ${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${ABS_FIL} + DEPENDS ${ABS_FIL} ${Protobuf_PROTOC_EXECUTABLE} + COMMENT "Running C++ protocol buffer compiler on ${FIL}" + VERBATIM ) + endforeach() + + set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE) + set(${SRCS} ${${SRCS}} PARENT_SCOPE) + set(${HDRS} ${${HDRS}} PARENT_SCOPE) + endfunction() +endif() + +message (STATUS "Using protobuf: ${Protobuf_INCLUDE_DIR} : ${Protobuf_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index efc975689c5..fc4413aff26 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -192,6 +192,24 @@ if (USE_INTERNAL_LLVM_LIBRARY) add_subdirectory (llvm/llvm) endif () +if (USE_INTERNAL_LIBGSASL_LIBRARY) + add_subdirectory(libgsasl) +endif() + +if (USE_INTERNAL_LIBXML2_LIBRARY) + add_subdirectory(libxml2-cmake) +endif () + +if (USE_INTERNAL_HDFS3_LIBRARY) + include(${CMAKE_SOURCE_DIR}/cmake/find_protobuf.cmake) + if (USE_INTERNAL_PROTOBUF_LIBRARY) + set(protobuf_BUILD_TESTS OFF CACHE INTERNAL "" FORCE) + set(protobuf_BUILD_SHARED_LIBS OFF CACHE INTERNAL "" FORCE) + add_subdirectory(protobuf/cmake) + endif () + add_subdirectory(libhdfs3-cmake) +endif () + if (USE_BASE64) add_subdirectory (base64-cmake) endif() diff --git a/contrib/boost b/contrib/boost index 2d5cb2c86f6..6883b40449f 160000 --- a/contrib/boost +++ b/contrib/boost @@ -1 +1 @@ -Subproject commit 2d5cb2c86f61126f4e1efe9ab97332efd44e7dea +Subproject commit 6883b40449f378019aec792f9983ce3afc7ff16e diff --git a/contrib/boost-cmake/CMakeLists.txt b/contrib/boost-cmake/CMakeLists.txt index 49d50036e4b..acb76ad0328 100644 --- a/contrib/boost-cmake/CMakeLists.txt +++ b/contrib/boost-cmake/CMakeLists.txt @@ -42,12 +42,17 @@ ${LIBRARY_DIR}/libs/filesystem/src/windows_file_codecvt.cpp) add_library(boost_system_internal ${LINK_MODE} ${LIBRARY_DIR}/libs/system/src/error_code.cpp) +add_library(boost_random_internal ${LINK_MODE} +${LIBRARY_DIR}/libs/random/src/random_device.cpp) + target_link_libraries (boost_filesystem_internal PUBLIC boost_system_internal) target_include_directories (boost_program_options_internal SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS}) target_include_directories (boost_filesystem_internal SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS}) target_include_directories (boost_system_internal SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS}) +target_include_directories (boost_random_internal SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS}) target_compile_definitions (boost_program_options_internal PUBLIC BOOST_SYSTEM_NO_DEPRECATED) target_compile_definitions (boost_filesystem_internal PUBLIC BOOST_SYSTEM_NO_DEPRECATED) target_compile_definitions (boost_system_internal PUBLIC BOOST_SYSTEM_NO_DEPRECATED) +target_compile_definitions (boost_random_internal PUBLIC BOOST_SYSTEM_NO_DEPRECATED) diff --git a/contrib/libgsasl b/contrib/libgsasl new file mode 160000 index 00000000000..3b8948a4042 --- /dev/null +++ b/contrib/libgsasl @@ -0,0 +1 @@ +Subproject commit 3b8948a4042e34fb00b4fb987535dc9e02e39040 diff --git a/contrib/libhdfs3 b/contrib/libhdfs3 new file mode 160000 index 00000000000..bd6505cbb0c --- /dev/null +++ b/contrib/libhdfs3 @@ -0,0 +1 @@ +Subproject commit bd6505cbb0c130b0db695305b9a38546fa880e5a diff --git a/contrib/libhdfs3-cmake/CMake/CMakeTestCompileNestedException.cpp b/contrib/libhdfs3-cmake/CMake/CMakeTestCompileNestedException.cpp new file mode 100644 index 00000000000..66918ca516e --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/CMakeTestCompileNestedException.cpp @@ -0,0 +1,10 @@ +#include +#include + +int main() { + try { + throw 2; + } catch (int) { + std::throw_with_nested(std::runtime_error("test")); + } +} diff --git a/contrib/libhdfs3-cmake/CMake/CMakeTestCompileSteadyClock.cpp b/contrib/libhdfs3-cmake/CMake/CMakeTestCompileSteadyClock.cpp new file mode 100644 index 00000000000..afcbe1b83b2 --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/CMakeTestCompileSteadyClock.cpp @@ -0,0 +1,7 @@ +#include + +using std::chrono::steady_clock; + +void foo(const steady_clock &clock) { + return; +} diff --git a/contrib/libhdfs3-cmake/CMake/CMakeTestCompileStrerror.cpp b/contrib/libhdfs3-cmake/CMake/CMakeTestCompileStrerror.cpp new file mode 100644 index 00000000000..0ef4eda583e --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/CMakeTestCompileStrerror.cpp @@ -0,0 +1,10 @@ +#include + +int main() +{ + // We can't test "char *p = strerror_r()" because that only causes a + // compiler warning when strerror_r returns an integer. + char *buf = 0; + int i = strerror_r(0, buf, 100); + return i; +} diff --git a/contrib/libhdfs3-cmake/CMake/CodeCoverage.cmake b/contrib/libhdfs3-cmake/CMake/CodeCoverage.cmake new file mode 100644 index 00000000000..ce997925fcc --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/CodeCoverage.cmake @@ -0,0 +1,48 @@ +# Check prereqs +FIND_PROGRAM(GCOV_PATH gcov) +FIND_PROGRAM(LCOV_PATH lcov) +FIND_PROGRAM(GENHTML_PATH genhtml) + +IF(NOT GCOV_PATH) + MESSAGE(FATAL_ERROR "gcov not found! Aborting...") +ENDIF(NOT GCOV_PATH) + +IF(NOT CMAKE_BUILD_TYPE STREQUAL Debug) + MESSAGE(WARNING "Code coverage results with an optimised (non-Debug) build may be misleading") +ENDIF(NOT CMAKE_BUILD_TYPE STREQUAL Debug) + +#Setup compiler options +ADD_DEFINITIONS(-fprofile-arcs -ftest-coverage) + +SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs ") +SET(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fprofile-arcs ") + +IF(NOT LCOV_PATH) + MESSAGE(FATAL_ERROR "lcov not found! Aborting...") +ENDIF(NOT LCOV_PATH) + +IF(NOT GENHTML_PATH) + MESSAGE(FATAL_ERROR "genhtml not found! Aborting...") +ENDIF(NOT GENHTML_PATH) + +#Setup target +ADD_CUSTOM_TARGET(ShowCoverage + #Capturing lcov counters and generating report + COMMAND ${LCOV_PATH} --directory . --capture --output-file CodeCoverage.info + COMMAND ${LCOV_PATH} --remove CodeCoverage.info '${CMAKE_CURRENT_BINARY_DIR}/*' 'test/*' 'mock/*' '/usr/*' '/opt/*' '*ext/rhel5_x86_64*' '*ext/osx*' --output-file CodeCoverage.info.cleaned + COMMAND ${GENHTML_PATH} -o CodeCoverageReport CodeCoverage.info.cleaned +) + + +ADD_CUSTOM_TARGET(ShowAllCoverage + #Capturing lcov counters and generating report + COMMAND ${LCOV_PATH} -a CodeCoverage.info.cleaned -a CodeCoverage.info.cleaned_withoutHA -o AllCodeCoverage.info + COMMAND sed -e 's|/.*/src|${CMAKE_SOURCE_DIR}/src|' -ig AllCodeCoverage.info + COMMAND ${GENHTML_PATH} -o AllCodeCoverageReport AllCodeCoverage.info +) + +ADD_CUSTOM_TARGET(ResetCoverage + #Cleanup lcov + COMMAND ${LCOV_PATH} --directory . --zerocounters +) + diff --git a/contrib/libhdfs3-cmake/CMake/FindBoost.cmake b/contrib/libhdfs3-cmake/CMake/FindBoost.cmake new file mode 100644 index 00000000000..914a0a5b5cd --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/FindBoost.cmake @@ -0,0 +1,1162 @@ +# - Find Boost include dirs and libraries +# Use this module by invoking find_package with the form: +# find_package(Boost +# [version] [EXACT] # Minimum or EXACT version e.g. 1.36.0 +# [REQUIRED] # Fail with error if Boost is not found +# [COMPONENTS ...] # Boost libraries by their canonical name +# ) # e.g. "date_time" for "libboost_date_time" +# This module finds headers and requested component libraries OR a CMake +# package configuration file provided by a "Boost CMake" build. For the +# latter case skip to the "Boost CMake" section below. For the former +# case results are reported in variables: +# Boost_FOUND - True if headers and requested libraries were found +# Boost_INCLUDE_DIRS - Boost include directories +# Boost_LIBRARY_DIRS - Link directories for Boost libraries +# Boost_LIBRARIES - Boost component libraries to be linked +# Boost__FOUND - True if component was found ( is upper-case) +# Boost__LIBRARY - Libraries to link for component (may include +# target_link_libraries debug/optimized keywords) +# Boost_VERSION - BOOST_VERSION value from boost/version.hpp +# Boost_LIB_VERSION - Version string appended to library filenames +# Boost_MAJOR_VERSION - Boost major version number (X in X.y.z) +# Boost_MINOR_VERSION - Boost minor version number (Y in x.Y.z) +# Boost_SUBMINOR_VERSION - Boost subminor version number (Z in x.y.Z) +# Boost_LIB_DIAGNOSTIC_DEFINITIONS (Windows) +# - Pass to add_definitions() to have diagnostic +# information about Boost's automatic linking +# displayed during compilation +# +# This module reads hints about search locations from variables: +# BOOST_ROOT - Preferred installation prefix +# (or BOOSTROOT) +# BOOST_INCLUDEDIR - Preferred include directory e.g. /include +# BOOST_LIBRARYDIR - Preferred library directory e.g. /lib +# Boost_NO_SYSTEM_PATHS - Set to ON to disable searching in locations not +# specified by these hint variables. Default is OFF. +# Boost_ADDITIONAL_VERSIONS +# - List of Boost versions not known to this module +# (Boost install locations may contain the version) +# and saves search results persistently in CMake cache entries: +# Boost_INCLUDE_DIR - Directory containing Boost headers +# Boost_LIBRARY_DIR - Directory containing Boost libraries +# Boost__LIBRARY_DEBUG - Component library debug variant +# Boost__LIBRARY_RELEASE - Component library release variant +# Users may set these hints or results as cache entries. Projects should +# not read these entries directly but instead use the above result variables. +# Note that some hint names start in upper-case "BOOST". One may specify +# these as environment variables if they are not specified as CMake variables +# or cache entries. +# +# This module first searches for the Boost header files using the above hint +# variables (excluding BOOST_LIBRARYDIR) and saves the result in +# Boost_INCLUDE_DIR. Then it searches for requested component libraries using +# the above hints (excluding BOOST_INCLUDEDIR and Boost_ADDITIONAL_VERSIONS), +# "lib" directories near Boost_INCLUDE_DIR, and the library name configuration +# settings below. It saves the library directory in Boost_LIBRARY_DIR and +# individual library locations in Boost__LIBRARY_DEBUG and +# Boost__LIBRARY_RELEASE. When one changes settings used by previous +# searches in the same build tree (excluding environment variables) this +# module discards previous search results affected by the changes and searches +# again. +# +# Boost libraries come in many variants encoded in their file name. Users or +# projects may tell this module which variant to find by setting variables: +# Boost_USE_MULTITHREADED - Set to OFF to use the non-multithreaded +# libraries ('mt' tag). Default is ON. +# Boost_USE_STATIC_LIBS - Set to ON to force the use of the static +# libraries. Default is OFF. +# Boost_USE_STATIC_RUNTIME - Set to ON or OFF to specify whether to use +# libraries linked statically to the C++ runtime +# ('s' tag). Default is platform dependent. +# Boost_USE_DEBUG_PYTHON - Set to ON to use libraries compiled with a +# debug Python build ('y' tag). Default is OFF. +# Boost_USE_STLPORT - Set to ON to use libraries compiled with +# STLPort ('p' tag). Default is OFF. +# Boost_USE_STLPORT_DEPRECATED_NATIVE_IOSTREAMS +# - Set to ON to use libraries compiled with +# STLPort deprecated "native iostreams" +# ('n' tag). Default is OFF. +# Boost_COMPILER - Set to the compiler-specific library suffix +# (e.g. "-gcc43"). Default is auto-computed +# for the C++ compiler in use. +# Boost_THREADAPI - Suffix for "thread" component library name, +# such as "pthread" or "win32". Names with +# and without this suffix will both be tried. +# Other variables one may set to control this module are: +# Boost_DEBUG - Set to ON to enable debug output from FindBoost. +# Please enable this before filing any bug report. +# Boost_DETAILED_FAILURE_MSG +# - Set to ON to add detailed information to the +# failure message even when the REQUIRED option +# is not given to the find_package call. +# Boost_REALPATH - Set to ON to resolve symlinks for discovered +# libraries to assist with packaging. For example, +# the "system" component library may be resolved to +# "/usr/lib/libboost_system.so.1.42.0" instead of +# "/usr/lib/libboost_system.so". This does not +# affect linking and should not be enabled unless +# the user needs this information. +# On Visual Studio and Borland compilers Boost headers request automatic +# linking to corresponding libraries. This requires matching libraries to be +# linked explicitly or available in the link library search path. In this +# case setting Boost_USE_STATIC_LIBS to OFF may not achieve dynamic linking. +# Boost automatic linking typically requests static libraries with a few +# exceptions (such as Boost.Python). Use +# add_definitions(${Boost_LIB_DIAGNOSTIC_DEFINITIONS}) +# to ask Boost to report information about automatic linking requests. +# +# Example to find Boost headers only: +# find_package(Boost 1.36.0) +# if(Boost_FOUND) +# include_directories(${Boost_INCLUDE_DIRS}) +# add_executable(foo foo.cc) +# endif() +# Example to find Boost headers and some libraries: +# set(Boost_USE_STATIC_LIBS ON) +# set(Boost_USE_MULTITHREADED ON) +# set(Boost_USE_STATIC_RUNTIME OFF) +# find_package(Boost 1.36.0 COMPONENTS date_time filesystem system ...) +# if(Boost_FOUND) +# include_directories(${Boost_INCLUDE_DIRS}) +# add_executable(foo foo.cc) +# target_link_libraries(foo ${Boost_LIBRARIES}) +# endif() +# +# Boost CMake ---------------------------------------------------------- +# +# If Boost was built using the boost-cmake project it provides a package +# configuration file for use with find_package's Config mode. This module +# looks for the package configuration file called BoostConfig.cmake or +# boost-config.cmake and stores the result in cache entry "Boost_DIR". If +# found, the package configuration file is loaded and this module returns with +# no further action. See documentation of the Boost CMake package +# configuration for details on what it provides. +# +# Set Boost_NO_BOOST_CMAKE to ON to disable the search for boost-cmake. + +#============================================================================= +# Copyright 2006-2012 Kitware, Inc. +# Copyright 2006-2008 Andreas Schneider +# Copyright 2007 Wengo +# Copyright 2007 Mike Jackson +# Copyright 2008 Andreas Pakulat +# Copyright 2008-2012 Philip Lowman +# +# Distributed under the OSI-approved BSD License (the "License"); +# see accompanying file Copyright.txt for details. +# +# This software is distributed WITHOUT ANY WARRANTY; without even the +# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the License for more information. +#============================================================================= +# (To distribute this file outside of CMake, substitute the full +# License text for the above reference.) + + +#------------------------------------------------------------------------------- +# Before we go searching, check whether boost-cmake is available, unless the +# user specifically asked NOT to search for boost-cmake. +# +# If Boost_DIR is set, this behaves as any find_package call would. If not, +# it looks at BOOST_ROOT and BOOSTROOT to find Boost. +# +if (NOT Boost_NO_BOOST_CMAKE) + # If Boost_DIR is not set, look for BOOSTROOT and BOOST_ROOT as alternatives, + # since these are more conventional for Boost. + if ("$ENV{Boost_DIR}" STREQUAL "") + if (NOT "$ENV{BOOST_ROOT}" STREQUAL "") + set(ENV{Boost_DIR} $ENV{BOOST_ROOT}) + elseif (NOT "$ENV{BOOSTROOT}" STREQUAL "") + set(ENV{Boost_DIR} $ENV{BOOSTROOT}) + endif() + endif() + + # Do the same find_package call but look specifically for the CMake version. + # Note that args are passed in the Boost_FIND_xxxxx variables, so there is no + # need to delegate them to this find_package call. + find_package(Boost QUIET NO_MODULE) + mark_as_advanced(Boost_DIR) + + # If we found boost-cmake, then we're done. Print out what we found. + # Otherwise let the rest of the module try to find it. + if (Boost_FOUND) + message("Boost ${Boost_FIND_VERSION} found.") + if (Boost_FIND_COMPONENTS) + message("Found Boost components:") + message(" ${Boost_FIND_COMPONENTS}") + endif() + return() + endif() +endif() + + +#------------------------------------------------------------------------------- +# FindBoost functions & macros +# + +############################################ +# +# Check the existence of the libraries. +# +############################################ +# This macro was taken directly from the FindQt4.cmake file that is included +# with the CMake distribution. This is NOT my work. All work was done by the +# original authors of the FindQt4.cmake file. Only minor modifications were +# made to remove references to Qt and make this file more generally applicable +# And ELSE/ENDIF pairs were removed for readability. +######################################################################### + +macro(_Boost_ADJUST_LIB_VARS basename) + if(Boost_INCLUDE_DIR ) + if(Boost_${basename}_LIBRARY_DEBUG AND Boost_${basename}_LIBRARY_RELEASE) + # if the generator supports configuration types then set + # optimized and debug libraries, or if the CMAKE_BUILD_TYPE has a value + if(CMAKE_CONFIGURATION_TYPES OR CMAKE_BUILD_TYPE) + set(Boost_${basename}_LIBRARY optimized ${Boost_${basename}_LIBRARY_RELEASE} debug ${Boost_${basename}_LIBRARY_DEBUG}) + else() + # if there are no configuration types and CMAKE_BUILD_TYPE has no value + # then just use the release libraries + set(Boost_${basename}_LIBRARY ${Boost_${basename}_LIBRARY_RELEASE} ) + endif() + # FIXME: This probably should be set for both cases + set(Boost_${basename}_LIBRARIES optimized ${Boost_${basename}_LIBRARY_RELEASE} debug ${Boost_${basename}_LIBRARY_DEBUG}) + endif() + + # if only the release version was found, set the debug variable also to the release version + if(Boost_${basename}_LIBRARY_RELEASE AND NOT Boost_${basename}_LIBRARY_DEBUG) + set(Boost_${basename}_LIBRARY_DEBUG ${Boost_${basename}_LIBRARY_RELEASE}) + set(Boost_${basename}_LIBRARY ${Boost_${basename}_LIBRARY_RELEASE}) + set(Boost_${basename}_LIBRARIES ${Boost_${basename}_LIBRARY_RELEASE}) + endif() + + # if only the debug version was found, set the release variable also to the debug version + if(Boost_${basename}_LIBRARY_DEBUG AND NOT Boost_${basename}_LIBRARY_RELEASE) + set(Boost_${basename}_LIBRARY_RELEASE ${Boost_${basename}_LIBRARY_DEBUG}) + set(Boost_${basename}_LIBRARY ${Boost_${basename}_LIBRARY_DEBUG}) + set(Boost_${basename}_LIBRARIES ${Boost_${basename}_LIBRARY_DEBUG}) + endif() + + # If the debug & release library ends up being the same, omit the keywords + if(${Boost_${basename}_LIBRARY_RELEASE} STREQUAL ${Boost_${basename}_LIBRARY_DEBUG}) + set(Boost_${basename}_LIBRARY ${Boost_${basename}_LIBRARY_RELEASE} ) + set(Boost_${basename}_LIBRARIES ${Boost_${basename}_LIBRARY_RELEASE} ) + endif() + + if(Boost_${basename}_LIBRARY) + set(Boost_${basename}_FOUND ON) + endif() + + endif() + # Make variables changeable to the advanced user + mark_as_advanced( + Boost_${basename}_LIBRARY_RELEASE + Boost_${basename}_LIBRARY_DEBUG + ) +endmacro() + +macro(_Boost_CHANGE_DETECT changed_var) + set(${changed_var} 0) + foreach(v ${ARGN}) + if(DEFINED _Boost_COMPONENTS_SEARCHED) + if(${v}) + if(_${v}_LAST) + string(COMPARE NOTEQUAL "${${v}}" "${_${v}_LAST}" _${v}_CHANGED) + else() + set(_${v}_CHANGED 1) + endif() + elseif(_${v}_LAST) + set(_${v}_CHANGED 1) + endif() + if(_${v}_CHANGED) + set(${changed_var} 1) + endif() + else() + set(_${v}_CHANGED 0) + endif() + endforeach() +endmacro() + +macro(_Boost_FIND_LIBRARY var) + find_library(${var} ${ARGN}) + + # If we found the first library save Boost_LIBRARY_DIR. + if(${var} AND NOT Boost_LIBRARY_DIR) + get_filename_component(_dir "${${var}}" PATH) + set(Boost_LIBRARY_DIR "${_dir}" CACHE PATH "Boost library directory" FORCE) + endif() + + # If Boost_LIBRARY_DIR is known then search only there. + if(Boost_LIBRARY_DIR) + set(_boost_LIBRARY_SEARCH_DIRS ${Boost_LIBRARY_DIR} NO_DEFAULT_PATH) + endif() +endmacro() + +#------------------------------------------------------------------------------- + +# +# Runs compiler with "-dumpversion" and parses major/minor +# version with a regex. +# +function(_Boost_COMPILER_DUMPVERSION _OUTPUT_VERSION) + + exec_program(${CMAKE_CXX_COMPILER} + ARGS ${CMAKE_CXX_COMPILER_ARG1} -dumpversion + OUTPUT_VARIABLE _boost_COMPILER_VERSION + ) + string(REGEX REPLACE "([0-9])\\.([0-9])(\\.[0-9])?" "\\1\\2" + _boost_COMPILER_VERSION ${_boost_COMPILER_VERSION}) + + set(${_OUTPUT_VERSION} ${_boost_COMPILER_VERSION} PARENT_SCOPE) +endfunction() + +# +# Take a list of libraries with "thread" in it +# and prepend duplicates with "thread_${Boost_THREADAPI}" +# at the front of the list +# +function(_Boost_PREPEND_LIST_WITH_THREADAPI _output) + set(_orig_libnames ${ARGN}) + string(REPLACE "thread" "thread_${Boost_THREADAPI}" _threadapi_libnames "${_orig_libnames}") + set(${_output} ${_threadapi_libnames} ${_orig_libnames} PARENT_SCOPE) +endfunction() + +# +# If a library is found, replace its cache entry with its REALPATH +# +function(_Boost_SWAP_WITH_REALPATH _library _docstring) + if(${_library}) + get_filename_component(_boost_filepathreal ${${_library}} REALPATH) + unset(${_library} CACHE) + set(${_library} ${_boost_filepathreal} CACHE FILEPATH "${_docstring}") + endif() +endfunction() + +function(_Boost_CHECK_SPELLING _var) + if(${_var}) + string(TOUPPER ${_var} _var_UC) + message(FATAL_ERROR "ERROR: ${_var} is not the correct spelling. The proper spelling is ${_var_UC}.") + endif() +endfunction() + +# Guesses Boost's compiler prefix used in built library names +# Returns the guess by setting the variable pointed to by _ret +function(_Boost_GUESS_COMPILER_PREFIX _ret) + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel" + OR "${CMAKE_CXX_COMPILER}" MATCHES "icl" + OR "${CMAKE_CXX_COMPILER}" MATCHES "icpc") + if(WIN32) + set (_boost_COMPILER "-iw") + else() + set (_boost_COMPILER "-il") + endif() + elseif (MSVC12) + set(_boost_COMPILER "-vc120") + elseif (MSVC11) + set(_boost_COMPILER "-vc110") + elseif (MSVC10) + set(_boost_COMPILER "-vc100") + elseif (MSVC90) + set(_boost_COMPILER "-vc90") + elseif (MSVC80) + set(_boost_COMPILER "-vc80") + elseif (MSVC71) + set(_boost_COMPILER "-vc71") + elseif (MSVC70) # Good luck! + set(_boost_COMPILER "-vc7") # yes, this is correct + elseif (MSVC60) # Good luck! + set(_boost_COMPILER "-vc6") # yes, this is correct + elseif (BORLAND) + set(_boost_COMPILER "-bcb") + elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "SunPro") + set(_boost_COMPILER "-sw") + elseif (MINGW) + if(${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION} VERSION_LESS 1.34) + set(_boost_COMPILER "-mgw") # no GCC version encoding prior to 1.34 + else() + _Boost_COMPILER_DUMPVERSION(_boost_COMPILER_VERSION) + set(_boost_COMPILER "-mgw${_boost_COMPILER_VERSION}") + endif() + elseif (UNIX) + if (CMAKE_COMPILER_IS_GNUCXX) + if(${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION} VERSION_LESS 1.34) + set(_boost_COMPILER "-gcc") # no GCC version encoding prior to 1.34 + else() + _Boost_COMPILER_DUMPVERSION(_boost_COMPILER_VERSION) + # Determine which version of GCC we have. + if(APPLE) + if(Boost_MINOR_VERSION) + if(${Boost_MINOR_VERSION} GREATER 35) + # In Boost 1.36.0 and newer, the mangled compiler name used + # on Mac OS X/Darwin is "xgcc". + set(_boost_COMPILER "-xgcc${_boost_COMPILER_VERSION}") + else() + # In Boost <= 1.35.0, there is no mangled compiler name for + # the Mac OS X/Darwin version of GCC. + set(_boost_COMPILER "") + endif() + else() + # We don't know the Boost version, so assume it's + # pre-1.36.0. + set(_boost_COMPILER "") + endif() + else() + set(_boost_COMPILER "-gcc${_boost_COMPILER_VERSION}") + endif() + endif() + endif () + else() + # TODO at least Boost_DEBUG here? + set(_boost_COMPILER "") + endif() + set(${_ret} ${_boost_COMPILER} PARENT_SCOPE) +endfunction() + +# +# End functions/macros +# +#------------------------------------------------------------------------------- + +#------------------------------------------------------------------------------- +# main. +#------------------------------------------------------------------------------- + +if(NOT DEFINED Boost_USE_MULTITHREADED) + set(Boost_USE_MULTITHREADED TRUE) +endif() + +# Check the version of Boost against the requested version. +if(Boost_FIND_VERSION AND NOT Boost_FIND_VERSION_MINOR) + message(SEND_ERROR "When requesting a specific version of Boost, you must provide at least the major and minor version numbers, e.g., 1.34") +endif() + +if(Boost_FIND_VERSION_EXACT) + # The version may appear in a directory with or without the patch + # level, even when the patch level is non-zero. + set(_boost_TEST_VERSIONS + "${Boost_FIND_VERSION_MAJOR}.${Boost_FIND_VERSION_MINOR}.${Boost_FIND_VERSION_PATCH}" + "${Boost_FIND_VERSION_MAJOR}.${Boost_FIND_VERSION_MINOR}") +else() + # The user has not requested an exact version. Among known + # versions, find those that are acceptable to the user request. + set(_Boost_KNOWN_VERSIONS ${Boost_ADDITIONAL_VERSIONS} + "1.56.0" "1.56" "1.55.0" "1.55" "1.54.0" "1.54" + "1.53.0" "1.53" "1.52.0" "1.52" "1.51.0" "1.51" + "1.50.0" "1.50" "1.49.0" "1.49" "1.48.0" "1.48" "1.47.0" "1.47" "1.46.1" + "1.46.0" "1.46" "1.45.0" "1.45" "1.44.0" "1.44" "1.43.0" "1.43" "1.42.0" "1.42" + "1.41.0" "1.41" "1.40.0" "1.40" "1.39.0" "1.39" "1.38.0" "1.38" "1.37.0" "1.37" + "1.36.1" "1.36.0" "1.36" "1.35.1" "1.35.0" "1.35" "1.34.1" "1.34.0" + "1.34" "1.33.1" "1.33.0" "1.33") + set(_boost_TEST_VERSIONS) + if(Boost_FIND_VERSION) + set(_Boost_FIND_VERSION_SHORT "${Boost_FIND_VERSION_MAJOR}.${Boost_FIND_VERSION_MINOR}") + # Select acceptable versions. + foreach(version ${_Boost_KNOWN_VERSIONS}) + if(NOT "${version}" VERSION_LESS "${Boost_FIND_VERSION}") + # This version is high enough. + list(APPEND _boost_TEST_VERSIONS "${version}") + elseif("${version}.99" VERSION_EQUAL "${_Boost_FIND_VERSION_SHORT}.99") + # This version is a short-form for the requested version with + # the patch level dropped. + list(APPEND _boost_TEST_VERSIONS "${version}") + endif() + endforeach() + else() + # Any version is acceptable. + set(_boost_TEST_VERSIONS "${_Boost_KNOWN_VERSIONS}") + endif() +endif() + +# The reason that we failed to find Boost. This will be set to a +# user-friendly message when we fail to find some necessary piece of +# Boost. +set(Boost_ERROR_REASON) + +if(Boost_DEBUG) + # Output some of their choices + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "_boost_TEST_VERSIONS = ${_boost_TEST_VERSIONS}") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "Boost_USE_MULTITHREADED = ${Boost_USE_MULTITHREADED}") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "Boost_USE_STATIC_LIBS = ${Boost_USE_STATIC_LIBS}") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "Boost_USE_STATIC_RUNTIME = ${Boost_USE_STATIC_RUNTIME}") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "Boost_ADDITIONAL_VERSIONS = ${Boost_ADDITIONAL_VERSIONS}") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "Boost_NO_SYSTEM_PATHS = ${Boost_NO_SYSTEM_PATHS}") +endif() + +if(WIN32) + # In windows, automatic linking is performed, so you do not have + # to specify the libraries. If you are linking to a dynamic + # runtime, then you can choose to link to either a static or a + # dynamic Boost library, the default is to do a static link. You + # can alter this for a specific library "whatever" by defining + # BOOST_WHATEVER_DYN_LINK to force Boost library "whatever" to be + # linked dynamically. Alternatively you can force all Boost + # libraries to dynamic link by defining BOOST_ALL_DYN_LINK. + + # This feature can be disabled for Boost library "whatever" by + # defining BOOST_WHATEVER_NO_LIB, or for all of Boost by defining + # BOOST_ALL_NO_LIB. + + # If you want to observe which libraries are being linked against + # then defining BOOST_LIB_DIAGNOSTIC will cause the auto-linking + # code to emit a #pragma message each time a library is selected + # for linking. + set(Boost_LIB_DIAGNOSTIC_DEFINITIONS "-DBOOST_LIB_DIAGNOSTIC") +endif() + +_Boost_CHECK_SPELLING(Boost_ROOT) +_Boost_CHECK_SPELLING(Boost_LIBRARYDIR) +_Boost_CHECK_SPELLING(Boost_INCLUDEDIR) + +# Collect environment variable inputs as hints. Do not consider changes. +foreach(v BOOSTROOT BOOST_ROOT BOOST_INCLUDEDIR BOOST_LIBRARYDIR) + set(_env $ENV{${v}}) + if(_env) + file(TO_CMAKE_PATH "${_env}" _ENV_${v}) + else() + set(_ENV_${v} "") + endif() +endforeach() +if(NOT _ENV_BOOST_ROOT AND _ENV_BOOSTROOT) + set(_ENV_BOOST_ROOT "${_ENV_BOOSTROOT}") +endif() + +# Collect inputs and cached results. Detect changes since the last run. +if(NOT BOOST_ROOT AND BOOSTROOT) + set(BOOST_ROOT "${BOOSTROOT}") +endif() +set(_Boost_VARS_DIR + BOOST_ROOT + Boost_NO_SYSTEM_PATHS + ) + +if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "Declared as CMake or Environmental Variables:") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + " BOOST_ROOT = ${BOOST_ROOT}") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + " BOOST_INCLUDEDIR = ${BOOST_INCLUDEDIR}") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + " BOOST_LIBRARYDIR = ${BOOST_LIBRARYDIR}") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "_boost_TEST_VERSIONS = ${_boost_TEST_VERSIONS}") +endif() + +# ------------------------------------------------------------------------ +# Search for Boost include DIR +# ------------------------------------------------------------------------ + +set(_Boost_VARS_INC BOOST_INCLUDEDIR Boost_INCLUDE_DIR Boost_ADDITIONAL_VERSIONS) +_Boost_CHANGE_DETECT(_Boost_CHANGE_INCDIR ${_Boost_VARS_DIR} ${_Boost_VARS_INC}) +# Clear Boost_INCLUDE_DIR if it did not change but other input affecting the +# location did. We will find a new one based on the new inputs. +if(_Boost_CHANGE_INCDIR AND NOT _Boost_INCLUDE_DIR_CHANGED) + unset(Boost_INCLUDE_DIR CACHE) +endif() + +if(NOT Boost_INCLUDE_DIR) + set(_boost_INCLUDE_SEARCH_DIRS "") + if(BOOST_INCLUDEDIR) + list(APPEND _boost_INCLUDE_SEARCH_DIRS ${BOOST_INCLUDEDIR}) + elseif(_ENV_BOOST_INCLUDEDIR) + list(APPEND _boost_INCLUDE_SEARCH_DIRS ${_ENV_BOOST_INCLUDEDIR}) + endif() + + if( BOOST_ROOT ) + list(APPEND _boost_INCLUDE_SEARCH_DIRS ${BOOST_ROOT}/include ${BOOST_ROOT}) + elseif( _ENV_BOOST_ROOT ) + list(APPEND _boost_INCLUDE_SEARCH_DIRS ${_ENV_BOOST_ROOT}/include ${_ENV_BOOST_ROOT}) + endif() + + if( Boost_NO_SYSTEM_PATHS) + list(APPEND _boost_INCLUDE_SEARCH_DIRS NO_CMAKE_SYSTEM_PATH) + else() + list(APPEND _boost_INCLUDE_SEARCH_DIRS PATHS + C:/boost/include + C:/boost + /sw/local/include + ) + endif() + + # Try to find Boost by stepping backwards through the Boost versions + # we know about. + # Build a list of path suffixes for each version. + set(_boost_PATH_SUFFIXES) + foreach(_boost_VER ${_boost_TEST_VERSIONS}) + # Add in a path suffix, based on the required version, ideally + # we could read this from version.hpp, but for that to work we'd + # need to know the include dir already + set(_boost_BOOSTIFIED_VERSION) + + # Transform 1.35 => 1_35 and 1.36.0 => 1_36_0 + if(_boost_VER MATCHES "[0-9]+\\.[0-9]+\\.[0-9]+") + string(REGEX REPLACE "([0-9]+)\\.([0-9]+)\\.([0-9]+)" "\\1_\\2_\\3" + _boost_BOOSTIFIED_VERSION ${_boost_VER}) + elseif(_boost_VER MATCHES "[0-9]+\\.[0-9]+") + string(REGEX REPLACE "([0-9]+)\\.([0-9]+)" "\\1_\\2" + _boost_BOOSTIFIED_VERSION ${_boost_VER}) + endif() + + list(APPEND _boost_PATH_SUFFIXES + "boost-${_boost_BOOSTIFIED_VERSION}" + "boost_${_boost_BOOSTIFIED_VERSION}" + "boost/boost-${_boost_BOOSTIFIED_VERSION}" + "boost/boost_${_boost_BOOSTIFIED_VERSION}" + ) + + endforeach() + + if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "Include debugging info:") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + " _boost_INCLUDE_SEARCH_DIRS = ${_boost_INCLUDE_SEARCH_DIRS}") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + " _boost_PATH_SUFFIXES = ${_boost_PATH_SUFFIXES}") + endif() + + # Look for a standard boost header file. + find_path(Boost_INCLUDE_DIR + NAMES boost/config.hpp + HINTS ${_boost_INCLUDE_SEARCH_DIRS} + PATH_SUFFIXES ${_boost_PATH_SUFFIXES} + ) +endif() + +# ------------------------------------------------------------------------ +# Extract version information from version.hpp +# ------------------------------------------------------------------------ + +# Set Boost_FOUND based only on header location and version. +# It will be updated below for component libraries. +if(Boost_INCLUDE_DIR) + if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "location of version.hpp: ${Boost_INCLUDE_DIR}/boost/version.hpp") + endif() + + # Extract Boost_VERSION and Boost_LIB_VERSION from version.hpp + set(Boost_VERSION 0) + set(Boost_LIB_VERSION "") + file(STRINGS "${Boost_INCLUDE_DIR}/boost/version.hpp" _boost_VERSION_HPP_CONTENTS REGEX "#define BOOST_(LIB_)?VERSION ") + set(_Boost_VERSION_REGEX "([0-9]+)") + set(_Boost_LIB_VERSION_REGEX "\"([0-9_]+)\"") + foreach(v VERSION LIB_VERSION) + if("${_boost_VERSION_HPP_CONTENTS}" MATCHES ".*#define BOOST_${v} ${_Boost_${v}_REGEX}.*") + set(Boost_${v} "${CMAKE_MATCH_1}") + endif() + endforeach() + unset(_boost_VERSION_HPP_CONTENTS) + + math(EXPR Boost_MAJOR_VERSION "${Boost_VERSION} / 100000") + math(EXPR Boost_MINOR_VERSION "${Boost_VERSION} / 100 % 1000") + math(EXPR Boost_SUBMINOR_VERSION "${Boost_VERSION} % 100") + + set(Boost_ERROR_REASON + "${Boost_ERROR_REASON}Boost version: ${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}\nBoost include path: ${Boost_INCLUDE_DIR}") + if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "version.hpp reveals boost " + "${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}") + endif() + + if(Boost_FIND_VERSION) + # Set Boost_FOUND based on requested version. + set(_Boost_VERSION "${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}") + if("${_Boost_VERSION}" VERSION_LESS "${Boost_FIND_VERSION}") + set(Boost_FOUND 0) + set(_Boost_VERSION_AGE "old") + elseif(Boost_FIND_VERSION_EXACT AND + NOT "${_Boost_VERSION}" VERSION_EQUAL "${Boost_FIND_VERSION}") + set(Boost_FOUND 0) + set(_Boost_VERSION_AGE "new") + else() + set(Boost_FOUND 1) + endif() + if(NOT Boost_FOUND) + # State that we found a version of Boost that is too new or too old. + set(Boost_ERROR_REASON + "${Boost_ERROR_REASON}\nDetected version of Boost is too ${_Boost_VERSION_AGE}. Requested version was ${Boost_FIND_VERSION_MAJOR}.${Boost_FIND_VERSION_MINOR}") + if (Boost_FIND_VERSION_PATCH) + set(Boost_ERROR_REASON + "${Boost_ERROR_REASON}.${Boost_FIND_VERSION_PATCH}") + endif () + if (NOT Boost_FIND_VERSION_EXACT) + set(Boost_ERROR_REASON "${Boost_ERROR_REASON} (or newer)") + endif () + set(Boost_ERROR_REASON "${Boost_ERROR_REASON}.") + endif () + else() + # Caller will accept any Boost version. + set(Boost_FOUND 1) + endif() +else() + set(Boost_FOUND 0) + set(Boost_ERROR_REASON + "${Boost_ERROR_REASON}Unable to find the Boost header files. Please set BOOST_ROOT to the root directory containing Boost or BOOST_INCLUDEDIR to the directory containing Boost's headers.") +endif() + +# ------------------------------------------------------------------------ +# Suffix initialization and compiler suffix detection. +# ------------------------------------------------------------------------ + +set(_Boost_VARS_NAME + Boost_COMPILER + Boost_THREADAPI + Boost_USE_DEBUG_PYTHON + Boost_USE_MULTITHREADED + Boost_USE_STATIC_LIBS + Boost_USE_STATIC_RUNTIME + Boost_USE_STLPORT + Boost_USE_STLPORT_DEPRECATED_NATIVE_IOSTREAMS + ) +_Boost_CHANGE_DETECT(_Boost_CHANGE_LIBNAME ${_Boost_VARS_NAME}) + +# Setting some more suffixes for the library +set(Boost_LIB_PREFIX "") +if ( WIN32 AND Boost_USE_STATIC_LIBS AND NOT CYGWIN) + set(Boost_LIB_PREFIX "lib") +endif() + +if (Boost_COMPILER) + set(_boost_COMPILER ${Boost_COMPILER}) + if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "using user-specified Boost_COMPILER = ${_boost_COMPILER}") + endif() +else() + # Attempt to guess the compiler suffix + # NOTE: this is not perfect yet, if you experience any issues + # please report them and use the Boost_COMPILER variable + # to work around the problems. + _Boost_GUESS_COMPILER_PREFIX(_boost_COMPILER) + if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "guessed _boost_COMPILER = ${_boost_COMPILER}") + endif() +endif() + +set (_boost_MULTITHREADED "-mt") +if( NOT Boost_USE_MULTITHREADED ) + set (_boost_MULTITHREADED "") +endif() +if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "_boost_MULTITHREADED = ${_boost_MULTITHREADED}") +endif() + +#====================== +# Systematically build up the Boost ABI tag +# http://boost.org/doc/libs/1_41_0/more/getting_started/windows.html#library-naming +set( _boost_RELEASE_ABI_TAG "-") +set( _boost_DEBUG_ABI_TAG "-") +# Key Use this library when: +# s linking statically to the C++ standard library and +# compiler runtime support libraries. +if(Boost_USE_STATIC_RUNTIME) + set( _boost_RELEASE_ABI_TAG "${_boost_RELEASE_ABI_TAG}s") + set( _boost_DEBUG_ABI_TAG "${_boost_DEBUG_ABI_TAG}s") +endif() +# g using debug versions of the standard and runtime +# support libraries +if(WIN32) + if(MSVC OR "${CMAKE_CXX_COMPILER}" MATCHES "icl" + OR "${CMAKE_CXX_COMPILER}" MATCHES "icpc") + set(_boost_DEBUG_ABI_TAG "${_boost_DEBUG_ABI_TAG}g") + endif() +endif() +# y using special debug build of python +if(Boost_USE_DEBUG_PYTHON) + set(_boost_DEBUG_ABI_TAG "${_boost_DEBUG_ABI_TAG}y") +endif() +# d using a debug version of your code +set(_boost_DEBUG_ABI_TAG "${_boost_DEBUG_ABI_TAG}d") +# p using the STLport standard library rather than the +# default one supplied with your compiler +if(Boost_USE_STLPORT) + set( _boost_RELEASE_ABI_TAG "${_boost_RELEASE_ABI_TAG}p") + set( _boost_DEBUG_ABI_TAG "${_boost_DEBUG_ABI_TAG}p") +endif() +# n using the STLport deprecated "native iostreams" feature +if(Boost_USE_STLPORT_DEPRECATED_NATIVE_IOSTREAMS) + set( _boost_RELEASE_ABI_TAG "${_boost_RELEASE_ABI_TAG}n") + set( _boost_DEBUG_ABI_TAG "${_boost_DEBUG_ABI_TAG}n") +endif() + +if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "_boost_RELEASE_ABI_TAG = ${_boost_RELEASE_ABI_TAG}") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "_boost_DEBUG_ABI_TAG = ${_boost_DEBUG_ABI_TAG}") +endif() + +# ------------------------------------------------------------------------ +# Begin finding boost libraries +# ------------------------------------------------------------------------ +set(_Boost_VARS_LIB BOOST_LIBRARYDIR Boost_LIBRARY_DIR) +_Boost_CHANGE_DETECT(_Boost_CHANGE_LIBDIR ${_Boost_VARS_DIR} ${_Boost_VARS_LIB} Boost_INCLUDE_DIR) +# Clear Boost_LIBRARY_DIR if it did not change but other input affecting the +# location did. We will find a new one based on the new inputs. +if(_Boost_CHANGE_LIBDIR AND NOT _Boost_LIBRARY_DIR_CHANGED) + unset(Boost_LIBRARY_DIR CACHE) +endif() + +if(Boost_LIBRARY_DIR) + set(_boost_LIBRARY_SEARCH_DIRS ${Boost_LIBRARY_DIR} NO_DEFAULT_PATH) +else() + set(_boost_LIBRARY_SEARCH_DIRS "") + if(BOOST_LIBRARYDIR) + list(APPEND _boost_LIBRARY_SEARCH_DIRS ${BOOST_LIBRARYDIR}) + elseif(_ENV_BOOST_LIBRARYDIR) + list(APPEND _boost_LIBRARY_SEARCH_DIRS ${_ENV_BOOST_LIBRARYDIR}) + endif() + + if(BOOST_ROOT) + list(APPEND _boost_LIBRARY_SEARCH_DIRS ${BOOST_ROOT}/lib ${BOOST_ROOT}/stage/lib) + elseif(_ENV_BOOST_ROOT) + list(APPEND _boost_LIBRARY_SEARCH_DIRS ${_ENV_BOOST_ROOT}/lib ${_ENV_BOOST_ROOT}/stage/lib) + endif() + + list(APPEND _boost_LIBRARY_SEARCH_DIRS + ${Boost_INCLUDE_DIR}/lib + ${Boost_INCLUDE_DIR}/../lib + ${Boost_INCLUDE_DIR}/stage/lib + ) + if( Boost_NO_SYSTEM_PATHS ) + list(APPEND _boost_LIBRARY_SEARCH_DIRS NO_CMAKE_SYSTEM_PATH) + else() + list(APPEND _boost_LIBRARY_SEARCH_DIRS PATHS + C:/boost/lib + C:/boost + /sw/local/lib + ) + endif() +endif() + +if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "_boost_LIBRARY_SEARCH_DIRS = ${_boost_LIBRARY_SEARCH_DIRS}") +endif() + +# Support preference of static libs by adjusting CMAKE_FIND_LIBRARY_SUFFIXES +if( Boost_USE_STATIC_LIBS ) + set( _boost_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) + if(WIN32) + set(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a ${CMAKE_FIND_LIBRARY_SUFFIXES}) + else() + set(CMAKE_FIND_LIBRARY_SUFFIXES .a ) + endif() +endif() + +# We want to use the tag inline below without risking double dashes +if(_boost_RELEASE_ABI_TAG) + if(${_boost_RELEASE_ABI_TAG} STREQUAL "-") + set(_boost_RELEASE_ABI_TAG "") + endif() +endif() +if(_boost_DEBUG_ABI_TAG) + if(${_boost_DEBUG_ABI_TAG} STREQUAL "-") + set(_boost_DEBUG_ABI_TAG "") + endif() +endif() + +# The previous behavior of FindBoost when Boost_USE_STATIC_LIBS was enabled +# on WIN32 was to: +# 1. Search for static libs compiled against a SHARED C++ standard runtime library (use if found) +# 2. Search for static libs compiled against a STATIC C++ standard runtime library (use if found) +# We maintain this behavior since changing it could break people's builds. +# To disable the ambiguous behavior, the user need only +# set Boost_USE_STATIC_RUNTIME either ON or OFF. +set(_boost_STATIC_RUNTIME_WORKAROUND false) +if(WIN32 AND Boost_USE_STATIC_LIBS) + if(NOT DEFINED Boost_USE_STATIC_RUNTIME) + set(_boost_STATIC_RUNTIME_WORKAROUND true) + endif() +endif() + +# On versions < 1.35, remove the System library from the considered list +# since it wasn't added until 1.35. +if(Boost_VERSION AND Boost_FIND_COMPONENTS) + if(Boost_VERSION LESS 103500) + list(REMOVE_ITEM Boost_FIND_COMPONENTS system) + endif() +endif() + +# If the user changed any of our control inputs flush previous results. +if(_Boost_CHANGE_LIBDIR OR _Boost_CHANGE_LIBNAME) + foreach(COMPONENT ${_Boost_COMPONENTS_SEARCHED}) + string(TOUPPER ${COMPONENT} UPPERCOMPONENT) + foreach(c DEBUG RELEASE) + set(_var Boost_${UPPERCOMPONENT}_LIBRARY_${c}) + unset(${_var} CACHE) + set(${_var} "${_var}-NOTFOUND") + endforeach() + endforeach() + set(_Boost_COMPONENTS_SEARCHED "") +endif() + +foreach(COMPONENT ${Boost_FIND_COMPONENTS}) + string(TOUPPER ${COMPONENT} UPPERCOMPONENT) + + set( _boost_docstring_release "Boost ${COMPONENT} library (release)") + set( _boost_docstring_debug "Boost ${COMPONENT} library (debug)") + + # + # Find RELEASE libraries + # + set(_boost_RELEASE_NAMES + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_RELEASE_ABI_TAG}-${Boost_LIB_VERSION} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_RELEASE_ABI_TAG} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_RELEASE_ABI_TAG}-${Boost_LIB_VERSION} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_RELEASE_ABI_TAG} + ${Boost_LIB_PREFIX}boost_${COMPONENT} ) + if(_boost_STATIC_RUNTIME_WORKAROUND) + set(_boost_RELEASE_STATIC_ABI_TAG "-s${_boost_RELEASE_ABI_TAG}") + list(APPEND _boost_RELEASE_NAMES + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_RELEASE_STATIC_ABI_TAG}-${Boost_LIB_VERSION} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_RELEASE_STATIC_ABI_TAG} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_RELEASE_STATIC_ABI_TAG}-${Boost_LIB_VERSION} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_RELEASE_STATIC_ABI_TAG} ) + endif() + if(Boost_THREADAPI AND ${COMPONENT} STREQUAL "thread") + _Boost_PREPEND_LIST_WITH_THREADAPI(_boost_RELEASE_NAMES ${_boost_RELEASE_NAMES}) + endif() + if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "Searching for ${UPPERCOMPONENT}_LIBRARY_RELEASE: ${_boost_RELEASE_NAMES}") + endif() + + # Avoid passing backslashes to _Boost_FIND_LIBRARY due to macro re-parsing. + string(REPLACE "\\" "/" _boost_LIBRARY_SEARCH_DIRS_tmp "${_boost_LIBRARY_SEARCH_DIRS}") + + _Boost_FIND_LIBRARY(Boost_${UPPERCOMPONENT}_LIBRARY_RELEASE + NAMES ${_boost_RELEASE_NAMES} + HINTS ${_boost_LIBRARY_SEARCH_DIRS_tmp} + NAMES_PER_DIR + DOC "${_boost_docstring_release}" + ) + + # + # Find DEBUG libraries + # + set(_boost_DEBUG_NAMES + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_DEBUG_ABI_TAG}-${Boost_LIB_VERSION} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_DEBUG_ABI_TAG} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_DEBUG_ABI_TAG}-${Boost_LIB_VERSION} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_DEBUG_ABI_TAG} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED} + ${Boost_LIB_PREFIX}boost_${COMPONENT} ) + if(_boost_STATIC_RUNTIME_WORKAROUND) + set(_boost_DEBUG_STATIC_ABI_TAG "-s${_boost_DEBUG_ABI_TAG}") + list(APPEND _boost_DEBUG_NAMES + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_DEBUG_STATIC_ABI_TAG}-${Boost_LIB_VERSION} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_DEBUG_STATIC_ABI_TAG} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_DEBUG_STATIC_ABI_TAG}-${Boost_LIB_VERSION} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_DEBUG_STATIC_ABI_TAG} ) + endif() + if(Boost_THREADAPI AND ${COMPONENT} STREQUAL "thread") + _Boost_PREPEND_LIST_WITH_THREADAPI(_boost_DEBUG_NAMES ${_boost_DEBUG_NAMES}) + endif() + if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "Searching for ${UPPERCOMPONENT}_LIBRARY_DEBUG: ${_boost_DEBUG_NAMES}") + endif() + + # Avoid passing backslashes to _Boost_FIND_LIBRARY due to macro re-parsing. + string(REPLACE "\\" "/" _boost_LIBRARY_SEARCH_DIRS_tmp "${_boost_LIBRARY_SEARCH_DIRS}") + + _Boost_FIND_LIBRARY(Boost_${UPPERCOMPONENT}_LIBRARY_DEBUG + NAMES ${_boost_DEBUG_NAMES} + HINTS ${_boost_LIBRARY_SEARCH_DIRS_tmp} + NAMES_PER_DIR + DOC "${_boost_docstring_debug}" + ) + + if(Boost_REALPATH) + _Boost_SWAP_WITH_REALPATH(Boost_${UPPERCOMPONENT}_LIBRARY_RELEASE "${_boost_docstring_release}") + _Boost_SWAP_WITH_REALPATH(Boost_${UPPERCOMPONENT}_LIBRARY_DEBUG "${_boost_docstring_debug}" ) + endif() + + _Boost_ADJUST_LIB_VARS(${UPPERCOMPONENT}) + +endforeach() + +# Restore the original find library ordering +if( Boost_USE_STATIC_LIBS ) + set(CMAKE_FIND_LIBRARY_SUFFIXES ${_boost_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES}) +endif() + +# ------------------------------------------------------------------------ +# End finding boost libraries +# ------------------------------------------------------------------------ + +set(Boost_INCLUDE_DIRS ${Boost_INCLUDE_DIR}) +set(Boost_LIBRARY_DIRS ${Boost_LIBRARY_DIR}) + +# The above setting of Boost_FOUND was based only on the header files. +# Update it for the requested component libraries. +if(Boost_FOUND) + # The headers were found. Check for requested component libs. + set(_boost_CHECKED_COMPONENT FALSE) + set(_Boost_MISSING_COMPONENTS "") + foreach(COMPONENT ${Boost_FIND_COMPONENTS}) + string(TOUPPER ${COMPONENT} COMPONENT) + set(_boost_CHECKED_COMPONENT TRUE) + if(NOT Boost_${COMPONENT}_FOUND) + string(TOLOWER ${COMPONENT} COMPONENT) + list(APPEND _Boost_MISSING_COMPONENTS ${COMPONENT}) + endif() + endforeach() + + if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] Boost_FOUND = ${Boost_FOUND}") + endif() + + if (_Boost_MISSING_COMPONENTS) + set(Boost_FOUND 0) + # We were unable to find some libraries, so generate a sensible + # error message that lists the libraries we were unable to find. + set(Boost_ERROR_REASON + "${Boost_ERROR_REASON}\nCould not find the following") + if(Boost_USE_STATIC_LIBS) + set(Boost_ERROR_REASON "${Boost_ERROR_REASON} static") + endif() + set(Boost_ERROR_REASON + "${Boost_ERROR_REASON} Boost libraries:\n") + foreach(COMPONENT ${_Boost_MISSING_COMPONENTS}) + set(Boost_ERROR_REASON + "${Boost_ERROR_REASON} boost_${COMPONENT}\n") + endforeach() + + list(LENGTH Boost_FIND_COMPONENTS Boost_NUM_COMPONENTS_WANTED) + list(LENGTH _Boost_MISSING_COMPONENTS Boost_NUM_MISSING_COMPONENTS) + if (${Boost_NUM_COMPONENTS_WANTED} EQUAL ${Boost_NUM_MISSING_COMPONENTS}) + set(Boost_ERROR_REASON + "${Boost_ERROR_REASON}No Boost libraries were found. You may need to set BOOST_LIBRARYDIR to the directory containing Boost libraries or BOOST_ROOT to the location of Boost.") + else () + set(Boost_ERROR_REASON + "${Boost_ERROR_REASON}Some (but not all) of the required Boost libraries were found. You may need to install these additional Boost libraries. Alternatively, set BOOST_LIBRARYDIR to the directory containing Boost libraries or BOOST_ROOT to the location of Boost.") + endif () + endif () + + if( NOT Boost_LIBRARY_DIRS AND NOT _boost_CHECKED_COMPONENT ) + # Compatibility Code for backwards compatibility with CMake + # 2.4's FindBoost module. + + # Look for the boost library path. + # Note that the user may not have installed any libraries + # so it is quite possible the Boost_LIBRARY_DIRS may not exist. + set(_boost_LIB_DIR ${Boost_INCLUDE_DIR}) + + if("${_boost_LIB_DIR}" MATCHES "boost-[0-9]+") + get_filename_component(_boost_LIB_DIR ${_boost_LIB_DIR} PATH) + endif() + + if("${_boost_LIB_DIR}" MATCHES "/include$") + # Strip off the trailing "/include" in the path. + get_filename_component(_boost_LIB_DIR ${_boost_LIB_DIR} PATH) + endif() + + if(EXISTS "${_boost_LIB_DIR}/lib") + set(_boost_LIB_DIR ${_boost_LIB_DIR}/lib) + else() + if(EXISTS "${_boost_LIB_DIR}/stage/lib") + set(_boost_LIB_DIR ${_boost_LIB_DIR}/stage/lib) + else() + set(_boost_LIB_DIR "") + endif() + endif() + + if(_boost_LIB_DIR AND EXISTS "${_boost_LIB_DIR}") + set(Boost_LIBRARY_DIRS ${_boost_LIB_DIR}) + endif() + + endif() +else() + # Boost headers were not found so no components were found. + foreach(COMPONENT ${Boost_FIND_COMPONENTS}) + string(TOUPPER ${COMPONENT} UPPERCOMPONENT) + set(Boost_${UPPERCOMPONENT}_FOUND 0) + endforeach() +endif() + +# ------------------------------------------------------------------------ +# Notification to end user about what was found +# ------------------------------------------------------------------------ + +set(Boost_LIBRARIES "") +if(Boost_FOUND) + if(NOT Boost_FIND_QUIETLY) + message(STATUS "Boost version: ${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}") + if(Boost_FIND_COMPONENTS) + message(STATUS "Found the following Boost libraries:") + endif() + endif() + foreach( COMPONENT ${Boost_FIND_COMPONENTS} ) + string( TOUPPER ${COMPONENT} UPPERCOMPONENT ) + if( Boost_${UPPERCOMPONENT}_FOUND ) + if(NOT Boost_FIND_QUIETLY) + message (STATUS " ${COMPONENT}") + endif() + list(APPEND Boost_LIBRARIES ${Boost_${UPPERCOMPONENT}_LIBRARY}) + endif() + endforeach() +else() + if(Boost_FIND_REQUIRED) + message(SEND_ERROR "Unable to find the requested Boost libraries.\n${Boost_ERROR_REASON}") + else() + if(NOT Boost_FIND_QUIETLY) + # we opt not to automatically output Boost_ERROR_REASON here as + # it could be quite lengthy and somewhat imposing in its requests + # Since Boost is not always a required dependency we'll leave this + # up to the end-user. + if(Boost_DEBUG OR Boost_DETAILED_FAILURE_MSG) + message(STATUS "Could NOT find Boost\n${Boost_ERROR_REASON}") + else() + message(STATUS "Could NOT find Boost") + endif() + endif() + endif() +endif() + +# Configure display of cache entries in GUI. +foreach(v BOOSTROOT BOOST_ROOT ${_Boost_VARS_INC} ${_Boost_VARS_LIB}) + get_property(_type CACHE ${v} PROPERTY TYPE) + if(_type) + set_property(CACHE ${v} PROPERTY ADVANCED 1) + if("x${_type}" STREQUAL "xUNINITIALIZED") + if("x${v}" STREQUAL "xBoost_ADDITIONAL_VERSIONS") + set_property(CACHE ${v} PROPERTY TYPE STRING) + else() + set_property(CACHE ${v} PROPERTY TYPE PATH) + endif() + endif() + endif() +endforeach() + +# Record last used values of input variables so we can +# detect on the next run if the user changed them. +foreach(v + ${_Boost_VARS_INC} ${_Boost_VARS_LIB} + ${_Boost_VARS_DIR} ${_Boost_VARS_NAME} + ) + if(DEFINED ${v}) + set(_${v}_LAST "${${v}}" CACHE INTERNAL "Last used ${v} value.") + else() + unset(_${v}_LAST CACHE) + endif() +endforeach() + +# Maintain a persistent list of components requested anywhere since +# the last flush. +set(_Boost_COMPONENTS_SEARCHED "${_Boost_COMPONENTS_SEARCHED}") +list(APPEND _Boost_COMPONENTS_SEARCHED ${Boost_FIND_COMPONENTS}) +list(REMOVE_DUPLICATES _Boost_COMPONENTS_SEARCHED) +list(SORT _Boost_COMPONENTS_SEARCHED) +set(_Boost_COMPONENTS_SEARCHED "${_Boost_COMPONENTS_SEARCHED}" + CACHE INTERNAL "Components requested for this build tree.") diff --git a/contrib/libhdfs3-cmake/CMake/FindGSasl.cmake b/contrib/libhdfs3-cmake/CMake/FindGSasl.cmake new file mode 100644 index 00000000000..19ca7c30d1e --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/FindGSasl.cmake @@ -0,0 +1,26 @@ +# - Try to find the GNU sasl library (gsasl) +# +# Once done this will define +# +# GSASL_FOUND - System has gnutls +# GSASL_INCLUDE_DIR - The gnutls include directory +# GSASL_LIBRARIES - The libraries needed to use gnutls +# GSASL_DEFINITIONS - Compiler switches required for using gnutls + + +IF (GSASL_INCLUDE_DIR AND GSASL_LIBRARIES) + # in cache already + SET(GSasl_FIND_QUIETLY TRUE) +ENDIF (GSASL_INCLUDE_DIR AND GSASL_LIBRARIES) + +FIND_PATH(GSASL_INCLUDE_DIR gsasl.h) + +FIND_LIBRARY(GSASL_LIBRARIES gsasl) + +INCLUDE(FindPackageHandleStandardArgs) + +# handle the QUIETLY and REQUIRED arguments and set GSASL_FOUND to TRUE if +# all listed variables are TRUE +FIND_PACKAGE_HANDLE_STANDARD_ARGS(GSASL DEFAULT_MSG GSASL_LIBRARIES GSASL_INCLUDE_DIR) + +MARK_AS_ADVANCED(GSASL_INCLUDE_DIR GSASL_LIBRARIES) \ No newline at end of file diff --git a/contrib/libhdfs3-cmake/CMake/FindGoogleTest.cmake b/contrib/libhdfs3-cmake/CMake/FindGoogleTest.cmake new file mode 100644 index 00000000000..fd57c1e2abd --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/FindGoogleTest.cmake @@ -0,0 +1,65 @@ +include(CheckCXXSourceRuns) + +find_path(GTest_INCLUDE_DIR gtest/gtest.h + NO_DEFAULT_PATH + PATHS + "${PROJECT_SOURCE_DIR}/../thirdparty/googletest/googletest/include" + "/usr/local/include" + "/usr/include") + +find_path(GMock_INCLUDE_DIR gmock/gmock.h + NO_DEFAULT_PATH + PATHS + "${PROJECT_SOURCE_DIR}/../thirdparty/googletest/googlemock/include" + "/usr/local/include" + "/usr/include") + +find_library(Gtest_LIBRARY + NAMES libgtest.a + HINTS + "${PROJECT_SOURCE_DIR}/../thirdparty/googletest/build/googlemock/gtest" + "/usr/local/lib" + "/usr/lib") + +find_library(Gmock_LIBRARY + NAMES libgmock.a + HINTS + "${PROJECT_SOURCE_DIR}/../thirdparty/googletest/build/googlemock" + "/usr/local/lib" + "/usr/lib") + +message(STATUS "Find GoogleTest include path: ${GTest_INCLUDE_DIR}") +message(STATUS "Find GoogleMock include path: ${GMock_INCLUDE_DIR}") +message(STATUS "Find Gtest library path: ${Gtest_LIBRARY}") +message(STATUS "Find Gmock library path: ${Gmock_LIBRARY}") + +set(CMAKE_REQUIRED_INCLUDES ${GTest_INCLUDE_DIR} ${GMock_INCLUDE_DIR}) +set(CMAKE_REQUIRED_LIBRARIES ${Gtest_LIBRARY} ${Gmock_LIBRARY} -lpthread) +set(CMAKE_REQUIRED_FLAGS) +check_cxx_source_runs(" +#include +#include +int main(int argc, char *argv[]) +{ + double pi = 3.14; + EXPECT_EQ(pi, 3.14); + return 0; +} +" GoogleTest_CHECK_FINE) +message(STATUS "GoogleTest check: ${GoogleTest_CHECK_FINE}") + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + GoogleTest + REQUIRED_VARS + GTest_INCLUDE_DIR + GMock_INCLUDE_DIR + Gtest_LIBRARY + Gmock_LIBRARY + GoogleTest_CHECK_FINE) + +set(GoogleTest_INCLUDE_DIR ${GTest_INCLUDE_DIR} ${GMock_INCLUDE_DIR}) +set(GoogleTest_LIBRARIES ${Gtest_LIBRARY} ${Gmock_LIBRARY}) +mark_as_advanced( + GoogleTest_INCLUDE_DIR + GoogleTest_LIBRARIES) diff --git a/contrib/libhdfs3-cmake/CMake/FindKERBEROS.cmake b/contrib/libhdfs3-cmake/CMake/FindKERBEROS.cmake new file mode 100644 index 00000000000..5fc58235a3f --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/FindKERBEROS.cmake @@ -0,0 +1,23 @@ +# - Find kerberos +# Find the native KERBEROS includes and library +# +# KERBEROS_INCLUDE_DIRS - where to find krb5.h, etc. +# KERBEROS_LIBRARIES - List of libraries when using krb5. +# KERBEROS_FOUND - True if krb5 found. + +IF (KERBEROS_INCLUDE_DIRS) + # Already in cache, be silent + SET(KERBEROS_FIND_QUIETLY TRUE) +ENDIF (KERBEROS_INCLUDE_DIRS) + +FIND_PATH(KERBEROS_INCLUDE_DIRS krb5.h) + +SET(KERBEROS_NAMES krb5 k5crypto com_err) +FIND_LIBRARY(KERBEROS_LIBRARIES NAMES ${KERBEROS_NAMES}) + +# handle the QUIETLY and REQUIRED arguments and set KERBEROS_FOUND to TRUE if +# all listed variables are TRUE +INCLUDE(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(KERBEROS DEFAULT_MSG KERBEROS_LIBRARIES KERBEROS_INCLUDE_DIRS) + +MARK_AS_ADVANCED(KERBEROS_LIBRARIES KERBEROS_INCLUDE_DIRS) diff --git a/contrib/libhdfs3-cmake/CMake/FindSSL.cmake b/contrib/libhdfs3-cmake/CMake/FindSSL.cmake new file mode 100644 index 00000000000..bcbc5d89653 --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/FindSSL.cmake @@ -0,0 +1,26 @@ +# - Try to find the Open ssl library (ssl) +# +# Once done this will define +# +# SSL_FOUND - System has gnutls +# SSL_INCLUDE_DIR - The gnutls include directory +# SSL_LIBRARIES - The libraries needed to use gnutls +# SSL_DEFINITIONS - Compiler switches required for using gnutls + + +IF (SSL_INCLUDE_DIR AND SSL_LIBRARIES) + # in cache already + SET(SSL_FIND_QUIETLY TRUE) +ENDIF (SSL_INCLUDE_DIR AND SSL_LIBRARIES) + +FIND_PATH(SSL_INCLUDE_DIR openssl/opensslv.h) + +FIND_LIBRARY(SSL_LIBRARIES crypto) + +INCLUDE(FindPackageHandleStandardArgs) + +# handle the QUIETLY and REQUIRED arguments and set SSL_FOUND to TRUE if +# all listed variables are TRUE +FIND_PACKAGE_HANDLE_STANDARD_ARGS(SSL DEFAULT_MSG SSL_LIBRARIES SSL_INCLUDE_DIR) + +MARK_AS_ADVANCED(SSL_INCLUDE_DIR SSL_LIBRARIES) \ No newline at end of file diff --git a/contrib/libhdfs3-cmake/CMake/Functions.cmake b/contrib/libhdfs3-cmake/CMake/Functions.cmake new file mode 100644 index 00000000000..a771b6043fb --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/Functions.cmake @@ -0,0 +1,46 @@ +FUNCTION(AUTO_SOURCES RETURN_VALUE PATTERN SOURCE_SUBDIRS) + + IF ("${SOURCE_SUBDIRS}" STREQUAL "RECURSE") + SET(PATH ".") + IF (${ARGC} EQUAL 4) + LIST(GET ARGV 3 PATH) + ENDIF () + ENDIF() + + IF ("${SOURCE_SUBDIRS}" STREQUAL "RECURSE") + UNSET(${RETURN_VALUE}) + FILE(GLOB SUBDIR_FILES "${PATH}/${PATTERN}") + LIST(APPEND ${RETURN_VALUE} ${SUBDIR_FILES}) + + FILE(GLOB SUBDIRS RELATIVE ${PATH} ${PATH}/*) + + FOREACH(DIR ${SUBDIRS}) + IF (IS_DIRECTORY ${PATH}/${DIR}) + IF (NOT "${DIR}" STREQUAL "CMAKEFILES") + FILE(GLOB_RECURSE SUBDIR_FILES "${PATH}/${DIR}/${PATTERN}") + LIST(APPEND ${RETURN_VALUE} ${SUBDIR_FILES}) + ENDIF() + ENDIF() + ENDFOREACH() + ELSE () + FILE(GLOB ${RETURN_VALUE} "${PATTERN}") + + FOREACH (PATH ${SOURCE_SUBDIRS}) + FILE(GLOB SUBDIR_FILES "${PATH}/${PATTERN}") + LIST(APPEND ${RETURN_VALUE} ${SUBDIR_FILES}) + ENDFOREACH(PATH ${SOURCE_SUBDIRS}) + ENDIF () + + IF (${FILTER_OUT}) + LIST(REMOVE_ITEM ${RETURN_VALUE} ${FILTER_OUT}) + ENDIF() + + SET(${RETURN_VALUE} ${${RETURN_VALUE}} PARENT_SCOPE) +ENDFUNCTION(AUTO_SOURCES) + +FUNCTION(CONTAINS_STRING FILE SEARCH RETURN_VALUE) + FILE(STRINGS ${FILE} FILE_CONTENTS REGEX ".*${SEARCH}.*") + IF (FILE_CONTENTS) + SET(${RETURN_VALUE} TRUE PARENT_SCOPE) + ENDIF() +ENDFUNCTION(CONTAINS_STRING) diff --git a/contrib/libhdfs3-cmake/CMake/Options.cmake b/contrib/libhdfs3-cmake/CMake/Options.cmake new file mode 100644 index 00000000000..5561f3ccc1e --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/Options.cmake @@ -0,0 +1,169 @@ +OPTION(ENABLE_COVERAGE "enable code coverage" OFF) +OPTION(ENABLE_DEBUG "enable debug build" OFF) +OPTION(ENABLE_SSE "enable SSE4.2 buildin function" ON) +OPTION(ENABLE_FRAME_POINTER "enable frame pointer on 64bit system with flag -fno-omit-frame-pointer, on 32bit system, it is always enabled" ON) +OPTION(ENABLE_LIBCPP "using libc++ instead of libstdc++, only valid for clang compiler" OFF) +OPTION(ENABLE_BOOST "using boost instead of native compiler c++0x support" OFF) + +INCLUDE (CheckFunctionExists) +CHECK_FUNCTION_EXISTS(dladdr HAVE_DLADDR) +CHECK_FUNCTION_EXISTS(nanosleep HAVE_NANOSLEEP) + +IF(ENABLE_DEBUG STREQUAL ON) + SET(CMAKE_BUILD_TYPE Debug CACHE + STRING "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel." FORCE) + SET(CMAKE_CXX_FLAGS_DEBUG "-g -O0" CACHE STRING "compiler flags for debug" FORCE) + SET(CMAKE_C_FLAGS_DEBUG "-g -O0" CACHE STRING "compiler flags for debug" FORCE) +ELSE(ENABLE_DEBUG STREQUAL ON) + SET(CMAKE_BUILD_TYPE RelWithDebInfo CACHE + STRING "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel." FORCE) +ENDIF(ENABLE_DEBUG STREQUAL ON) + +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-strict-aliasing") +SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-strict-aliasing") + +IF(ENABLE_COVERAGE STREQUAL ON) + INCLUDE(CodeCoverage) +ENDIF(ENABLE_COVERAGE STREQUAL ON) + +IF(ENABLE_FRAME_POINTER STREQUAL ON) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer") +ENDIF(ENABLE_FRAME_POINTER STREQUAL ON) + +IF(ENABLE_SSE STREQUAL ON) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2") +ENDIF(ENABLE_SSE STREQUAL ON) + +IF(NOT TEST_HDFS_PREFIX) +SET(TEST_HDFS_PREFIX "./" CACHE STRING "default directory prefix used for test." FORCE) +ENDIF(NOT TEST_HDFS_PREFIX) + +ADD_DEFINITIONS(-DTEST_HDFS_PREFIX="${TEST_HDFS_PREFIX}") +ADD_DEFINITIONS(-D__STDC_FORMAT_MACROS) +ADD_DEFINITIONS(-D_GNU_SOURCE) + +IF(OS_MACOSX AND CMAKE_COMPILER_IS_GNUCXX) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-bind_at_load") +ENDIF(OS_MACOSX AND CMAKE_COMPILER_IS_GNUCXX) + +IF(OS_LINUX) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--export-dynamic") +ENDIF(OS_LINUX) + +SET(BOOST_ROOT ${CMAKE_PREFIX_PATH}) +IF(ENABLE_BOOST STREQUAL ON) + MESSAGE(STATUS "using boost instead of native compiler c++0x support.") + FIND_PACKAGE(Boost 1.50 REQUIRED) + SET(NEED_BOOST true CACHE INTERNAL "boost is required") +ELSE(ENABLE_BOOST STREQUAL ON) + SET(NEED_BOOST false CACHE INTERNAL "boost is required") +ENDIF(ENABLE_BOOST STREQUAL ON) + +IF(CMAKE_COMPILER_IS_GNUCXX) + IF(ENABLE_LIBCPP STREQUAL ON) + MESSAGE(FATAL_ERROR "Unsupport using GCC compiler with libc++") + ENDIF(ENABLE_LIBCPP STREQUAL ON) + + IF((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR EQUAL 4) AND OS_MACOSX) + SET(NEED_GCCEH true CACHE INTERNAL "Explicitly link with gcc_eh") + MESSAGE(STATUS "link with -lgcc_eh for TLS") + ENDIF((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR EQUAL 4) AND OS_MACOSX) + + IF((GCC_COMPILER_VERSION_MAJOR LESS 4) OR ((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 4))) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall") + IF(NOT ENABLE_BOOST STREQUAL ON) + MESSAGE(STATUS "gcc version is older than 4.6.0, boost is required.") + FIND_PACKAGE(Boost 1.50 REQUIRED) + SET(NEED_BOOST true CACHE INTERNAL "boost is required") + ENDIF(NOT ENABLE_BOOST STREQUAL ON) + ELSEIF((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 7)) + IF(NOT ENABLE_BOOST STREQUAL ON) + MESSAGE(STATUS "gcc version is older than 4.6.0, boost is required.") + FIND_PACKAGE(Boost 1.50 REQUIRED) + SET(NEED_BOOST true CACHE INTERNAL "boost is required") + ENDIF(NOT ENABLE_BOOST STREQUAL ON) + MESSAGE(STATUS "adding c++0x support for gcc compiler") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x") + ELSE((GCC_COMPILER_VERSION_MAJOR LESS 4) OR ((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 4))) + MESSAGE(STATUS "adding c++0x support for gcc compiler") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x") + ENDIF((GCC_COMPILER_VERSION_MAJOR LESS 4) OR ((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 4))) + + IF(NEED_BOOST) + IF((Boost_MAJOR_VERSION LESS 1) OR ((Boost_MAJOR_VERSION EQUAL 1) AND (Boost_MINOR_VERSION LESS 50))) + MESSAGE(FATAL_ERROR "boost 1.50+ is required") + ENDIF() + ELSE(NEED_BOOST) + IF(HAVE_NANOSLEEP) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_NANOSLEEP") + ELSE(HAVE_NANOSLEEP) + MESSAGE(FATAL_ERROR "nanosleep() is required") + ENDIF(HAVE_NANOSLEEP) + ENDIF(NEED_BOOST) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall") +ELSEIF(CMAKE_COMPILER_IS_CLANG) + MESSAGE(STATUS "adding c++0x support for clang compiler") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x") + SET(CMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LANGUAGE_STANDARD "c++0x") + IF(ENABLE_LIBCPP STREQUAL ON) + MESSAGE(STATUS "using libc++ instead of libstdc++") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") + SET(CMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LIBRARY "libc++") + ENDIF(ENABLE_LIBCPP STREQUAL ON) +ENDIF(CMAKE_COMPILER_IS_GNUCXX) + +TRY_COMPILE(STRERROR_R_RETURN_INT + ${CMAKE_CURRENT_BINARY_DIR} + ${HDFS3_ROOT_DIR}/CMake/CMakeTestCompileStrerror.cpp + CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'" + OUTPUT_VARIABLE OUTPUT) + +MESSAGE(STATUS "Checking whether strerror_r returns an int") + +IF(STRERROR_R_RETURN_INT) + MESSAGE(STATUS "Checking whether strerror_r returns an int -- yes") +ELSE(STRERROR_R_RETURN_INT) + MESSAGE(STATUS "Checking whether strerror_r returns an int -- no") +ENDIF(STRERROR_R_RETURN_INT) + +TRY_COMPILE(HAVE_STEADY_CLOCK + ${CMAKE_CURRENT_BINARY_DIR} + ${HDFS3_ROOT_DIR}/CMake/CMakeTestCompileSteadyClock.cpp + CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'" + OUTPUT_VARIABLE OUTPUT) + +TRY_COMPILE(HAVE_NESTED_EXCEPTION + ${CMAKE_CURRENT_BINARY_DIR} + ${HDFS3_ROOT_DIR}/CMake/CMakeTestCompileNestedException.cpp + CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'" + OUTPUT_VARIABLE OUTPUT) + +FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include ") +TRY_COMPILE(HAVE_BOOST_CHRONO + ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_BINARY_DIR}/test.cpp + CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'" + -DINCLUDE_DIRECTORIES=${Boost_INCLUDE_DIR} + OUTPUT_VARIABLE OUTPUT) + +FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include ") +TRY_COMPILE(HAVE_STD_CHRONO + ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_BINARY_DIR}/test.cpp + CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'" + OUTPUT_VARIABLE OUTPUT) + +FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include ") +TRY_COMPILE(HAVE_BOOST_ATOMIC + ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_BINARY_DIR}/test.cpp + CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'" + -DINCLUDE_DIRECTORIES=${Boost_INCLUDE_DIR} + OUTPUT_VARIABLE OUTPUT) + +FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include ") +TRY_COMPILE(HAVE_STD_ATOMIC + ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_BINARY_DIR}/test.cpp + CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'" + OUTPUT_VARIABLE OUTPUT) diff --git a/contrib/libhdfs3-cmake/CMake/Platform.cmake b/contrib/libhdfs3-cmake/CMake/Platform.cmake new file mode 100644 index 00000000000..55fbf646589 --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/Platform.cmake @@ -0,0 +1,33 @@ +IF(CMAKE_SYSTEM_NAME STREQUAL "Linux") + SET(OS_LINUX true CACHE INTERNAL "Linux operating system") +ELSEIF(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + SET(OS_MACOSX true CACHE INTERNAL "Mac Darwin operating system") +ELSE(CMAKE_SYSTEM_NAME STREQUAL "Linux") + MESSAGE(FATAL_ERROR "Unsupported OS: \"${CMAKE_SYSTEM_NAME}\"") +ENDIF(CMAKE_SYSTEM_NAME STREQUAL "Linux") + +IF(CMAKE_COMPILER_IS_GNUCXX) + EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_COMPILER_VERSION) + + IF (NOT GCC_COMPILER_VERSION) + MESSAGE(FATAL_ERROR "Cannot get gcc version") + ENDIF (NOT GCC_COMPILER_VERSION) + + STRING(REGEX MATCHALL "[0-9]+" GCC_COMPILER_VERSION ${GCC_COMPILER_VERSION}) + + LIST(GET GCC_COMPILER_VERSION 0 GCC_COMPILER_VERSION_MAJOR) + LIST(GET GCC_COMPILER_VERSION 0 GCC_COMPILER_VERSION_MINOR) + + SET(GCC_COMPILER_VERSION_MAJOR ${GCC_COMPILER_VERSION_MAJOR} CACHE INTERNAL "gcc major version") + SET(GCC_COMPILER_VERSION_MINOR ${GCC_COMPILER_VERSION_MINOR} CACHE INTERNAL "gcc minor version") + + MESSAGE(STATUS "checking compiler: GCC (${GCC_COMPILER_VERSION_MAJOR}.${GCC_COMPILER_VERSION_MINOR}.${GCC_COMPILER_VERSION_PATCH})") +ELSE(CMAKE_COMPILER_IS_GNUCXX) + EXECUTE_PROCESS(COMMAND ${CMAKE_C_COMPILER} --version OUTPUT_VARIABLE COMPILER_OUTPUT) + IF(COMPILER_OUTPUT MATCHES "clang") + SET(CMAKE_COMPILER_IS_CLANG true CACHE INTERNAL "using clang as compiler") + MESSAGE(STATUS "checking compiler: CLANG") + ELSE(COMPILER_OUTPUT MATCHES "clang") + MESSAGE(FATAL_ERROR "Unsupported compiler: \"${CMAKE_CXX_COMPILER}\"") + ENDIF(COMPILER_OUTPUT MATCHES "clang") +ENDIF(CMAKE_COMPILER_IS_GNUCXX) diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt new file mode 100644 index 00000000000..1e9e36ecd08 --- /dev/null +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -0,0 +1,212 @@ +if (NOT USE_INTERNAL_PROTOBUF_LIBRARY) + # compatiable with protobuf which was compiled old C++ ABI + set(CMAKE_CXX_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=0") + set(CMAKE_C_FLAGS "") + if (NOT (CMAKE_VERSION VERSION_LESS "3.8.0")) + unset(CMAKE_CXX_STANDARD) + endif () +endif() + +SET(WITH_KERBEROS false) +# project and source dir +set(HDFS3_ROOT_DIR ${CMAKE_SOURCE_DIR}/contrib/libhdfs3) +set(HDFS3_SOURCE_DIR ${HDFS3_ROOT_DIR}/src) +set(HDFS3_COMMON_DIR ${HDFS3_SOURCE_DIR}/common) + +# module +set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH}) +include(Platform) +include(Options) + +# prefer shared libraries +if (WITH_KERBEROS) + find_package(KERBEROS REQUIRED) +endif() + +# source +set(PROTO_FILES + #${HDFS3_SOURCE_DIR}/proto/encryption.proto + ${HDFS3_SOURCE_DIR}/proto/ClientDatanodeProtocol.proto + ${HDFS3_SOURCE_DIR}/proto/hdfs.proto + ${HDFS3_SOURCE_DIR}/proto/Security.proto + ${HDFS3_SOURCE_DIR}/proto/ProtobufRpcEngine.proto + ${HDFS3_SOURCE_DIR}/proto/ClientNamenodeProtocol.proto + ${HDFS3_SOURCE_DIR}/proto/IpcConnectionContext.proto + ${HDFS3_SOURCE_DIR}/proto/RpcHeader.proto + ${HDFS3_SOURCE_DIR}/proto/datatransfer.proto + ) + +PROTOBUF_GENERATE_CPP(PROTO_SOURCES PROTO_HEADERS ${PROTO_FILES}) + +configure_file(${HDFS3_SOURCE_DIR}/platform.h.in ${CMAKE_CURRENT_BINARY_DIR}/platform.h) + +set(SRCS + ${HDFS3_SOURCE_DIR}/network/TcpSocket.cpp + ${HDFS3_SOURCE_DIR}/network/DomainSocket.cpp + ${HDFS3_SOURCE_DIR}/network/BufferedSocketReader.cpp + ${HDFS3_SOURCE_DIR}/client/ReadShortCircuitInfo.cpp + ${HDFS3_SOURCE_DIR}/client/Pipeline.cpp + ${HDFS3_SOURCE_DIR}/client/Hdfs.cpp + ${HDFS3_SOURCE_DIR}/client/Packet.cpp + ${HDFS3_SOURCE_DIR}/client/OutputStreamImpl.cpp + ${HDFS3_SOURCE_DIR}/client/KerberosName.cpp + ${HDFS3_SOURCE_DIR}/client/PacketHeader.cpp + ${HDFS3_SOURCE_DIR}/client/LocalBlockReader.cpp + ${HDFS3_SOURCE_DIR}/client/UserInfo.cpp + ${HDFS3_SOURCE_DIR}/client/RemoteBlockReader.cpp + ${HDFS3_SOURCE_DIR}/client/Permission.cpp + ${HDFS3_SOURCE_DIR}/client/FileSystemImpl.cpp + ${HDFS3_SOURCE_DIR}/client/DirectoryIterator.cpp + ${HDFS3_SOURCE_DIR}/client/FileSystemKey.cpp + ${HDFS3_SOURCE_DIR}/client/DataTransferProtocolSender.cpp + ${HDFS3_SOURCE_DIR}/client/LeaseRenewer.cpp + ${HDFS3_SOURCE_DIR}/client/PeerCache.cpp + ${HDFS3_SOURCE_DIR}/client/InputStream.cpp + ${HDFS3_SOURCE_DIR}/client/FileSystem.cpp + ${HDFS3_SOURCE_DIR}/client/InputStreamImpl.cpp + ${HDFS3_SOURCE_DIR}/client/Token.cpp + ${HDFS3_SOURCE_DIR}/client/PacketPool.cpp + ${HDFS3_SOURCE_DIR}/client/OutputStream.cpp + ${HDFS3_SOURCE_DIR}/rpc/RpcChannelKey.cpp + ${HDFS3_SOURCE_DIR}/rpc/RpcProtocolInfo.cpp + ${HDFS3_SOURCE_DIR}/rpc/RpcClient.cpp + ${HDFS3_SOURCE_DIR}/rpc/RpcRemoteCall.cpp + ${HDFS3_SOURCE_DIR}/rpc/RpcChannel.cpp + ${HDFS3_SOURCE_DIR}/rpc/RpcAuth.cpp + ${HDFS3_SOURCE_DIR}/rpc/RpcContentWrapper.cpp + ${HDFS3_SOURCE_DIR}/rpc/RpcConfig.cpp + ${HDFS3_SOURCE_DIR}/rpc/RpcServerInfo.cpp + ${HDFS3_SOURCE_DIR}/rpc/SaslClient.cpp + ${HDFS3_SOURCE_DIR}/server/Datanode.cpp + ${HDFS3_SOURCE_DIR}/server/LocatedBlocks.cpp + ${HDFS3_SOURCE_DIR}/server/NamenodeProxy.cpp + ${HDFS3_SOURCE_DIR}/server/NamenodeImpl.cpp + ${HDFS3_SOURCE_DIR}/server/NamenodeInfo.cpp + ${HDFS3_SOURCE_DIR}/common/WritableUtils.cpp + ${HDFS3_SOURCE_DIR}/common/ExceptionInternal.cpp + ${HDFS3_SOURCE_DIR}/common/SessionConfig.cpp + ${HDFS3_SOURCE_DIR}/common/StackPrinter.cpp + ${HDFS3_SOURCE_DIR}/common/Exception.cpp + ${HDFS3_SOURCE_DIR}/common/Logger.cpp + ${HDFS3_SOURCE_DIR}/common/CFileWrapper.cpp + ${HDFS3_SOURCE_DIR}/common/XmlConfig.cpp + ${HDFS3_SOURCE_DIR}/common/WriteBuffer.cpp + ${HDFS3_SOURCE_DIR}/common/HWCrc32c.cpp + ${HDFS3_SOURCE_DIR}/common/MappedFileWrapper.cpp + ${HDFS3_SOURCE_DIR}/common/Hash.cpp + ${HDFS3_SOURCE_DIR}/common/SWCrc32c.cpp + ${HDFS3_SOURCE_DIR}/common/Thread.cpp + + ${HDFS3_SOURCE_DIR}/network/TcpSocket.h + ${HDFS3_SOURCE_DIR}/network/BufferedSocketReader.h + ${HDFS3_SOURCE_DIR}/network/Socket.h + ${HDFS3_SOURCE_DIR}/network/DomainSocket.h + ${HDFS3_SOURCE_DIR}/network/Syscall.h + ${HDFS3_SOURCE_DIR}/client/InputStreamImpl.h + ${HDFS3_SOURCE_DIR}/client/FileSystem.h + ${HDFS3_SOURCE_DIR}/client/ReadShortCircuitInfo.h + ${HDFS3_SOURCE_DIR}/client/InputStreamInter.h + ${HDFS3_SOURCE_DIR}/client/FileSystemImpl.h + ${HDFS3_SOURCE_DIR}/client/PacketPool.h + ${HDFS3_SOURCE_DIR}/client/Pipeline.h + ${HDFS3_SOURCE_DIR}/client/OutputStreamInter.h + ${HDFS3_SOURCE_DIR}/client/RemoteBlockReader.h + ${HDFS3_SOURCE_DIR}/client/Token.h + ${HDFS3_SOURCE_DIR}/client/KerberosName.h + ${HDFS3_SOURCE_DIR}/client/DirectoryIterator.h + ${HDFS3_SOURCE_DIR}/client/hdfs.h + ${HDFS3_SOURCE_DIR}/client/FileSystemStats.h + ${HDFS3_SOURCE_DIR}/client/FileSystemKey.h + ${HDFS3_SOURCE_DIR}/client/DataTransferProtocolSender.h + ${HDFS3_SOURCE_DIR}/client/Packet.h + ${HDFS3_SOURCE_DIR}/client/PacketHeader.h + ${HDFS3_SOURCE_DIR}/client/FileSystemInter.h + ${HDFS3_SOURCE_DIR}/client/LocalBlockReader.h + ${HDFS3_SOURCE_DIR}/client/TokenInternal.h + ${HDFS3_SOURCE_DIR}/client/InputStream.h + ${HDFS3_SOURCE_DIR}/client/PipelineAck.h + ${HDFS3_SOURCE_DIR}/client/BlockReader.h + ${HDFS3_SOURCE_DIR}/client/Permission.h + ${HDFS3_SOURCE_DIR}/client/OutputStreamImpl.h + ${HDFS3_SOURCE_DIR}/client/LeaseRenewer.h + ${HDFS3_SOURCE_DIR}/client/UserInfo.h + ${HDFS3_SOURCE_DIR}/client/PeerCache.h + ${HDFS3_SOURCE_DIR}/client/OutputStream.h + ${HDFS3_SOURCE_DIR}/client/FileStatus.h + ${HDFS3_SOURCE_DIR}/client/DataTransferProtocol.h + ${HDFS3_SOURCE_DIR}/client/BlockLocation.h + ${HDFS3_SOURCE_DIR}/rpc/RpcConfig.h + ${HDFS3_SOURCE_DIR}/rpc/SaslClient.h + ${HDFS3_SOURCE_DIR}/rpc/RpcAuth.h + ${HDFS3_SOURCE_DIR}/rpc/RpcClient.h + ${HDFS3_SOURCE_DIR}/rpc/RpcCall.h + ${HDFS3_SOURCE_DIR}/rpc/RpcContentWrapper.h + ${HDFS3_SOURCE_DIR}/rpc/RpcProtocolInfo.h + ${HDFS3_SOURCE_DIR}/rpc/RpcRemoteCall.h + ${HDFS3_SOURCE_DIR}/rpc/RpcServerInfo.h + ${HDFS3_SOURCE_DIR}/rpc/RpcChannel.h + ${HDFS3_SOURCE_DIR}/rpc/RpcChannelKey.h + ${HDFS3_SOURCE_DIR}/server/BlockLocalPathInfo.h + ${HDFS3_SOURCE_DIR}/server/LocatedBlocks.h + ${HDFS3_SOURCE_DIR}/server/DatanodeInfo.h + ${HDFS3_SOURCE_DIR}/server/RpcHelper.h + ${HDFS3_SOURCE_DIR}/server/ExtendedBlock.h + ${HDFS3_SOURCE_DIR}/server/NamenodeInfo.h + ${HDFS3_SOURCE_DIR}/server/NamenodeImpl.h + ${HDFS3_SOURCE_DIR}/server/LocatedBlock.h + ${HDFS3_SOURCE_DIR}/server/NamenodeProxy.h + ${HDFS3_SOURCE_DIR}/server/Datanode.h + ${HDFS3_SOURCE_DIR}/server/Namenode.h + ${HDFS3_SOURCE_DIR}/common/XmlConfig.h + ${HDFS3_SOURCE_DIR}/common/Logger.h + ${HDFS3_SOURCE_DIR}/common/WriteBuffer.h + ${HDFS3_SOURCE_DIR}/common/HWCrc32c.h + ${HDFS3_SOURCE_DIR}/common/Checksum.h + ${HDFS3_SOURCE_DIR}/common/SessionConfig.h + ${HDFS3_SOURCE_DIR}/common/Unordered.h + ${HDFS3_SOURCE_DIR}/common/BigEndian.h + ${HDFS3_SOURCE_DIR}/common/Thread.h + ${HDFS3_SOURCE_DIR}/common/StackPrinter.h + ${HDFS3_SOURCE_DIR}/common/Exception.h + ${HDFS3_SOURCE_DIR}/common/WritableUtils.h + ${HDFS3_SOURCE_DIR}/common/StringUtil.h + ${HDFS3_SOURCE_DIR}/common/LruMap.h + ${HDFS3_SOURCE_DIR}/common/Function.h + ${HDFS3_SOURCE_DIR}/common/DateTime.h + ${HDFS3_SOURCE_DIR}/common/Hash.h + ${HDFS3_SOURCE_DIR}/common/SWCrc32c.h + ${HDFS3_SOURCE_DIR}/common/ExceptionInternal.h + ${HDFS3_SOURCE_DIR}/common/Memory.h + ${HDFS3_SOURCE_DIR}/common/FileWrapper.h + ) + +# target +add_library(hdfs3 STATIC ${SRCS} ${PROTO_SOURCES} ${PROTO_HEADERS}) + +if (USE_INTERNAL_PROTOBUF_LIBRARY) + add_dependencies(hdfs3 protoc) +endif() + +target_include_directories(hdfs3 PRIVATE ${HDFS3_SOURCE_DIR}) +target_include_directories(hdfs3 PRIVATE ${HDFS3_COMMON_DIR}) +target_include_directories(hdfs3 PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) + +target_include_directories(hdfs3 PRIVATE ${LIBGSASL_INCLUDE_DIR}) +if (WITH_KERBEROS) + target_include_directories(hdfs3 PRIVATE ${KERBEROS_INCLUDE_DIRS}) +endif() +target_include_directories(hdfs3 PRIVATE ${LIBXML2_INCLUDE_DIR}) + +target_link_libraries(hdfs3 ${LIBGSASL_LIBRARY}) +if (WITH_KERBEROS) + target_link_libraries(hdfs3 ${KERBEROS_LIBRARIES}) +endif() +target_link_libraries(hdfs3 ${LIBXML2_LIBRARY}) + +# inherit from parent cmake +target_include_directories(hdfs3 PRIVATE ${Boost_INCLUDE_DIRS}) +target_include_directories(hdfs3 PRIVATE ${Protobuf_INCLUDE_DIR}) +target_include_directories(hdfs3 PRIVATE ${OPENSSL_INCLUDE_DIR}) + +target_link_libraries(hdfs3 ${Protobuf_LIBRARY}) +target_link_libraries(hdfs3 ${OPENSSL_LIBRARIES}) diff --git a/contrib/libxml2 b/contrib/libxml2 new file mode 160000 index 00000000000..18890f471c4 --- /dev/null +++ b/contrib/libxml2 @@ -0,0 +1 @@ +Subproject commit 18890f471c420411aa3c989e104d090966ec9dbf diff --git a/contrib/libxml2-cmake/CMakeLists.txt b/contrib/libxml2-cmake/CMakeLists.txt new file mode 100644 index 00000000000..c4b7f39cc8f --- /dev/null +++ b/contrib/libxml2-cmake/CMakeLists.txt @@ -0,0 +1,63 @@ +set(LIBXML2_SOURCE_DIR ${CMAKE_SOURCE_DIR}/contrib/libxml2) +set(LIBXML2_BINARY_DIR ${CMAKE_BINARY_DIR}/contrib/libxml2) + + +set(SRCS + ${LIBXML2_SOURCE_DIR}/parser.c + ${LIBXML2_SOURCE_DIR}/HTMLparser.c + ${LIBXML2_SOURCE_DIR}/buf.c + ${LIBXML2_SOURCE_DIR}/xzlib.c + ${LIBXML2_SOURCE_DIR}/xmlregexp.c + ${LIBXML2_SOURCE_DIR}/entities.c + ${LIBXML2_SOURCE_DIR}/rngparser.c + ${LIBXML2_SOURCE_DIR}/encoding.c + ${LIBXML2_SOURCE_DIR}/legacy.c + ${LIBXML2_SOURCE_DIR}/error.c + ${LIBXML2_SOURCE_DIR}/debugXML.c + ${LIBXML2_SOURCE_DIR}/xpointer.c + ${LIBXML2_SOURCE_DIR}/DOCBparser.c + ${LIBXML2_SOURCE_DIR}/xmlcatalog.c + ${LIBXML2_SOURCE_DIR}/c14n.c + ${LIBXML2_SOURCE_DIR}/xmlreader.c + ${LIBXML2_SOURCE_DIR}/xmlstring.c + ${LIBXML2_SOURCE_DIR}/dict.c + ${LIBXML2_SOURCE_DIR}/xpath.c + ${LIBXML2_SOURCE_DIR}/tree.c + ${LIBXML2_SOURCE_DIR}/trionan.c + ${LIBXML2_SOURCE_DIR}/pattern.c + ${LIBXML2_SOURCE_DIR}/globals.c + ${LIBXML2_SOURCE_DIR}/xmllint.c + ${LIBXML2_SOURCE_DIR}/chvalid.c + ${LIBXML2_SOURCE_DIR}/relaxng.c + ${LIBXML2_SOURCE_DIR}/list.c + ${LIBXML2_SOURCE_DIR}/xinclude.c + ${LIBXML2_SOURCE_DIR}/xmlIO.c + ${LIBXML2_SOURCE_DIR}/triostr.c + ${LIBXML2_SOURCE_DIR}/hash.c + ${LIBXML2_SOURCE_DIR}/xmlsave.c + ${LIBXML2_SOURCE_DIR}/HTMLtree.c + ${LIBXML2_SOURCE_DIR}/SAX.c + ${LIBXML2_SOURCE_DIR}/xmlschemas.c + ${LIBXML2_SOURCE_DIR}/SAX2.c + ${LIBXML2_SOURCE_DIR}/threads.c + ${LIBXML2_SOURCE_DIR}/runsuite.c + ${LIBXML2_SOURCE_DIR}/catalog.c + ${LIBXML2_SOURCE_DIR}/uri.c + ${LIBXML2_SOURCE_DIR}/xmlmodule.c + ${LIBXML2_SOURCE_DIR}/xlink.c + ${LIBXML2_SOURCE_DIR}/parserInternals.c + ${LIBXML2_SOURCE_DIR}/xmlwriter.c + ${LIBXML2_SOURCE_DIR}/xmlunicode.c + ${LIBXML2_SOURCE_DIR}/runxmlconf.c + ${LIBXML2_SOURCE_DIR}/xmlmemory.c + ${LIBXML2_SOURCE_DIR}/nanoftp.c + ${LIBXML2_SOURCE_DIR}/xmlschemastypes.c + ${LIBXML2_SOURCE_DIR}/valid.c + ${LIBXML2_SOURCE_DIR}/nanohttp.c + ${LIBXML2_SOURCE_DIR}/schematron.c +) +add_library(libxml2 STATIC ${SRCS}) + +target_include_directories(libxml2 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/include) + +target_include_directories(libxml2 PUBLIC ${LIBXML2_SOURCE_DIR}/include) diff --git a/contrib/libxml2-cmake/linux_x86_64/include/config.h b/contrib/libxml2-cmake/linux_x86_64/include/config.h new file mode 100644 index 00000000000..7969b377dc3 --- /dev/null +++ b/contrib/libxml2-cmake/linux_x86_64/include/config.h @@ -0,0 +1,285 @@ +/* config.h. Generated from config.h.in by configure. */ +/* config.h.in. Generated from configure.ac by autoheader. */ + +/* Type cast for the gethostbyname() argument */ +#define GETHOSTBYNAME_ARG_CAST /**/ + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_INET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_ARPA_NAMESER_H 1 + +/* Whether struct sockaddr::__ss_family exists */ +/* #undef HAVE_BROKEN_SS_FAMILY */ + +/* Define to 1 if you have the header file. */ +#define HAVE_CTYPE_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_DIRENT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_DLFCN_H 1 + +/* Have dlopen based dso */ +#define HAVE_DLOPEN /**/ + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_DL_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_ERRNO_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_FCNTL_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_FLOAT_H 1 + +/* Define to 1 if you have the `fprintf' function. */ +#define HAVE_FPRINTF 1 + +/* Define to 1 if you have the `ftime' function. */ +#define HAVE_FTIME 1 + +/* Define if getaddrinfo is there */ +#define HAVE_GETADDRINFO /**/ + +/* Define to 1 if you have the `gettimeofday' function. */ +#define HAVE_GETTIMEOFDAY 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_INTTYPES_H 1 + +/* Define to 1 if you have the `isascii' function. */ +#define HAVE_ISASCII 1 + +/* Define if isinf is there */ +#define HAVE_ISINF /**/ + +/* Define if isnan is there */ +#define HAVE_ISNAN /**/ + +/* Define if history library is there (-lhistory) */ +/* #undef HAVE_LIBHISTORY */ + +/* Define if pthread library is there (-lpthread) */ +#define HAVE_LIBPTHREAD /**/ + +/* Define if readline library is there (-lreadline) */ +/* #undef HAVE_LIBREADLINE */ + +/* Define to 1 if you have the header file. */ +#define HAVE_LIMITS_H 1 + +/* Define to 1 if you have the `localtime' function. */ +#define HAVE_LOCALTIME 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_LZMA_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_MALLOC_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MATH_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_MEMORY_H 1 + +/* Define to 1 if you have the `mmap' function. */ +#define HAVE_MMAP 1 + +/* Define to 1 if you have the `munmap' function. */ +#define HAVE_MUNMAP 1 + +/* mmap() is no good without munmap() */ +#if defined(HAVE_MMAP) && !defined(HAVE_MUNMAP) +# undef /**/ HAVE_MMAP +#endif + +/* Define to 1 if you have the header file, and it defines `DIR'. */ +/* #undef HAVE_NDIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_NETDB_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_NETINET_IN_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_POLL_H 1 + +/* Define to 1 if you have the `printf' function. */ +#define HAVE_PRINTF 1 + +/* Define if is there */ +#define HAVE_PTHREAD_H /**/ + +/* Define to 1 if you have the `putenv' function. */ +#define HAVE_PUTENV 1 + +/* Define to 1 if you have the `rand' function. */ +#define HAVE_RAND 1 + +/* Define to 1 if you have the `rand_r' function. */ +#define HAVE_RAND_R 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_RESOLV_H 1 + +/* Have shl_load based dso */ +/* #undef HAVE_SHLLOAD */ + +/* Define to 1 if you have the `signal' function. */ +#define HAVE_SIGNAL 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SIGNAL_H 1 + +/* Define to 1 if you have the `snprintf' function. */ +#define HAVE_SNPRINTF 1 + +/* Define to 1 if you have the `sprintf' function. */ +#define HAVE_SPRINTF 1 + +/* Define to 1 if you have the `srand' function. */ +#define HAVE_SRAND 1 + +/* Define to 1 if you have the `sscanf' function. */ +#define HAVE_SSCANF 1 + +/* Define to 1 if you have the `stat' function. */ +#define HAVE_STAT 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDARG_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDINT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STDLIB_H 1 + +/* Define to 1 if you have the `strftime' function. */ +#define HAVE_STRFTIME 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRINGS_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_STRING_H 1 + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_DIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_MMAN_H 1 + +/* Define to 1 if you have the header file, and it defines `DIR'. + */ +/* #undef HAVE_SYS_NDIR_H */ + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SELECT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_SOCKET_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_STAT_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIMEB_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_SYS_TYPES_H 1 + +/* Define to 1 if you have the `time' function. */ +#define HAVE_TIME 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_TIME_H 1 + +/* Define to 1 if you have the header file. */ +#define HAVE_UNISTD_H 1 + +/* Whether va_copy() is available */ +#define HAVE_VA_COPY 1 + +/* Define to 1 if you have the `vfprintf' function. */ +#define HAVE_VFPRINTF 1 + +/* Define to 1 if you have the `vsnprintf' function. */ +#define HAVE_VSNPRINTF 1 + +/* Define to 1 if you have the `vsprintf' function. */ +#define HAVE_VSPRINTF 1 + +/* Define to 1 if you have the header file. */ +/* #undef HAVE_ZLIB_H */ + +/* Whether __va_copy() is available */ +/* #undef HAVE___VA_COPY */ + +/* Define as const if the declaration of iconv() needs const. */ +#define ICONV_CONST + +/* Define to the sub-directory where libtool stores uninstalled libraries. */ +#define LT_OBJDIR ".libs/" + +/* Name of package */ +#define PACKAGE "libxml2" + +/* Define to the address where bug reports for this package should be sent. */ +#define PACKAGE_BUGREPORT "" + +/* Define to the full name of this package. */ +#define PACKAGE_NAME "" + +/* Define to the full name and version of this package. */ +#define PACKAGE_STRING "" + +/* Define to the one symbol short name of this package. */ +#define PACKAGE_TARNAME "" + +/* Define to the home page for this package. */ +#define PACKAGE_URL "" + +/* Define to the version of this package. */ +#define PACKAGE_VERSION "" + +/* Type cast for the send() function 2nd arg */ +#define SEND_ARG2_CAST /**/ + +/* Define to 1 if you have the ANSI C header files. */ +#define STDC_HEADERS 1 + +/* Support for IPv6 */ +#define SUPPORT_IP6 /**/ + +/* Define if va_list is an array type */ +#define VA_LIST_IS_ARRAY 1 + +/* Version number of package */ +#define VERSION "2.9.8" + +/* Determine what socket length (socklen_t) data type is */ +#define XML_SOCKLEN_T socklen_t + +/* Define for Solaris 2.5.1 so the uint32_t typedef from , + , or is not used. If the typedef were allowed, the + #define below would cause a syntax error. */ +/* #undef _UINT32_T */ + +/* ss_family is not defined here, use __ss_family instead */ +/* #undef ss_family */ + +/* Define to the type of an unsigned integer type of width exactly 32 bits if + such a type exists and the standard includes do not define it. */ +/* #undef uint32_t */ diff --git a/contrib/libxml2-cmake/linux_x86_64/include/libxml/xmlversion.h b/contrib/libxml2-cmake/linux_x86_64/include/libxml/xmlversion.h new file mode 100644 index 00000000000..92d3414fdac --- /dev/null +++ b/contrib/libxml2-cmake/linux_x86_64/include/libxml/xmlversion.h @@ -0,0 +1,481 @@ +/* + * Summary: compile-time version informations + * Description: compile-time version informations for the XML library + * + * Copy: See Copyright for the status of this software. + * + * Author: Daniel Veillard + */ + +#ifndef __XML_VERSION_H__ +#define __XML_VERSION_H__ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * use those to be sure nothing nasty will happen if + * your library and includes mismatch + */ +#ifndef LIBXML2_COMPILING_MSCCDEF +XMLPUBFUN void XMLCALL xmlCheckVersion(int version); +#endif /* LIBXML2_COMPILING_MSCCDEF */ + +/** + * LIBXML_DOTTED_VERSION: + * + * the version string like "1.2.3" + */ +#define LIBXML_DOTTED_VERSION "2.9.8" + +/** + * LIBXML_VERSION: + * + * the version number: 1.2.3 value is 10203 + */ +#define LIBXML_VERSION 20908 + +/** + * LIBXML_VERSION_STRING: + * + * the version number string, 1.2.3 value is "10203" + */ +#define LIBXML_VERSION_STRING "20908" + +/** + * LIBXML_VERSION_EXTRA: + * + * extra version information, used to show a CVS compilation + */ +#define LIBXML_VERSION_EXTRA "-GITv2.9.9-rc2-1-g6fc04d71" + +/** + * LIBXML_TEST_VERSION: + * + * Macro to check that the libxml version in use is compatible with + * the version the software has been compiled against + */ +#define LIBXML_TEST_VERSION xmlCheckVersion(20908); + +#ifndef VMS +#if 0 +/** + * WITH_TRIO: + * + * defined if the trio support need to be configured in + */ +#define WITH_TRIO +#else +/** + * WITHOUT_TRIO: + * + * defined if the trio support should not be configured in + */ +#define WITHOUT_TRIO +#endif +#else /* VMS */ +/** + * WITH_TRIO: + * + * defined if the trio support need to be configured in + */ +#define WITH_TRIO 1 +#endif /* VMS */ + +/** + * LIBXML_THREAD_ENABLED: + * + * Whether the thread support is configured in + */ +#define LIBXML_THREAD_ENABLED 1 + +/** + * LIBXML_THREAD_ALLOC_ENABLED: + * + * Whether the allocation hooks are per-thread + */ +#if 0 +#define LIBXML_THREAD_ALLOC_ENABLED +#endif + +/** + * LIBXML_TREE_ENABLED: + * + * Whether the DOM like tree manipulation API support is configured in + */ +#if 1 +#define LIBXML_TREE_ENABLED +#endif + +/** + * LIBXML_OUTPUT_ENABLED: + * + * Whether the serialization/saving support is configured in + */ +#if 1 +#define LIBXML_OUTPUT_ENABLED +#endif + +/** + * LIBXML_PUSH_ENABLED: + * + * Whether the push parsing interfaces are configured in + */ +#if 1 +#define LIBXML_PUSH_ENABLED +#endif + +/** + * LIBXML_READER_ENABLED: + * + * Whether the xmlReader parsing interface is configured in + */ +#if 1 +#define LIBXML_READER_ENABLED +#endif + +/** + * LIBXML_PATTERN_ENABLED: + * + * Whether the xmlPattern node selection interface is configured in + */ +#if 1 +#define LIBXML_PATTERN_ENABLED +#endif + +/** + * LIBXML_WRITER_ENABLED: + * + * Whether the xmlWriter saving interface is configured in + */ +#if 1 +#define LIBXML_WRITER_ENABLED +#endif + +/** + * LIBXML_SAX1_ENABLED: + * + * Whether the older SAX1 interface is configured in + */ +#if 1 +#define LIBXML_SAX1_ENABLED +#endif + +/** + * LIBXML_FTP_ENABLED: + * + * Whether the FTP support is configured in + */ +#if 1 +#define LIBXML_FTP_ENABLED +#endif + +/** + * LIBXML_HTTP_ENABLED: + * + * Whether the HTTP support is configured in + */ +#if 1 +#define LIBXML_HTTP_ENABLED +#endif + +/** + * LIBXML_VALID_ENABLED: + * + * Whether the DTD validation support is configured in + */ +#if 1 +#define LIBXML_VALID_ENABLED +#endif + +/** + * LIBXML_HTML_ENABLED: + * + * Whether the HTML support is configured in + */ +#if 1 +#define LIBXML_HTML_ENABLED +#endif + +/** + * LIBXML_LEGACY_ENABLED: + * + * Whether the deprecated APIs are compiled in for compatibility + */ +#if 1 +#define LIBXML_LEGACY_ENABLED +#endif + +/** + * LIBXML_C14N_ENABLED: + * + * Whether the Canonicalization support is configured in + */ +#if 1 +#define LIBXML_C14N_ENABLED +#endif + +/** + * LIBXML_CATALOG_ENABLED: + * + * Whether the Catalog support is configured in + */ +#if 1 +#define LIBXML_CATALOG_ENABLED +#endif + +/** + * LIBXML_DOCB_ENABLED: + * + * Whether the SGML Docbook support is configured in + */ +#if 1 +#define LIBXML_DOCB_ENABLED +#endif + +/** + * LIBXML_XPATH_ENABLED: + * + * Whether XPath is configured in + */ +#if 1 +#define LIBXML_XPATH_ENABLED +#endif + +/** + * LIBXML_XPTR_ENABLED: + * + * Whether XPointer is configured in + */ +#if 1 +#define LIBXML_XPTR_ENABLED +#endif + +/** + * LIBXML_XINCLUDE_ENABLED: + * + * Whether XInclude is configured in + */ +#if 1 +#define LIBXML_XINCLUDE_ENABLED +#endif + +/** + * LIBXML_ICONV_ENABLED: + * + * Whether iconv support is available + */ +#if 1 +#define LIBXML_ICONV_ENABLED +#endif + +/** + * LIBXML_ICU_ENABLED: + * + * Whether icu support is available + */ +#if 0 +#define LIBXML_ICU_ENABLED +#endif + +/** + * LIBXML_ISO8859X_ENABLED: + * + * Whether ISO-8859-* support is made available in case iconv is not + */ +#if 1 +#define LIBXML_ISO8859X_ENABLED +#endif + +/** + * LIBXML_DEBUG_ENABLED: + * + * Whether Debugging module is configured in + */ +#if 1 +#define LIBXML_DEBUG_ENABLED +#endif + +/** + * DEBUG_MEMORY_LOCATION: + * + * Whether the memory debugging is configured in + */ +#if 0 +#define DEBUG_MEMORY_LOCATION +#endif + +/** + * LIBXML_DEBUG_RUNTIME: + * + * Whether the runtime debugging is configured in + */ +#if 0 +#define LIBXML_DEBUG_RUNTIME +#endif + +/** + * LIBXML_UNICODE_ENABLED: + * + * Whether the Unicode related interfaces are compiled in + */ +#if 1 +#define LIBXML_UNICODE_ENABLED +#endif + +/** + * LIBXML_REGEXP_ENABLED: + * + * Whether the regular expressions interfaces are compiled in + */ +#if 1 +#define LIBXML_REGEXP_ENABLED +#endif + +/** + * LIBXML_AUTOMATA_ENABLED: + * + * Whether the automata interfaces are compiled in + */ +#if 1 +#define LIBXML_AUTOMATA_ENABLED +#endif + +/** + * LIBXML_EXPR_ENABLED: + * + * Whether the formal expressions interfaces are compiled in + */ +#if 1 +#define LIBXML_EXPR_ENABLED +#endif + +/** + * LIBXML_SCHEMAS_ENABLED: + * + * Whether the Schemas validation interfaces are compiled in + */ +#if 1 +#define LIBXML_SCHEMAS_ENABLED +#endif + +/** + * LIBXML_SCHEMATRON_ENABLED: + * + * Whether the Schematron validation interfaces are compiled in + */ +#if 1 +#define LIBXML_SCHEMATRON_ENABLED +#endif + +/** + * LIBXML_MODULES_ENABLED: + * + * Whether the module interfaces are compiled in + */ +#if 1 +#define LIBXML_MODULES_ENABLED +/** + * LIBXML_MODULE_EXTENSION: + * + * the string suffix used by dynamic modules (usually shared libraries) + */ +#define LIBXML_MODULE_EXTENSION ".so" +#endif + +/** + * LIBXML_ZLIB_ENABLED: + * + * Whether the Zlib support is compiled in + */ +#if 1 +#define LIBXML_ZLIB_ENABLED +#endif + +/** + * LIBXML_LZMA_ENABLED: + * + * Whether the Lzma support is compiled in + */ +#if 0 +#define LIBXML_LZMA_ENABLED +#endif + +#ifdef __GNUC__ + +/** + * ATTRIBUTE_UNUSED: + * + * Macro used to signal to GCC unused function parameters + */ + +#ifndef ATTRIBUTE_UNUSED +# if ((__GNUC__ > 2) || ((__GNUC__ == 2) && (__GNUC_MINOR__ >= 7))) +# define ATTRIBUTE_UNUSED __attribute__((unused)) +# else +# define ATTRIBUTE_UNUSED +# endif +#endif + +/** + * LIBXML_ATTR_ALLOC_SIZE: + * + * Macro used to indicate to GCC this is an allocator function + */ + +#ifndef LIBXML_ATTR_ALLOC_SIZE +# if (!defined(__clang__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && (__GNUC_MINOR__ >= 3)))) +# define LIBXML_ATTR_ALLOC_SIZE(x) __attribute__((alloc_size(x))) +# else +# define LIBXML_ATTR_ALLOC_SIZE(x) +# endif +#else +# define LIBXML_ATTR_ALLOC_SIZE(x) +#endif + +/** + * LIBXML_ATTR_FORMAT: + * + * Macro used to indicate to GCC the parameter are printf like + */ + +#ifndef LIBXML_ATTR_FORMAT +# if ((__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 3))) +# define LIBXML_ATTR_FORMAT(fmt,args) __attribute__((__format__(__printf__,fmt,args))) +# else +# define LIBXML_ATTR_FORMAT(fmt,args) +# endif +#else +# define LIBXML_ATTR_FORMAT(fmt,args) +#endif + +#else /* ! __GNUC__ */ +/** + * ATTRIBUTE_UNUSED: + * + * Macro used to signal to GCC unused function parameters + */ +#define ATTRIBUTE_UNUSED +/** + * LIBXML_ATTR_ALLOC_SIZE: + * + * Macro used to indicate to GCC this is an allocator function + */ +#define LIBXML_ATTR_ALLOC_SIZE(x) +/** + * LIBXML_ATTR_FORMAT: + * + * Macro used to indicate to GCC the parameter are printf like + */ +#define LIBXML_ATTR_FORMAT(fmt,args) +#endif /* __GNUC__ */ + +#ifdef __cplusplus +} +#endif /* __cplusplus */ +#endif + + diff --git a/contrib/protobuf b/contrib/protobuf new file mode 160000 index 00000000000..12735370922 --- /dev/null +++ b/contrib/protobuf @@ -0,0 +1 @@ +Subproject commit 12735370922a35f03999afff478e1c6d7aa917a4 diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index cce97e4a57e..f285a65356a 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -264,6 +264,11 @@ target_link_libraries(dbms PRIVATE ${OPENSSL_CRYPTO_LIBRARY} Threads::Threads) target_include_directories (dbms SYSTEM BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR}) target_include_directories (dbms SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR}) +if (USE_HDFS) + target_link_libraries (dbms PRIVATE ${HDFS3_LIBRARY}) + target_include_directories (dbms SYSTEM BEFORE PRIVATE ${HDFS3_INCLUDE_DIR}) +endif() + if (NOT USE_INTERNAL_LZ4_LIBRARY) target_include_directories (dbms SYSTEM BEFORE PRIVATE ${LZ4_INCLUDE_DIR}) endif () diff --git a/dbms/src/Common/config.h.in b/dbms/src/Common/config.h.in index 44aa9e9773d..302fc33c6b4 100644 --- a/dbms/src/Common/config.h.in +++ b/dbms/src/Common/config.h.in @@ -16,3 +16,4 @@ #cmakedefine01 USE_POCO_NETSSL #cmakedefine01 CLICKHOUSE_SPLIT_BINARY #cmakedefine01 USE_BASE64 +#cmakedefine01 USE_HDFS diff --git a/dbms/src/IO/ReadBufferFromHDFS.h b/dbms/src/IO/ReadBufferFromHDFS.h new file mode 100644 index 00000000000..21705c1ef30 --- /dev/null +++ b/dbms/src/IO/ReadBufferFromHDFS.h @@ -0,0 +1,96 @@ +#pragma once + +#include + +#if USE_HDFS +#include +#include +#include +#include + +#ifndef O_DIRECT +#define O_DIRECT 00040000 +#endif + +namespace DB +{ + namespace ErrorCodes + { + extern const int BAD_ARGUMENTS; + extern const int NETWORK_ERROR; + } + /** Accepts path to file and opens it, or pre-opened file descriptor. + * Closes file by himself (thus "owns" a file descriptor). + */ + class ReadBufferFromHDFS : public BufferWithOwnMemory + { + protected: + std::string hdfs_uri; + struct hdfsBuilder *builder; + hdfsFS fs; + hdfsFile fin; + public: + ReadBufferFromHDFS(const std::string & hdfs_name_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE) + : BufferWithOwnMemory(buf_size), hdfs_uri(hdfs_name_) , builder(hdfsNewBuilder()) + { + Poco::URI uri(hdfs_name_); + auto & host = uri.getHost(); + auto port = uri.getPort(); + auto & path = uri.getPath(); + if (host.empty() || port == 0 || path.empty()) + { + throw Exception("Illegal HDFS URI: " + hdfs_uri, ErrorCodes::BAD_ARGUMENTS); + } + // set read/connect timeout, default value in libhdfs3 is about 1 hour, and too large + /// TODO Allow to tune from query Settings. + hdfsBuilderConfSetStr(builder, "input.read.timeout", "60000"); // 1 min + hdfsBuilderConfSetStr(builder, "input.connect.timeout", "60000"); // 1 min + + hdfsBuilderSetNameNode(builder, host.c_str()); + hdfsBuilderSetNameNodePort(builder, port); + fs = hdfsBuilderConnect(builder); + + if (fs == nullptr) + { + throw Exception("Unable to connect to HDFS: " + String(hdfsGetLastError()), ErrorCodes::NETWORK_ERROR); + } + + fin = hdfsOpenFile(fs, path.c_str(), O_RDONLY, 0, 0, 0); + } + + ReadBufferFromHDFS(ReadBufferFromHDFS &&) = default; + + ~ReadBufferFromHDFS() override + { + close(); + hdfsFreeBuilder(builder); + } + + /// Close HDFS connection before destruction of object. + void close() + { + hdfsCloseFile(fs, fin); + } + + bool nextImpl() override + { + int bytes_read = hdfsRead(fs, fin, internal_buffer.begin(), internal_buffer.size()); + if (bytes_read < 0) + { + throw Exception("Fail to read HDFS file: " + hdfs_uri + " " + String(hdfsGetLastError()), ErrorCodes::NETWORK_ERROR); + } + + if (bytes_read) + working_buffer.resize(bytes_read); + else + return false; + return true; + } + + const std::string & getHDFSUri() const + { + return hdfs_uri; + } + }; +} +#endif diff --git a/dbms/src/Storages/StorageHDFS.cpp b/dbms/src/Storages/StorageHDFS.cpp new file mode 100644 index 00000000000..03c9626a582 --- /dev/null +++ b/dbms/src/Storages/StorageHDFS.cpp @@ -0,0 +1,179 @@ +#include + +#if USE_HDFS + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int NOT_IMPLEMENTED; + extern const int BAD_ARGUMENTS; +} + +StorageHDFS::StorageHDFS(const String & uri_, + const std::string & table_name_, + const String & format_name_, + const ColumnsDescription & columns_, + Context &) + : IStorage(columns_), uri(uri_), format_name(format_name_), table_name(table_name_) +{ +} + +namespace +{ + class StorageHDFSBlockInputStream : public IProfilingBlockInputStream + { + public: + StorageHDFSBlockInputStream(const String & uri, + const String & format, + const String & name_, + const Block & sample_block, + const Context & context, + size_t max_block_size) + : name(name_) + { + // Assume no query and fragment in uri, todo, add sanity check + String fuzzyFileNames; + String uriPrefix = uri.substr(0, uri.find_last_of('/')); + if (uriPrefix.length() == uri.length()) + { + fuzzyFileNames = uri; + uriPrefix.clear(); + } + else + { + uriPrefix += "/"; + fuzzyFileNames = uri.substr(uriPrefix.length()); + } + + std::vector fuzzyNameList = parseRemoteDescription(fuzzyFileNames, 0, fuzzyFileNames.length(), ',' , 100/* hard coded max files */); + + BlockInputStreams inputs; + + for (auto & name: fuzzyNameList) + { + std::unique_ptr read_buf = std::make_unique(uriPrefix + name); + + inputs.emplace_back( + std::make_shared>( + FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size), + std::move(read_buf))); + } + + if (inputs.size() == 0) + throw Exception("StorageHDFS inputs interpreter error", ErrorCodes::BAD_ARGUMENTS); + + if (inputs.size() == 1) + { + reader = inputs[0]; + } + else + { + reader = std::make_shared(inputs, nullptr, context.getSettingsRef().max_distributed_connections); + } + } + + String getName() const override + { + return name; + } + + Block readImpl() override + { + return reader->read(); + } + + Block getHeader() const override + { + return reader->getHeader(); + } + + void readPrefixImpl() override + { + reader->readPrefix(); + } + + void readSuffixImpl() override + { + auto explicitReader = dynamic_cast(reader.get()); + if (explicitReader) explicitReader->cancel(false); // skip Union read suffix assertion + + reader->readSuffix(); + } + + private: + String name; + BlockInputStreamPtr reader; + }; + +} + + +BlockInputStreams StorageHDFS::read( + const Names & /*column_names*/, + const SelectQueryInfo & /*query_info*/, + const Context & context, + QueryProcessingStage::Enum /*processed_stage*/, + size_t max_block_size, + unsigned /*num_streams*/) +{ + return {std::make_shared( + uri, + format_name, + getName(), + getSampleBlock(), + context, + max_block_size)}; +} + +void StorageHDFS::rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & /*new_table_name*/) {} + +BlockOutputStreamPtr StorageHDFS::write(const ASTPtr & /*query*/, const Settings & /*settings*/) +{ + throw Exception("StorageHDFS write is not supported yet", ErrorCodes::NOT_IMPLEMENTED); + return {}; +} + +void registerStorageHDFS(StorageFactory & factory) +{ + factory.registerStorage("HDFS", [](const StorageFactory::Arguments & args) + { + ASTs & engine_args = args.engine_args; + + if (!(engine_args.size() == 1 || engine_args.size() == 2)) + throw Exception( + "Storage HDFS requires exactly 2 arguments: url and name of used format.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + engine_args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[0], args.local_context); + + String url = static_cast(*engine_args[0]).value.safeGet(); + + engine_args[1] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[1], args.local_context); + + String format_name = static_cast(*engine_args[1]).value.safeGet(); + + return StorageHDFS::create(url, args.table_name, format_name, args.columns, args.context); + }); +} + +} + +#endif diff --git a/dbms/src/Storages/StorageHDFS.h b/dbms/src/Storages/StorageHDFS.h new file mode 100644 index 00000000000..44ff23c4d67 --- /dev/null +++ b/dbms/src/Storages/StorageHDFS.h @@ -0,0 +1,56 @@ +#pragma once +#include +#if USE_HDFS + +#include +#include +#include +#include + +namespace DB +{ +/** + * This class represents table engine for external hdfs files. + * Read method is supported for now. + */ +class StorageHDFS : public ext::shared_ptr_helper, public IStorage +{ +public: + String getName() const override + { + return "HDFS"; + } + + String getTableName() const override + { + return table_name; + } + + BlockInputStreams read(const Names & column_names, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + unsigned num_streams) override; + + BlockOutputStreamPtr write(const ASTPtr & query, const Settings & settings) override; + + void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override; + +protected: + StorageHDFS(const String & uri_, + const String & table_name_, + const String & format_name_, + const ColumnsDescription & columns_, + Context & context_); + +private: + String uri; + String format_name; + String table_name; + + Logger * log = &Logger::get("StorageHDFS"); +}; +} + +#endif diff --git a/dbms/src/Storages/registerStorages.cpp b/dbms/src/Storages/registerStorages.cpp index ce831fbb758..4bd2d995104 100644 --- a/dbms/src/Storages/registerStorages.cpp +++ b/dbms/src/Storages/registerStorages.cpp @@ -24,6 +24,10 @@ void registerStorageJoin(StorageFactory & factory); void registerStorageView(StorageFactory & factory); void registerStorageMaterializedView(StorageFactory & factory); +#if USE_HDFS +void registerStorageHDFS(StorageFactory & factory); +#endif + #if USE_POCO_SQLODBC || USE_POCO_DATAODBC void registerStorageODBC(StorageFactory & factory); #endif @@ -60,6 +64,10 @@ void registerStorages() registerStorageView(factory); registerStorageMaterializedView(factory); + #if USE_HDFS + registerStorageHDFS(factory); + #endif + #if USE_POCO_SQLODBC || USE_POCO_DATAODBC registerStorageODBC(factory); #endif diff --git a/dbms/src/TableFunctions/ITableFunction.h b/dbms/src/TableFunctions/ITableFunction.h index a3885f16152..ddf900fa65c 100644 --- a/dbms/src/TableFunctions/ITableFunction.h +++ b/dbms/src/TableFunctions/ITableFunction.h @@ -3,7 +3,6 @@ #include #include - namespace DB { diff --git a/dbms/src/TableFunctions/TableFunctionHDFS.cpp b/dbms/src/TableFunctions/TableFunctionHDFS.cpp new file mode 100644 index 00000000000..9c09ad9313c --- /dev/null +++ b/dbms/src/TableFunctions/TableFunctionHDFS.cpp @@ -0,0 +1,25 @@ +#include + +#if USE_HDFS +#include +#include +#include + +namespace DB +{ +StoragePtr TableFunctionHDFS::getStorage( + const String & source, const String & format, const Block & sample_block, Context & global_context) const +{ + return StorageHDFS::create(source, + getName(), + format, + ColumnsDescription{sample_block.getNamesAndTypesList()}, + global_context); +} + +void registerTableFunctionHDFS(TableFunctionFactory & factory) +{ + factory.registerFunction(); +} +} +#endif diff --git a/dbms/src/TableFunctions/TableFunctionHDFS.h b/dbms/src/TableFunctions/TableFunctionHDFS.h new file mode 100644 index 00000000000..8033034deb8 --- /dev/null +++ b/dbms/src/TableFunctions/TableFunctionHDFS.h @@ -0,0 +1,32 @@ +#pragma once + +#include + +#if USE_HDFS + +#include +#include +#include + + +namespace DB +{ +/* hdfs(name_node_ip:name_node_port, format, structure) - creates a temporary storage from hdfs file + * + */ +class TableFunctionHDFS : public ITableFunctionFileLike +{ +public: + static constexpr auto name = "hdfs"; + std::string getName() const override + { + return name; + } + +private: + StoragePtr getStorage( + const String & source, const String & format, const Block & sample_block, Context & global_context) const override; +}; +} + +#endif diff --git a/dbms/src/TableFunctions/TableFunctionRemote.cpp b/dbms/src/TableFunctions/TableFunctionRemote.cpp index 1478f8960a8..b93a1638d48 100644 --- a/dbms/src/TableFunctions/TableFunctionRemote.cpp +++ b/dbms/src/TableFunctions/TableFunctionRemote.cpp @@ -11,6 +11,7 @@ #include #include +#include namespace DB @@ -22,165 +23,6 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } - -/// The Cartesian product of two sets of rows, the result is written in place of the first argument -static void append(std::vector & to, const std::vector & what, size_t max_addresses) -{ - if (what.empty()) - return; - - if (to.empty()) - { - to = what; - return; - } - - if (what.size() * to.size() > max_addresses) - throw Exception("Table function 'remote': first argument generates too many result addresses", - ErrorCodes::BAD_ARGUMENTS); - std::vector res; - for (size_t i = 0; i < to.size(); ++i) - for (size_t j = 0; j < what.size(); ++j) - res.push_back(to[i] + what[j]); - - to.swap(res); -} - - -/// Parse number from substring -static bool parseNumber(const String & description, size_t l, size_t r, size_t & res) -{ - res = 0; - for (size_t pos = l; pos < r; pos ++) - { - if (!isNumericASCII(description[pos])) - return false; - res = res * 10 + description[pos] - '0'; - if (res > 1e15) - return false; - } - return true; -} - - - -/* Parse a string that generates shards and replicas. Separator - one of two characters | or , - * depending on whether shards or replicas are generated. - * For example: - * host1,host2,... - generates set of shards from host1, host2, ... - * host1|host2|... - generates set of replicas from host1, host2, ... - * abc{8..10}def - generates set of shards abc8def, abc9def, abc10def. - * abc{08..10}def - generates set of shards abc08def, abc09def, abc10def. - * abc{x,yy,z}def - generates set of shards abcxdef, abcyydef, abczdef. - * abc{x|yy|z} def - generates set of replicas abcxdef, abcyydef, abczdef. - * abc{1..9}de{f,g,h} - is a direct product, 27 shards. - * abc{1..9}de{0|1} - is a direct product, 9 shards, in each 2 replicas. - */ -static std::vector parseDescription(const String & description, size_t l, size_t r, char separator, size_t max_addresses) -{ - std::vector res; - std::vector cur; - - /// An empty substring means a set of an empty string - if (l >= r) - { - res.push_back(""); - return res; - } - - for (size_t i = l; i < r; ++i) - { - /// Either the numeric interval (8..10) or equivalent expression in brackets - if (description[i] == '{') - { - int cnt = 1; - int last_dot = -1; /// The rightmost pair of points, remember the index of the right of the two - size_t m; - std::vector buffer; - bool have_splitter = false; - - /// Look for the corresponding closing bracket - for (m = i + 1; m < r; ++m) - { - if (description[m] == '{') ++cnt; - if (description[m] == '}') --cnt; - if (description[m] == '.' && description[m-1] == '.') last_dot = m; - if (description[m] == separator) have_splitter = true; - if (cnt == 0) break; - } - if (cnt != 0) - throw Exception("Table function 'remote': incorrect brace sequence in first argument", - ErrorCodes::BAD_ARGUMENTS); - /// The presence of a dot - numeric interval - if (last_dot != -1) - { - size_t left, right; - if (description[last_dot - 1] != '.') - throw Exception("Table function 'remote': incorrect argument in braces (only one dot): " + description.substr(i, m - i + 1), - ErrorCodes::BAD_ARGUMENTS); - if (!parseNumber(description, i + 1, last_dot - 1, left)) - throw Exception("Table function 'remote': incorrect argument in braces (Incorrect left number): " - + description.substr(i, m - i + 1), - ErrorCodes::BAD_ARGUMENTS); - if (!parseNumber(description, last_dot + 1, m, right)) - throw Exception("Table function 'remote': incorrect argument in braces (Incorrect right number): " - + description.substr(i, m - i + 1), - ErrorCodes::BAD_ARGUMENTS); - if (left > right) - throw Exception("Table function 'remote': incorrect argument in braces (left number is greater then right): " - + description.substr(i, m - i + 1), - ErrorCodes::BAD_ARGUMENTS); - if (right - left + 1 > max_addresses) - throw Exception("Table function 'remote': first argument generates too many result addresses", - ErrorCodes::BAD_ARGUMENTS); - bool add_leading_zeroes = false; - size_t len = last_dot - 1 - (i + 1); - /// If the left and right borders have equal numbers, then you must add leading zeros. - if (last_dot - 1 - (i + 1) == m - (last_dot + 1)) - add_leading_zeroes = true; - for (size_t id = left; id <= right; ++id) - { - String cur = toString(id); - if (add_leading_zeroes) - { - while (cur.size() < len) - cur = "0" + cur; - } - buffer.push_back(cur); - } - } - else if (have_splitter) /// If there is a current delimiter inside, then generate a set of resulting rows - buffer = parseDescription(description, i + 1, m, separator, max_addresses); - else /// Otherwise just copy, spawn will occur when you call with the correct delimiter - buffer.push_back(description.substr(i, m - i + 1)); - /// Add all possible received extensions to the current set of lines - append(cur, buffer, max_addresses); - i = m; - } - else if (description[i] == separator) - { - /// If the delimiter, then add found rows - res.insert(res.end(), cur.begin(), cur.end()); - cur.clear(); - } - else - { - /// Otherwise, simply append the character to current lines - std::vector buffer; - buffer.push_back(description.substr(i, 1)); - append(cur, buffer, max_addresses); - } - } - - res.insert(res.end(), cur.begin(), cur.end()); - if (res.size() > max_addresses) - throw Exception("Table function 'remote': first argument generates too many result addresses", - ErrorCodes::BAD_ARGUMENTS); - - return res; -} - - StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & ast_function, const Context & context) const { ASTs & args_func = typeid_cast(*ast_function).children; @@ -304,11 +146,11 @@ StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & ast_function, const C { /// Create new cluster from the scratch size_t max_addresses = context.getSettingsRef().table_function_remote_max_addresses; - std::vector shards = parseDescription(cluster_description, 0, cluster_description.size(), ',', max_addresses); + std::vector shards = parseRemoteDescription(cluster_description, 0, cluster_description.size(), ',', max_addresses); std::vector> names; for (size_t i = 0; i < shards.size(); ++i) - names.push_back(parseDescription(shards[i], 0, shards[i].size(), '|', max_addresses)); + names.push_back(parseRemoteDescription(shards[i], 0, shards[i].size(), '|', max_addresses)); if (names.empty()) throw Exception("Shard list is empty after parsing first argument", ErrorCodes::BAD_ARGUMENTS); diff --git a/dbms/src/TableFunctions/parseRemoteDescription.cpp b/dbms/src/TableFunctions/parseRemoteDescription.cpp new file mode 100644 index 00000000000..d903fe72f03 --- /dev/null +++ b/dbms/src/TableFunctions/parseRemoteDescription.cpp @@ -0,0 +1,171 @@ +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int BAD_ARGUMENTS; +} + +/// The Cartesian product of two sets of rows, the result is written in place of the first argument +static void append(std::vector & to, const std::vector & what, size_t max_addresses) +{ + if (what.empty()) + return; + + if (to.empty()) + { + to = what; + return; + } + + if (what.size() * to.size() > max_addresses) + throw Exception("Table function 'remote': first argument generates too many result addresses", + ErrorCodes::BAD_ARGUMENTS); + std::vector res; + for (size_t i = 0; i < to.size(); ++i) + for (size_t j = 0; j < what.size(); ++j) + res.push_back(to[i] + what[j]); + + to.swap(res); +} + + +/// Parse number from substring +static bool parseNumber(const String & description, size_t l, size_t r, size_t & res) +{ + res = 0; + for (size_t pos = l; pos < r; pos ++) + { + if (!isNumericASCII(description[pos])) + return false; + res = res * 10 + description[pos] - '0'; + if (res > 1e15) + return false; + } + return true; +} + + + +/* Parse a string that generates shards and replicas. Separator - one of two characters | or , + * depending on whether shards or replicas are generated. + * For example: + * host1,host2,... - generates set of shards from host1, host2, ... + * host1|host2|... - generates set of replicas from host1, host2, ... + * abc{8..10}def - generates set of shards abc8def, abc9def, abc10def. + * abc{08..10}def - generates set of shards abc08def, abc09def, abc10def. + * abc{x,yy,z}def - generates set of shards abcxdef, abcyydef, abczdef. + * abc{x|yy|z} def - generates set of replicas abcxdef, abcyydef, abczdef. + * abc{1..9}de{f,g,h} - is a direct product, 27 shards. + * abc{1..9}de{0|1} - is a direct product, 9 shards, in each 2 replicas. + */ +std::vector parseRemoteDescription(const String & description, size_t l, size_t r, char separator, size_t max_addresses) +{ + std::vector res; + std::vector cur; + + /// An empty substring means a set of an empty string + if (l >= r) + { + res.push_back(""); + return res; + } + + for (size_t i = l; i < r; ++i) + { + /// Either the numeric interval (8..10) or equivalent expression in brackets + if (description[i] == '{') + { + int cnt = 1; + int last_dot = -1; /// The rightmost pair of points, remember the index of the right of the two + size_t m; + std::vector buffer; + bool have_splitter = false; + + /// Look for the corresponding closing bracket + for (m = i + 1; m < r; ++m) + { + if (description[m] == '{') ++cnt; + if (description[m] == '}') --cnt; + if (description[m] == '.' && description[m-1] == '.') last_dot = m; + if (description[m] == separator) have_splitter = true; + if (cnt == 0) break; + } + if (cnt != 0) + throw Exception("Table function 'remote': incorrect brace sequence in first argument", + ErrorCodes::BAD_ARGUMENTS); + /// The presence of a dot - numeric interval + if (last_dot != -1) + { + size_t left, right; + if (description[last_dot - 1] != '.') + throw Exception("Table function 'remote': incorrect argument in braces (only one dot): " + description.substr(i, m - i + 1), + ErrorCodes::BAD_ARGUMENTS); + if (!parseNumber(description, i + 1, last_dot - 1, left)) + throw Exception("Table function 'remote': incorrect argument in braces (Incorrect left number): " + + description.substr(i, m - i + 1), + ErrorCodes::BAD_ARGUMENTS); + if (!parseNumber(description, last_dot + 1, m, right)) + throw Exception("Table function 'remote': incorrect argument in braces (Incorrect right number): " + + description.substr(i, m - i + 1), + ErrorCodes::BAD_ARGUMENTS); + if (left > right) + throw Exception("Table function 'remote': incorrect argument in braces (left number is greater then right): " + + description.substr(i, m - i + 1), + ErrorCodes::BAD_ARGUMENTS); + if (right - left + 1 > max_addresses) + throw Exception("Table function 'remote': first argument generates too many result addresses", + ErrorCodes::BAD_ARGUMENTS); + bool add_leading_zeroes = false; + size_t len = last_dot - 1 - (i + 1); + /// If the left and right borders have equal numbers, then you must add leading zeros. + if (last_dot - 1 - (i + 1) == m - (last_dot + 1)) + add_leading_zeroes = true; + for (size_t id = left; id <= right; ++id) + { + String cur = toString(id); + if (add_leading_zeroes) + { + while (cur.size() < len) + cur = "0" + cur; + } + buffer.push_back(cur); + } + } + else if (have_splitter) /// If there is a current delimiter inside, then generate a set of resulting rows + buffer = parseRemoteDescription(description, i + 1, m, separator, max_addresses); + else /// Otherwise just copy, spawn will occur when you call with the correct delimiter + buffer.push_back(description.substr(i, m - i + 1)); + /// Add all possible received extensions to the current set of lines + append(cur, buffer, max_addresses); + i = m; + } + else if (description[i] == separator) + { + /// If the delimiter, then add found rows + res.insert(res.end(), cur.begin(), cur.end()); + cur.clear(); + } + else + { + /// Otherwise, simply append the character to current lines + std::vector buffer; + buffer.push_back(description.substr(i, 1)); + append(cur, buffer, max_addresses); + } + } + + res.insert(res.end(), cur.begin(), cur.end()); + if (res.size() > max_addresses) + throw Exception("Table function 'remote': first argument generates too many result addresses", + ErrorCodes::BAD_ARGUMENTS); + + return res; +} + +} diff --git a/dbms/src/TableFunctions/parseRemoteDescription.h b/dbms/src/TableFunctions/parseRemoteDescription.h new file mode 100644 index 00000000000..cbc73380628 --- /dev/null +++ b/dbms/src/TableFunctions/parseRemoteDescription.h @@ -0,0 +1,20 @@ +#pragma once +#include +#include +namespace DB +{ +/* Parse a string that generates shards and replicas. Separator - one of two characters | or , + * depending on whether shards or replicas are generated. + * For example: + * host1,host2,... - generates set of shards from host1, host2, ... + * host1|host2|... - generates set of replicas from host1, host2, ... + * abc{8..10}def - generates set of shards abc8def, abc9def, abc10def. + * abc{08..10}def - generates set of shards abc08def, abc09def, abc10def. + * abc{x,yy,z}def - generates set of shards abcxdef, abcyydef, abczdef. + * abc{x|yy|z} def - generates set of replicas abcxdef, abcyydef, abczdef. + * abc{1..9}de{f,g,h} - is a direct product, 27 shards. + * abc{1..9}de{0|1} - is a direct product, 9 shards, in each 2 replicas. + */ +std::vector parseRemoteDescription(const String & description, size_t l, size_t r, char separator, size_t max_addresses); + +} diff --git a/dbms/src/TableFunctions/registerTableFunctions.cpp b/dbms/src/TableFunctions/registerTableFunctions.cpp index d0afeff0b17..8974dcd53fe 100644 --- a/dbms/src/TableFunctions/registerTableFunctions.cpp +++ b/dbms/src/TableFunctions/registerTableFunctions.cpp @@ -14,6 +14,10 @@ void registerTableFunctionCatBoostPool(TableFunctionFactory & factory); void registerTableFunctionFile(TableFunctionFactory & factory); void registerTableFunctionURL(TableFunctionFactory & factory); +#if USE_HDFS +void registerTableFunctionHDFS(TableFunctionFactory & factory); +#endif + #if USE_POCO_SQLODBC || USE_POCO_DATAODBC void registerTableFunctionODBC(TableFunctionFactory & factory); #endif @@ -37,6 +41,10 @@ void registerTableFunctions() registerTableFunctionFile(factory); registerTableFunctionURL(factory); +#if USE_HDFS + registerTableFunctionHDFS(factory); +#endif + #if USE_POCO_SQLODBC || USE_POCO_DATAODBC registerTableFunctionODBC(factory); #endif diff --git a/dbms/tests/integration/helpers/cluster.py b/dbms/tests/integration/helpers/cluster.py index 39227369c2a..22d34d05844 100644 --- a/dbms/tests/integration/helpers/cluster.py +++ b/dbms/tests/integration/helpers/cluster.py @@ -14,11 +14,13 @@ import xml.dom.minidom from kazoo.client import KazooClient from kazoo.exceptions import KazooException import psycopg2 +import requests import docker from docker.errors import ContainerError from .client import Client, CommandRequest +from .hdfs_api import HDFSApi HELPERS_DIR = p.dirname(__file__) @@ -83,6 +85,7 @@ class ClickHouseCluster: self.with_postgres = False self.with_kafka = False self.with_odbc_drivers = False + self.with_hdfs = False self.docker_client = None self.is_up = False @@ -94,7 +97,7 @@ class ClickHouseCluster: cmd += " client" return cmd - def add_instance(self, name, config_dir=None, main_configs=[], user_configs=[], macros={}, with_zookeeper=False, with_mysql=False, with_kafka=False, clickhouse_path_dir=None, with_odbc_drivers=False, with_postgres=False, hostname=None, env_variables={}, image="yandex/clickhouse-integration-test", stay_alive=False): + def add_instance(self, name, config_dir=None, main_configs=[], user_configs=[], macros={}, with_zookeeper=False, with_mysql=False, with_kafka=False, clickhouse_path_dir=None, with_odbc_drivers=False, with_postgres=False, with_hdfs=False, hostname=None, env_variables={}, image="yandex/clickhouse-integration-test", stay_alive=False): """Add an instance to the cluster. name - the name of the instance directory and the value of the 'instance' macro in ClickHouse. @@ -148,13 +151,19 @@ class ClickHouseCluster: self.base_postgres_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', self.project_name, '--file', p.join(HELPERS_DIR, 'docker_compose_postgres.yml')] - if with_kafka and not self.with_kafka: self.with_kafka = True self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_kafka.yml')]) self.base_kafka_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', self.project_name, '--file', p.join(HELPERS_DIR, 'docker_compose_kafka.yml')] + if with_hdfs and not self.with_hdfs: + self.with_hdfs = True + self.base_cmd.extend(['--file', p.join(HELPERS_DIR, 'docker_compose_hdfs.yml')]) + self.base_hdfs_cmd = ['docker-compose', '--project-directory', self.base_dir, '--project-name', + self.project_name, '--file', p.join(HELPERS_DIR, 'docker_compose_hdfs.yml')] + + return instance @@ -212,6 +221,20 @@ class ClickHouseCluster: raise Exception("Cannot wait ZooKeeper container") + def wait_hdfs_to_start(self, timeout=60): + hdfs_api = HDFSApi("root") + start = time.time() + while time.time() - start < timeout: + try: + hdfs_api.write_data("/somefilewithrandomname222", "1") + print "Connected to HDFS and SafeMode disabled! " + return + except Exception as ex: + print "Can't connect to HDFS " + str(ex) + time.sleep(1) + + raise Exception("Can't wait HDFS to start") + def start(self, destroy_dirs=True): if self.is_up: return @@ -250,7 +273,11 @@ class ClickHouseCluster: subprocess_check_call(self.base_kafka_cmd + ['up', '-d', '--force-recreate']) self.kafka_docker_id = self.get_instance_docker_id('kafka1') - subprocess_check_call(self.base_cmd + ['up', '-d', '--force-recreate']) + if self.with_hdfs and self.base_hdfs_cmd: + subprocess_check_call(self.base_hdfs_cmd + ['up', '-d', '--force-recreate']) + self.wait_hdfs_to_start(120) + + subprocess_check_call(self.base_cmd + ['up', '-d', '--no-recreate']) start_deadline = time.time() + 20.0 # seconds for instance in self.instances.itervalues(): @@ -310,7 +337,6 @@ services: {name}: image: {image} hostname: {hostname} - user: '{uid}' volumes: - {binary_path}:/usr/bin/clickhouse:ro - {configs_dir}:/etc/clickhouse-server/ @@ -588,7 +614,6 @@ class ClickHouseInstance: image=self.image, name=self.name, hostname=self.hostname, - uid=os.getuid(), binary_path=self.server_bin_path, configs_dir=configs_dir, config_d_dir=config_d_dir, diff --git a/dbms/tests/integration/helpers/docker_compose_hdfs.yml b/dbms/tests/integration/helpers/docker_compose_hdfs.yml new file mode 100644 index 00000000000..ecfb0c329b3 --- /dev/null +++ b/dbms/tests/integration/helpers/docker_compose_hdfs.yml @@ -0,0 +1,9 @@ +version: '2' +services: + hdfs1: + image: sequenceiq/hadoop-docker:2.7.0 + restart: always + ports: + - 50075:50075 + - 50070:50070 + entrypoint: /etc/bootstrap.sh -d diff --git a/dbms/tests/integration/helpers/hdfs_api.py b/dbms/tests/integration/helpers/hdfs_api.py new file mode 100644 index 00000000000..989d66ee1e3 --- /dev/null +++ b/dbms/tests/integration/helpers/hdfs_api.py @@ -0,0 +1,46 @@ +#-*- coding: utf-8 -*- +import requests +import subprocess +from tempfile import NamedTemporaryFile + +class HDFSApi(object): + def __init__(self, user): + self.host = "localhost" + self.http_proxy_port = "50070" + self.http_data_port = "50075" + self.user = user + + def read_data(self, path): + response = requests.get("http://{host}:{port}/webhdfs/v1{path}?op=OPEN".format(host=self.host, port=self.http_proxy_port, path=path), allow_redirects=False) + if response.status_code != 307: + response.raise_for_status() + additional_params = '&'.join(response.headers['Location'].split('&')[1:2]) + response_data = requests.get("http://{host}:{port}/webhdfs/v1{path}?op=OPEN&{params}".format(host=self.host, port=self.http_data_port, path=path, params=additional_params)) + if response_data.status_code != 200: + response_data.raise_for_status() + + return response_data.text + + # Requests can't put file + def _curl_to_put(self, filename, path, params): + url = "http://{host}:{port}/webhdfs/v1{path}?op=CREATE&{params}".format(host=self.host, port=self.http_data_port, path=path, params=params) + cmd = "curl -s -i -X PUT -T {fname} '{url}'".format(fname=filename, url=url) + output = subprocess.check_output(cmd, shell=True) + return output + + def write_data(self, path, content): + named_file = NamedTemporaryFile() + fpath = named_file.name + named_file.write(content) + named_file.flush() + response = requests.put( + "http://{host}:{port}/webhdfs/v1{path}?op=CREATE".format(host=self.host, port=self.http_proxy_port, path=path, user=self.user), + allow_redirects=False + ) + if response.status_code != 307: + response.raise_for_status() + + additional_params = '&'.join(response.headers['Location'].split('&')[1:2] + ["user.name={}".format(self.user), "overwrite=true"]) + output = self._curl_to_put(fpath, path, additional_params) + if "201 Created" not in output: + raise Exception("Can't create file on hdfs:\n {}".format(output)) diff --git a/dbms/tests/integration/test_storage_hdfs/__init__.py b/dbms/tests/integration/test_storage_hdfs/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_storage_hdfs/configs/log_conf.xml b/dbms/tests/integration/test_storage_hdfs/configs/log_conf.xml new file mode 100644 index 00000000000..0de2745ca4c --- /dev/null +++ b/dbms/tests/integration/test_storage_hdfs/configs/log_conf.xml @@ -0,0 +1,11 @@ + + + trace + /var/log/clickhouse-server/log.log + /var/log/clickhouse-server/log.err.log + 1000M + 10 + /var/log/clickhouse-server/stderr.log + /var/log/clickhouse-server/stdout.log + + diff --git a/dbms/tests/integration/test_storage_hdfs/test.py b/dbms/tests/integration/test_storage_hdfs/test.py new file mode 100644 index 00000000000..76b35cc7bed --- /dev/null +++ b/dbms/tests/integration/test_storage_hdfs/test.py @@ -0,0 +1,47 @@ +import time +import pytest +import requests +from tempfile import NamedTemporaryFile +from helpers.hdfs_api import HDFSApi + +import os + +from helpers.cluster import ClickHouseCluster +import subprocess + + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance('node1', with_hdfs=True, image='withlibsimage', config_dir="configs", main_configs=['configs/log_conf.xml']) + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + yield cluster + + except Exception as ex: + print(ex) + raise ex + finally: + cluster.shutdown() + +def test_read_write_storage(started_cluster): + + hdfs_api = HDFSApi("root") + hdfs_api.write_data("/simple_storage", "1\tMark\t72.53\n") + + assert hdfs_api.read_data("/simple_storage") == "1\tMark\t72.53\n" + + node1.query("create table SimpleHDFSStorage (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs1:9000/simple_storage', 'TSV')") + assert node1.query("select * from SimpleHDFSStorage") == "1\tMark\t72.53\n" + +def test_read_write_table(started_cluster): + hdfs_api = HDFSApi("root") + data = "1\tSerialize\t555.222\n2\tData\t777.333\n" + hdfs_api.write_data("/simple_table_function", data) + + assert hdfs_api.read_data("/simple_table_function") == data + + assert node1.query("select * from hdfs('hdfs://hdfs1:9000/simple_table_function', 'TSV', 'id UInt64, text String, number Float64')") == data diff --git a/release b/release index 23bfd6f2dd6..4c536609571 100755 --- a/release +++ b/release @@ -64,7 +64,7 @@ do shift elif [[ $1 == '--fast' ]]; then # Wrong but fast pbuilder mode: create base package with all depends - EXTRAPACKAGES="$EXTRAPACKAGES debhelper cmake ninja-build gcc-7 g++-7 libc6-dev libicu-dev libreadline-dev psmisc bash expect python python-lxml python-termcolor python-requests curl perl sudo openssl netcat-openbsd" + EXTRAPACKAGES="$EXTRAPACKAGES debhelper cmake ninja-build gcc-7 g++-7 libc6-dev libicu-dev libreadline-dev psmisc bash expect python python-lxml python-termcolor python-requests curl perl sudo openssl netcat-openbsd uuid xml2 krb5 gsasl" shift else echo "Unknown option $1"