From 76843c2dc02197640d43c0b98723c2d72795bf2b Mon Sep 17 00:00:00 2001 From: "chenxing.xc" Date: Mon, 19 Nov 2018 16:17:09 +0800 Subject: [PATCH] hdfs read support --- .gitmodules | 12 +- CMakeLists.txt | 1 + cmake/find_hdfs3.cmake | 13 + cmake/find_llvm.cmake | 2 +- cmake/find_protobuf.cmake | 80 ++ contrib/CMakeLists.txt | 9 + contrib/boost | 2 +- contrib/libhdfs3 | 1 + .../CMake/CMakeTestCompileNestedException.cpp | 10 + .../CMake/CMakeTestCompileSteadyClock.cpp | 7 + .../CMake/CMakeTestCompileStrerror.cpp | 10 + .../libhdfs3-cmake/CMake/CodeCoverage.cmake | 48 + contrib/libhdfs3-cmake/CMake/FindBoost.cmake | 1162 +++++++++++++++++ contrib/libhdfs3-cmake/CMake/FindCurl.cmake | 26 + contrib/libhdfs3-cmake/CMake/FindGSasl.cmake | 26 + .../libhdfs3-cmake/CMake/FindGoogleTest.cmake | 65 + .../libhdfs3-cmake/CMake/FindKERBEROS.cmake | 23 + .../libhdfs3-cmake/CMake/FindLibUUID.cmake | 23 + contrib/libhdfs3-cmake/CMake/FindSSL.cmake | 26 + contrib/libhdfs3-cmake/CMake/Functions.cmake | 46 + contrib/libhdfs3-cmake/CMake/Options.cmake | 169 +++ contrib/libhdfs3-cmake/CMake/Platform.cmake | 33 + contrib/libhdfs3-cmake/CMakeLists.txt | 224 ++++ contrib/protobuf | 1 + dbms/CMakeLists.txt | 1 + dbms/programs/client/Client.cpp | 2 +- dbms/src/IO/ReadBufferFromHDFS.h | 87 ++ .../Interpreters/InterpreterInsertQuery.cpp | 85 ++ dbms/src/Parsers/ASTInsertQuery.cpp | 15 +- dbms/src/Parsers/ASTInsertQuery.h | 6 + dbms/src/Parsers/ParserInsertQuery.cpp | 68 +- dbms/src/Parsers/ParserInsertQuery.h | 9 +- dbms/src/Storages/StorageHDFS.cpp | 178 +++ dbms/src/Storages/StorageHDFS.h | 53 + dbms/src/TableFunctions/ITableFunction.cpp | 166 +++ dbms/src/TableFunctions/ITableFunction.h | 3 + dbms/src/TableFunctions/TableFunctionHDFS.cpp | 21 + dbms/src/TableFunctions/TableFunctionHDFS.h | 26 + .../TableFunctions/TableFunctionRemote.cpp | 159 --- .../TableFunctions/registerTableFunctions.cpp | 2 + release | 2 +- 41 files changed, 2713 insertions(+), 189 deletions(-) create mode 100644 cmake/find_hdfs3.cmake create mode 100644 cmake/find_protobuf.cmake create mode 160000 contrib/libhdfs3 create mode 100644 contrib/libhdfs3-cmake/CMake/CMakeTestCompileNestedException.cpp create mode 100644 contrib/libhdfs3-cmake/CMake/CMakeTestCompileSteadyClock.cpp create mode 100644 contrib/libhdfs3-cmake/CMake/CMakeTestCompileStrerror.cpp create mode 100644 contrib/libhdfs3-cmake/CMake/CodeCoverage.cmake create mode 100644 contrib/libhdfs3-cmake/CMake/FindBoost.cmake create mode 100644 contrib/libhdfs3-cmake/CMake/FindCurl.cmake create mode 100644 contrib/libhdfs3-cmake/CMake/FindGSasl.cmake create mode 100644 contrib/libhdfs3-cmake/CMake/FindGoogleTest.cmake create mode 100644 contrib/libhdfs3-cmake/CMake/FindKERBEROS.cmake create mode 100644 contrib/libhdfs3-cmake/CMake/FindLibUUID.cmake create mode 100644 contrib/libhdfs3-cmake/CMake/FindSSL.cmake create mode 100644 contrib/libhdfs3-cmake/CMake/Functions.cmake create mode 100644 contrib/libhdfs3-cmake/CMake/Options.cmake create mode 100644 contrib/libhdfs3-cmake/CMake/Platform.cmake create mode 100644 contrib/libhdfs3-cmake/CMakeLists.txt create mode 160000 contrib/protobuf create mode 100644 dbms/src/IO/ReadBufferFromHDFS.h create mode 100644 dbms/src/Storages/StorageHDFS.cpp create mode 100644 dbms/src/Storages/StorageHDFS.h create mode 100644 dbms/src/TableFunctions/TableFunctionHDFS.cpp create mode 100644 dbms/src/TableFunctions/TableFunctionHDFS.h diff --git a/.gitmodules b/.gitmodules index c43b754dba8..bff1289e443 100644 --- a/.gitmodules +++ b/.gitmodules @@ -31,9 +31,6 @@ [submodule "contrib/ssl"] path = contrib/ssl url = https://github.com/ClickHouse-Extras/ssl.git -[submodule "contrib/boost"] - path = contrib/boost - url = https://github.com/ClickHouse-Extras/boost.git [submodule "contrib/llvm"] path = contrib/llvm url = https://github.com/ClickHouse-Extras/llvm @@ -46,3 +43,12 @@ [submodule "contrib/unixodbc"] path = contrib/unixodbc url = https://github.com/ClickHouse-Extras/UnixODBC.git +[submodule "contrib/libhdfs3"] + path = contrib/libhdfs3 + url = https://github.com/chenxing-xc/ClickHouse-Extras-libhdfs3.git +[submodule "contrib/protobuf"] + path = contrib/protobuf + url = https://github.com/chenxing-xc/ClickHouse-Extras-protobuf.git +[submodule "contrib/boost"] + path = contrib/boost + url = https://github.com/chenxing-xc/ClickHouse-Extras-boost.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 552ec57c8fd..833cddef772 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -255,6 +255,7 @@ include (cmake/find_rdkafka.cmake) include (cmake/find_capnp.cmake) include (cmake/find_llvm.cmake) include (cmake/find_cpuid.cmake) +include (cmake/find_hdfs3.cmake) include (cmake/find_consistent-hashing.cmake) if (ENABLE_TESTS) include (cmake/find_gtest.cmake) diff --git a/cmake/find_hdfs3.cmake b/cmake/find_hdfs3.cmake new file mode 100644 index 00000000000..5d9c3289c22 --- /dev/null +++ b/cmake/find_hdfs3.cmake @@ -0,0 +1,13 @@ +option (USE_INTERNAL_HDFS3_LIBRARY "Set to FALSE to use system HDFS3 instead of bundled" ON) + +if (NOT USE_INTERNAL_HDFS3_LIBRARY) + find_package(hdfs3) +endif () + +if (HDFS3_LIBRARY AND HDFS3_INCLUDE_DIR) +else () + set(HDFS3_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/src/client") + set(HDFS3_LIBRARY hdfs3) +endif() + +message (STATUS "Using hdfs3: ${HDFS3_INCLUDE_DIR} : ${HDFS3_LIBRARY}") diff --git a/cmake/find_llvm.cmake b/cmake/find_llvm.cmake index 81866b720c9..9780ae15c7d 100644 --- a/cmake/find_llvm.cmake +++ b/cmake/find_llvm.cmake @@ -1,6 +1,6 @@ # Broken in macos. TODO: update clang, re-test, enable if (NOT APPLE) - option (ENABLE_EMBEDDED_COMPILER "Set to TRUE to enable support for 'compile' option for query execution" 1) + option (ENABLE_EMBEDDED_COMPILER "Set to TRUE to enable support for 'compile' option for query execution" 0) option (USE_INTERNAL_LLVM_LIBRARY "Use bundled or system LLVM library. Default: system library for quicker developer builds." ${APPLE}) endif () diff --git a/cmake/find_protobuf.cmake b/cmake/find_protobuf.cmake new file mode 100644 index 00000000000..5daf5a0c186 --- /dev/null +++ b/cmake/find_protobuf.cmake @@ -0,0 +1,80 @@ +option (USE_INTERNAL_PROTOBUF_LIBRARY "Set to FALSE to use system protobuf instead of bundled" ON) + +if (NOT USE_INTERNAL_PROTOBUF_LIBRARY) + find_package(Protobuf) +endif () + +if (Protobuf_LIBRARY AND Protobuf_INCLUDE_DIR) +else () + set(Protobuf_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/contrib/protobuf/src) + + set(Protobuf_LIBRARY libprotobuf) + set(Protobuf_PROTOC_LIBRARY libprotoc) + set(Protobuf_LITE_LIBRARY libprotobuf-lite) + + set(Protobuf_PROTOC_EXECUTABLE ${CMAKE_BINARY_DIR}/contrib/protobuf/cmake/protoc) + + if(NOT DEFINED PROTOBUF_GENERATE_CPP_APPEND_PATH) + set(PROTOBUF_GENERATE_CPP_APPEND_PATH TRUE) + endif() + + function(PROTOBUF_GENERATE_CPP SRCS HDRS) + if(NOT ARGN) + message(SEND_ERROR "Error: PROTOBUF_GENERATE_CPP() called without any proto files") + return() + endif() + + if(PROTOBUF_GENERATE_CPP_APPEND_PATH) + # Create an include path for each file specified + foreach(FIL ${ARGN}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(ABS_PATH ${ABS_FIL} PATH) + list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _protobuf_include_path -I ${ABS_PATH}) + endif() + endforeach() + else() + set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR}) + endif() + + if(DEFINED PROTOBUF_IMPORT_DIRS AND NOT DEFINED Protobuf_IMPORT_DIRS) + set(Protobuf_IMPORT_DIRS "${PROTOBUF_IMPORT_DIRS}") + endif() + + if(DEFINED Protobuf_IMPORT_DIRS) + foreach(DIR ${Protobuf_IMPORT_DIRS}) + get_filename_component(ABS_PATH ${DIR} ABSOLUTE) + list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _protobuf_include_path -I ${ABS_PATH}) + endif() + endforeach() + endif() + + set(${SRCS}) + set(${HDRS}) + foreach(FIL ${ARGN}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(FIL_WE ${FIL} NAME_WE) + + list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc") + list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h") + + add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc" + "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h" + COMMAND ${Protobuf_PROTOC_EXECUTABLE} + ARGS --cpp_out ${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${ABS_FIL} + DEPENDS ${ABS_FIL} ${Protobuf_PROTOC_EXECUTABLE} + COMMENT "Running C++ protocol buffer compiler on ${FIL}" + VERBATIM ) + endforeach() + + set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE) + set(${SRCS} ${${SRCS}} PARENT_SCOPE) + set(${HDRS} ${${HDRS}} PARENT_SCOPE) + endfunction() +endif() + +message (STATUS "Using protobuf: ${Protobuf_INCLUDE_DIR} : ${Protobuf_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 928c48e9334..20c00d3f549 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -191,3 +191,12 @@ if (USE_INTERNAL_LLVM_LIBRARY) add_subdirectory (llvm/llvm) endif () +if (USE_INTERNAL_HDFS3_LIBRARY) + include(${CMAKE_SOURCE_DIR}/cmake/find_protobuf.cmake) + if (USE_INTERNAL_PROTOBUF_LIBRARY) + set(protobuf_BUILD_TESTS OFF CACHE INTERNAL "" FORCE) + set(protobuf_BUILD_SHARED_LIBS OFF CACHE INTERNAL "" FORCE) + add_subdirectory(protobuf/cmake) + endif () + add_subdirectory(libhdfs3-cmake) +endif () diff --git a/contrib/boost b/contrib/boost index 2d5cb2c86f6..2e6b038865a 160000 --- a/contrib/boost +++ b/contrib/boost @@ -1 +1 @@ -Subproject commit 2d5cb2c86f61126f4e1efe9ab97332efd44e7dea +Subproject commit 2e6b038865aec18a9af199d0898069e9fa53eabf diff --git a/contrib/libhdfs3 b/contrib/libhdfs3 new file mode 160000 index 00000000000..695aefb67f4 --- /dev/null +++ b/contrib/libhdfs3 @@ -0,0 +1 @@ +Subproject commit 695aefb67f416059a13c463374422404eb0fcd99 diff --git a/contrib/libhdfs3-cmake/CMake/CMakeTestCompileNestedException.cpp b/contrib/libhdfs3-cmake/CMake/CMakeTestCompileNestedException.cpp new file mode 100644 index 00000000000..66918ca516e --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/CMakeTestCompileNestedException.cpp @@ -0,0 +1,10 @@ +#include +#include + +int main() { + try { + throw 2; + } catch (int) { + std::throw_with_nested(std::runtime_error("test")); + } +} diff --git a/contrib/libhdfs3-cmake/CMake/CMakeTestCompileSteadyClock.cpp b/contrib/libhdfs3-cmake/CMake/CMakeTestCompileSteadyClock.cpp new file mode 100644 index 00000000000..afcbe1b83b2 --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/CMakeTestCompileSteadyClock.cpp @@ -0,0 +1,7 @@ +#include + +using std::chrono::steady_clock; + +void foo(const steady_clock &clock) { + return; +} diff --git a/contrib/libhdfs3-cmake/CMake/CMakeTestCompileStrerror.cpp b/contrib/libhdfs3-cmake/CMake/CMakeTestCompileStrerror.cpp new file mode 100644 index 00000000000..0ef4eda583e --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/CMakeTestCompileStrerror.cpp @@ -0,0 +1,10 @@ +#include + +int main() +{ + // We can't test "char *p = strerror_r()" because that only causes a + // compiler warning when strerror_r returns an integer. + char *buf = 0; + int i = strerror_r(0, buf, 100); + return i; +} diff --git a/contrib/libhdfs3-cmake/CMake/CodeCoverage.cmake b/contrib/libhdfs3-cmake/CMake/CodeCoverage.cmake new file mode 100644 index 00000000000..ce997925fcc --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/CodeCoverage.cmake @@ -0,0 +1,48 @@ +# Check prereqs +FIND_PROGRAM(GCOV_PATH gcov) +FIND_PROGRAM(LCOV_PATH lcov) +FIND_PROGRAM(GENHTML_PATH genhtml) + +IF(NOT GCOV_PATH) + MESSAGE(FATAL_ERROR "gcov not found! Aborting...") +ENDIF(NOT GCOV_PATH) + +IF(NOT CMAKE_BUILD_TYPE STREQUAL Debug) + MESSAGE(WARNING "Code coverage results with an optimised (non-Debug) build may be misleading") +ENDIF(NOT CMAKE_BUILD_TYPE STREQUAL Debug) + +#Setup compiler options +ADD_DEFINITIONS(-fprofile-arcs -ftest-coverage) + +SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs ") +SET(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fprofile-arcs ") + +IF(NOT LCOV_PATH) + MESSAGE(FATAL_ERROR "lcov not found! Aborting...") +ENDIF(NOT LCOV_PATH) + +IF(NOT GENHTML_PATH) + MESSAGE(FATAL_ERROR "genhtml not found! Aborting...") +ENDIF(NOT GENHTML_PATH) + +#Setup target +ADD_CUSTOM_TARGET(ShowCoverage + #Capturing lcov counters and generating report + COMMAND ${LCOV_PATH} --directory . --capture --output-file CodeCoverage.info + COMMAND ${LCOV_PATH} --remove CodeCoverage.info '${CMAKE_CURRENT_BINARY_DIR}/*' 'test/*' 'mock/*' '/usr/*' '/opt/*' '*ext/rhel5_x86_64*' '*ext/osx*' --output-file CodeCoverage.info.cleaned + COMMAND ${GENHTML_PATH} -o CodeCoverageReport CodeCoverage.info.cleaned +) + + +ADD_CUSTOM_TARGET(ShowAllCoverage + #Capturing lcov counters and generating report + COMMAND ${LCOV_PATH} -a CodeCoverage.info.cleaned -a CodeCoverage.info.cleaned_withoutHA -o AllCodeCoverage.info + COMMAND sed -e 's|/.*/src|${CMAKE_SOURCE_DIR}/src|' -ig AllCodeCoverage.info + COMMAND ${GENHTML_PATH} -o AllCodeCoverageReport AllCodeCoverage.info +) + +ADD_CUSTOM_TARGET(ResetCoverage + #Cleanup lcov + COMMAND ${LCOV_PATH} --directory . --zerocounters +) + diff --git a/contrib/libhdfs3-cmake/CMake/FindBoost.cmake b/contrib/libhdfs3-cmake/CMake/FindBoost.cmake new file mode 100644 index 00000000000..914a0a5b5cd --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/FindBoost.cmake @@ -0,0 +1,1162 @@ +# - Find Boost include dirs and libraries +# Use this module by invoking find_package with the form: +# find_package(Boost +# [version] [EXACT] # Minimum or EXACT version e.g. 1.36.0 +# [REQUIRED] # Fail with error if Boost is not found +# [COMPONENTS ...] # Boost libraries by their canonical name +# ) # e.g. "date_time" for "libboost_date_time" +# This module finds headers and requested component libraries OR a CMake +# package configuration file provided by a "Boost CMake" build. For the +# latter case skip to the "Boost CMake" section below. For the former +# case results are reported in variables: +# Boost_FOUND - True if headers and requested libraries were found +# Boost_INCLUDE_DIRS - Boost include directories +# Boost_LIBRARY_DIRS - Link directories for Boost libraries +# Boost_LIBRARIES - Boost component libraries to be linked +# Boost__FOUND - True if component was found ( is upper-case) +# Boost__LIBRARY - Libraries to link for component (may include +# target_link_libraries debug/optimized keywords) +# Boost_VERSION - BOOST_VERSION value from boost/version.hpp +# Boost_LIB_VERSION - Version string appended to library filenames +# Boost_MAJOR_VERSION - Boost major version number (X in X.y.z) +# Boost_MINOR_VERSION - Boost minor version number (Y in x.Y.z) +# Boost_SUBMINOR_VERSION - Boost subminor version number (Z in x.y.Z) +# Boost_LIB_DIAGNOSTIC_DEFINITIONS (Windows) +# - Pass to add_definitions() to have diagnostic +# information about Boost's automatic linking +# displayed during compilation +# +# This module reads hints about search locations from variables: +# BOOST_ROOT - Preferred installation prefix +# (or BOOSTROOT) +# BOOST_INCLUDEDIR - Preferred include directory e.g. /include +# BOOST_LIBRARYDIR - Preferred library directory e.g. /lib +# Boost_NO_SYSTEM_PATHS - Set to ON to disable searching in locations not +# specified by these hint variables. Default is OFF. +# Boost_ADDITIONAL_VERSIONS +# - List of Boost versions not known to this module +# (Boost install locations may contain the version) +# and saves search results persistently in CMake cache entries: +# Boost_INCLUDE_DIR - Directory containing Boost headers +# Boost_LIBRARY_DIR - Directory containing Boost libraries +# Boost__LIBRARY_DEBUG - Component library debug variant +# Boost__LIBRARY_RELEASE - Component library release variant +# Users may set these hints or results as cache entries. Projects should +# not read these entries directly but instead use the above result variables. +# Note that some hint names start in upper-case "BOOST". One may specify +# these as environment variables if they are not specified as CMake variables +# or cache entries. +# +# This module first searches for the Boost header files using the above hint +# variables (excluding BOOST_LIBRARYDIR) and saves the result in +# Boost_INCLUDE_DIR. Then it searches for requested component libraries using +# the above hints (excluding BOOST_INCLUDEDIR and Boost_ADDITIONAL_VERSIONS), +# "lib" directories near Boost_INCLUDE_DIR, and the library name configuration +# settings below. It saves the library directory in Boost_LIBRARY_DIR and +# individual library locations in Boost__LIBRARY_DEBUG and +# Boost__LIBRARY_RELEASE. When one changes settings used by previous +# searches in the same build tree (excluding environment variables) this +# module discards previous search results affected by the changes and searches +# again. +# +# Boost libraries come in many variants encoded in their file name. Users or +# projects may tell this module which variant to find by setting variables: +# Boost_USE_MULTITHREADED - Set to OFF to use the non-multithreaded +# libraries ('mt' tag). Default is ON. +# Boost_USE_STATIC_LIBS - Set to ON to force the use of the static +# libraries. Default is OFF. +# Boost_USE_STATIC_RUNTIME - Set to ON or OFF to specify whether to use +# libraries linked statically to the C++ runtime +# ('s' tag). Default is platform dependent. +# Boost_USE_DEBUG_PYTHON - Set to ON to use libraries compiled with a +# debug Python build ('y' tag). Default is OFF. +# Boost_USE_STLPORT - Set to ON to use libraries compiled with +# STLPort ('p' tag). Default is OFF. +# Boost_USE_STLPORT_DEPRECATED_NATIVE_IOSTREAMS +# - Set to ON to use libraries compiled with +# STLPort deprecated "native iostreams" +# ('n' tag). Default is OFF. +# Boost_COMPILER - Set to the compiler-specific library suffix +# (e.g. "-gcc43"). Default is auto-computed +# for the C++ compiler in use. +# Boost_THREADAPI - Suffix for "thread" component library name, +# such as "pthread" or "win32". Names with +# and without this suffix will both be tried. +# Other variables one may set to control this module are: +# Boost_DEBUG - Set to ON to enable debug output from FindBoost. +# Please enable this before filing any bug report. +# Boost_DETAILED_FAILURE_MSG +# - Set to ON to add detailed information to the +# failure message even when the REQUIRED option +# is not given to the find_package call. +# Boost_REALPATH - Set to ON to resolve symlinks for discovered +# libraries to assist with packaging. For example, +# the "system" component library may be resolved to +# "/usr/lib/libboost_system.so.1.42.0" instead of +# "/usr/lib/libboost_system.so". This does not +# affect linking and should not be enabled unless +# the user needs this information. +# On Visual Studio and Borland compilers Boost headers request automatic +# linking to corresponding libraries. This requires matching libraries to be +# linked explicitly or available in the link library search path. In this +# case setting Boost_USE_STATIC_LIBS to OFF may not achieve dynamic linking. +# Boost automatic linking typically requests static libraries with a few +# exceptions (such as Boost.Python). Use +# add_definitions(${Boost_LIB_DIAGNOSTIC_DEFINITIONS}) +# to ask Boost to report information about automatic linking requests. +# +# Example to find Boost headers only: +# find_package(Boost 1.36.0) +# if(Boost_FOUND) +# include_directories(${Boost_INCLUDE_DIRS}) +# add_executable(foo foo.cc) +# endif() +# Example to find Boost headers and some libraries: +# set(Boost_USE_STATIC_LIBS ON) +# set(Boost_USE_MULTITHREADED ON) +# set(Boost_USE_STATIC_RUNTIME OFF) +# find_package(Boost 1.36.0 COMPONENTS date_time filesystem system ...) +# if(Boost_FOUND) +# include_directories(${Boost_INCLUDE_DIRS}) +# add_executable(foo foo.cc) +# target_link_libraries(foo ${Boost_LIBRARIES}) +# endif() +# +# Boost CMake ---------------------------------------------------------- +# +# If Boost was built using the boost-cmake project it provides a package +# configuration file for use with find_package's Config mode. This module +# looks for the package configuration file called BoostConfig.cmake or +# boost-config.cmake and stores the result in cache entry "Boost_DIR". If +# found, the package configuration file is loaded and this module returns with +# no further action. See documentation of the Boost CMake package +# configuration for details on what it provides. +# +# Set Boost_NO_BOOST_CMAKE to ON to disable the search for boost-cmake. + +#============================================================================= +# Copyright 2006-2012 Kitware, Inc. +# Copyright 2006-2008 Andreas Schneider +# Copyright 2007 Wengo +# Copyright 2007 Mike Jackson +# Copyright 2008 Andreas Pakulat +# Copyright 2008-2012 Philip Lowman +# +# Distributed under the OSI-approved BSD License (the "License"); +# see accompanying file Copyright.txt for details. +# +# This software is distributed WITHOUT ANY WARRANTY; without even the +# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the License for more information. +#============================================================================= +# (To distribute this file outside of CMake, substitute the full +# License text for the above reference.) + + +#------------------------------------------------------------------------------- +# Before we go searching, check whether boost-cmake is available, unless the +# user specifically asked NOT to search for boost-cmake. +# +# If Boost_DIR is set, this behaves as any find_package call would. If not, +# it looks at BOOST_ROOT and BOOSTROOT to find Boost. +# +if (NOT Boost_NO_BOOST_CMAKE) + # If Boost_DIR is not set, look for BOOSTROOT and BOOST_ROOT as alternatives, + # since these are more conventional for Boost. + if ("$ENV{Boost_DIR}" STREQUAL "") + if (NOT "$ENV{BOOST_ROOT}" STREQUAL "") + set(ENV{Boost_DIR} $ENV{BOOST_ROOT}) + elseif (NOT "$ENV{BOOSTROOT}" STREQUAL "") + set(ENV{Boost_DIR} $ENV{BOOSTROOT}) + endif() + endif() + + # Do the same find_package call but look specifically for the CMake version. + # Note that args are passed in the Boost_FIND_xxxxx variables, so there is no + # need to delegate them to this find_package call. + find_package(Boost QUIET NO_MODULE) + mark_as_advanced(Boost_DIR) + + # If we found boost-cmake, then we're done. Print out what we found. + # Otherwise let the rest of the module try to find it. + if (Boost_FOUND) + message("Boost ${Boost_FIND_VERSION} found.") + if (Boost_FIND_COMPONENTS) + message("Found Boost components:") + message(" ${Boost_FIND_COMPONENTS}") + endif() + return() + endif() +endif() + + +#------------------------------------------------------------------------------- +# FindBoost functions & macros +# + +############################################ +# +# Check the existence of the libraries. +# +############################################ +# This macro was taken directly from the FindQt4.cmake file that is included +# with the CMake distribution. This is NOT my work. All work was done by the +# original authors of the FindQt4.cmake file. Only minor modifications were +# made to remove references to Qt and make this file more generally applicable +# And ELSE/ENDIF pairs were removed for readability. +######################################################################### + +macro(_Boost_ADJUST_LIB_VARS basename) + if(Boost_INCLUDE_DIR ) + if(Boost_${basename}_LIBRARY_DEBUG AND Boost_${basename}_LIBRARY_RELEASE) + # if the generator supports configuration types then set + # optimized and debug libraries, or if the CMAKE_BUILD_TYPE has a value + if(CMAKE_CONFIGURATION_TYPES OR CMAKE_BUILD_TYPE) + set(Boost_${basename}_LIBRARY optimized ${Boost_${basename}_LIBRARY_RELEASE} debug ${Boost_${basename}_LIBRARY_DEBUG}) + else() + # if there are no configuration types and CMAKE_BUILD_TYPE has no value + # then just use the release libraries + set(Boost_${basename}_LIBRARY ${Boost_${basename}_LIBRARY_RELEASE} ) + endif() + # FIXME: This probably should be set for both cases + set(Boost_${basename}_LIBRARIES optimized ${Boost_${basename}_LIBRARY_RELEASE} debug ${Boost_${basename}_LIBRARY_DEBUG}) + endif() + + # if only the release version was found, set the debug variable also to the release version + if(Boost_${basename}_LIBRARY_RELEASE AND NOT Boost_${basename}_LIBRARY_DEBUG) + set(Boost_${basename}_LIBRARY_DEBUG ${Boost_${basename}_LIBRARY_RELEASE}) + set(Boost_${basename}_LIBRARY ${Boost_${basename}_LIBRARY_RELEASE}) + set(Boost_${basename}_LIBRARIES ${Boost_${basename}_LIBRARY_RELEASE}) + endif() + + # if only the debug version was found, set the release variable also to the debug version + if(Boost_${basename}_LIBRARY_DEBUG AND NOT Boost_${basename}_LIBRARY_RELEASE) + set(Boost_${basename}_LIBRARY_RELEASE ${Boost_${basename}_LIBRARY_DEBUG}) + set(Boost_${basename}_LIBRARY ${Boost_${basename}_LIBRARY_DEBUG}) + set(Boost_${basename}_LIBRARIES ${Boost_${basename}_LIBRARY_DEBUG}) + endif() + + # If the debug & release library ends up being the same, omit the keywords + if(${Boost_${basename}_LIBRARY_RELEASE} STREQUAL ${Boost_${basename}_LIBRARY_DEBUG}) + set(Boost_${basename}_LIBRARY ${Boost_${basename}_LIBRARY_RELEASE} ) + set(Boost_${basename}_LIBRARIES ${Boost_${basename}_LIBRARY_RELEASE} ) + endif() + + if(Boost_${basename}_LIBRARY) + set(Boost_${basename}_FOUND ON) + endif() + + endif() + # Make variables changeable to the advanced user + mark_as_advanced( + Boost_${basename}_LIBRARY_RELEASE + Boost_${basename}_LIBRARY_DEBUG + ) +endmacro() + +macro(_Boost_CHANGE_DETECT changed_var) + set(${changed_var} 0) + foreach(v ${ARGN}) + if(DEFINED _Boost_COMPONENTS_SEARCHED) + if(${v}) + if(_${v}_LAST) + string(COMPARE NOTEQUAL "${${v}}" "${_${v}_LAST}" _${v}_CHANGED) + else() + set(_${v}_CHANGED 1) + endif() + elseif(_${v}_LAST) + set(_${v}_CHANGED 1) + endif() + if(_${v}_CHANGED) + set(${changed_var} 1) + endif() + else() + set(_${v}_CHANGED 0) + endif() + endforeach() +endmacro() + +macro(_Boost_FIND_LIBRARY var) + find_library(${var} ${ARGN}) + + # If we found the first library save Boost_LIBRARY_DIR. + if(${var} AND NOT Boost_LIBRARY_DIR) + get_filename_component(_dir "${${var}}" PATH) + set(Boost_LIBRARY_DIR "${_dir}" CACHE PATH "Boost library directory" FORCE) + endif() + + # If Boost_LIBRARY_DIR is known then search only there. + if(Boost_LIBRARY_DIR) + set(_boost_LIBRARY_SEARCH_DIRS ${Boost_LIBRARY_DIR} NO_DEFAULT_PATH) + endif() +endmacro() + +#------------------------------------------------------------------------------- + +# +# Runs compiler with "-dumpversion" and parses major/minor +# version with a regex. +# +function(_Boost_COMPILER_DUMPVERSION _OUTPUT_VERSION) + + exec_program(${CMAKE_CXX_COMPILER} + ARGS ${CMAKE_CXX_COMPILER_ARG1} -dumpversion + OUTPUT_VARIABLE _boost_COMPILER_VERSION + ) + string(REGEX REPLACE "([0-9])\\.([0-9])(\\.[0-9])?" "\\1\\2" + _boost_COMPILER_VERSION ${_boost_COMPILER_VERSION}) + + set(${_OUTPUT_VERSION} ${_boost_COMPILER_VERSION} PARENT_SCOPE) +endfunction() + +# +# Take a list of libraries with "thread" in it +# and prepend duplicates with "thread_${Boost_THREADAPI}" +# at the front of the list +# +function(_Boost_PREPEND_LIST_WITH_THREADAPI _output) + set(_orig_libnames ${ARGN}) + string(REPLACE "thread" "thread_${Boost_THREADAPI}" _threadapi_libnames "${_orig_libnames}") + set(${_output} ${_threadapi_libnames} ${_orig_libnames} PARENT_SCOPE) +endfunction() + +# +# If a library is found, replace its cache entry with its REALPATH +# +function(_Boost_SWAP_WITH_REALPATH _library _docstring) + if(${_library}) + get_filename_component(_boost_filepathreal ${${_library}} REALPATH) + unset(${_library} CACHE) + set(${_library} ${_boost_filepathreal} CACHE FILEPATH "${_docstring}") + endif() +endfunction() + +function(_Boost_CHECK_SPELLING _var) + if(${_var}) + string(TOUPPER ${_var} _var_UC) + message(FATAL_ERROR "ERROR: ${_var} is not the correct spelling. The proper spelling is ${_var_UC}.") + endif() +endfunction() + +# Guesses Boost's compiler prefix used in built library names +# Returns the guess by setting the variable pointed to by _ret +function(_Boost_GUESS_COMPILER_PREFIX _ret) + if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel" + OR "${CMAKE_CXX_COMPILER}" MATCHES "icl" + OR "${CMAKE_CXX_COMPILER}" MATCHES "icpc") + if(WIN32) + set (_boost_COMPILER "-iw") + else() + set (_boost_COMPILER "-il") + endif() + elseif (MSVC12) + set(_boost_COMPILER "-vc120") + elseif (MSVC11) + set(_boost_COMPILER "-vc110") + elseif (MSVC10) + set(_boost_COMPILER "-vc100") + elseif (MSVC90) + set(_boost_COMPILER "-vc90") + elseif (MSVC80) + set(_boost_COMPILER "-vc80") + elseif (MSVC71) + set(_boost_COMPILER "-vc71") + elseif (MSVC70) # Good luck! + set(_boost_COMPILER "-vc7") # yes, this is correct + elseif (MSVC60) # Good luck! + set(_boost_COMPILER "-vc6") # yes, this is correct + elseif (BORLAND) + set(_boost_COMPILER "-bcb") + elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "SunPro") + set(_boost_COMPILER "-sw") + elseif (MINGW) + if(${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION} VERSION_LESS 1.34) + set(_boost_COMPILER "-mgw") # no GCC version encoding prior to 1.34 + else() + _Boost_COMPILER_DUMPVERSION(_boost_COMPILER_VERSION) + set(_boost_COMPILER "-mgw${_boost_COMPILER_VERSION}") + endif() + elseif (UNIX) + if (CMAKE_COMPILER_IS_GNUCXX) + if(${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION} VERSION_LESS 1.34) + set(_boost_COMPILER "-gcc") # no GCC version encoding prior to 1.34 + else() + _Boost_COMPILER_DUMPVERSION(_boost_COMPILER_VERSION) + # Determine which version of GCC we have. + if(APPLE) + if(Boost_MINOR_VERSION) + if(${Boost_MINOR_VERSION} GREATER 35) + # In Boost 1.36.0 and newer, the mangled compiler name used + # on Mac OS X/Darwin is "xgcc". + set(_boost_COMPILER "-xgcc${_boost_COMPILER_VERSION}") + else() + # In Boost <= 1.35.0, there is no mangled compiler name for + # the Mac OS X/Darwin version of GCC. + set(_boost_COMPILER "") + endif() + else() + # We don't know the Boost version, so assume it's + # pre-1.36.0. + set(_boost_COMPILER "") + endif() + else() + set(_boost_COMPILER "-gcc${_boost_COMPILER_VERSION}") + endif() + endif() + endif () + else() + # TODO at least Boost_DEBUG here? + set(_boost_COMPILER "") + endif() + set(${_ret} ${_boost_COMPILER} PARENT_SCOPE) +endfunction() + +# +# End functions/macros +# +#------------------------------------------------------------------------------- + +#------------------------------------------------------------------------------- +# main. +#------------------------------------------------------------------------------- + +if(NOT DEFINED Boost_USE_MULTITHREADED) + set(Boost_USE_MULTITHREADED TRUE) +endif() + +# Check the version of Boost against the requested version. +if(Boost_FIND_VERSION AND NOT Boost_FIND_VERSION_MINOR) + message(SEND_ERROR "When requesting a specific version of Boost, you must provide at least the major and minor version numbers, e.g., 1.34") +endif() + +if(Boost_FIND_VERSION_EXACT) + # The version may appear in a directory with or without the patch + # level, even when the patch level is non-zero. + set(_boost_TEST_VERSIONS + "${Boost_FIND_VERSION_MAJOR}.${Boost_FIND_VERSION_MINOR}.${Boost_FIND_VERSION_PATCH}" + "${Boost_FIND_VERSION_MAJOR}.${Boost_FIND_VERSION_MINOR}") +else() + # The user has not requested an exact version. Among known + # versions, find those that are acceptable to the user request. + set(_Boost_KNOWN_VERSIONS ${Boost_ADDITIONAL_VERSIONS} + "1.56.0" "1.56" "1.55.0" "1.55" "1.54.0" "1.54" + "1.53.0" "1.53" "1.52.0" "1.52" "1.51.0" "1.51" + "1.50.0" "1.50" "1.49.0" "1.49" "1.48.0" "1.48" "1.47.0" "1.47" "1.46.1" + "1.46.0" "1.46" "1.45.0" "1.45" "1.44.0" "1.44" "1.43.0" "1.43" "1.42.0" "1.42" + "1.41.0" "1.41" "1.40.0" "1.40" "1.39.0" "1.39" "1.38.0" "1.38" "1.37.0" "1.37" + "1.36.1" "1.36.0" "1.36" "1.35.1" "1.35.0" "1.35" "1.34.1" "1.34.0" + "1.34" "1.33.1" "1.33.0" "1.33") + set(_boost_TEST_VERSIONS) + if(Boost_FIND_VERSION) + set(_Boost_FIND_VERSION_SHORT "${Boost_FIND_VERSION_MAJOR}.${Boost_FIND_VERSION_MINOR}") + # Select acceptable versions. + foreach(version ${_Boost_KNOWN_VERSIONS}) + if(NOT "${version}" VERSION_LESS "${Boost_FIND_VERSION}") + # This version is high enough. + list(APPEND _boost_TEST_VERSIONS "${version}") + elseif("${version}.99" VERSION_EQUAL "${_Boost_FIND_VERSION_SHORT}.99") + # This version is a short-form for the requested version with + # the patch level dropped. + list(APPEND _boost_TEST_VERSIONS "${version}") + endif() + endforeach() + else() + # Any version is acceptable. + set(_boost_TEST_VERSIONS "${_Boost_KNOWN_VERSIONS}") + endif() +endif() + +# The reason that we failed to find Boost. This will be set to a +# user-friendly message when we fail to find some necessary piece of +# Boost. +set(Boost_ERROR_REASON) + +if(Boost_DEBUG) + # Output some of their choices + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "_boost_TEST_VERSIONS = ${_boost_TEST_VERSIONS}") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "Boost_USE_MULTITHREADED = ${Boost_USE_MULTITHREADED}") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "Boost_USE_STATIC_LIBS = ${Boost_USE_STATIC_LIBS}") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "Boost_USE_STATIC_RUNTIME = ${Boost_USE_STATIC_RUNTIME}") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "Boost_ADDITIONAL_VERSIONS = ${Boost_ADDITIONAL_VERSIONS}") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "Boost_NO_SYSTEM_PATHS = ${Boost_NO_SYSTEM_PATHS}") +endif() + +if(WIN32) + # In windows, automatic linking is performed, so you do not have + # to specify the libraries. If you are linking to a dynamic + # runtime, then you can choose to link to either a static or a + # dynamic Boost library, the default is to do a static link. You + # can alter this for a specific library "whatever" by defining + # BOOST_WHATEVER_DYN_LINK to force Boost library "whatever" to be + # linked dynamically. Alternatively you can force all Boost + # libraries to dynamic link by defining BOOST_ALL_DYN_LINK. + + # This feature can be disabled for Boost library "whatever" by + # defining BOOST_WHATEVER_NO_LIB, or for all of Boost by defining + # BOOST_ALL_NO_LIB. + + # If you want to observe which libraries are being linked against + # then defining BOOST_LIB_DIAGNOSTIC will cause the auto-linking + # code to emit a #pragma message each time a library is selected + # for linking. + set(Boost_LIB_DIAGNOSTIC_DEFINITIONS "-DBOOST_LIB_DIAGNOSTIC") +endif() + +_Boost_CHECK_SPELLING(Boost_ROOT) +_Boost_CHECK_SPELLING(Boost_LIBRARYDIR) +_Boost_CHECK_SPELLING(Boost_INCLUDEDIR) + +# Collect environment variable inputs as hints. Do not consider changes. +foreach(v BOOSTROOT BOOST_ROOT BOOST_INCLUDEDIR BOOST_LIBRARYDIR) + set(_env $ENV{${v}}) + if(_env) + file(TO_CMAKE_PATH "${_env}" _ENV_${v}) + else() + set(_ENV_${v} "") + endif() +endforeach() +if(NOT _ENV_BOOST_ROOT AND _ENV_BOOSTROOT) + set(_ENV_BOOST_ROOT "${_ENV_BOOSTROOT}") +endif() + +# Collect inputs and cached results. Detect changes since the last run. +if(NOT BOOST_ROOT AND BOOSTROOT) + set(BOOST_ROOT "${BOOSTROOT}") +endif() +set(_Boost_VARS_DIR + BOOST_ROOT + Boost_NO_SYSTEM_PATHS + ) + +if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "Declared as CMake or Environmental Variables:") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + " BOOST_ROOT = ${BOOST_ROOT}") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + " BOOST_INCLUDEDIR = ${BOOST_INCLUDEDIR}") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + " BOOST_LIBRARYDIR = ${BOOST_LIBRARYDIR}") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "_boost_TEST_VERSIONS = ${_boost_TEST_VERSIONS}") +endif() + +# ------------------------------------------------------------------------ +# Search for Boost include DIR +# ------------------------------------------------------------------------ + +set(_Boost_VARS_INC BOOST_INCLUDEDIR Boost_INCLUDE_DIR Boost_ADDITIONAL_VERSIONS) +_Boost_CHANGE_DETECT(_Boost_CHANGE_INCDIR ${_Boost_VARS_DIR} ${_Boost_VARS_INC}) +# Clear Boost_INCLUDE_DIR if it did not change but other input affecting the +# location did. We will find a new one based on the new inputs. +if(_Boost_CHANGE_INCDIR AND NOT _Boost_INCLUDE_DIR_CHANGED) + unset(Boost_INCLUDE_DIR CACHE) +endif() + +if(NOT Boost_INCLUDE_DIR) + set(_boost_INCLUDE_SEARCH_DIRS "") + if(BOOST_INCLUDEDIR) + list(APPEND _boost_INCLUDE_SEARCH_DIRS ${BOOST_INCLUDEDIR}) + elseif(_ENV_BOOST_INCLUDEDIR) + list(APPEND _boost_INCLUDE_SEARCH_DIRS ${_ENV_BOOST_INCLUDEDIR}) + endif() + + if( BOOST_ROOT ) + list(APPEND _boost_INCLUDE_SEARCH_DIRS ${BOOST_ROOT}/include ${BOOST_ROOT}) + elseif( _ENV_BOOST_ROOT ) + list(APPEND _boost_INCLUDE_SEARCH_DIRS ${_ENV_BOOST_ROOT}/include ${_ENV_BOOST_ROOT}) + endif() + + if( Boost_NO_SYSTEM_PATHS) + list(APPEND _boost_INCLUDE_SEARCH_DIRS NO_CMAKE_SYSTEM_PATH) + else() + list(APPEND _boost_INCLUDE_SEARCH_DIRS PATHS + C:/boost/include + C:/boost + /sw/local/include + ) + endif() + + # Try to find Boost by stepping backwards through the Boost versions + # we know about. + # Build a list of path suffixes for each version. + set(_boost_PATH_SUFFIXES) + foreach(_boost_VER ${_boost_TEST_VERSIONS}) + # Add in a path suffix, based on the required version, ideally + # we could read this from version.hpp, but for that to work we'd + # need to know the include dir already + set(_boost_BOOSTIFIED_VERSION) + + # Transform 1.35 => 1_35 and 1.36.0 => 1_36_0 + if(_boost_VER MATCHES "[0-9]+\\.[0-9]+\\.[0-9]+") + string(REGEX REPLACE "([0-9]+)\\.([0-9]+)\\.([0-9]+)" "\\1_\\2_\\3" + _boost_BOOSTIFIED_VERSION ${_boost_VER}) + elseif(_boost_VER MATCHES "[0-9]+\\.[0-9]+") + string(REGEX REPLACE "([0-9]+)\\.([0-9]+)" "\\1_\\2" + _boost_BOOSTIFIED_VERSION ${_boost_VER}) + endif() + + list(APPEND _boost_PATH_SUFFIXES + "boost-${_boost_BOOSTIFIED_VERSION}" + "boost_${_boost_BOOSTIFIED_VERSION}" + "boost/boost-${_boost_BOOSTIFIED_VERSION}" + "boost/boost_${_boost_BOOSTIFIED_VERSION}" + ) + + endforeach() + + if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "Include debugging info:") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + " _boost_INCLUDE_SEARCH_DIRS = ${_boost_INCLUDE_SEARCH_DIRS}") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + " _boost_PATH_SUFFIXES = ${_boost_PATH_SUFFIXES}") + endif() + + # Look for a standard boost header file. + find_path(Boost_INCLUDE_DIR + NAMES boost/config.hpp + HINTS ${_boost_INCLUDE_SEARCH_DIRS} + PATH_SUFFIXES ${_boost_PATH_SUFFIXES} + ) +endif() + +# ------------------------------------------------------------------------ +# Extract version information from version.hpp +# ------------------------------------------------------------------------ + +# Set Boost_FOUND based only on header location and version. +# It will be updated below for component libraries. +if(Boost_INCLUDE_DIR) + if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "location of version.hpp: ${Boost_INCLUDE_DIR}/boost/version.hpp") + endif() + + # Extract Boost_VERSION and Boost_LIB_VERSION from version.hpp + set(Boost_VERSION 0) + set(Boost_LIB_VERSION "") + file(STRINGS "${Boost_INCLUDE_DIR}/boost/version.hpp" _boost_VERSION_HPP_CONTENTS REGEX "#define BOOST_(LIB_)?VERSION ") + set(_Boost_VERSION_REGEX "([0-9]+)") + set(_Boost_LIB_VERSION_REGEX "\"([0-9_]+)\"") + foreach(v VERSION LIB_VERSION) + if("${_boost_VERSION_HPP_CONTENTS}" MATCHES ".*#define BOOST_${v} ${_Boost_${v}_REGEX}.*") + set(Boost_${v} "${CMAKE_MATCH_1}") + endif() + endforeach() + unset(_boost_VERSION_HPP_CONTENTS) + + math(EXPR Boost_MAJOR_VERSION "${Boost_VERSION} / 100000") + math(EXPR Boost_MINOR_VERSION "${Boost_VERSION} / 100 % 1000") + math(EXPR Boost_SUBMINOR_VERSION "${Boost_VERSION} % 100") + + set(Boost_ERROR_REASON + "${Boost_ERROR_REASON}Boost version: ${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}\nBoost include path: ${Boost_INCLUDE_DIR}") + if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "version.hpp reveals boost " + "${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}") + endif() + + if(Boost_FIND_VERSION) + # Set Boost_FOUND based on requested version. + set(_Boost_VERSION "${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}") + if("${_Boost_VERSION}" VERSION_LESS "${Boost_FIND_VERSION}") + set(Boost_FOUND 0) + set(_Boost_VERSION_AGE "old") + elseif(Boost_FIND_VERSION_EXACT AND + NOT "${_Boost_VERSION}" VERSION_EQUAL "${Boost_FIND_VERSION}") + set(Boost_FOUND 0) + set(_Boost_VERSION_AGE "new") + else() + set(Boost_FOUND 1) + endif() + if(NOT Boost_FOUND) + # State that we found a version of Boost that is too new or too old. + set(Boost_ERROR_REASON + "${Boost_ERROR_REASON}\nDetected version of Boost is too ${_Boost_VERSION_AGE}. Requested version was ${Boost_FIND_VERSION_MAJOR}.${Boost_FIND_VERSION_MINOR}") + if (Boost_FIND_VERSION_PATCH) + set(Boost_ERROR_REASON + "${Boost_ERROR_REASON}.${Boost_FIND_VERSION_PATCH}") + endif () + if (NOT Boost_FIND_VERSION_EXACT) + set(Boost_ERROR_REASON "${Boost_ERROR_REASON} (or newer)") + endif () + set(Boost_ERROR_REASON "${Boost_ERROR_REASON}.") + endif () + else() + # Caller will accept any Boost version. + set(Boost_FOUND 1) + endif() +else() + set(Boost_FOUND 0) + set(Boost_ERROR_REASON + "${Boost_ERROR_REASON}Unable to find the Boost header files. Please set BOOST_ROOT to the root directory containing Boost or BOOST_INCLUDEDIR to the directory containing Boost's headers.") +endif() + +# ------------------------------------------------------------------------ +# Suffix initialization and compiler suffix detection. +# ------------------------------------------------------------------------ + +set(_Boost_VARS_NAME + Boost_COMPILER + Boost_THREADAPI + Boost_USE_DEBUG_PYTHON + Boost_USE_MULTITHREADED + Boost_USE_STATIC_LIBS + Boost_USE_STATIC_RUNTIME + Boost_USE_STLPORT + Boost_USE_STLPORT_DEPRECATED_NATIVE_IOSTREAMS + ) +_Boost_CHANGE_DETECT(_Boost_CHANGE_LIBNAME ${_Boost_VARS_NAME}) + +# Setting some more suffixes for the library +set(Boost_LIB_PREFIX "") +if ( WIN32 AND Boost_USE_STATIC_LIBS AND NOT CYGWIN) + set(Boost_LIB_PREFIX "lib") +endif() + +if (Boost_COMPILER) + set(_boost_COMPILER ${Boost_COMPILER}) + if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "using user-specified Boost_COMPILER = ${_boost_COMPILER}") + endif() +else() + # Attempt to guess the compiler suffix + # NOTE: this is not perfect yet, if you experience any issues + # please report them and use the Boost_COMPILER variable + # to work around the problems. + _Boost_GUESS_COMPILER_PREFIX(_boost_COMPILER) + if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "guessed _boost_COMPILER = ${_boost_COMPILER}") + endif() +endif() + +set (_boost_MULTITHREADED "-mt") +if( NOT Boost_USE_MULTITHREADED ) + set (_boost_MULTITHREADED "") +endif() +if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "_boost_MULTITHREADED = ${_boost_MULTITHREADED}") +endif() + +#====================== +# Systematically build up the Boost ABI tag +# http://boost.org/doc/libs/1_41_0/more/getting_started/windows.html#library-naming +set( _boost_RELEASE_ABI_TAG "-") +set( _boost_DEBUG_ABI_TAG "-") +# Key Use this library when: +# s linking statically to the C++ standard library and +# compiler runtime support libraries. +if(Boost_USE_STATIC_RUNTIME) + set( _boost_RELEASE_ABI_TAG "${_boost_RELEASE_ABI_TAG}s") + set( _boost_DEBUG_ABI_TAG "${_boost_DEBUG_ABI_TAG}s") +endif() +# g using debug versions of the standard and runtime +# support libraries +if(WIN32) + if(MSVC OR "${CMAKE_CXX_COMPILER}" MATCHES "icl" + OR "${CMAKE_CXX_COMPILER}" MATCHES "icpc") + set(_boost_DEBUG_ABI_TAG "${_boost_DEBUG_ABI_TAG}g") + endif() +endif() +# y using special debug build of python +if(Boost_USE_DEBUG_PYTHON) + set(_boost_DEBUG_ABI_TAG "${_boost_DEBUG_ABI_TAG}y") +endif() +# d using a debug version of your code +set(_boost_DEBUG_ABI_TAG "${_boost_DEBUG_ABI_TAG}d") +# p using the STLport standard library rather than the +# default one supplied with your compiler +if(Boost_USE_STLPORT) + set( _boost_RELEASE_ABI_TAG "${_boost_RELEASE_ABI_TAG}p") + set( _boost_DEBUG_ABI_TAG "${_boost_DEBUG_ABI_TAG}p") +endif() +# n using the STLport deprecated "native iostreams" feature +if(Boost_USE_STLPORT_DEPRECATED_NATIVE_IOSTREAMS) + set( _boost_RELEASE_ABI_TAG "${_boost_RELEASE_ABI_TAG}n") + set( _boost_DEBUG_ABI_TAG "${_boost_DEBUG_ABI_TAG}n") +endif() + +if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "_boost_RELEASE_ABI_TAG = ${_boost_RELEASE_ABI_TAG}") + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "_boost_DEBUG_ABI_TAG = ${_boost_DEBUG_ABI_TAG}") +endif() + +# ------------------------------------------------------------------------ +# Begin finding boost libraries +# ------------------------------------------------------------------------ +set(_Boost_VARS_LIB BOOST_LIBRARYDIR Boost_LIBRARY_DIR) +_Boost_CHANGE_DETECT(_Boost_CHANGE_LIBDIR ${_Boost_VARS_DIR} ${_Boost_VARS_LIB} Boost_INCLUDE_DIR) +# Clear Boost_LIBRARY_DIR if it did not change but other input affecting the +# location did. We will find a new one based on the new inputs. +if(_Boost_CHANGE_LIBDIR AND NOT _Boost_LIBRARY_DIR_CHANGED) + unset(Boost_LIBRARY_DIR CACHE) +endif() + +if(Boost_LIBRARY_DIR) + set(_boost_LIBRARY_SEARCH_DIRS ${Boost_LIBRARY_DIR} NO_DEFAULT_PATH) +else() + set(_boost_LIBRARY_SEARCH_DIRS "") + if(BOOST_LIBRARYDIR) + list(APPEND _boost_LIBRARY_SEARCH_DIRS ${BOOST_LIBRARYDIR}) + elseif(_ENV_BOOST_LIBRARYDIR) + list(APPEND _boost_LIBRARY_SEARCH_DIRS ${_ENV_BOOST_LIBRARYDIR}) + endif() + + if(BOOST_ROOT) + list(APPEND _boost_LIBRARY_SEARCH_DIRS ${BOOST_ROOT}/lib ${BOOST_ROOT}/stage/lib) + elseif(_ENV_BOOST_ROOT) + list(APPEND _boost_LIBRARY_SEARCH_DIRS ${_ENV_BOOST_ROOT}/lib ${_ENV_BOOST_ROOT}/stage/lib) + endif() + + list(APPEND _boost_LIBRARY_SEARCH_DIRS + ${Boost_INCLUDE_DIR}/lib + ${Boost_INCLUDE_DIR}/../lib + ${Boost_INCLUDE_DIR}/stage/lib + ) + if( Boost_NO_SYSTEM_PATHS ) + list(APPEND _boost_LIBRARY_SEARCH_DIRS NO_CMAKE_SYSTEM_PATH) + else() + list(APPEND _boost_LIBRARY_SEARCH_DIRS PATHS + C:/boost/lib + C:/boost + /sw/local/lib + ) + endif() +endif() + +if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "_boost_LIBRARY_SEARCH_DIRS = ${_boost_LIBRARY_SEARCH_DIRS}") +endif() + +# Support preference of static libs by adjusting CMAKE_FIND_LIBRARY_SUFFIXES +if( Boost_USE_STATIC_LIBS ) + set( _boost_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES}) + if(WIN32) + set(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a ${CMAKE_FIND_LIBRARY_SUFFIXES}) + else() + set(CMAKE_FIND_LIBRARY_SUFFIXES .a ) + endif() +endif() + +# We want to use the tag inline below without risking double dashes +if(_boost_RELEASE_ABI_TAG) + if(${_boost_RELEASE_ABI_TAG} STREQUAL "-") + set(_boost_RELEASE_ABI_TAG "") + endif() +endif() +if(_boost_DEBUG_ABI_TAG) + if(${_boost_DEBUG_ABI_TAG} STREQUAL "-") + set(_boost_DEBUG_ABI_TAG "") + endif() +endif() + +# The previous behavior of FindBoost when Boost_USE_STATIC_LIBS was enabled +# on WIN32 was to: +# 1. Search for static libs compiled against a SHARED C++ standard runtime library (use if found) +# 2. Search for static libs compiled against a STATIC C++ standard runtime library (use if found) +# We maintain this behavior since changing it could break people's builds. +# To disable the ambiguous behavior, the user need only +# set Boost_USE_STATIC_RUNTIME either ON or OFF. +set(_boost_STATIC_RUNTIME_WORKAROUND false) +if(WIN32 AND Boost_USE_STATIC_LIBS) + if(NOT DEFINED Boost_USE_STATIC_RUNTIME) + set(_boost_STATIC_RUNTIME_WORKAROUND true) + endif() +endif() + +# On versions < 1.35, remove the System library from the considered list +# since it wasn't added until 1.35. +if(Boost_VERSION AND Boost_FIND_COMPONENTS) + if(Boost_VERSION LESS 103500) + list(REMOVE_ITEM Boost_FIND_COMPONENTS system) + endif() +endif() + +# If the user changed any of our control inputs flush previous results. +if(_Boost_CHANGE_LIBDIR OR _Boost_CHANGE_LIBNAME) + foreach(COMPONENT ${_Boost_COMPONENTS_SEARCHED}) + string(TOUPPER ${COMPONENT} UPPERCOMPONENT) + foreach(c DEBUG RELEASE) + set(_var Boost_${UPPERCOMPONENT}_LIBRARY_${c}) + unset(${_var} CACHE) + set(${_var} "${_var}-NOTFOUND") + endforeach() + endforeach() + set(_Boost_COMPONENTS_SEARCHED "") +endif() + +foreach(COMPONENT ${Boost_FIND_COMPONENTS}) + string(TOUPPER ${COMPONENT} UPPERCOMPONENT) + + set( _boost_docstring_release "Boost ${COMPONENT} library (release)") + set( _boost_docstring_debug "Boost ${COMPONENT} library (debug)") + + # + # Find RELEASE libraries + # + set(_boost_RELEASE_NAMES + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_RELEASE_ABI_TAG}-${Boost_LIB_VERSION} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_RELEASE_ABI_TAG} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_RELEASE_ABI_TAG}-${Boost_LIB_VERSION} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_RELEASE_ABI_TAG} + ${Boost_LIB_PREFIX}boost_${COMPONENT} ) + if(_boost_STATIC_RUNTIME_WORKAROUND) + set(_boost_RELEASE_STATIC_ABI_TAG "-s${_boost_RELEASE_ABI_TAG}") + list(APPEND _boost_RELEASE_NAMES + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_RELEASE_STATIC_ABI_TAG}-${Boost_LIB_VERSION} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_RELEASE_STATIC_ABI_TAG} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_RELEASE_STATIC_ABI_TAG}-${Boost_LIB_VERSION} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_RELEASE_STATIC_ABI_TAG} ) + endif() + if(Boost_THREADAPI AND ${COMPONENT} STREQUAL "thread") + _Boost_PREPEND_LIST_WITH_THREADAPI(_boost_RELEASE_NAMES ${_boost_RELEASE_NAMES}) + endif() + if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "Searching for ${UPPERCOMPONENT}_LIBRARY_RELEASE: ${_boost_RELEASE_NAMES}") + endif() + + # Avoid passing backslashes to _Boost_FIND_LIBRARY due to macro re-parsing. + string(REPLACE "\\" "/" _boost_LIBRARY_SEARCH_DIRS_tmp "${_boost_LIBRARY_SEARCH_DIRS}") + + _Boost_FIND_LIBRARY(Boost_${UPPERCOMPONENT}_LIBRARY_RELEASE + NAMES ${_boost_RELEASE_NAMES} + HINTS ${_boost_LIBRARY_SEARCH_DIRS_tmp} + NAMES_PER_DIR + DOC "${_boost_docstring_release}" + ) + + # + # Find DEBUG libraries + # + set(_boost_DEBUG_NAMES + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_DEBUG_ABI_TAG}-${Boost_LIB_VERSION} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_DEBUG_ABI_TAG} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_DEBUG_ABI_TAG}-${Boost_LIB_VERSION} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_DEBUG_ABI_TAG} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED} + ${Boost_LIB_PREFIX}boost_${COMPONENT} ) + if(_boost_STATIC_RUNTIME_WORKAROUND) + set(_boost_DEBUG_STATIC_ABI_TAG "-s${_boost_DEBUG_ABI_TAG}") + list(APPEND _boost_DEBUG_NAMES + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_DEBUG_STATIC_ABI_TAG}-${Boost_LIB_VERSION} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_COMPILER}${_boost_MULTITHREADED}${_boost_DEBUG_STATIC_ABI_TAG} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_DEBUG_STATIC_ABI_TAG}-${Boost_LIB_VERSION} + ${Boost_LIB_PREFIX}boost_${COMPONENT}${_boost_MULTITHREADED}${_boost_DEBUG_STATIC_ABI_TAG} ) + endif() + if(Boost_THREADAPI AND ${COMPONENT} STREQUAL "thread") + _Boost_PREPEND_LIST_WITH_THREADAPI(_boost_DEBUG_NAMES ${_boost_DEBUG_NAMES}) + endif() + if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] " + "Searching for ${UPPERCOMPONENT}_LIBRARY_DEBUG: ${_boost_DEBUG_NAMES}") + endif() + + # Avoid passing backslashes to _Boost_FIND_LIBRARY due to macro re-parsing. + string(REPLACE "\\" "/" _boost_LIBRARY_SEARCH_DIRS_tmp "${_boost_LIBRARY_SEARCH_DIRS}") + + _Boost_FIND_LIBRARY(Boost_${UPPERCOMPONENT}_LIBRARY_DEBUG + NAMES ${_boost_DEBUG_NAMES} + HINTS ${_boost_LIBRARY_SEARCH_DIRS_tmp} + NAMES_PER_DIR + DOC "${_boost_docstring_debug}" + ) + + if(Boost_REALPATH) + _Boost_SWAP_WITH_REALPATH(Boost_${UPPERCOMPONENT}_LIBRARY_RELEASE "${_boost_docstring_release}") + _Boost_SWAP_WITH_REALPATH(Boost_${UPPERCOMPONENT}_LIBRARY_DEBUG "${_boost_docstring_debug}" ) + endif() + + _Boost_ADJUST_LIB_VARS(${UPPERCOMPONENT}) + +endforeach() + +# Restore the original find library ordering +if( Boost_USE_STATIC_LIBS ) + set(CMAKE_FIND_LIBRARY_SUFFIXES ${_boost_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES}) +endif() + +# ------------------------------------------------------------------------ +# End finding boost libraries +# ------------------------------------------------------------------------ + +set(Boost_INCLUDE_DIRS ${Boost_INCLUDE_DIR}) +set(Boost_LIBRARY_DIRS ${Boost_LIBRARY_DIR}) + +# The above setting of Boost_FOUND was based only on the header files. +# Update it for the requested component libraries. +if(Boost_FOUND) + # The headers were found. Check for requested component libs. + set(_boost_CHECKED_COMPONENT FALSE) + set(_Boost_MISSING_COMPONENTS "") + foreach(COMPONENT ${Boost_FIND_COMPONENTS}) + string(TOUPPER ${COMPONENT} COMPONENT) + set(_boost_CHECKED_COMPONENT TRUE) + if(NOT Boost_${COMPONENT}_FOUND) + string(TOLOWER ${COMPONENT} COMPONENT) + list(APPEND _Boost_MISSING_COMPONENTS ${COMPONENT}) + endif() + endforeach() + + if(Boost_DEBUG) + message(STATUS "[ ${CMAKE_CURRENT_LIST_FILE}:${CMAKE_CURRENT_LIST_LINE} ] Boost_FOUND = ${Boost_FOUND}") + endif() + + if (_Boost_MISSING_COMPONENTS) + set(Boost_FOUND 0) + # We were unable to find some libraries, so generate a sensible + # error message that lists the libraries we were unable to find. + set(Boost_ERROR_REASON + "${Boost_ERROR_REASON}\nCould not find the following") + if(Boost_USE_STATIC_LIBS) + set(Boost_ERROR_REASON "${Boost_ERROR_REASON} static") + endif() + set(Boost_ERROR_REASON + "${Boost_ERROR_REASON} Boost libraries:\n") + foreach(COMPONENT ${_Boost_MISSING_COMPONENTS}) + set(Boost_ERROR_REASON + "${Boost_ERROR_REASON} boost_${COMPONENT}\n") + endforeach() + + list(LENGTH Boost_FIND_COMPONENTS Boost_NUM_COMPONENTS_WANTED) + list(LENGTH _Boost_MISSING_COMPONENTS Boost_NUM_MISSING_COMPONENTS) + if (${Boost_NUM_COMPONENTS_WANTED} EQUAL ${Boost_NUM_MISSING_COMPONENTS}) + set(Boost_ERROR_REASON + "${Boost_ERROR_REASON}No Boost libraries were found. You may need to set BOOST_LIBRARYDIR to the directory containing Boost libraries or BOOST_ROOT to the location of Boost.") + else () + set(Boost_ERROR_REASON + "${Boost_ERROR_REASON}Some (but not all) of the required Boost libraries were found. You may need to install these additional Boost libraries. Alternatively, set BOOST_LIBRARYDIR to the directory containing Boost libraries or BOOST_ROOT to the location of Boost.") + endif () + endif () + + if( NOT Boost_LIBRARY_DIRS AND NOT _boost_CHECKED_COMPONENT ) + # Compatibility Code for backwards compatibility with CMake + # 2.4's FindBoost module. + + # Look for the boost library path. + # Note that the user may not have installed any libraries + # so it is quite possible the Boost_LIBRARY_DIRS may not exist. + set(_boost_LIB_DIR ${Boost_INCLUDE_DIR}) + + if("${_boost_LIB_DIR}" MATCHES "boost-[0-9]+") + get_filename_component(_boost_LIB_DIR ${_boost_LIB_DIR} PATH) + endif() + + if("${_boost_LIB_DIR}" MATCHES "/include$") + # Strip off the trailing "/include" in the path. + get_filename_component(_boost_LIB_DIR ${_boost_LIB_DIR} PATH) + endif() + + if(EXISTS "${_boost_LIB_DIR}/lib") + set(_boost_LIB_DIR ${_boost_LIB_DIR}/lib) + else() + if(EXISTS "${_boost_LIB_DIR}/stage/lib") + set(_boost_LIB_DIR ${_boost_LIB_DIR}/stage/lib) + else() + set(_boost_LIB_DIR "") + endif() + endif() + + if(_boost_LIB_DIR AND EXISTS "${_boost_LIB_DIR}") + set(Boost_LIBRARY_DIRS ${_boost_LIB_DIR}) + endif() + + endif() +else() + # Boost headers were not found so no components were found. + foreach(COMPONENT ${Boost_FIND_COMPONENTS}) + string(TOUPPER ${COMPONENT} UPPERCOMPONENT) + set(Boost_${UPPERCOMPONENT}_FOUND 0) + endforeach() +endif() + +# ------------------------------------------------------------------------ +# Notification to end user about what was found +# ------------------------------------------------------------------------ + +set(Boost_LIBRARIES "") +if(Boost_FOUND) + if(NOT Boost_FIND_QUIETLY) + message(STATUS "Boost version: ${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}") + if(Boost_FIND_COMPONENTS) + message(STATUS "Found the following Boost libraries:") + endif() + endif() + foreach( COMPONENT ${Boost_FIND_COMPONENTS} ) + string( TOUPPER ${COMPONENT} UPPERCOMPONENT ) + if( Boost_${UPPERCOMPONENT}_FOUND ) + if(NOT Boost_FIND_QUIETLY) + message (STATUS " ${COMPONENT}") + endif() + list(APPEND Boost_LIBRARIES ${Boost_${UPPERCOMPONENT}_LIBRARY}) + endif() + endforeach() +else() + if(Boost_FIND_REQUIRED) + message(SEND_ERROR "Unable to find the requested Boost libraries.\n${Boost_ERROR_REASON}") + else() + if(NOT Boost_FIND_QUIETLY) + # we opt not to automatically output Boost_ERROR_REASON here as + # it could be quite lengthy and somewhat imposing in its requests + # Since Boost is not always a required dependency we'll leave this + # up to the end-user. + if(Boost_DEBUG OR Boost_DETAILED_FAILURE_MSG) + message(STATUS "Could NOT find Boost\n${Boost_ERROR_REASON}") + else() + message(STATUS "Could NOT find Boost") + endif() + endif() + endif() +endif() + +# Configure display of cache entries in GUI. +foreach(v BOOSTROOT BOOST_ROOT ${_Boost_VARS_INC} ${_Boost_VARS_LIB}) + get_property(_type CACHE ${v} PROPERTY TYPE) + if(_type) + set_property(CACHE ${v} PROPERTY ADVANCED 1) + if("x${_type}" STREQUAL "xUNINITIALIZED") + if("x${v}" STREQUAL "xBoost_ADDITIONAL_VERSIONS") + set_property(CACHE ${v} PROPERTY TYPE STRING) + else() + set_property(CACHE ${v} PROPERTY TYPE PATH) + endif() + endif() + endif() +endforeach() + +# Record last used values of input variables so we can +# detect on the next run if the user changed them. +foreach(v + ${_Boost_VARS_INC} ${_Boost_VARS_LIB} + ${_Boost_VARS_DIR} ${_Boost_VARS_NAME} + ) + if(DEFINED ${v}) + set(_${v}_LAST "${${v}}" CACHE INTERNAL "Last used ${v} value.") + else() + unset(_${v}_LAST CACHE) + endif() +endforeach() + +# Maintain a persistent list of components requested anywhere since +# the last flush. +set(_Boost_COMPONENTS_SEARCHED "${_Boost_COMPONENTS_SEARCHED}") +list(APPEND _Boost_COMPONENTS_SEARCHED ${Boost_FIND_COMPONENTS}) +list(REMOVE_DUPLICATES _Boost_COMPONENTS_SEARCHED) +list(SORT _Boost_COMPONENTS_SEARCHED) +set(_Boost_COMPONENTS_SEARCHED "${_Boost_COMPONENTS_SEARCHED}" + CACHE INTERNAL "Components requested for this build tree.") diff --git a/contrib/libhdfs3-cmake/CMake/FindCurl.cmake b/contrib/libhdfs3-cmake/CMake/FindCurl.cmake new file mode 100644 index 00000000000..e93b01de8c6 --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/FindCurl.cmake @@ -0,0 +1,26 @@ +# - Try to find the CURL library (curl) +# +# Once done this will define +# +# CURL_FOUND - System has gnutls +# CURL_INCLUDE_DIR - The gnutls include directory +# CURL_LIBRARIES - The libraries needed to use gnutls +# CURL_DEFINITIONS - Compiler switches required for using gnutls + + +IF (CURL_INCLUDE_DIR AND CURL_LIBRARIES) + # in cache already + SET(CURL_FIND_QUIETLY TRUE) +ENDIF (CURL_INCLUDE_DIR AND CURL_LIBRARIES) + +FIND_PATH(CURL_INCLUDE_DIR curl/curl.h) + +FIND_LIBRARY(CURL_LIBRARIES curl) + +INCLUDE(FindPackageHandleStandardArgs) + +# handle the QUIETLY and REQUIRED arguments and set CURL_FOUND to TRUE if +# all listed variables are TRUE +FIND_PACKAGE_HANDLE_STANDARD_ARGS(CURL DEFAULT_MSG CURL_LIBRARIES CURL_INCLUDE_DIR) + +MARK_AS_ADVANCED(CURL_INCLUDE_DIR CURL_LIBRARIES) diff --git a/contrib/libhdfs3-cmake/CMake/FindGSasl.cmake b/contrib/libhdfs3-cmake/CMake/FindGSasl.cmake new file mode 100644 index 00000000000..19ca7c30d1e --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/FindGSasl.cmake @@ -0,0 +1,26 @@ +# - Try to find the GNU sasl library (gsasl) +# +# Once done this will define +# +# GSASL_FOUND - System has gnutls +# GSASL_INCLUDE_DIR - The gnutls include directory +# GSASL_LIBRARIES - The libraries needed to use gnutls +# GSASL_DEFINITIONS - Compiler switches required for using gnutls + + +IF (GSASL_INCLUDE_DIR AND GSASL_LIBRARIES) + # in cache already + SET(GSasl_FIND_QUIETLY TRUE) +ENDIF (GSASL_INCLUDE_DIR AND GSASL_LIBRARIES) + +FIND_PATH(GSASL_INCLUDE_DIR gsasl.h) + +FIND_LIBRARY(GSASL_LIBRARIES gsasl) + +INCLUDE(FindPackageHandleStandardArgs) + +# handle the QUIETLY and REQUIRED arguments and set GSASL_FOUND to TRUE if +# all listed variables are TRUE +FIND_PACKAGE_HANDLE_STANDARD_ARGS(GSASL DEFAULT_MSG GSASL_LIBRARIES GSASL_INCLUDE_DIR) + +MARK_AS_ADVANCED(GSASL_INCLUDE_DIR GSASL_LIBRARIES) \ No newline at end of file diff --git a/contrib/libhdfs3-cmake/CMake/FindGoogleTest.cmake b/contrib/libhdfs3-cmake/CMake/FindGoogleTest.cmake new file mode 100644 index 00000000000..fd57c1e2abd --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/FindGoogleTest.cmake @@ -0,0 +1,65 @@ +include(CheckCXXSourceRuns) + +find_path(GTest_INCLUDE_DIR gtest/gtest.h + NO_DEFAULT_PATH + PATHS + "${PROJECT_SOURCE_DIR}/../thirdparty/googletest/googletest/include" + "/usr/local/include" + "/usr/include") + +find_path(GMock_INCLUDE_DIR gmock/gmock.h + NO_DEFAULT_PATH + PATHS + "${PROJECT_SOURCE_DIR}/../thirdparty/googletest/googlemock/include" + "/usr/local/include" + "/usr/include") + +find_library(Gtest_LIBRARY + NAMES libgtest.a + HINTS + "${PROJECT_SOURCE_DIR}/../thirdparty/googletest/build/googlemock/gtest" + "/usr/local/lib" + "/usr/lib") + +find_library(Gmock_LIBRARY + NAMES libgmock.a + HINTS + "${PROJECT_SOURCE_DIR}/../thirdparty/googletest/build/googlemock" + "/usr/local/lib" + "/usr/lib") + +message(STATUS "Find GoogleTest include path: ${GTest_INCLUDE_DIR}") +message(STATUS "Find GoogleMock include path: ${GMock_INCLUDE_DIR}") +message(STATUS "Find Gtest library path: ${Gtest_LIBRARY}") +message(STATUS "Find Gmock library path: ${Gmock_LIBRARY}") + +set(CMAKE_REQUIRED_INCLUDES ${GTest_INCLUDE_DIR} ${GMock_INCLUDE_DIR}) +set(CMAKE_REQUIRED_LIBRARIES ${Gtest_LIBRARY} ${Gmock_LIBRARY} -lpthread) +set(CMAKE_REQUIRED_FLAGS) +check_cxx_source_runs(" +#include +#include +int main(int argc, char *argv[]) +{ + double pi = 3.14; + EXPECT_EQ(pi, 3.14); + return 0; +} +" GoogleTest_CHECK_FINE) +message(STATUS "GoogleTest check: ${GoogleTest_CHECK_FINE}") + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args( + GoogleTest + REQUIRED_VARS + GTest_INCLUDE_DIR + GMock_INCLUDE_DIR + Gtest_LIBRARY + Gmock_LIBRARY + GoogleTest_CHECK_FINE) + +set(GoogleTest_INCLUDE_DIR ${GTest_INCLUDE_DIR} ${GMock_INCLUDE_DIR}) +set(GoogleTest_LIBRARIES ${Gtest_LIBRARY} ${Gmock_LIBRARY}) +mark_as_advanced( + GoogleTest_INCLUDE_DIR + GoogleTest_LIBRARIES) diff --git a/contrib/libhdfs3-cmake/CMake/FindKERBEROS.cmake b/contrib/libhdfs3-cmake/CMake/FindKERBEROS.cmake new file mode 100644 index 00000000000..5fc58235a3f --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/FindKERBEROS.cmake @@ -0,0 +1,23 @@ +# - Find kerberos +# Find the native KERBEROS includes and library +# +# KERBEROS_INCLUDE_DIRS - where to find krb5.h, etc. +# KERBEROS_LIBRARIES - List of libraries when using krb5. +# KERBEROS_FOUND - True if krb5 found. + +IF (KERBEROS_INCLUDE_DIRS) + # Already in cache, be silent + SET(KERBEROS_FIND_QUIETLY TRUE) +ENDIF (KERBEROS_INCLUDE_DIRS) + +FIND_PATH(KERBEROS_INCLUDE_DIRS krb5.h) + +SET(KERBEROS_NAMES krb5 k5crypto com_err) +FIND_LIBRARY(KERBEROS_LIBRARIES NAMES ${KERBEROS_NAMES}) + +# handle the QUIETLY and REQUIRED arguments and set KERBEROS_FOUND to TRUE if +# all listed variables are TRUE +INCLUDE(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(KERBEROS DEFAULT_MSG KERBEROS_LIBRARIES KERBEROS_INCLUDE_DIRS) + +MARK_AS_ADVANCED(KERBEROS_LIBRARIES KERBEROS_INCLUDE_DIRS) diff --git a/contrib/libhdfs3-cmake/CMake/FindLibUUID.cmake b/contrib/libhdfs3-cmake/CMake/FindLibUUID.cmake new file mode 100644 index 00000000000..ac6e7267ef7 --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/FindLibUUID.cmake @@ -0,0 +1,23 @@ +# - Find libuuid +# Find the native LIBUUID includes and library +# +# LIBUUID_INCLUDE_DIRS - where to find uuid/uuid.h, etc. +# LIBUUID_LIBRARIES - List of libraries when using uuid. +# LIBUUID_FOUND - True if uuid found. + +IF (LIBUUID_INCLUDE_DIRS) + # Already in cache, be silent + SET(LIBUUID_FIND_QUIETLY TRUE) +ENDIF (LIBUUID_INCLUDE_DIRS) + +FIND_PATH(LIBUUID_INCLUDE_DIRS uuid/uuid.h) + +SET(LIBUUID_NAMES uuid) +FIND_LIBRARY(LIBUUID_LIBRARIES NAMES ${LIBUUID_NAMES}) + +# handle the QUIETLY and REQUIRED arguments and set LIBUUID_FOUND to TRUE if +# all listed variables are TRUE +INCLUDE(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(LIBUUID DEFAULT_MSG LIBUUID_LIBRARIES LIBUUID_INCLUDE_DIRS) + +MARK_AS_ADVANCED(LIBUUID_LIBRARIES LIBUUID_INCLUDE_DIRS) diff --git a/contrib/libhdfs3-cmake/CMake/FindSSL.cmake b/contrib/libhdfs3-cmake/CMake/FindSSL.cmake new file mode 100644 index 00000000000..bcbc5d89653 --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/FindSSL.cmake @@ -0,0 +1,26 @@ +# - Try to find the Open ssl library (ssl) +# +# Once done this will define +# +# SSL_FOUND - System has gnutls +# SSL_INCLUDE_DIR - The gnutls include directory +# SSL_LIBRARIES - The libraries needed to use gnutls +# SSL_DEFINITIONS - Compiler switches required for using gnutls + + +IF (SSL_INCLUDE_DIR AND SSL_LIBRARIES) + # in cache already + SET(SSL_FIND_QUIETLY TRUE) +ENDIF (SSL_INCLUDE_DIR AND SSL_LIBRARIES) + +FIND_PATH(SSL_INCLUDE_DIR openssl/opensslv.h) + +FIND_LIBRARY(SSL_LIBRARIES crypto) + +INCLUDE(FindPackageHandleStandardArgs) + +# handle the QUIETLY and REQUIRED arguments and set SSL_FOUND to TRUE if +# all listed variables are TRUE +FIND_PACKAGE_HANDLE_STANDARD_ARGS(SSL DEFAULT_MSG SSL_LIBRARIES SSL_INCLUDE_DIR) + +MARK_AS_ADVANCED(SSL_INCLUDE_DIR SSL_LIBRARIES) \ No newline at end of file diff --git a/contrib/libhdfs3-cmake/CMake/Functions.cmake b/contrib/libhdfs3-cmake/CMake/Functions.cmake new file mode 100644 index 00000000000..a771b6043fb --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/Functions.cmake @@ -0,0 +1,46 @@ +FUNCTION(AUTO_SOURCES RETURN_VALUE PATTERN SOURCE_SUBDIRS) + + IF ("${SOURCE_SUBDIRS}" STREQUAL "RECURSE") + SET(PATH ".") + IF (${ARGC} EQUAL 4) + LIST(GET ARGV 3 PATH) + ENDIF () + ENDIF() + + IF ("${SOURCE_SUBDIRS}" STREQUAL "RECURSE") + UNSET(${RETURN_VALUE}) + FILE(GLOB SUBDIR_FILES "${PATH}/${PATTERN}") + LIST(APPEND ${RETURN_VALUE} ${SUBDIR_FILES}) + + FILE(GLOB SUBDIRS RELATIVE ${PATH} ${PATH}/*) + + FOREACH(DIR ${SUBDIRS}) + IF (IS_DIRECTORY ${PATH}/${DIR}) + IF (NOT "${DIR}" STREQUAL "CMAKEFILES") + FILE(GLOB_RECURSE SUBDIR_FILES "${PATH}/${DIR}/${PATTERN}") + LIST(APPEND ${RETURN_VALUE} ${SUBDIR_FILES}) + ENDIF() + ENDIF() + ENDFOREACH() + ELSE () + FILE(GLOB ${RETURN_VALUE} "${PATTERN}") + + FOREACH (PATH ${SOURCE_SUBDIRS}) + FILE(GLOB SUBDIR_FILES "${PATH}/${PATTERN}") + LIST(APPEND ${RETURN_VALUE} ${SUBDIR_FILES}) + ENDFOREACH(PATH ${SOURCE_SUBDIRS}) + ENDIF () + + IF (${FILTER_OUT}) + LIST(REMOVE_ITEM ${RETURN_VALUE} ${FILTER_OUT}) + ENDIF() + + SET(${RETURN_VALUE} ${${RETURN_VALUE}} PARENT_SCOPE) +ENDFUNCTION(AUTO_SOURCES) + +FUNCTION(CONTAINS_STRING FILE SEARCH RETURN_VALUE) + FILE(STRINGS ${FILE} FILE_CONTENTS REGEX ".*${SEARCH}.*") + IF (FILE_CONTENTS) + SET(${RETURN_VALUE} TRUE PARENT_SCOPE) + ENDIF() +ENDFUNCTION(CONTAINS_STRING) diff --git a/contrib/libhdfs3-cmake/CMake/Options.cmake b/contrib/libhdfs3-cmake/CMake/Options.cmake new file mode 100644 index 00000000000..5561f3ccc1e --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/Options.cmake @@ -0,0 +1,169 @@ +OPTION(ENABLE_COVERAGE "enable code coverage" OFF) +OPTION(ENABLE_DEBUG "enable debug build" OFF) +OPTION(ENABLE_SSE "enable SSE4.2 buildin function" ON) +OPTION(ENABLE_FRAME_POINTER "enable frame pointer on 64bit system with flag -fno-omit-frame-pointer, on 32bit system, it is always enabled" ON) +OPTION(ENABLE_LIBCPP "using libc++ instead of libstdc++, only valid for clang compiler" OFF) +OPTION(ENABLE_BOOST "using boost instead of native compiler c++0x support" OFF) + +INCLUDE (CheckFunctionExists) +CHECK_FUNCTION_EXISTS(dladdr HAVE_DLADDR) +CHECK_FUNCTION_EXISTS(nanosleep HAVE_NANOSLEEP) + +IF(ENABLE_DEBUG STREQUAL ON) + SET(CMAKE_BUILD_TYPE Debug CACHE + STRING "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel." FORCE) + SET(CMAKE_CXX_FLAGS_DEBUG "-g -O0" CACHE STRING "compiler flags for debug" FORCE) + SET(CMAKE_C_FLAGS_DEBUG "-g -O0" CACHE STRING "compiler flags for debug" FORCE) +ELSE(ENABLE_DEBUG STREQUAL ON) + SET(CMAKE_BUILD_TYPE RelWithDebInfo CACHE + STRING "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel." FORCE) +ENDIF(ENABLE_DEBUG STREQUAL ON) + +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-strict-aliasing") +SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-strict-aliasing") + +IF(ENABLE_COVERAGE STREQUAL ON) + INCLUDE(CodeCoverage) +ENDIF(ENABLE_COVERAGE STREQUAL ON) + +IF(ENABLE_FRAME_POINTER STREQUAL ON) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer") +ENDIF(ENABLE_FRAME_POINTER STREQUAL ON) + +IF(ENABLE_SSE STREQUAL ON) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2") +ENDIF(ENABLE_SSE STREQUAL ON) + +IF(NOT TEST_HDFS_PREFIX) +SET(TEST_HDFS_PREFIX "./" CACHE STRING "default directory prefix used for test." FORCE) +ENDIF(NOT TEST_HDFS_PREFIX) + +ADD_DEFINITIONS(-DTEST_HDFS_PREFIX="${TEST_HDFS_PREFIX}") +ADD_DEFINITIONS(-D__STDC_FORMAT_MACROS) +ADD_DEFINITIONS(-D_GNU_SOURCE) + +IF(OS_MACOSX AND CMAKE_COMPILER_IS_GNUCXX) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-bind_at_load") +ENDIF(OS_MACOSX AND CMAKE_COMPILER_IS_GNUCXX) + +IF(OS_LINUX) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--export-dynamic") +ENDIF(OS_LINUX) + +SET(BOOST_ROOT ${CMAKE_PREFIX_PATH}) +IF(ENABLE_BOOST STREQUAL ON) + MESSAGE(STATUS "using boost instead of native compiler c++0x support.") + FIND_PACKAGE(Boost 1.50 REQUIRED) + SET(NEED_BOOST true CACHE INTERNAL "boost is required") +ELSE(ENABLE_BOOST STREQUAL ON) + SET(NEED_BOOST false CACHE INTERNAL "boost is required") +ENDIF(ENABLE_BOOST STREQUAL ON) + +IF(CMAKE_COMPILER_IS_GNUCXX) + IF(ENABLE_LIBCPP STREQUAL ON) + MESSAGE(FATAL_ERROR "Unsupport using GCC compiler with libc++") + ENDIF(ENABLE_LIBCPP STREQUAL ON) + + IF((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR EQUAL 4) AND OS_MACOSX) + SET(NEED_GCCEH true CACHE INTERNAL "Explicitly link with gcc_eh") + MESSAGE(STATUS "link with -lgcc_eh for TLS") + ENDIF((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR EQUAL 4) AND OS_MACOSX) + + IF((GCC_COMPILER_VERSION_MAJOR LESS 4) OR ((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 4))) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall") + IF(NOT ENABLE_BOOST STREQUAL ON) + MESSAGE(STATUS "gcc version is older than 4.6.0, boost is required.") + FIND_PACKAGE(Boost 1.50 REQUIRED) + SET(NEED_BOOST true CACHE INTERNAL "boost is required") + ENDIF(NOT ENABLE_BOOST STREQUAL ON) + ELSEIF((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 7)) + IF(NOT ENABLE_BOOST STREQUAL ON) + MESSAGE(STATUS "gcc version is older than 4.6.0, boost is required.") + FIND_PACKAGE(Boost 1.50 REQUIRED) + SET(NEED_BOOST true CACHE INTERNAL "boost is required") + ENDIF(NOT ENABLE_BOOST STREQUAL ON) + MESSAGE(STATUS "adding c++0x support for gcc compiler") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x") + ELSE((GCC_COMPILER_VERSION_MAJOR LESS 4) OR ((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 4))) + MESSAGE(STATUS "adding c++0x support for gcc compiler") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x") + ENDIF((GCC_COMPILER_VERSION_MAJOR LESS 4) OR ((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 4))) + + IF(NEED_BOOST) + IF((Boost_MAJOR_VERSION LESS 1) OR ((Boost_MAJOR_VERSION EQUAL 1) AND (Boost_MINOR_VERSION LESS 50))) + MESSAGE(FATAL_ERROR "boost 1.50+ is required") + ENDIF() + ELSE(NEED_BOOST) + IF(HAVE_NANOSLEEP) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_NANOSLEEP") + ELSE(HAVE_NANOSLEEP) + MESSAGE(FATAL_ERROR "nanosleep() is required") + ENDIF(HAVE_NANOSLEEP) + ENDIF(NEED_BOOST) + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall") +ELSEIF(CMAKE_COMPILER_IS_CLANG) + MESSAGE(STATUS "adding c++0x support for clang compiler") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x") + SET(CMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LANGUAGE_STANDARD "c++0x") + IF(ENABLE_LIBCPP STREQUAL ON) + MESSAGE(STATUS "using libc++ instead of libstdc++") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++") + SET(CMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LIBRARY "libc++") + ENDIF(ENABLE_LIBCPP STREQUAL ON) +ENDIF(CMAKE_COMPILER_IS_GNUCXX) + +TRY_COMPILE(STRERROR_R_RETURN_INT + ${CMAKE_CURRENT_BINARY_DIR} + ${HDFS3_ROOT_DIR}/CMake/CMakeTestCompileStrerror.cpp + CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'" + OUTPUT_VARIABLE OUTPUT) + +MESSAGE(STATUS "Checking whether strerror_r returns an int") + +IF(STRERROR_R_RETURN_INT) + MESSAGE(STATUS "Checking whether strerror_r returns an int -- yes") +ELSE(STRERROR_R_RETURN_INT) + MESSAGE(STATUS "Checking whether strerror_r returns an int -- no") +ENDIF(STRERROR_R_RETURN_INT) + +TRY_COMPILE(HAVE_STEADY_CLOCK + ${CMAKE_CURRENT_BINARY_DIR} + ${HDFS3_ROOT_DIR}/CMake/CMakeTestCompileSteadyClock.cpp + CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'" + OUTPUT_VARIABLE OUTPUT) + +TRY_COMPILE(HAVE_NESTED_EXCEPTION + ${CMAKE_CURRENT_BINARY_DIR} + ${HDFS3_ROOT_DIR}/CMake/CMakeTestCompileNestedException.cpp + CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'" + OUTPUT_VARIABLE OUTPUT) + +FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include ") +TRY_COMPILE(HAVE_BOOST_CHRONO + ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_BINARY_DIR}/test.cpp + CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'" + -DINCLUDE_DIRECTORIES=${Boost_INCLUDE_DIR} + OUTPUT_VARIABLE OUTPUT) + +FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include ") +TRY_COMPILE(HAVE_STD_CHRONO + ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_BINARY_DIR}/test.cpp + CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'" + OUTPUT_VARIABLE OUTPUT) + +FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include ") +TRY_COMPILE(HAVE_BOOST_ATOMIC + ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_BINARY_DIR}/test.cpp + CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'" + -DINCLUDE_DIRECTORIES=${Boost_INCLUDE_DIR} + OUTPUT_VARIABLE OUTPUT) + +FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include ") +TRY_COMPILE(HAVE_STD_ATOMIC + ${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_BINARY_DIR}/test.cpp + CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'" + OUTPUT_VARIABLE OUTPUT) diff --git a/contrib/libhdfs3-cmake/CMake/Platform.cmake b/contrib/libhdfs3-cmake/CMake/Platform.cmake new file mode 100644 index 00000000000..ea00fa3f401 --- /dev/null +++ b/contrib/libhdfs3-cmake/CMake/Platform.cmake @@ -0,0 +1,33 @@ +IF(CMAKE_SYSTEM_NAME STREQUAL "Linux") + SET(OS_LINUX true CACHE INTERNAL "Linux operating system") +ELSEIF(CMAKE_SYSTEM_NAME STREQUAL "Darwin") + SET(OS_MACOSX true CACHE INTERNAL "Mac Darwin operating system") +ELSE(CMAKE_SYSTEM_NAME STREQUAL "Linux") + MESSAGE(FATAL_ERROR "Unsupported OS: \"${CMAKE_SYSTEM_NAME}\"") +ENDIF(CMAKE_SYSTEM_NAME STREQUAL "Linux") + +IF(CMAKE_COMPILER_IS_GNUCXX) + EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} -dumpversion OUTPUT_VARIABLE GCC_COMPILER_VERSION) + + IF (NOT GCC_COMPILER_VERSION) + MESSAGE(FATAL_ERROR "Cannot get gcc version") + ENDIF (NOT GCC_COMPILER_VERSION) + + STRING(REGEX MATCHALL "[0-9]+" GCC_COMPILER_VERSION ${GCC_COMPILER_VERSION}) + + LIST(GET GCC_COMPILER_VERSION 0 GCC_COMPILER_VERSION_MAJOR) + LIST(GET GCC_COMPILER_VERSION 1 GCC_COMPILER_VERSION_MINOR) + + SET(GCC_COMPILER_VERSION_MAJOR ${GCC_COMPILER_VERSION_MAJOR} CACHE INTERNAL "gcc major version") + SET(GCC_COMPILER_VERSION_MINOR ${GCC_COMPILER_VERSION_MINOR} CACHE INTERNAL "gcc minor version") + + MESSAGE(STATUS "checking compiler: GCC (${GCC_COMPILER_VERSION_MAJOR}.${GCC_COMPILER_VERSION_MINOR}.${GCC_COMPILER_VERSION_PATCH})") +ELSE(CMAKE_COMPILER_IS_GNUCXX) + EXECUTE_PROCESS(COMMAND ${CMAKE_C_COMPILER} --version OUTPUT_VARIABLE COMPILER_OUTPUT) + IF(COMPILER_OUTPUT MATCHES "clang") + SET(CMAKE_COMPILER_IS_CLANG true CACHE INTERNAL "using clang as compiler") + MESSAGE(STATUS "checking compiler: CLANG") + ELSE(COMPILER_OUTPUT MATCHES "clang") + MESSAGE(FATAL_ERROR "Unsupported compiler: \"${CMAKE_CXX_COMPILER}\"") + ENDIF(COMPILER_OUTPUT MATCHES "clang") +ENDIF(CMAKE_COMPILER_IS_GNUCXX) diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt new file mode 100644 index 00000000000..209f907e967 --- /dev/null +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -0,0 +1,224 @@ +if (NOT USE_INTERNAL_PROTOBUF_LIBRARY) + # compatiable with protobuf which was compiled old C++ ABI + set(CMAKE_CXX_FLAGS "-D_GLIBCXX_USE_CXX11_ABI=0") + set(CMAKE_C_FLAGS "") + if (NOT (CMAKE_VERSION VERSION_LESS "3.8.0")) + unset(CMAKE_CXX_STANDARD) + endif () +endif() + +# project and source dir +set(HDFS3_ROOT_DIR ${CMAKE_SOURCE_DIR}/contrib/libhdfs3) +set(HDFS3_SOURCE_DIR ${HDFS3_ROOT_DIR}/src) +set(HDFS3_COMMON_DIR ${HDFS3_SOURCE_DIR}/common) + +# module +set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH}) +include(Platform) +include(Options) + +# prefer shared libraries +set(CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_SHARED_LIBRARY_SUFFIX}) +find_package(CURL REQUIRED) +find_package(GSasl REQUIRED) +find_package(KERBEROS REQUIRED) +find_package(LibXml2 REQUIRED) +find_package(LibUUID REQUIRED) + +# source +set(PROTO_FILES + ${HDFS3_SOURCE_DIR}/proto/encryption.proto + ${HDFS3_SOURCE_DIR}/proto/ClientDatanodeProtocol.proto + ${HDFS3_SOURCE_DIR}/proto/hdfs.proto + ${HDFS3_SOURCE_DIR}/proto/Security.proto + ${HDFS3_SOURCE_DIR}/proto/ProtobufRpcEngine.proto + ${HDFS3_SOURCE_DIR}/proto/ClientNamenodeProtocol.proto + ${HDFS3_SOURCE_DIR}/proto/IpcConnectionContext.proto + ${HDFS3_SOURCE_DIR}/proto/RpcHeader.proto + ${HDFS3_SOURCE_DIR}/proto/datatransfer.proto + ) + +PROTOBUF_GENERATE_CPP(PROTO_SOURCES PROTO_HEADERS ${PROTO_FILES}) + +configure_file(${HDFS3_SOURCE_DIR}/platform.h.in ${CMAKE_CURRENT_BINARY_DIR}/platform.h) + +set(SRCS + ${HDFS3_SOURCE_DIR}/network/TcpSocket.cpp + ${HDFS3_SOURCE_DIR}/network/DomainSocket.cpp + ${HDFS3_SOURCE_DIR}/network/BufferedSocketReader.cpp + ${HDFS3_SOURCE_DIR}/client/EncryptionZoneIterator.cpp + ${HDFS3_SOURCE_DIR}/client/ReadShortCircuitInfo.cpp + ${HDFS3_SOURCE_DIR}/client/Pipeline.cpp + ${HDFS3_SOURCE_DIR}/client/Hdfs.cpp + ${HDFS3_SOURCE_DIR}/client/Packet.cpp + ${HDFS3_SOURCE_DIR}/client/OutputStreamImpl.cpp + ${HDFS3_SOURCE_DIR}/client/KerberosName.cpp + ${HDFS3_SOURCE_DIR}/client/PacketHeader.cpp + ${HDFS3_SOURCE_DIR}/client/LocalBlockReader.cpp + ${HDFS3_SOURCE_DIR}/client/UserInfo.cpp + ${HDFS3_SOURCE_DIR}/client/RemoteBlockReader.cpp + ${HDFS3_SOURCE_DIR}/client/Permission.cpp + ${HDFS3_SOURCE_DIR}/client/FileSystemImpl.cpp + ${HDFS3_SOURCE_DIR}/client/CryptoCodec.cpp + ${HDFS3_SOURCE_DIR}/client/DirectoryIterator.cpp + ${HDFS3_SOURCE_DIR}/client/FileSystemKey.cpp + ${HDFS3_SOURCE_DIR}/client/DataTransferProtocolSender.cpp + ${HDFS3_SOURCE_DIR}/client/LeaseRenewer.cpp + ${HDFS3_SOURCE_DIR}/client/HttpClient.cpp + ${HDFS3_SOURCE_DIR}/client/PeerCache.cpp + ${HDFS3_SOURCE_DIR}/client/InputStream.cpp + ${HDFS3_SOURCE_DIR}/client/FileSystem.cpp + ${HDFS3_SOURCE_DIR}/client/InputStreamImpl.cpp + ${HDFS3_SOURCE_DIR}/client/Token.cpp + ${HDFS3_SOURCE_DIR}/client/PacketPool.cpp + ${HDFS3_SOURCE_DIR}/client/KmsClientProvider.cpp + ${HDFS3_SOURCE_DIR}/client/OutputStream.cpp + ${HDFS3_SOURCE_DIR}/rpc/RpcChannelKey.cpp + ${HDFS3_SOURCE_DIR}/rpc/RpcProtocolInfo.cpp + ${HDFS3_SOURCE_DIR}/rpc/RpcClient.cpp + ${HDFS3_SOURCE_DIR}/rpc/RpcRemoteCall.cpp + ${HDFS3_SOURCE_DIR}/rpc/RpcChannel.cpp + ${HDFS3_SOURCE_DIR}/rpc/RpcAuth.cpp + ${HDFS3_SOURCE_DIR}/rpc/RpcContentWrapper.cpp + ${HDFS3_SOURCE_DIR}/rpc/RpcConfig.cpp + ${HDFS3_SOURCE_DIR}/rpc/RpcServerInfo.cpp + ${HDFS3_SOURCE_DIR}/rpc/SaslClient.cpp + ${HDFS3_SOURCE_DIR}/server/Datanode.cpp + ${HDFS3_SOURCE_DIR}/server/LocatedBlocks.cpp + ${HDFS3_SOURCE_DIR}/server/NamenodeProxy.cpp + ${HDFS3_SOURCE_DIR}/server/NamenodeImpl.cpp + ${HDFS3_SOURCE_DIR}/server/NamenodeInfo.cpp + ${HDFS3_SOURCE_DIR}/common/WritableUtils.cpp + ${HDFS3_SOURCE_DIR}/common/ExceptionInternal.cpp + ${HDFS3_SOURCE_DIR}/common/SessionConfig.cpp + ${HDFS3_SOURCE_DIR}/common/StackPrinter.cpp + ${HDFS3_SOURCE_DIR}/common/Exception.cpp + ${HDFS3_SOURCE_DIR}/common/Logger.cpp + ${HDFS3_SOURCE_DIR}/common/CFileWrapper.cpp + ${HDFS3_SOURCE_DIR}/common/XmlConfig.cpp + ${HDFS3_SOURCE_DIR}/common/WriteBuffer.cpp + ${HDFS3_SOURCE_DIR}/common/HWCrc32c.cpp + ${HDFS3_SOURCE_DIR}/common/MappedFileWrapper.cpp + ${HDFS3_SOURCE_DIR}/common/Hash.cpp + ${HDFS3_SOURCE_DIR}/common/SWCrc32c.cpp + ${HDFS3_SOURCE_DIR}/common/Thread.cpp + + ${HDFS3_SOURCE_DIR}/network/TcpSocket.h + ${HDFS3_SOURCE_DIR}/network/BufferedSocketReader.h + ${HDFS3_SOURCE_DIR}/network/Socket.h + ${HDFS3_SOURCE_DIR}/network/DomainSocket.h + ${HDFS3_SOURCE_DIR}/network/Syscall.h + ${HDFS3_SOURCE_DIR}/client/InputStreamImpl.h + ${HDFS3_SOURCE_DIR}/client/FileSystem.h + ${HDFS3_SOURCE_DIR}/client/ReadShortCircuitInfo.h + ${HDFS3_SOURCE_DIR}/client/InputStreamInter.h + ${HDFS3_SOURCE_DIR}/client/HttpClient.h + ${HDFS3_SOURCE_DIR}/client/FileSystemImpl.h + ${HDFS3_SOURCE_DIR}/client/FileEncryptionInfo.h + ${HDFS3_SOURCE_DIR}/client/PacketPool.h + ${HDFS3_SOURCE_DIR}/client/Pipeline.h + ${HDFS3_SOURCE_DIR}/client/EncryptionZoneInfo.h + ${HDFS3_SOURCE_DIR}/client/OutputStreamInter.h + ${HDFS3_SOURCE_DIR}/client/RemoteBlockReader.h + ${HDFS3_SOURCE_DIR}/client/CryptoCodec.h + ${HDFS3_SOURCE_DIR}/client/Token.h + ${HDFS3_SOURCE_DIR}/client/EncryptionZoneIterator.h + ${HDFS3_SOURCE_DIR}/client/KerberosName.h + ${HDFS3_SOURCE_DIR}/client/DirectoryIterator.h + ${HDFS3_SOURCE_DIR}/client/hdfs.h + ${HDFS3_SOURCE_DIR}/client/FileSystemStats.h + ${HDFS3_SOURCE_DIR}/client/FileSystemKey.h + ${HDFS3_SOURCE_DIR}/client/DataTransferProtocolSender.h + ${HDFS3_SOURCE_DIR}/client/Packet.h + ${HDFS3_SOURCE_DIR}/client/PacketHeader.h + ${HDFS3_SOURCE_DIR}/client/FileSystemInter.h + ${HDFS3_SOURCE_DIR}/client/LocalBlockReader.h + ${HDFS3_SOURCE_DIR}/client/TokenInternal.h + ${HDFS3_SOURCE_DIR}/client/InputStream.h + ${HDFS3_SOURCE_DIR}/client/PipelineAck.h + ${HDFS3_SOURCE_DIR}/client/BlockReader.h + ${HDFS3_SOURCE_DIR}/client/Permission.h + ${HDFS3_SOURCE_DIR}/client/OutputStreamImpl.h + ${HDFS3_SOURCE_DIR}/client/LeaseRenewer.h + ${HDFS3_SOURCE_DIR}/client/UserInfo.h + ${HDFS3_SOURCE_DIR}/client/PeerCache.h + ${HDFS3_SOURCE_DIR}/client/OutputStream.h + ${HDFS3_SOURCE_DIR}/client/FileStatus.h + ${HDFS3_SOURCE_DIR}/client/KmsClientProvider.h + ${HDFS3_SOURCE_DIR}/client/DataTransferProtocol.h + ${HDFS3_SOURCE_DIR}/client/BlockLocation.h + ${HDFS3_SOURCE_DIR}/rpc/RpcConfig.h + ${HDFS3_SOURCE_DIR}/rpc/SaslClient.h + ${HDFS3_SOURCE_DIR}/rpc/RpcAuth.h + ${HDFS3_SOURCE_DIR}/rpc/RpcClient.h + ${HDFS3_SOURCE_DIR}/rpc/RpcCall.h + ${HDFS3_SOURCE_DIR}/rpc/RpcContentWrapper.h + ${HDFS3_SOURCE_DIR}/rpc/RpcProtocolInfo.h + ${HDFS3_SOURCE_DIR}/rpc/RpcRemoteCall.h + ${HDFS3_SOURCE_DIR}/rpc/RpcServerInfo.h + ${HDFS3_SOURCE_DIR}/rpc/RpcChannel.h + ${HDFS3_SOURCE_DIR}/rpc/RpcChannelKey.h + ${HDFS3_SOURCE_DIR}/server/BlockLocalPathInfo.h + ${HDFS3_SOURCE_DIR}/server/LocatedBlocks.h + ${HDFS3_SOURCE_DIR}/server/DatanodeInfo.h + ${HDFS3_SOURCE_DIR}/server/RpcHelper.h + ${HDFS3_SOURCE_DIR}/server/ExtendedBlock.h + ${HDFS3_SOURCE_DIR}/server/NamenodeInfo.h + ${HDFS3_SOURCE_DIR}/server/NamenodeImpl.h + ${HDFS3_SOURCE_DIR}/server/LocatedBlock.h + ${HDFS3_SOURCE_DIR}/server/NamenodeProxy.h + ${HDFS3_SOURCE_DIR}/server/Datanode.h + ${HDFS3_SOURCE_DIR}/server/Namenode.h + ${HDFS3_SOURCE_DIR}/common/XmlConfig.h + ${HDFS3_SOURCE_DIR}/common/Logger.h + ${HDFS3_SOURCE_DIR}/common/WriteBuffer.h + ${HDFS3_SOURCE_DIR}/common/HWCrc32c.h + ${HDFS3_SOURCE_DIR}/common/Checksum.h + ${HDFS3_SOURCE_DIR}/common/SessionConfig.h + ${HDFS3_SOURCE_DIR}/common/Unordered.h + ${HDFS3_SOURCE_DIR}/common/BigEndian.h + ${HDFS3_SOURCE_DIR}/common/Thread.h + ${HDFS3_SOURCE_DIR}/common/StackPrinter.h + ${HDFS3_SOURCE_DIR}/common/Exception.h + ${HDFS3_SOURCE_DIR}/common/WritableUtils.h + ${HDFS3_SOURCE_DIR}/common/StringUtil.h + ${HDFS3_SOURCE_DIR}/common/LruMap.h + ${HDFS3_SOURCE_DIR}/common/Function.h + ${HDFS3_SOURCE_DIR}/common/DateTime.h + ${HDFS3_SOURCE_DIR}/common/Hash.h + ${HDFS3_SOURCE_DIR}/common/SWCrc32c.h + ${HDFS3_SOURCE_DIR}/common/ExceptionInternal.h + ${HDFS3_SOURCE_DIR}/common/Memory.h + ${HDFS3_SOURCE_DIR}/common/FileWrapper.h + ) + +# target +add_library(hdfs3 STATIC ${SRCS} ${PROTO_SOURCES} ${PROTO_HEADERS}) + +if (USE_INTERNAL_PROTOBUF_LIBRARY) + add_dependencies(hdfs3 protoc) +endif() + +target_include_directories(hdfs3 PRIVATE ${HDFS3_SOURCE_DIR}) +target_include_directories(hdfs3 PRIVATE ${HDFS3_COMMON_DIR}) +target_include_directories(hdfs3 PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) + +target_include_directories(hdfs3 PRIVATE ${CURL_INCLUDE_DIR}) +target_include_directories(hdfs3 PRIVATE ${GSASL_INCLUDE_DIR}) +target_include_directories(hdfs3 PRIVATE ${KERBEROS_INCLUDE_DIRS}) +target_include_directories(hdfs3 PRIVATE ${LIBXML2_INCLUDE_DIR}) +target_include_directories(hdfs3 PRIVATE ${LIBUUID_INCLUDE_DIRS}) + +target_link_libraries(hdfs3 ${CURL_LIBRARIES}) +target_link_libraries(hdfs3 ${GSASL_LIBRARIES}) +target_link_libraries(hdfs3 ${KERBEROS_LIBRARIES}) +target_link_libraries(hdfs3 ${LIBXML2_LIBRARIES}) +target_link_libraries(hdfs3 ${LIBUUID_LIBRARIES}) + +# inherit from parent cmake +target_include_directories(hdfs3 PRIVATE ${Boost_INCLUDE_DIRS}) +target_include_directories(hdfs3 PRIVATE ${Protobuf_INCLUDE_DIR}) +target_include_directories(hdfs3 PRIVATE ${OPENSSL_INCLUDE_DIR}) + +target_link_libraries(hdfs3 ${Protobuf_LIBRARY}) +target_link_libraries(hdfs3 ${OPENSSL_LIBRARIES}) diff --git a/contrib/protobuf b/contrib/protobuf new file mode 160000 index 00000000000..12735370922 --- /dev/null +++ b/contrib/protobuf @@ -0,0 +1 @@ +Subproject commit 12735370922a35f03999afff478e1c6d7aa917a4 diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 9bb852db923..94d1e693849 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -167,6 +167,7 @@ target_link_libraries (clickhouse_common_io ${Boost_SYSTEM_LIBRARY} apple_rt ${CMAKE_DL_LIBS} + ${HDFS3_LIBRARY} ) target_link_libraries (dbms diff --git a/dbms/programs/client/Client.cpp b/dbms/programs/client/Client.cpp index 516547de5d1..0845b05a45c 100644 --- a/dbms/programs/client/Client.cpp +++ b/dbms/programs/client/Client.cpp @@ -802,7 +802,7 @@ private: connection->forceConnected(); - if (insert && !insert->select) + if (insert && !insert->select && !insert->in_file) processInsertQuery(); else processOrdinaryQuery(); diff --git a/dbms/src/IO/ReadBufferFromHDFS.h b/dbms/src/IO/ReadBufferFromHDFS.h new file mode 100644 index 00000000000..5dc4efc3b82 --- /dev/null +++ b/dbms/src/IO/ReadBufferFromHDFS.h @@ -0,0 +1,87 @@ +#pragma once +#include +#include +#include +#include + +#ifndef O_DIRECT +#define O_DIRECT 00040000 +#endif + +namespace DB +{ + + /** Accepts path to file and opens it, or pre-opened file descriptor. + * Closes file by himself (thus "owns" a file descriptor). + */ + class ReadBufferFromHDFS : public BufferWithOwnMemory + { + protected: + std::string hdfs_uri; + // std::unique_ptr builder; + struct hdfsBuilder *builder; + hdfsFS fs; + hdfsFile fin; + public: + ReadBufferFromHDFS(const std::string & hdfs_name_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE) + : BufferWithOwnMemory(buf_size), hdfs_uri(hdfs_name_) , builder(hdfsNewBuilder()) + { + Poco::URI uri(hdfs_name_); + auto& host = uri.getHost(); + auto port = uri.getPort(); + auto& path = uri.getPath(); + if (host.empty() || port == 0 || path.empty()) + { + throw Exception("Illegal HDFS URI : " + hdfs_uri); + } + // set read/connect timeout, default value in libhdfs3 is about 1 hour, and too large + hdfsBuilderConfSetStr(builder, "input.read.timeout", "60000"); // 1 min + hdfsBuilderConfSetStr(builder, "input.connect.timeout", "60000"); // 1 min + + hdfsBuilderSetNameNode(builder, host.c_str()); + hdfsBuilderSetNameNodePort(builder, port); + fs = hdfsBuilderConnect(builder); + + if (fs == nullptr) + { + throw Exception("Unable to connect to HDFS:" + String(hdfsGetLastError())); + } + + fin = hdfsOpenFile(fs, path.c_str(), O_RDONLY, 0, 0, 0); + } + + ReadBufferFromHDFS(ReadBufferFromHDFS &&) = default; + + ~ReadBufferFromHDFS() override + { + close(); + hdfsFreeBuilder(builder); + } + + /// Close HDFS connection before destruction of object. + void close() + { + hdfsCloseFile(fs, fin); + } + + bool nextImpl() override + { + int done = hdfsRead(fs, fin, internal_buffer.begin(), internal_buffer.size()); + if (done <0) + { + throw Exception("Fail to read HDFS file: " + hdfs_uri + " " + String(hdfsGetLastError())); + } + + if (done) + working_buffer.resize(done); + else + return false; + return true; + } + + const std::string& getHDFSUri() const + { + return hdfs_uri; + } + }; +} diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index f058550a441..28ca86d76ff 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -9,16 +9,23 @@ #include #include #include +#include +#include + #include #include +#include #include #include #include #include +#include +#include +#include namespace DB { @@ -141,6 +148,84 @@ BlockIO InterpreterInsertQuery::execute() throw Exception("Cannot insert column " + name_type.name + ", because it is MATERIALIZED column.", ErrorCodes::ILLEGAL_COLUMN); } } + else if (query.in_file) + { + // read data stream from in_file, and copy it to out + // Special handling in_file based on url type: + String uristr = typeid_cast(*query.in_file).value.safeGet(); + // create Datastream based on Format: + String format = query.format; + if (format.empty()) format = "Values"; + + auto & settings = context.getSettingsRef(); + + // Assume no query and fragment in uri, todo, add sanity check + String fuzzyFileNames; + String uriPrefix = uristr.substr(0, uristr.find_last_of('/')); + if (uriPrefix.length() == uristr.length()) + { + fuzzyFileNames = uristr; + uriPrefix.clear(); + } + else + { + uriPrefix += "/"; + fuzzyFileNames = uristr.substr(uriPrefix.length()); + } + + Poco::URI uri(uriPrefix); + String scheme = uri.getScheme(); + + std::vector fuzzyNameList = parseDescription(fuzzyFileNames, 0, fuzzyFileNames.length(), ',' , 100/* hard coded max files */); + + std::vector > fileNames; + + for(auto fuzzyName : fuzzyNameList) + fileNames.push_back(parseDescription(fuzzyName, 0, fuzzyName.length(), '|', 100)); + + BlockInputStreams inputs; + + for (auto & vecNames : fileNames) + { + for (auto & name: vecNames) + { + std::unique_ptr read_buf = nullptr; + + if (scheme.empty() || scheme == "file") + { + read_buf = std::make_unique(Poco::URI(uriPrefix + name).getPath()); + } + else if (scheme == "hdfs") + { + read_buf = std::make_unique(uriPrefix + name); + } + else + { + throw Exception("URI scheme " + scheme + " is not supported with insert statement yet"); + } + + inputs.emplace_back( + std::make_shared>( + context.getInputFormat(format, *read_buf, + res.out->getHeader(), // sample_block + settings.max_insert_block_size), + std::move(read_buf))); + } + } + + if (inputs.size() == 0) + throw Exception("Inputs interpreter error"); + + auto stream = inputs[0]; + if (inputs.size() > 1) + { + stream = std::make_shared >(inputs, nullptr, settings.max_distributed_connections); + } + + res.in = std::make_shared(stream, res.out); + + res.out = nullptr; + } return res; } diff --git a/dbms/src/Parsers/ASTInsertQuery.cpp b/dbms/src/Parsers/ASTInsertQuery.cpp index 1ecf4f9daef..7e1538bd810 100644 --- a/dbms/src/Parsers/ASTInsertQuery.cpp +++ b/dbms/src/Parsers/ASTInsertQuery.cpp @@ -36,10 +36,23 @@ void ASTInsertQuery::formatImpl(const FormatSettings & settings, FormatState & s if (!format.empty()) { settings.ostr << (settings.hilite ? hilite_keyword : "") << " FORMAT " << (settings.hilite ? hilite_none : "") << format; + if (in_file) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << " INFILE " << (settings.hilite ? hilite_none : ""); + in_file->formatImpl(settings, state, frame); + } } else { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " VALUES" << (settings.hilite ? hilite_none : ""); + if (in_file) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << " INFILE " << (settings.hilite ? hilite_none : ""); + in_file->formatImpl(settings, state, frame); + } + else + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << " VALUES" << (settings.hilite ? hilite_none : ""); + } } } } diff --git a/dbms/src/Parsers/ASTInsertQuery.h b/dbms/src/Parsers/ASTInsertQuery.h index 9e72174470c..dd93b84cbbd 100644 --- a/dbms/src/Parsers/ASTInsertQuery.h +++ b/dbms/src/Parsers/ASTInsertQuery.h @@ -18,6 +18,7 @@ public: String format; ASTPtr select; ASTPtr table_function; + ASTPtr in_file; // Set to true if the data should only be inserted into attached views bool no_destination = false; @@ -41,6 +42,11 @@ public: res->table_function = table_function->clone(); res->children.push_back(res->table_function); } + if (in_file) + { + res->in_file = in_file->clone(); res->children.push_back(res->in_file); + } + return res; } diff --git a/dbms/src/Parsers/ParserInsertQuery.cpp b/dbms/src/Parsers/ParserInsertQuery.cpp index 73aca09c210..f9d9c2762a7 100644 --- a/dbms/src/Parsers/ParserInsertQuery.cpp +++ b/dbms/src/Parsers/ParserInsertQuery.cpp @@ -31,6 +31,7 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserKeyword s_format("FORMAT"); ParserKeyword s_select("SELECT"); ParserKeyword s_with("WITH"); + ParserKeyword s_infile("INFILE"); ParserToken s_lparen(TokenType::OpeningRoundBracket); ParserToken s_rparen(TokenType::ClosingRoundBracket); ParserIdentifier name_p; @@ -43,6 +44,7 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ASTPtr format; ASTPtr select; ASTPtr table_function; + ASTPtr in_file; /// Insertion data const char * data = nullptr; @@ -81,7 +83,7 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) Pos before_select = pos; - /// VALUES or FORMAT or SELECT + /// VALUES or FORMAT or SELECT or INFILE if (s_values.ignore(pos, expected)) { data = pos->begin; @@ -93,28 +95,42 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (!name_p.parse(pos, format, expected)) return false; - data = name_pos->end; + // there are two case after FORMAT xx: + // case 1: data_set. + // case 2: INFILE xx clause. + if (s_infile.ignore(pos, expected)) + { + ParserStringLiteral in_file_p; - if (data < end && *data == ';') - throw Exception("You have excessive ';' symbol before data for INSERT.\n" - "Example:\n\n" - "INSERT INTO t (x, y) FORMAT TabSeparated\n" - ";\tHello\n" - "2\tWorld\n" - "\n" - "Note that there is no ';' just after format name, " - "you need to put at least one whitespace symbol before the data.", ErrorCodes::SYNTAX_ERROR); + if (!in_file_p.parse(pos, in_file, expected)) + return false; - while (data < end && (*data == ' ' || *data == '\t' || *data == '\f')) - ++data; + } + else + { + data = name_pos->end; - /// Data starts after the first newline, if there is one, or after all the whitespace characters, otherwise. + if (data < end && *data == ';') + throw Exception("You have excessive ';' symbol before data for INSERT.\n" + "Example:\n\n" + "INSERT INTO t (x, y) FORMAT TabSeparated\n" + ";\tHello\n" + "2\tWorld\n" + "\n" + "Note that there is no ';' just after format name, " + "you need to put at least one whitespace symbol before the data.", ErrorCodes::SYNTAX_ERROR); - if (data < end && *data == '\r') - ++data; + while (data < end && (*data == ' ' || *data == '\t' || *data == '\f')) + ++data; - if (data < end && *data == '\n') - ++data; + /// Data starts after the first newline, if there is one, or after all the whitespace characters, otherwise. + + if (data < end && *data == '\r') + ++data; + + if (data < end && *data == '\n') + ++data; + } } else if (s_select.ignore(pos, expected) || s_with.ignore(pos,expected)) { @@ -122,6 +138,12 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserSelectWithUnionQuery select_p; select_p.parse(pos, select, expected); } + else if (s_infile.ignore(pos, expected)) + { + ParserStringLiteral in_file_p; + if (!in_file_p.parse(pos, in_file, expected)) + return false; + } else { return false; @@ -147,7 +169,12 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) query->columns = columns; query->select = select; - query->data = data != end ? data : nullptr; + query->in_file = in_file; + + if (query->in_file) + query->data = nullptr; + else + query->data = data != end ? data : nullptr; query->end = end; if (columns) @@ -155,6 +182,9 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (select) query->children.push_back(select); + if (in_file) + query->children.push_back(in_file); + return true; } diff --git a/dbms/src/Parsers/ParserInsertQuery.h b/dbms/src/Parsers/ParserInsertQuery.h index 86198365edc..65d04e17d8f 100644 --- a/dbms/src/Parsers/ParserInsertQuery.h +++ b/dbms/src/Parsers/ParserInsertQuery.h @@ -9,18 +9,21 @@ namespace DB /** Cases: * - * Normal case: + * #1 Normal case: * INSERT INTO [db.]table (c1, c2, c3) VALUES (v11, v12, v13), (v21, v22, v23), ... * INSERT INTO [db.]table VALUES (v11, v12, v13), (v21, v22, v23), ... * - * Insert of data in an arbitrary format. + * #2 Insert of data in an arbitrary format. * The data itself comes after LF(line feed), if it exists, or after all the whitespace characters, otherwise. * INSERT INTO [db.]table (c1, c2, c3) FORMAT format \n ... * INSERT INTO [db.]table FORMAT format \n ... * - * Insert the result of the SELECT query. + * #3 Insert the result of the SELECT query. * INSERT INTO [db.]table (c1, c2, c3) SELECT ... * INSERT INTO [db.]table SELECT ... + + * #4 Insert of data in an arbitrary form from file(a bit variant of #2) + * INSERT INTO [db.]table (c1, c2, c3) FORMAT format INFILE 'url' */ class ParserInsertQuery : public IParserBase { diff --git a/dbms/src/Storages/StorageHDFS.cpp b/dbms/src/Storages/StorageHDFS.cpp new file mode 100644 index 00000000000..b8a818fb7ce --- /dev/null +++ b/dbms/src/Storages/StorageHDFS.cpp @@ -0,0 +1,178 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + +StorageHDFS::StorageHDFS(const String & uri_, + const std::string & table_name_, + const String & format_name_, + const ColumnsDescription & columns_, + Context & context_) + : IStorage(columns_), uri(uri_), format_name(format_name_), table_name(table_name_), context_global(context_) +{ +} + +namespace +{ + class StorageHDFSBlockInputStream : public IProfilingBlockInputStream + { + public: + StorageHDFSBlockInputStream(const String & uri, + const String & format, + const String & name_, + const Block & sample_block, + const Context & context, + size_t max_block_size) + : name(name_) + { + // Assume no query and fragment in uri, todo, add sanity check + String fuzzyFileNames; + String uriPrefix = uri.substr(0, uri.find_last_of('/')); + if (uriPrefix.length() == uri.length()) + { + fuzzyFileNames = uri; + uriPrefix.clear(); + } + else + { + uriPrefix += "/"; + fuzzyFileNames = uri.substr(uriPrefix.length()); + } + + std::vector fuzzyNameList = parseDescription(fuzzyFileNames, 0, fuzzyFileNames.length(), ',' , 100/* hard coded max files */); + + std::vector > fileNames; + + for(auto fuzzyName : fuzzyNameList) + fileNames.push_back(parseDescription(fuzzyName, 0, fuzzyName.length(), '|', 100)); + + BlockInputStreams inputs; + + for (auto & vecNames : fileNames) + { + for (auto & name: vecNames) + { + std::unique_ptr read_buf = std::make_unique(uriPrefix + name); + + inputs.emplace_back( + std::make_shared>( + FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size), + std::move(read_buf))); + } + } + + if (inputs.size() == 0) + throw Exception("StorageHDFS inputs interpreter error"); + + if (inputs.size() == 1) + { + reader = inputs[0]; + } + else + { + reader = std::make_shared >(inputs, nullptr, context.getSettingsRef().max_distributed_connections); + } + } + + String getName() const override + { + return name; + } + + Block readImpl() override + { + return reader->read(); + } + + Block getHeader() const override + { + return reader->getHeader(); + } + + void readPrefixImpl() override + { + reader->readPrefix(); + } + + void readSuffixImpl() override + { + auto explicitReader = dynamic_cast *>(reader.get()); + if (explicitReader) explicitReader->cancel(false); // skip Union read suffix assertion + + reader->readSuffix(); + } + + private: + String name; + BlockInputStreamPtr reader; + }; + +} + + +BlockInputStreams StorageHDFS::read( + const Names & /*column_names*/, + const SelectQueryInfo & /*query_info*/, + const Context & context, + QueryProcessingStage::Enum /*processed_stage*/, + size_t max_block_size, + unsigned /*num_streams*/) +{ + return {std::make_shared( + uri, + format_name, + getName(), + getSampleBlock(), + context, + max_block_size)}; +} + +void StorageHDFS::rename(const String & /*new_path_to_db*/, const String & /*new_database_name*/, const String & /*new_table_name*/) {} + +BlockOutputStreamPtr StorageHDFS::write(const ASTPtr & /*query*/, const Settings & /*settings*/) +{ + throw Exception("StorageHDFS write is not supported yet"); + return {}; +} + +void registerStorageHDFS(StorageFactory & factory) +{ + factory.registerStorage("HDFS", [](const StorageFactory::Arguments & args) + { + ASTs & engine_args = args.engine_args; + + if (!(engine_args.size() == 1 || engine_args.size() == 2)) + throw Exception( + "Storage HDFS requires exactly 2 arguments: url and name of used format.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + engine_args[0] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[0], args.local_context); + + String url = static_cast(*engine_args[0]).value.safeGet(); + + engine_args[1] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[1], args.local_context); + + String format_name = static_cast(*engine_args[1]).value.safeGet(); + + return StorageHDFS::create(url, args.table_name, format_name, args.columns, args.context); + }); +} + +} diff --git a/dbms/src/Storages/StorageHDFS.h b/dbms/src/Storages/StorageHDFS.h new file mode 100644 index 00000000000..1fcee97eb86 --- /dev/null +++ b/dbms/src/Storages/StorageHDFS.h @@ -0,0 +1,53 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB +{ +/** + * This class represents table engine for external hdfs files. + * Read method is supported for now. + */ +class StorageHDFS : public ext::shared_ptr_helper, public IStorage +{ +public: + String getName() const override + { + return "HDFS"; + } + + String getTableName() const override + { + return table_name; + } + + BlockInputStreams read(const Names & column_names, + const SelectQueryInfo & query_info, + const Context & context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + unsigned num_streams) override; + + BlockOutputStreamPtr write(const ASTPtr & query, const Settings & settings) override; + + void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name) override; + +protected: + StorageHDFS(const String & uri_, + const String & table_name_, + const String & format_name_, + const ColumnsDescription & columns_, + Context & context_); + +private: + String uri; + String format_name; + String table_name; + Context & context_global; + + Logger * log = &Logger::get("StorageHDFS"); +}; +} diff --git a/dbms/src/TableFunctions/ITableFunction.cpp b/dbms/src/TableFunctions/ITableFunction.cpp index b15cbbc9fd9..b1d84a5ecac 100644 --- a/dbms/src/TableFunctions/ITableFunction.cpp +++ b/dbms/src/TableFunctions/ITableFunction.cpp @@ -1,5 +1,7 @@ #include #include +#include +#include namespace ProfileEvents @@ -16,4 +18,168 @@ StoragePtr ITableFunction::execute(const ASTPtr & ast_function, const Context & return executeImpl(ast_function, context); } +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int BAD_ARGUMENTS; +} + +/// The Cartesian product of two sets of rows, the result is written in place of the first argument +static void append(std::vector & to, const std::vector & what, size_t max_addresses) +{ + if (what.empty()) + return; + + if (to.empty()) + { + to = what; + return; + } + + if (what.size() * to.size() > max_addresses) + throw Exception("Table function 'remote': first argument generates too many result addresses", + ErrorCodes::BAD_ARGUMENTS); + std::vector res; + for (size_t i = 0; i < to.size(); ++i) + for (size_t j = 0; j < what.size(); ++j) + res.push_back(to[i] + what[j]); + + to.swap(res); +} + + +/// Parse number from substring +static bool parseNumber(const String & description, size_t l, size_t r, size_t & res) +{ + res = 0; + for (size_t pos = l; pos < r; pos ++) + { + if (!isNumericASCII(description[pos])) + return false; + res = res * 10 + description[pos] - '0'; + if (res > 1e15) + return false; + } + return true; +} + + + +/* Parse a string that generates shards and replicas. Separator - one of two characters | or , + * depending on whether shards or replicas are generated. + * For example: + * host1,host2,... - generates set of shards from host1, host2, ... + * host1|host2|... - generates set of replicas from host1, host2, ... + * abc{8..10}def - generates set of shards abc8def, abc9def, abc10def. + * abc{08..10}def - generates set of shards abc08def, abc09def, abc10def. + * abc{x,yy,z}def - generates set of shards abcxdef, abcyydef, abczdef. + * abc{x|yy|z} def - generates set of replicas abcxdef, abcyydef, abczdef. + * abc{1..9}de{f,g,h} - is a direct product, 27 shards. + * abc{1..9}de{0|1} - is a direct product, 9 shards, in each 2 replicas. + */ +std::vector parseDescription(const String & description, size_t l, size_t r, char separator, size_t max_addresses) +{ + std::vector res; + std::vector cur; + + /// An empty substring means a set of an empty string + if (l >= r) + { + res.push_back(""); + return res; + } + + for (size_t i = l; i < r; ++i) + { + /// Either the numeric interval (8..10) or equivalent expression in brackets + if (description[i] == '{') + { + int cnt = 1; + int last_dot = -1; /// The rightmost pair of points, remember the index of the right of the two + size_t m; + std::vector buffer; + bool have_splitter = false; + + /// Look for the corresponding closing bracket + for (m = i + 1; m < r; ++m) + { + if (description[m] == '{') ++cnt; + if (description[m] == '}') --cnt; + if (description[m] == '.' && description[m-1] == '.') last_dot = m; + if (description[m] == separator) have_splitter = true; + if (cnt == 0) break; + } + if (cnt != 0) + throw Exception("Table function 'remote': incorrect brace sequence in first argument", + ErrorCodes::BAD_ARGUMENTS); + /// The presence of a dot - numeric interval + if (last_dot != -1) + { + size_t left, right; + if (description[last_dot - 1] != '.') + throw Exception("Table function 'remote': incorrect argument in braces (only one dot): " + description.substr(i, m - i + 1), + ErrorCodes::BAD_ARGUMENTS); + if (!parseNumber(description, i + 1, last_dot - 1, left)) + throw Exception("Table function 'remote': incorrect argument in braces (Incorrect left number): " + + description.substr(i, m - i + 1), + ErrorCodes::BAD_ARGUMENTS); + if (!parseNumber(description, last_dot + 1, m, right)) + throw Exception("Table function 'remote': incorrect argument in braces (Incorrect right number): " + + description.substr(i, m - i + 1), + ErrorCodes::BAD_ARGUMENTS); + if (left > right) + throw Exception("Table function 'remote': incorrect argument in braces (left number is greater then right): " + + description.substr(i, m - i + 1), + ErrorCodes::BAD_ARGUMENTS); + if (right - left + 1 > max_addresses) + throw Exception("Table function 'remote': first argument generates too many result addresses", + ErrorCodes::BAD_ARGUMENTS); + bool add_leading_zeroes = false; + size_t len = last_dot - 1 - (i + 1); + /// If the left and right borders have equal numbers, then you must add leading zeros. + if (last_dot - 1 - (i + 1) == m - (last_dot + 1)) + add_leading_zeroes = true; + for (size_t id = left; id <= right; ++id) + { + String cur = toString(id); + if (add_leading_zeroes) + { + while (cur.size() < len) + cur = "0" + cur; + } + buffer.push_back(cur); + } + } + else if (have_splitter) /// If there is a current delimiter inside, then generate a set of resulting rows + buffer = parseDescription(description, i + 1, m, separator, max_addresses); + else /// Otherwise just copy, spawn will occur when you call with the correct delimiter + buffer.push_back(description.substr(i, m - i + 1)); + /// Add all possible received extensions to the current set of lines + append(cur, buffer, max_addresses); + i = m; + } + else if (description[i] == separator) + { + /// If the delimiter, then add found rows + res.insert(res.end(), cur.begin(), cur.end()); + cur.clear(); + } + else + { + /// Otherwise, simply append the character to current lines + std::vector buffer; + buffer.push_back(description.substr(i, 1)); + append(cur, buffer, max_addresses); + } + } + + res.insert(res.end(), cur.begin(), cur.end()); + if (res.size() > max_addresses) + throw Exception("Table function 'remote': first argument generates too many result addresses", + ErrorCodes::BAD_ARGUMENTS); + + return res; +} + + } diff --git a/dbms/src/TableFunctions/ITableFunction.h b/dbms/src/TableFunctions/ITableFunction.h index a3885f16152..52f72ec74e8 100644 --- a/dbms/src/TableFunctions/ITableFunction.h +++ b/dbms/src/TableFunctions/ITableFunction.h @@ -2,6 +2,8 @@ #include #include +#include +#include namespace DB @@ -42,5 +44,6 @@ private: using TableFunctionPtr = std::shared_ptr; +std::vector parseDescription(const String & description, size_t l, size_t r, char separator, size_t max_addresses); } diff --git a/dbms/src/TableFunctions/TableFunctionHDFS.cpp b/dbms/src/TableFunctions/TableFunctionHDFS.cpp new file mode 100644 index 00000000000..5f2efd3b8fc --- /dev/null +++ b/dbms/src/TableFunctions/TableFunctionHDFS.cpp @@ -0,0 +1,21 @@ +#include +#include +#include + +namespace DB +{ +StoragePtr TableFunctionHDFS::getStorage( + const String & source, const String & format, const Block & sample_block, Context & global_context) const +{ + return StorageHDFS::create(source, + getName(), + format, + ColumnsDescription{sample_block.getNamesAndTypesList()}, + global_context); +} + +void registerTableFunctionHDFS(TableFunctionFactory & factory) +{ + factory.registerFunction(); +} +} diff --git a/dbms/src/TableFunctions/TableFunctionHDFS.h b/dbms/src/TableFunctions/TableFunctionHDFS.h new file mode 100644 index 00000000000..79e8c74ccfc --- /dev/null +++ b/dbms/src/TableFunctions/TableFunctionHDFS.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ +/* hdfs(name_node_ip:name_node_port, format, structure) - creates a temporary storage from hdfs file + * + */ +class TableFunctionHDFS : public ITableFunctionFileLike +{ +public: + static constexpr auto name = "hdfs"; + std::string getName() const override + { + return name; + } + +private: + StoragePtr getStorage( + const String & source, const String & format, const Block & sample_block, Context & global_context) const override; +}; +} diff --git a/dbms/src/TableFunctions/TableFunctionRemote.cpp b/dbms/src/TableFunctions/TableFunctionRemote.cpp index acc094408f8..d29ffcc7b4f 100644 --- a/dbms/src/TableFunctions/TableFunctionRemote.cpp +++ b/dbms/src/TableFunctions/TableFunctionRemote.cpp @@ -22,165 +22,6 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } - -/// The Cartesian product of two sets of rows, the result is written in place of the first argument -static void append(std::vector & to, const std::vector & what, size_t max_addresses) -{ - if (what.empty()) - return; - - if (to.empty()) - { - to = what; - return; - } - - if (what.size() * to.size() > max_addresses) - throw Exception("Table function 'remote': first argument generates too many result addresses", - ErrorCodes::BAD_ARGUMENTS); - std::vector res; - for (size_t i = 0; i < to.size(); ++i) - for (size_t j = 0; j < what.size(); ++j) - res.push_back(to[i] + what[j]); - - to.swap(res); -} - - -/// Parse number from substring -static bool parseNumber(const String & description, size_t l, size_t r, size_t & res) -{ - res = 0; - for (size_t pos = l; pos < r; pos ++) - { - if (!isNumericASCII(description[pos])) - return false; - res = res * 10 + description[pos] - '0'; - if (res > 1e15) - return false; - } - return true; -} - - - -/* Parse a string that generates shards and replicas. Separator - one of two characters | or , - * depending on whether shards or replicas are generated. - * For example: - * host1,host2,... - generates set of shards from host1, host2, ... - * host1|host2|... - generates set of replicas from host1, host2, ... - * abc{8..10}def - generates set of shards abc8def, abc9def, abc10def. - * abc{08..10}def - generates set of shards abc08def, abc09def, abc10def. - * abc{x,yy,z}def - generates set of shards abcxdef, abcyydef, abczdef. - * abc{x|yy|z} def - generates set of replicas abcxdef, abcyydef, abczdef. - * abc{1..9}de{f,g,h} - is a direct product, 27 shards. - * abc{1..9}de{0|1} - is a direct product, 9 shards, in each 2 replicas. - */ -static std::vector parseDescription(const String & description, size_t l, size_t r, char separator, size_t max_addresses) -{ - std::vector res; - std::vector cur; - - /// An empty substring means a set of an empty string - if (l >= r) - { - res.push_back(""); - return res; - } - - for (size_t i = l; i < r; ++i) - { - /// Either the numeric interval (8..10) or equivalent expression in brackets - if (description[i] == '{') - { - int cnt = 1; - int last_dot = -1; /// The rightmost pair of points, remember the index of the right of the two - size_t m; - std::vector buffer; - bool have_splitter = false; - - /// Look for the corresponding closing bracket - for (m = i + 1; m < r; ++m) - { - if (description[m] == '{') ++cnt; - if (description[m] == '}') --cnt; - if (description[m] == '.' && description[m-1] == '.') last_dot = m; - if (description[m] == separator) have_splitter = true; - if (cnt == 0) break; - } - if (cnt != 0) - throw Exception("Table function 'remote': incorrect brace sequence in first argument", - ErrorCodes::BAD_ARGUMENTS); - /// The presence of a dot - numeric interval - if (last_dot != -1) - { - size_t left, right; - if (description[last_dot - 1] != '.') - throw Exception("Table function 'remote': incorrect argument in braces (only one dot): " + description.substr(i, m - i + 1), - ErrorCodes::BAD_ARGUMENTS); - if (!parseNumber(description, i + 1, last_dot - 1, left)) - throw Exception("Table function 'remote': incorrect argument in braces (Incorrect left number): " - + description.substr(i, m - i + 1), - ErrorCodes::BAD_ARGUMENTS); - if (!parseNumber(description, last_dot + 1, m, right)) - throw Exception("Table function 'remote': incorrect argument in braces (Incorrect right number): " - + description.substr(i, m - i + 1), - ErrorCodes::BAD_ARGUMENTS); - if (left > right) - throw Exception("Table function 'remote': incorrect argument in braces (left number is greater then right): " - + description.substr(i, m - i + 1), - ErrorCodes::BAD_ARGUMENTS); - if (right - left + 1 > max_addresses) - throw Exception("Table function 'remote': first argument generates too many result addresses", - ErrorCodes::BAD_ARGUMENTS); - bool add_leading_zeroes = false; - size_t len = last_dot - 1 - (i + 1); - /// If the left and right borders have equal numbers, then you must add leading zeros. - if (last_dot - 1 - (i + 1) == m - (last_dot + 1)) - add_leading_zeroes = true; - for (size_t id = left; id <= right; ++id) - { - String cur = toString(id); - if (add_leading_zeroes) - { - while (cur.size() < len) - cur = "0" + cur; - } - buffer.push_back(cur); - } - } - else if (have_splitter) /// If there is a current delimiter inside, then generate a set of resulting rows - buffer = parseDescription(description, i + 1, m, separator, max_addresses); - else /// Otherwise just copy, spawn will occur when you call with the correct delimiter - buffer.push_back(description.substr(i, m - i + 1)); - /// Add all possible received extensions to the current set of lines - append(cur, buffer, max_addresses); - i = m; - } - else if (description[i] == separator) - { - /// If the delimiter, then add found rows - res.insert(res.end(), cur.begin(), cur.end()); - cur.clear(); - } - else - { - /// Otherwise, simply append the character to current lines - std::vector buffer; - buffer.push_back(description.substr(i, 1)); - append(cur, buffer, max_addresses); - } - } - - res.insert(res.end(), cur.begin(), cur.end()); - if (res.size() > max_addresses) - throw Exception("Table function 'remote': first argument generates too many result addresses", - ErrorCodes::BAD_ARGUMENTS); - - return res; -} - - StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & ast_function, const Context & context) const { ASTs & args_func = typeid_cast(*ast_function).children; diff --git a/dbms/src/TableFunctions/registerTableFunctions.cpp b/dbms/src/TableFunctions/registerTableFunctions.cpp index d0afeff0b17..e6bc80e48d2 100644 --- a/dbms/src/TableFunctions/registerTableFunctions.cpp +++ b/dbms/src/TableFunctions/registerTableFunctions.cpp @@ -13,6 +13,7 @@ void registerTableFunctionNumbers(TableFunctionFactory & factory); void registerTableFunctionCatBoostPool(TableFunctionFactory & factory); void registerTableFunctionFile(TableFunctionFactory & factory); void registerTableFunctionURL(TableFunctionFactory & factory); +void registerTableFunctionHDFS(TableFunctionFactory & factory); #if USE_POCO_SQLODBC || USE_POCO_DATAODBC void registerTableFunctionODBC(TableFunctionFactory & factory); @@ -36,6 +37,7 @@ void registerTableFunctions() registerTableFunctionCatBoostPool(factory); registerTableFunctionFile(factory); registerTableFunctionURL(factory); + registerTableFunctionHDFS(factory); #if USE_POCO_SQLODBC || USE_POCO_DATAODBC registerTableFunctionODBC(factory); diff --git a/release b/release index 23bfd6f2dd6..4c536609571 100755 --- a/release +++ b/release @@ -64,7 +64,7 @@ do shift elif [[ $1 == '--fast' ]]; then # Wrong but fast pbuilder mode: create base package with all depends - EXTRAPACKAGES="$EXTRAPACKAGES debhelper cmake ninja-build gcc-7 g++-7 libc6-dev libicu-dev libreadline-dev psmisc bash expect python python-lxml python-termcolor python-requests curl perl sudo openssl netcat-openbsd" + EXTRAPACKAGES="$EXTRAPACKAGES debhelper cmake ninja-build gcc-7 g++-7 libc6-dev libicu-dev libreadline-dev psmisc bash expect python python-lxml python-termcolor python-requests curl perl sudo openssl netcat-openbsd uuid xml2 krb5 gsasl" shift else echo "Unknown option $1"