From aef6668cb4ac67b38f12c594702e01c795d10e8e Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 17 Jan 2022 21:45:53 +0300 Subject: [PATCH] Remove unbundled hdfs3 support --- CMakeLists.txt | 1 - cmake/find/hdfs3.cmake | 45 ------------------- contrib/CMakeLists.txt | 5 +-- contrib/libhdfs3-cmake/CMakeLists.txt | 39 +++++++++++----- src/CMakeLists.txt | 7 ++- src/Common/config.h.in | 1 - src/Storages/HDFS/HDFSCommon.cpp | 9 ---- ...StorageSystemBuildOptions.generated.cpp.in | 1 - src/configure_config.cmake | 3 ++ 9 files changed, 34 insertions(+), 77 deletions(-) delete mode 100644 cmake/find/hdfs3.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index c24cd2cf155..6bc27d70219 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -507,7 +507,6 @@ include (cmake/find/libxml2.cmake) include (cmake/find/brotli.cmake) include (cmake/find/pdqsort.cmake) include (cmake/find/miniselect.cmake) -include (cmake/find/hdfs3.cmake) # uses protobuf include (cmake/find/poco.cmake) include (cmake/find/curl.cmake) include (cmake/find/s3.cmake) diff --git a/cmake/find/hdfs3.cmake b/cmake/find/hdfs3.cmake deleted file mode 100644 index 5f3f61254ed..00000000000 --- a/cmake/find/hdfs3.cmake +++ /dev/null @@ -1,45 +0,0 @@ -if(NOT ARCH_ARM AND NOT OS_FREEBSD AND NOT APPLE AND NOT ARCH_PPC64LE) # USE_PROTOBUF AND - option(ENABLE_HDFS "Enable HDFS" ${ENABLE_LIBRARIES}) -elseif(ENABLE_HDFS OR USE_INTERNAL_HDFS3_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use HDFS3 with current configuration") -endif() - -if(NOT ENABLE_HDFS) - if(USE_INTERNAL_HDFS3_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal HDFS3 library with ENABLE_HDFS3=OFF") - endif() - return() -endif() - -option(USE_INTERNAL_HDFS3_LIBRARY "Set to FALSE to use system HDFS3 instead of bundled (experimental - set to OFF on your own risk)" - ON) # We don't know any linux distribution with package for it - -if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include/hdfs/hdfs.h") - if(USE_INTERNAL_HDFS3_LIBRARY) - message(WARNING "submodule contrib/libhdfs3 is missing. to fix try run: \n git submodule update --init") - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal HDFS3 library") - set(USE_INTERNAL_HDFS3_LIBRARY 0) - endif() - set(MISSING_INTERNAL_HDFS3_LIBRARY 1) -endif() - -if(NOT USE_INTERNAL_HDFS3_LIBRARY) - find_library(HDFS3_LIBRARY hdfs3) - find_path(HDFS3_INCLUDE_DIR NAMES hdfs/hdfs.h PATHS ${HDFS3_INCLUDE_PATHS}) - if(NOT HDFS3_LIBRARY OR NOT HDFS3_INCLUDE_DIR) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot find system HDFS3 library") - endif() -endif() - -if(HDFS3_LIBRARY AND HDFS3_INCLUDE_DIR) - set(USE_HDFS 1) -elseif(NOT MISSING_INTERNAL_HDFS3_LIBRARY AND LIBGSASL_LIBRARY AND LIBXML2_LIBRARIES) - set(HDFS3_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include") - set(HDFS3_LIBRARY hdfs3) - set(USE_INTERNAL_HDFS3_LIBRARY 1) - set(USE_HDFS 1) -else() - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannout enable HDFS3") -endif() - -message(STATUS "Using hdfs3=${USE_HDFS}: ${HDFS3_INCLUDE_DIR} : ${HDFS3_LIBRARY}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index b64983f50a3..573ea3317e4 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -72,6 +72,7 @@ add_subdirectory (cppkafka-cmake) add_subdirectory (openldap-cmake) add_subdirectory (grpc-cmake) add_subdirectory (protobuf-cmake) +add_subdirectory (libhdfs3-cmake) # requires: protobuf # TODO: refactor the contrib libraries below this comment. @@ -147,10 +148,6 @@ if (USE_INTERNAL_BROTLI_LIBRARY) target_compile_definitions(brotli PRIVATE BROTLI_BUILD_PORTABLE=1) endif () -if (USE_INTERNAL_HDFS3_LIBRARY) - add_subdirectory(libhdfs3-cmake) -endif () - if (USE_INTERNAL_AWS_S3_LIBRARY) add_subdirectory(aws-s3-cmake) diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index 96a2b5ca50a..d9ed10282cb 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -1,3 +1,14 @@ +if(NOT ARCH_ARM AND NOT OS_FREEBSD AND NOT APPLE AND NOT ARCH_PPC64LE) + option(ENABLE_HDFS "Enable HDFS" ${ENABLE_LIBRARIES}) +elseif(ENABLE_HDFS) + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use HDFS3 with current configuration") +endif() + +if(NOT ENABLE_HDFS) + message(STATUS "Not using hdfs") + return() +endif() + if (${ENABLE_KRB5}) SET(WITH_KERBEROS 1) else() @@ -94,25 +105,29 @@ set(SRCS set_source_files_properties("${HDFS3_SOURCE_DIR}/rpc/RpcClient.cpp" PROPERTIES COMPILE_FLAGS "-DBOOST_UUID_RANDOM_PROVIDER_FORCE_POSIX=1") # target -add_library(hdfs3 ${SRCS}) +add_library(_hdfs3 ${SRCS}) -add_dependencies(hdfs3 protoc) +add_dependencies(_hdfs3 protoc) -target_include_directories(hdfs3 PRIVATE ${HDFS3_SOURCE_DIR}) -target_include_directories(hdfs3 PRIVATE ${HDFS3_COMMON_DIR}) -target_include_directories(hdfs3 PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) +target_include_directories(_hdfs3 PRIVATE ${HDFS3_SOURCE_DIR}) +target_include_directories(_hdfs3 PRIVATE ${HDFS3_COMMON_DIR}) +target_include_directories(_hdfs3 PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) -target_include_directories(hdfs3 PRIVATE ${LIBGSASL_INCLUDE_DIR}) -target_include_directories(hdfs3 PRIVATE ${LIBXML2_INCLUDE_DIR}) +target_include_directories(_hdfs3 PRIVATE ${LIBGSASL_INCLUDE_DIR}) +target_include_directories(_hdfs3 PRIVATE ${LIBXML2_INCLUDE_DIR}) -target_link_libraries(hdfs3 PRIVATE ${LIBGSASL_LIBRARY}) +target_include_directories(_hdfs3 SYSTEM BEFORE INTERFACE "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include") + +target_link_libraries(_hdfs3 PRIVATE ${LIBGSASL_LIBRARY}) if (WITH_KERBEROS) - target_link_libraries(hdfs3 PRIVATE ${KRB5_LIBRARY}) + target_link_libraries(_hdfs3 PRIVATE ${KRB5_LIBRARY}) endif() -target_link_libraries(hdfs3 PRIVATE ${LIBXML2_LIBRARIES}) +target_link_libraries(_hdfs3 PRIVATE ${LIBXML2_LIBRARIES}) # inherit from parent cmake -target_link_libraries(hdfs3 PRIVATE ch_contrib::protobuf boost::headers_only) +target_link_libraries(_hdfs3 PRIVATE ch_contrib::protobuf boost::headers_only) if (TARGET OpenSSL::SSL) - target_link_libraries(hdfs3 PRIVATE OpenSSL::Crypto OpenSSL::SSL) + target_link_libraries(_hdfs3 PRIVATE OpenSSL::Crypto OpenSSL::SSL) endif() + +add_library(ch_contrib::hdfs ALIAS _hdfs3) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9c90449560b..edf1809c49d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -114,7 +114,7 @@ if (USE_AZURE_BLOB_STORAGE) add_headers_and_sources(dbms Disks/AzureBlobStorage) endif() -if (USE_HDFS) +if (TARGET ch_contrib::hdfs) add_headers_and_sources(dbms Storages/HDFS) add_headers_and_sources(dbms Disks/HDFS) endif() @@ -434,9 +434,8 @@ if (TARGET clickhouse_grpc_protos) dbms_target_link_libraries (PUBLIC clickhouse_grpc_protos) endif() -if (USE_HDFS) - dbms_target_link_libraries(PRIVATE ${HDFS3_LIBRARY}) - dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${HDFS3_INCLUDE_DIR}) +if (TARGET ch_contrib::hdfs) + dbms_target_link_libraries(PRIVATE ch_contrib::hdfs) endif() if (TARGET ch_contrib::hivemetastore) diff --git a/src/Common/config.h.in b/src/Common/config.h.in index 28506a94581..03ea3ac2e7a 100644 --- a/src/Common/config.h.in +++ b/src/Common/config.h.in @@ -7,7 +7,6 @@ #cmakedefine01 USE_SSL #cmakedefine01 USE_INTERNAL_SSL_LIBRARY #cmakedefine01 USE_HDFS -#cmakedefine01 USE_INTERNAL_HDFS3_LIBRARY #cmakedefine01 USE_AWS_S3 #cmakedefine01 USE_AZURE_BLOB_STORAGE #cmakedefine01 USE_BROTLI diff --git a/src/Storages/HDFS/HDFSCommon.cpp b/src/Storages/HDFS/HDFSCommon.cpp index c99d100d6d4..85eb16d7de2 100644 --- a/src/Storages/HDFS/HDFSCommon.cpp +++ b/src/Storages/HDFS/HDFSCommon.cpp @@ -45,11 +45,7 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration { need_kinit = true; hadoop_kerberos_principal = config.getString(key_path); - -#if USE_INTERNAL_HDFS3_LIBRARY hdfsBuilderSetPrincipal(hdfs_builder, hadoop_kerberos_principal.c_str()); -#endif - continue; } else if (key == "hadoop_kerberos_kinit_command") @@ -170,12 +166,7 @@ HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::A String user_config_prefix = HDFSBuilderWrapper::CONFIG_PREFIX + "_" + user; if (config.has(user_config_prefix)) { -#if USE_INTERNAL_HDFS3_LIBRARY builder.loadFromConfig(config, user_config_prefix, true); -#else - throw Exception("Multi user HDFS configuration required internal libhdfs3", - ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG); -#endif } } diff --git a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in index 01a79e7c48d..01be3f8456d 100644 --- a/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in +++ b/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in @@ -63,7 +63,6 @@ const char * auto_config_build[] "USE_AWS_S3", "@USE_AWS_S3@", "USE_CASSANDRA", "@USE_CASSANDRA@", "USE_YAML_CPP", "@USE_YAML_CPP@", - "USE_INTERNAL_HDFS3_LIBRARY", "@USE_INTERNAL_HDFS3_LIBRARY@", "CLICKHOUSE_SPLIT_BINARY", "@CLICKHOUSE_SPLIT_BINARY@", "USE_SENTRY", "@USE_SENTRY@", "USE_DATASKETCHES", "@USE_DATASKETCHES@", diff --git a/src/configure_config.cmake b/src/configure_config.cmake index 18778595287..e216eaf088f 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -22,3 +22,6 @@ endif() if (TARGET ch_contrib::grpc) set(USE_GRPC 1) endif() +if (TARGET ch_contrib::hdfs) + set(USE_HDFS 1) +endif()