Remove unbundled hdfs3 support

CMakeLists.txt

@@ -507,7 +507,6 @@ include (cmake/find/libxml2.cmake)
 include (cmake/find/brotli.cmake)
 include (cmake/find/pdqsort.cmake)
 include (cmake/find/miniselect.cmake)
-include (cmake/find/hdfs3.cmake) # uses protobuf
 include (cmake/find/poco.cmake)
 include (cmake/find/curl.cmake)
 include (cmake/find/s3.cmake)

cmake/find/hdfs3.cmake

@@ -1,45 +0,0 @@
-if(NOT ARCH_ARM AND NOT OS_FREEBSD AND NOT APPLE AND NOT ARCH_PPC64LE) # USE_PROTOBUF AND 
-    option(ENABLE_HDFS "Enable HDFS" ${ENABLE_LIBRARIES})
-elseif(ENABLE_HDFS OR USE_INTERNAL_HDFS3_LIBRARY)
-    message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use HDFS3 with current configuration")
-endif()
-
-if(NOT ENABLE_HDFS)
-    if(USE_INTERNAL_HDFS3_LIBRARY)
-        message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal HDFS3 library with ENABLE_HDFS3=OFF")
-    endif()
-    return()
-endif()
-
-option(USE_INTERNAL_HDFS3_LIBRARY "Set to FALSE to use system HDFS3 instead of bundled (experimental - set to OFF on your own risk)"
-       ON) # We don't know any linux distribution with package for it
-
-if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include/hdfs/hdfs.h")
-    if(USE_INTERNAL_HDFS3_LIBRARY)
-        message(WARNING "submodule contrib/libhdfs3 is missing. to fix try run: \n git submodule update --init")
-        message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal HDFS3 library")
-        set(USE_INTERNAL_HDFS3_LIBRARY 0)
-    endif()
-    set(MISSING_INTERNAL_HDFS3_LIBRARY 1)
-endif()
-
-if(NOT USE_INTERNAL_HDFS3_LIBRARY)
-    find_library(HDFS3_LIBRARY hdfs3)
-    find_path(HDFS3_INCLUDE_DIR NAMES hdfs/hdfs.h PATHS ${HDFS3_INCLUDE_PATHS})
-    if(NOT HDFS3_LIBRARY OR NOT HDFS3_INCLUDE_DIR)
-        message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot find system HDFS3 library")
-    endif()
-endif()
-
-if(HDFS3_LIBRARY AND HDFS3_INCLUDE_DIR)
-    set(USE_HDFS 1)
-elseif(NOT MISSING_INTERNAL_HDFS3_LIBRARY AND LIBGSASL_LIBRARY AND LIBXML2_LIBRARIES)
-    set(HDFS3_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include")
-    set(HDFS3_LIBRARY hdfs3)
-    set(USE_INTERNAL_HDFS3_LIBRARY 1)
-    set(USE_HDFS 1)
-else()
-    message (${RECONFIGURE_MESSAGE_LEVEL} "Cannout enable HDFS3")
-endif()
-
-message(STATUS "Using hdfs3=${USE_HDFS}: ${HDFS3_INCLUDE_DIR} : ${HDFS3_LIBRARY}")

contrib/CMakeLists.txt

@@ -72,6 +72,7 @@ add_subdirectory (cppkafka-cmake)
 add_subdirectory (openldap-cmake)
 add_subdirectory (grpc-cmake)
 add_subdirectory (protobuf-cmake)
+add_subdirectory (libhdfs3-cmake) # requires: protobuf
 
 # TODO: refactor the contrib libraries below this comment.
 
@@ -147,10 +148,6 @@ if (USE_INTERNAL_BROTLI_LIBRARY)
     target_compile_definitions(brotli PRIVATE BROTLI_BUILD_PORTABLE=1)
 endif ()
 
-if (USE_INTERNAL_HDFS3_LIBRARY)
-    add_subdirectory(libhdfs3-cmake)
-endif ()
-
 if (USE_INTERNAL_AWS_S3_LIBRARY)
     add_subdirectory(aws-s3-cmake)
 

contrib/libhdfs3-cmake/CMakeLists.txt

@@ -1,3 +1,14 @@
+if(NOT ARCH_ARM AND NOT OS_FREEBSD AND NOT APPLE AND NOT ARCH_PPC64LE)
+    option(ENABLE_HDFS "Enable HDFS" ${ENABLE_LIBRARIES})
+elseif(ENABLE_HDFS)
+    message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use HDFS3 with current configuration")
+endif()
+
+if(NOT ENABLE_HDFS)
+    message(STATUS "Not using hdfs")
+    return()
+endif()
+
 if (${ENABLE_KRB5})
     SET(WITH_KERBEROS 1)
 else()
@@ -94,25 +105,29 @@ set(SRCS
 set_source_files_properties("${HDFS3_SOURCE_DIR}/rpc/RpcClient.cpp" PROPERTIES COMPILE_FLAGS "-DBOOST_UUID_RANDOM_PROVIDER_FORCE_POSIX=1")
 
 # target
-add_library(hdfs3 ${SRCS})
+add_library(_hdfs3 ${SRCS})
 
-add_dependencies(hdfs3 protoc)
+add_dependencies(_hdfs3 protoc)
 
-target_include_directories(hdfs3 PRIVATE ${HDFS3_SOURCE_DIR})
+target_include_directories(_hdfs3 PRIVATE ${HDFS3_SOURCE_DIR})
-target_include_directories(hdfs3 PRIVATE ${HDFS3_COMMON_DIR})
+target_include_directories(_hdfs3 PRIVATE ${HDFS3_COMMON_DIR})
-target_include_directories(hdfs3 PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
+target_include_directories(_hdfs3 PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
 
-target_include_directories(hdfs3 PRIVATE ${LIBGSASL_INCLUDE_DIR})
+target_include_directories(_hdfs3 PRIVATE ${LIBGSASL_INCLUDE_DIR})
-target_include_directories(hdfs3 PRIVATE ${LIBXML2_INCLUDE_DIR})
+target_include_directories(_hdfs3 PRIVATE ${LIBXML2_INCLUDE_DIR})
 
-target_link_libraries(hdfs3 PRIVATE ${LIBGSASL_LIBRARY})
+target_include_directories(_hdfs3 SYSTEM BEFORE INTERFACE "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include")
+
+target_link_libraries(_hdfs3 PRIVATE ${LIBGSASL_LIBRARY})
 if (WITH_KERBEROS)
-    target_link_libraries(hdfs3 PRIVATE ${KRB5_LIBRARY})
+    target_link_libraries(_hdfs3 PRIVATE ${KRB5_LIBRARY})
 endif()
-target_link_libraries(hdfs3 PRIVATE ${LIBXML2_LIBRARIES})
+target_link_libraries(_hdfs3 PRIVATE ${LIBXML2_LIBRARIES})
 
 # inherit from parent cmake
-target_link_libraries(hdfs3 PRIVATE ch_contrib::protobuf boost::headers_only)
+target_link_libraries(_hdfs3 PRIVATE ch_contrib::protobuf boost::headers_only)
 if (TARGET OpenSSL::SSL)
-    target_link_libraries(hdfs3 PRIVATE OpenSSL::Crypto OpenSSL::SSL)
+    target_link_libraries(_hdfs3 PRIVATE OpenSSL::Crypto OpenSSL::SSL)
 endif()
+
+add_library(ch_contrib::hdfs ALIAS _hdfs3)

src/CMakeLists.txt

@@ -114,7 +114,7 @@ if (USE_AZURE_BLOB_STORAGE)
     add_headers_and_sources(dbms Disks/AzureBlobStorage)
 endif()
 
-if (USE_HDFS)
+if (TARGET ch_contrib::hdfs)
     add_headers_and_sources(dbms Storages/HDFS)
     add_headers_and_sources(dbms Disks/HDFS)
 endif()
@@ -434,9 +434,8 @@ if (TARGET clickhouse_grpc_protos)
     dbms_target_link_libraries (PUBLIC clickhouse_grpc_protos)
 endif()
 
-if (USE_HDFS)
+if (TARGET ch_contrib::hdfs)
-    dbms_target_link_libraries(PRIVATE ${HDFS3_LIBRARY})
+    dbms_target_link_libraries(PRIVATE ch_contrib::hdfs)
-    dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${HDFS3_INCLUDE_DIR})
 endif()
 
 if (TARGET ch_contrib::hivemetastore)

src/Common/config.h.in

@@ -7,7 +7,6 @@
 #cmakedefine01 USE_SSL
 #cmakedefine01 USE_INTERNAL_SSL_LIBRARY
 #cmakedefine01 USE_HDFS
-#cmakedefine01 USE_INTERNAL_HDFS3_LIBRARY
 #cmakedefine01 USE_AWS_S3
 #cmakedefine01 USE_AZURE_BLOB_STORAGE
 #cmakedefine01 USE_BROTLI

src/Storages/HDFS/HDFSCommon.cpp

@@ -45,11 +45,7 @@ void HDFSBuilderWrapper::loadFromConfig(const Poco::Util::AbstractConfiguration
         {
             need_kinit = true;
             hadoop_kerberos_principal = config.getString(key_path);
-
-#if USE_INTERNAL_HDFS3_LIBRARY
             hdfsBuilderSetPrincipal(hdfs_builder, hadoop_kerberos_principal.c_str());
-#endif
-
             continue;
         }
         else if (key == "hadoop_kerberos_kinit_command")
@@ -170,12 +166,7 @@ HDFSBuilderWrapper createHDFSBuilder(const String & uri_str, const Poco::Util::A
         String user_config_prefix = HDFSBuilderWrapper::CONFIG_PREFIX + "_" + user;
         if (config.has(user_config_prefix))
         {
-#if USE_INTERNAL_HDFS3_LIBRARY
             builder.loadFromConfig(config, user_config_prefix, true);
-#else
-            throw Exception("Multi user HDFS configuration required internal libhdfs3",
-                ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG);
-#endif
         }
     }
 

src/Storages/System/StorageSystemBuildOptions.generated.cpp.in

@@ -63,7 +63,6 @@ const char * auto_config_build[]
     "USE_AWS_S3", "@USE_AWS_S3@",
     "USE_CASSANDRA", "@USE_CASSANDRA@",
     "USE_YAML_CPP", "@USE_YAML_CPP@",
-    "USE_INTERNAL_HDFS3_LIBRARY", "@USE_INTERNAL_HDFS3_LIBRARY@",
     "CLICKHOUSE_SPLIT_BINARY", "@CLICKHOUSE_SPLIT_BINARY@",
     "USE_SENTRY", "@USE_SENTRY@",
     "USE_DATASKETCHES", "@USE_DATASKETCHES@",

src/configure_config.cmake

@@ -22,3 +22,6 @@ endif()
 if (TARGET ch_contrib::grpc)
     set(USE_GRPC 1)
 endif()
+if (TARGET ch_contrib::hdfs)
+    set(USE_HDFS 1)
+endif()