Add brotli to contrib

.gitmodules

@@ -64,3 +64,6 @@
 [submodule "contrib/cppkafka"]
 	path = contrib/cppkafka
 	url = https://github.com/mfontanini/cppkafka.git
+[submodule "contrib/brotli"]
+	path = contrib/brotli
+	url = https://github.com/google/brotli.git

CMakeLists.txt

@@ -252,6 +252,7 @@ if (NOT USE_CPUID)
 endif()
 include (cmake/find_libgsasl.cmake)
 include (cmake/find_libxml2.cmake)
+include (cmake/find_brotli.cmake)
 include (cmake/find_protobuf.cmake)
 include (cmake/find_hdfs3.cmake)
 include (cmake/find_consistent-hashing.cmake)

cmake/find_brotli.cmake

@@ -0,0 +1,23 @@
+option (USE_INTERNAL_BROTLI_LIBRARY "Set to FALSE to use system libbrotli library instead of bundled" ${NOT_UNBUNDLED})
+
+if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/brotli/c/include/brotli/decode.h")
+    if (USE_INTERNAL_BROTLI_LIBRARY)
+        message (WARNING "submodule contrib/brotli is missing. to fix try run: \n git submodule update --init --recursive")
+        set (USE_INTERNAL_BROTLI_LIBRARY 0)
+    endif ()
+    set (MISSING_INTERNAL_BROTLI_LIBRARY 1)
+endif ()
+
+if (NOT USE_INTERNAL_BROTLI_LIBRARY)
+    find_library (BROTLI_LIBRARY libbrotli)
+    find_path (BROTLI_INCLUDE_DIR NAMES decode.h encode.h port.h types.h PATHS ${BROTLI_INCLUDE_PATHS})
+endif ()
+
+if (BROTLI_LIBRARY AND BROTLI_INCLUDE_DIR)
+elseif (NOT MISSING_INTERNAL_BROTLI_LIBRARY)
+    set (BROTLI_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/libbrotli/c/include)
+    set (USE_INTERNAL_BROTLI_LIBRARY 1)
+    set (BROTLI_LIBRARY libbrotli)
+endif ()
+
+message (STATUS "Using brotli: ${BROTLI_INCLUDE_DIR} : ${BROTLI_LIBRARY}")

contrib/CMakeLists.txt

@@ -215,6 +215,10 @@ if (USE_INTERNAL_LIBXML2_LIBRARY)
     add_subdirectory(libxml2-cmake)
 endif ()
 
+if (USE_INTERNAL_LIBBROTLI_LIBRARY)
+    add_subdirectory(libbrotli-cmake)
+endif ()
+
 if (USE_INTERNAL_PROTOBUF_LIBRARY)
     set(protobuf_BUILD_TESTS OFF CACHE INTERNAL "" FORCE)
     set(protobuf_BUILD_SHARED_LIBS OFF CACHE INTERNAL "" FORCE)

contrib/brotli

@@ -0,0 +1 @@
+Subproject commit 5805f99a533a8f8118699c0100d8c102f3605f65

contrib/brotli-cmake/CMakeLists.txt

@@ -0,0 +1,32 @@
+set(BROTLI_SOURCE_DIR ${CMAKE_SOURCE_DIR}/contrib/brotli)
+set(BROTLI_BINARY_DIR ${CMAKE_BINARY_DIR}/contrib/brotli)
+
+set(SRCS
+    ${BROTLI_SOURCE_DIR}/dec/bit_reader.c
+    ${BROTLI_SOURCE_DIR}/dec/state.c
+    ${BROTLI_SOURCE_DIR}/dec/huffman.c
+    ${BROTLI_SOURCE_DIR}/dec/decode.c
+    ${BROTLI_SOURCE_DIR}/enc/encode.c
+    ${BROTLI_SOURCE_DIR}/enc/dictionary_hash.c
+    ${BROTLI_SOURCE_DIR}/enc/cluster.c
+    ${BROTLI_SOURCE_DIR}/enc/entropy_encode.c
+    ${BROTLI_SOURCE_DIR}/enc/literal_cost.c
+    ${BROTLI_SOURCE_DIR}/enc/compress_fragment_two_pass.c
+    ${BROTLI_SOURCE_DIR}/enc/static_dict.c
+    ${BROTLI_SOURCE_DIR}/enc/compress_fragment.c
+    ${BROTLI_SOURCE_DIR}/enc/block_splitter.c
+    ${BROTLI_SOURCE_DIR}/enc/backward_references_hq.c
+    ${BROTLI_SOURCE_DIR}/enc/histogram.c
+    ${BROTLI_SOURCE_DIR}/enc/brotli_bit_stream.c
+    ${BROTLI_SOURCE_DIR}/enc/utf8_util.c
+    ${BROTLI_SOURCE_DIR}/enc/encoder_dict.c
+    ${BROTLI_SOURCE_DIR}/enc/backward_references.c
+    ${BROTLI_SOURCE_DIR}/enc/bit_cost.c
+    ${BROTLI_SOURCE_DIR}/enc/metablock.c
+    ${BROTLI_SOURCE_DIR}/enc/memory.c
+    ${BROTLI_SOURCE_DIR}/common/dictionary.c
+    ${BROTLI_SOURCE_DIR}/common/transform.c
+)
+add_library(libbrotli STATIC ${SRCS})
+
+target_include_directories(libbrotli PUBLIC ${BROTLI_SOURCE_DIR}/c/include)

dbms/CMakeLists.txt

@@ -314,6 +314,11 @@ if (USE_HDFS)
     target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${HDFS3_INCLUDE_DIR})
 endif()
 
+if (USE_BROTLI)
+    target_link_libraries (clickhouse_common_io PRIVATE ${BROTLI_LIBRARY})
+    target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${BROTLI_INCLUDE_DIR})
+endif()
+
 if (USE_JEMALLOC)
     target_include_directories (dbms SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR}) # used in Interpreters/AsynchronousMetrics.cpp
 endif ()