Fix build without protobuf, gtest, cppkafka (#4152)

CMakeLists.txt

@@ -81,7 +81,7 @@ option (ENABLE_TESTS "Enables tests" ON)
 if (CMAKE_SYSTEM_PROCESSOR MATCHES "amd64|x86_64")
     option (USE_INTERNAL_MEMCPY "Use internal implementation of 'memcpy' function instead of provided by libc. Only for x86_64." ON)
 
-    if (OS_LINUX AND NOT UNBUNDLED)
+    if (OS_LINUX AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES)
         option (GLIBC_COMPATIBILITY "Set to TRUE to enable compatibility with older glibc libraries. Only for x86_64, Linux. Implies USE_INTERNAL_MEMCPY." ON)
         if (GLIBC_COMPATIBILITY)
             message (STATUS "Some symbols from glibc will be replaced for compatibility")

cmake/find_gtest.cmake

@@ -15,6 +15,11 @@ endif ()
 if (NOT GTEST_INCLUDE_DIRS AND NOT MISSING_INTERNAL_GTEST_LIBRARY)
     set (USE_INTERNAL_GTEST_LIBRARY 1)
     set (GTEST_MAIN_LIBRARIES gtest_main)
+    set (GTEST_INCLUDE_DIRS ${ClickHouse_SOURCE_DIR}/contrib/googletest/googletest)
 endif ()
 
-message (STATUS "Using gtest: ${GTEST_INCLUDE_DIRS} : ${GTEST_MAIN_LIBRARIES}")
+if(GTEST_INCLUDE_DIRS AND GTEST_MAIN_LIBRARIES)
+    set(USE_GTEST 1)
+endif()
+
+message (STATUS "Using gtest=${USE_GTEST}: ${GTEST_INCLUDE_DIRS} : ${GTEST_MAIN_LIBRARIES}")

cmake/find_protobuf.cmake

@@ -1,18 +1,29 @@
-option (USE_INTERNAL_PROTOBUF_LIBRARY "Set to FALSE to use system protobuf instead of bundled" ON)
+option(USE_INTERNAL_PROTOBUF_LIBRARY "Set to FALSE to use system protobuf instead of bundled" ${NOT_UNBUNDLED})
+
+if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/protobuf/cmake/CMakeLists.txt")
+   if(USE_INTERNAL_PROTOBUF_LIBRARY)
+       message(WARNING "submodule contrib/protobuf is missing. to fix try run: \n git submodule update --init --recursive")
+       set(USE_INTERNAL_PROTOBUF_LIBRARY 0)
+   endif()
+   set(MISSING_INTERNAL_PROTOBUF_LIBRARY 1)
+endif()
 
 if(NOT USE_INTERNAL_PROTOBUF_LIBRARY)
     find_package(Protobuf)
 endif()
 
 if (Protobuf_LIBRARY AND Protobuf_INCLUDE_DIR)
-else ()
+    set(USE_PROTOBUF 1)
-    set(Protobuf_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/contrib/protobuf/src)
+elseif(NOT MISSING_INTERNAL_PROTOBUF_LIBRARY)
+    set(Protobuf_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/protobuf/src)
 
+    set(USE_PROTOBUF 1)
+    set(USE_INTERNAL_PROTOBUF_LIBRARY 1)
     set(Protobuf_LIBRARY libprotobuf)
     set(Protobuf_PROTOC_LIBRARY libprotoc)
     set(Protobuf_LITE_LIBRARY libprotobuf-lite)
 
-    set(Protobuf_PROTOC_EXECUTABLE ${CMAKE_BINARY_DIR}/contrib/protobuf/cmake/protoc)
+    set(Protobuf_PROTOC_EXECUTABLE ${ClickHouse_BINARY_DIR}/contrib/protobuf/cmake/protoc)
 
     if(NOT DEFINED PROTOBUF_GENERATE_CPP_APPEND_PATH)
         set(PROTOBUF_GENERATE_CPP_APPEND_PATH TRUE)
@@ -77,4 +88,4 @@ else ()
     endfunction()
 endif()
 
-message (STATUS "Using protobuf: ${Protobuf_INCLUDE_DIR} : ${Protobuf_LIBRARY}")
+message(STATUS "Using protobuf=${USE_PROTOBUF}: ${Protobuf_INCLUDE_DIR} : ${Protobuf_LIBRARY}")

cmake/find_rdkafka.cmake

@@ -2,6 +2,11 @@ if (NOT ARCH_ARM AND NOT ARCH_32 AND NOT APPLE)
     option (ENABLE_RDKAFKA "Enable kafka" ON)
 endif ()
 
+if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cppkafka/CMakeLists.txt")
+   message (WARNING "submodule contrib/cppkafka is missing. to fix try run: \n git submodule update --init --recursive")
+   set (ENABLE_RDKAFKA 0)
+endif ()
+
 if (ENABLE_RDKAFKA)
 
 if (OS_LINUX AND NOT ARCH_ARM)

dbms/CMakeLists.txt

@@ -298,8 +298,10 @@ target_link_libraries(dbms PRIVATE ${OPENSSL_CRYPTO_LIBRARY} Threads::Threads)
 target_include_directories (dbms SYSTEM BEFORE PRIVATE ${DIVIDE_INCLUDE_DIR})
 target_include_directories (dbms SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
 
+if (USE_PROTOBUF)
     target_link_libraries (dbms PRIVATE ${Protobuf_LIBRARY})
     target_include_directories (dbms SYSTEM BEFORE PRIVATE ${Protobuf_INCLUDE_DIR})
+endif ()
 
 if (USE_HDFS)
     target_link_libraries (clickhouse_common_io PRIVATE ${HDFS3_LIBRARY})
@@ -321,7 +323,7 @@ target_include_directories (clickhouse_common_io BEFORE PRIVATE ${COMMON_INCLUDE
 add_subdirectory (programs)
 add_subdirectory (tests)
 
-if (ENABLE_TESTS)
+if (ENABLE_TESTS AND USE_GTEST)
     macro (grep_gtest_sources BASE_DIR DST_VAR)
         # Cold match files that are not in tests/ directories
         file(GLOB_RECURSE "${DST_VAR}" RELATIVE "${BASE_DIR}" "gtest*.cpp")

dbms/programs/server/CMakeLists.txt

@@ -23,7 +23,7 @@ if (CLICKHOUSE_SPLIT_BINARY)
     install (TARGETS clickhouse-server ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
 endif ()
 
-if (OS_LINUX AND MAKE_STATIC_LIBRARIES)
+if (GLIBC_COMPATIBILITY)
     set (GLIBC_MAX_REQUIRED 2.4 CACHE INTERNAL "")
     # temporary disabled. to enable - change 'exit 0' to 'exit $a'
     add_test(NAME GLIBC_required_version COMMAND bash -c "readelf -s ${CMAKE_CURRENT_BINARY_DIR}/../clickhouse-server | perl -nE 'END {exit 0 if $a} ++$a, print if /\\x40GLIBC_(\\S+)/ and pack(q{C*}, split /\\./, \$1) gt pack q{C*}, split /\\./, q{${GLIBC_MAX_REQUIRED}}'")

dbms/src/Columns/tests/CMakeLists.txt

@@ -1,4 +1,4 @@
-set(SRCS)
+if(USE_GTEST)
-
+    add_executable(column_unique column_unique.cpp)
-add_executable (column_unique column_unique.cpp ${SRCS})
     target_link_libraries(column_unique PRIVATE dbms gtest_main)
+endif()

dbms/src/Common/config.h.in

@@ -17,6 +17,7 @@
 #cmakedefine01 USE_HDFS
 #cmakedefine01 USE_XXHASH
 #cmakedefine01 USE_INTERNAL_LLVM_LIBRARY
+#cmakedefine01 USE_PROTOBUF
 
 #cmakedefine01 CLICKHOUSE_SPLIT_BINARY
 #cmakedefine01 LLVM_HAS_RTTI

dbms/src/DataTypes/tests/CMakeLists.txt

@@ -1,10 +1,11 @@
-set(SRCS )
 
-add_executable (data_types_number_fixed data_types_number_fixed.cpp ${SRCS})
+add_executable (data_types_number_fixed data_types_number_fixed.cpp)
 target_link_libraries (data_types_number_fixed PRIVATE dbms)
 
-add_executable (data_type_string data_type_string.cpp ${SRCS})
+add_executable (data_type_string data_type_string.cpp)
 target_link_libraries (data_type_string PRIVATE dbms)
 
-add_executable (data_type_get_common_type data_type_get_common_type.cpp ${SRCS})
+if(USE_GTEST)
+    add_executable(data_type_get_common_type data_type_get_common_type.cpp)
     target_link_libraries(data_type_get_common_type PRIVATE dbms gtest_main)
+endif()

dbms/src/Formats/ProtobufBlockOutputStream.cpp

@@ -1,7 +1,12 @@
-#include <Core/Block.h>
 #include <Formats/FormatFactory.h>
+
+#include <Common/config.h>
+#if USE_PROTOBUF
+
+#include "ProtobufBlockOutputStream.h"
+
+#include <Core/Block.h>
 #include <Formats/FormatSchemaInfo.h>
-#include <Formats/ProtobufBlockOutputStream.h>
 #include <Formats/ProtobufSchemas.h>
 #include <Interpreters/Context.h>
 
@@ -73,3 +78,13 @@ void registerOutputFormatProtobuf(FormatFactory & factory)
 }
 
 }
+
+#else
+
+namespace DB
+{
+    class FormatFactory;
+    void registerOutputFormatProtobuf(FormatFactory &) {}
+}
+
+#endif

dbms/src/Formats/ProtobufSchemas.cpp

@@ -1,3 +1,6 @@
+#include <Common/config.h>
+#if USE_PROTOBUF
+
 #include <Common/Exception.h>
 #include <Core/Block.h>
 #include <Formats/FormatSchemaInfo.h>
@@ -68,3 +71,5 @@ void ProtobufSchemas::AddError(const String & filename, int line, int column, co
 }
 
 }
+
+#endif

dbms/src/Formats/ProtobufSchemas.h

@@ -1,5 +1,8 @@
 #pragma once
 
+#include <Common/config.h>
+#if USE_PROTOBUF
+
 #include <optional>
 #include <Core/Types.h>
 #include <google/protobuf/compiler/importer.h>
@@ -47,3 +50,5 @@ private:
 };
 
 }
+
+#endif

dbms/src/Formats/ProtobufWriter.cpp

@@ -1,3 +1,6 @@
+#include <Common/config.h>
+#if USE_PROTOBUF
+
 #include <cassert>
 #include <optional>
 #include <math.h>
@@ -795,3 +798,5 @@ void ProtobufWriter::writeAggregateFunction(const AggregateFunctionPtr & functio
 }
 
 }
+
+#endif

dbms/src/Storages/System/StorageSystemBuildOptions.generated.cpp.in

@@ -40,6 +40,7 @@ const char * auto_config_build[]
     "USE_POCO_MONGODB", "@USE_POCO_MONGODB@",
     "USE_POCO_NETSSL", "@USE_POCO_NETSSL@",
     "USE_BASE64", "@USE_BASE64@",
+    "USE_PROTOBUF", "@USE_PROTOBUF@",
 
     nullptr, nullptr
 };

dbms/tests/queries/bugs/fuzzy.sql

@@ -0,0 +1 @@
+SELECT joinGet(toDateTimeOrNull((CAST(([885455.14523]) AS String))));

libs/libcommon/src/tests/CMakeLists.txt

@@ -22,9 +22,11 @@ target_link_libraries (realloc-perf common)
 add_check(multi_version)
 add_check(local_date_time_comparison)
 
+if(USE_GTEST)
     add_executable(unit_tests_libcommon gtest_json_test.cpp gtest_strong_typedef.cpp gtest_find_symbols.cpp)
     target_link_libraries(unit_tests_libcommon common ${GTEST_MAIN_LIBRARIES})
     add_check(unit_tests_libcommon)
+endif()
 
 add_executable (dump_variable dump_variable.cpp)
 target_link_libraries (dump_variable clickhouse_common_io)

utils/test-data-generator/CMakeLists.txt

@@ -4,8 +4,10 @@ target_link_libraries(test-data-generator PRIVATE clickhouse_common_io ${Boost_P
 add_executable (markov-model markov-model.cpp)
 target_link_libraries(markov-model PRIVATE clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
 
+if(USE_PROTOBUF)
     protobuf_generate_cpp(ProtobufDelimitedMessagesSerializer_Srcs ProtobufDelimitedMessagesSerializer_Hdrs ${CMAKE_CURRENT_SOURCE_DIR}/../../dbms/tests/queries/0_stateless/00825_protobuf_format.proto)
     protobuf_generate_cpp(ProtobufDelimitedMessagesSerializer_Srcs2 ProtobufDelimitedMessagesSerializer_Hdrs2 ${CMAKE_CURRENT_SOURCE_DIR}/../../dbms/tests/queries/0_stateless/00825_protobuf_format_syntax2.proto)
     add_executable (ProtobufDelimitedMessagesSerializer ProtobufDelimitedMessagesSerializer.cpp ${ProtobufDelimitedMessagesSerializer_Srcs} ${ProtobufDelimitedMessagesSerializer_Hdrs} ${ProtobufDelimitedMessagesSerializer_Srcs2} ${ProtobufDelimitedMessagesSerializer_Hdrs2})
     target_include_directories (ProtobufDelimitedMessagesSerializer SYSTEM BEFORE PRIVATE ${Protobuf_INCLUDE_DIR} ${CMAKE_CURRENT_BINARY_DIR})
     target_link_libraries (ProtobufDelimitedMessagesSerializer PRIVATE ${Protobuf_LIBRARY})
+endif()