From 8a3b024a6db52d40fdfbe4ea111533d742442329 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 1 Oct 2020 01:26:02 +0300 Subject: [PATCH] Fix using external protobuf library for the build. --- cmake/find/protobuf.cmake | 81 +++---- cmake/protobuf_generate_cpp.cmake | 172 --------------- .../protobuf-cmake/protobuf_generate.cmake | 198 ++++++++++++++++++ 3 files changed, 241 insertions(+), 210 deletions(-) delete mode 100644 cmake/protobuf_generate_cpp.cmake create mode 100644 contrib/protobuf-cmake/protobuf_generate.cmake diff --git a/cmake/find/protobuf.cmake b/cmake/find/protobuf.cmake index a7769206e9f..eb9fbe3edef 100644 --- a/cmake/find/protobuf.cmake +++ b/cmake/find/protobuf.cmake @@ -1,57 +1,62 @@ option(ENABLE_PROTOBUF "Enable protobuf" ${ENABLE_LIBRARIES}) if(NOT ENABLE_PROTOBUF) - if(USE_INTERNAL_PROTOBUF_LIBRARY) - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal protobuf with ENABLE_PROTOBUF=OFF") - endif() - return() + if(USE_INTERNAL_PROTOBUF_LIBRARY) + message(${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal protobuf with ENABLE_PROTOBUF=OFF") + endif() + return() endif() -option(USE_INTERNAL_PROTOBUF_LIBRARY "Set to FALSE to use system protobuf instead of bundled" ${NOT_UNBUNDLED}) +# Normally we use the internal protobuf library. +# You can set USE_INTERNAL_PROTOBUF_LIBRARY to OFF to force using the external protobuf library, which should be installed in the system in this case. +# The external protobuf library can be installed in the system by running +# sudo apt-get install libprotobuf-dev protobuf-compiler libprotoc-dev +option(USE_INTERNAL_PROTOBUF_LIBRARY "Set to FALSE to use system protobuf instead of bundled. (Experimental. Set to OFF on your own risk)" ${NOT_UNBUNDLED}) if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/protobuf/cmake/CMakeLists.txt") - if(USE_INTERNAL_PROTOBUF_LIBRARY) - message(WARNING "submodule contrib/protobuf is missing. to fix try run: \n git submodule update --init --recursive") - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal protobuf") - set(USE_INTERNAL_PROTOBUF_LIBRARY 0) - endif() - set(MISSING_INTERNAL_PROTOBUF_LIBRARY 1) + if(USE_INTERNAL_PROTOBUF_LIBRARY) + message(WARNING "submodule contrib/protobuf is missing. to fix try run: \n git submodule update --init --recursive") + message(${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal protobuf") + set(USE_INTERNAL_PROTOBUF_LIBRARY 0) + endif() + set(MISSING_INTERNAL_PROTOBUF_LIBRARY 1) endif() if(NOT USE_INTERNAL_PROTOBUF_LIBRARY) - find_package(Protobuf) - if (Protobuf_LIBRARY AND Protobuf_INCLUDE_DIR AND Protobuf_PROTOC_EXECUTABLE) - set(EXTERNAL_PROTOBUF_LIBRARY_FOUND 1) - set(USE_PROTOBUF 1) - else() - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system protobuf") - set(EXTERNAL_PROTOBUF_LIBRARY_FOUND 0) - endif() + find_package(Protobuf) + if(NOT Protobuf_INCLUDE_DIR OR NOT Protobuf_LIBRARY) + message(${RECONFIGURE_MESSAGE_LEVEL} "Can't find system protobuf library") + set(EXTERNAL_PROTOBUF_LIBRARY_FOUND 0) + elseif(NOT Protobuf_PROTOC_EXECUTABLE) + message(${RECONFIGURE_MESSAGE_LEVEL} "Can't find system protobuf compiler") + set(EXTERNAL_PROTOBUF_LIBRARY_FOUND 0) + else() + set(EXTERNAL_PROTOBUF_LIBRARY_FOUND 1) + set(USE_PROTOBUF 1) + endif() endif() -if (NOT EXTERNAL_PROTOBUF_LIBRARY_FOUND AND NOT MISSING_INTERNAL_PROTOBUF_LIBRARY) - set(Protobuf_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/protobuf/src") +if(NOT EXTERNAL_PROTOBUF_LIBRARY_FOUND AND NOT MISSING_INTERNAL_PROTOBUF_LIBRARY) + set(Protobuf_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/protobuf/src") + set(Protobuf_LIBRARY libprotobuf) + set(Protobuf_PROTOC_EXECUTABLE "$") + set(Protobuf_PROTOC_LIBRARY libprotoc) - set(USE_PROTOBUF 1) - set(USE_INTERNAL_PROTOBUF_LIBRARY 1) - set(Protobuf_LIBRARY libprotobuf) - set(Protobuf_PROTOC_LIBRARY libprotoc) - set(Protobuf_LITE_LIBRARY libprotobuf-lite) + include("${ClickHouse_SOURCE_DIR}/contrib/protobuf-cmake/protobuf_generate.cmake") - set(Protobuf_PROTOC_EXECUTABLE "$") + set(USE_INTERNAL_PROTOBUF_LIBRARY 1) + set(USE_PROTOBUF 1) endif() if(OS_FREEBSD AND SANITIZE STREQUAL "address") - # ../contrib/protobuf/src/google/protobuf/arena_impl.h:45:10: fatal error: 'sanitizer/asan_interface.h' file not found - # #include - if(LLVM_INCLUDE_DIRS) - set(Protobuf_INCLUDE_DIR "${Protobuf_INCLUDE_DIR}" ${LLVM_INCLUDE_DIRS}) - else() - message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use protobuf on FreeBSD with address sanitizer without LLVM") - set(USE_PROTOBUF 0) - endif() + # ../contrib/protobuf/src/google/protobuf/arena_impl.h:45:10: fatal error: 'sanitizer/asan_interface.h' file not found + # #include + if(LLVM_INCLUDE_DIRS) + set(Protobuf_INCLUDE_DIR "${Protobuf_INCLUDE_DIR}" ${LLVM_INCLUDE_DIRS}) + else() + message(${RECONFIGURE_MESSAGE_LEVEL} "Can't use protobuf on FreeBSD with address sanitizer without LLVM") + set(USE_PROTOBUF 0) + endif() endif() -include ("${ClickHouse_SOURCE_DIR}/cmake/protobuf_generate_cpp.cmake") - -message(STATUS "Using protobuf=${USE_PROTOBUF}: ${Protobuf_INCLUDE_DIR} : ${Protobuf_LIBRARY} : ${Protobuf_PROTOC_EXECUTABLE}") +message(STATUS "Using protobuf=${USE_PROTOBUF}: ${Protobuf_INCLUDE_DIR} : ${Protobuf_LIBRARY} : ${Protobuf_PROTOC_EXECUTABLE} : ${Protobuf_PROTOC_LIBRARY}") diff --git a/cmake/protobuf_generate_cpp.cmake b/cmake/protobuf_generate_cpp.cmake deleted file mode 100644 index cc2502e5eeb..00000000000 --- a/cmake/protobuf_generate_cpp.cmake +++ /dev/null @@ -1,172 +0,0 @@ -# This file declares functions adding custom commands for generating C++ files from *.proto files: -# function (protobuf_generate_cpp SRCS HDRS) -# function (protobuf_generate_grpc_cpp SRCS HDRS) - -if (NOT USE_PROTOBUF) - message (WARNING "Could not use protobuf_generate_cpp() without the protobuf library") - return() -endif() - -if (NOT DEFINED PROTOBUF_PROTOC_EXECUTABLE) - set (PROTOBUF_PROTOC_EXECUTABLE "$") -endif() - -if (NOT DEFINED GRPC_CPP_PLUGIN_EXECUTABLE) - set (GRPC_CPP_PLUGIN_EXECUTABLE $) -endif() - -if (NOT DEFINED PROTOBUF_GENERATE_CPP_APPEND_PATH) - set (PROTOBUF_GENERATE_CPP_APPEND_PATH TRUE) -endif() - - -function(protobuf_generate_cpp_impl SRCS HDRS MODES OUTPUT_FILE_EXTS PLUGIN) - if(NOT ARGN) - message(SEND_ERROR "Error: protobuf_generate_cpp() called without any proto files") - return() - endif() - - if(PROTOBUF_GENERATE_CPP_APPEND_PATH) - # Create an include path for each file specified - foreach(FIL ${ARGN}) - get_filename_component(ABS_FIL ${FIL} ABSOLUTE) - get_filename_component(ABS_PATH ${ABS_FIL} PATH) - list(FIND protobuf_include_path ${ABS_PATH} _contains_already) - if(${_contains_already} EQUAL -1) - list(APPEND protobuf_include_path -I ${ABS_PATH}) - endif() - endforeach() - else() - set(protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR}) - endif() - - if(DEFINED PROTOBUF_IMPORT_DIRS AND NOT DEFINED Protobuf_IMPORT_DIRS) - set(Protobuf_IMPORT_DIRS "${PROTOBUF_IMPORT_DIRS}") - endif() - - if(DEFINED Protobuf_IMPORT_DIRS) - foreach(DIR ${Protobuf_IMPORT_DIRS}) - get_filename_component(ABS_PATH ${DIR} ABSOLUTE) - list(FIND protobuf_include_path ${ABS_PATH} _contains_already) - if(${_contains_already} EQUAL -1) - list(APPEND protobuf_include_path -I ${ABS_PATH}) - endif() - endforeach() - endif() - - set (intermediate_dir ${CMAKE_CURRENT_BINARY_DIR}/intermediate) - file (MAKE_DIRECTORY ${intermediate_dir}) - - set (protoc_args) - foreach (mode ${MODES}) - list (APPEND protoc_args "--${mode}_out" ${intermediate_dir}) - endforeach() - if (PLUGIN) - list (APPEND protoc_args "--plugin=${PLUGIN}") - endif() - - set(srcs) - set(hdrs) - set(all_intermediate_outputs) - - foreach(input_name ${ARGN}) - get_filename_component(abs_name ${input_name} ABSOLUTE) - get_filename_component(name ${input_name} NAME_WE) - - set (intermediate_outputs) - foreach (ext ${OUTPUT_FILE_EXTS}) - set (filename "${name}${ext}") - set (output "${CMAKE_CURRENT_BINARY_DIR}/${filename}") - set (intermediate_output "${intermediate_dir}/${filename}") - list (APPEND intermediate_outputs "${intermediate_output}") - list (APPEND all_intermediate_outputs "${intermediate_output}") - - if (${ext} MATCHES ".*\\.h") - list(APPEND hdrs "${output}") - else() - list(APPEND srcs "${output}") - endif() - - add_custom_command( - OUTPUT ${output} - COMMAND ${CMAKE_COMMAND} -DPROTOBUF_GENERATE_CPP_SCRIPT_MODE=1 -DUSE_PROTOBUF=1 -DDIR=${CMAKE_CURRENT_BINARY_DIR} -DFILENAME=${filename} -DCOMPILER_ID=${CMAKE_CXX_COMPILER_ID} -P ${ClickHouse_SOURCE_DIR}/cmake/protobuf_generate_cpp.cmake - DEPENDS ${intermediate_output}) - endforeach() - - add_custom_command( - OUTPUT ${intermediate_outputs} - COMMAND ${Protobuf_PROTOC_EXECUTABLE} - ARGS ${protobuf_include_path} ${protoc_args} ${abs_name} - DEPENDS ${abs_name} ${Protobuf_PROTOC_EXECUTABLE} ${PLUGIN} - COMMENT "Running C++ protocol buffer compiler on ${name}" - VERBATIM ) - endforeach() - - set_source_files_properties(${srcs} ${hdrs} ${all_intermediate_outputs} PROPERTIES GENERATED TRUE) - set(${SRCS} ${srcs} PARENT_SCOPE) - set(${HDRS} ${hdrs} PARENT_SCOPE) -endfunction() - - -if (PROTOBUF_GENERATE_CPP_SCRIPT_MODE) - set (output "${DIR}/${FILENAME}") - set (intermediate_dir ${DIR}/intermediate) - set (intermediate_output "${intermediate_dir}/${FILENAME}") - - if (COMPILER_ID MATCHES "Clang") - set (pragma_push "#pragma clang diagnostic push\n") - set (pragma_pop "#pragma clang diagnostic pop\n") - set (pragma_disable_warnings "#pragma clang diagnostic ignored \"-Weverything\"\n") - elseif (COMPILER_ID MATCHES "GNU") - set (pragma_push "#pragma GCC diagnostic push\n") - set (pragma_pop "#pragma GCC diagnostic pop\n") - set (pragma_disable_warnings "#pragma GCC diagnostic ignored \"-Wall\"\n" - "#pragma GCC diagnostic ignored \"-Wextra\"\n" - "#pragma GCC diagnostic ignored \"-Warray-bounds\"\n" - "#pragma GCC diagnostic ignored \"-Wold-style-cast\"\n" - "#pragma GCC diagnostic ignored \"-Wshadow\"\n" - "#pragma GCC diagnostic ignored \"-Wsuggest-override\"\n" - "#pragma GCC diagnostic ignored \"-Wcast-qual\"\n" - "#pragma GCC diagnostic ignored \"-Wunused-parameter\"\n") - endif() - - if (${FILENAME} MATCHES ".*\\.h") - file(WRITE "${output}" - "#pragma once\n" - ${pragma_push} - ${pragma_disable_warnings} - "#include \"${intermediate_output}\"\n" - ${pragma_pop} - ) - else() - file(WRITE "${output}" - ${pragma_disable_warnings} - "#include \"${intermediate_output}\"\n" - ) - endif() - return() -endif() - - -function(protobuf_generate_cpp SRCS HDRS) - set (modes cpp) - set (output_file_exts ".pb.cc" ".pb.h") - set (plugin) - - protobuf_generate_cpp_impl(srcs hdrs "${modes}" "${output_file_exts}" "${plugin}" ${ARGN}) - - set(${SRCS} ${srcs} PARENT_SCOPE) - set(${HDRS} ${hdrs} PARENT_SCOPE) -endfunction() - - -function(protobuf_generate_grpc_cpp SRCS HDRS) - set (modes cpp grpc) - set (output_file_exts ".pb.cc" ".pb.h" ".grpc.pb.cc" ".grpc.pb.h") - set (plugin "protoc-gen-grpc=${GRPC_CPP_PLUGIN_EXECUTABLE}") - - protobuf_generate_cpp_impl(srcs hdrs "${modes}" "${output_file_exts}" "${plugin}" ${ARGN}) - - set(${SRCS} ${srcs} PARENT_SCOPE) - set(${HDRS} ${hdrs} PARENT_SCOPE) -endfunction() diff --git a/contrib/protobuf-cmake/protobuf_generate.cmake b/contrib/protobuf-cmake/protobuf_generate.cmake new file mode 100644 index 00000000000..fc1dfd9cc11 --- /dev/null +++ b/contrib/protobuf-cmake/protobuf_generate.cmake @@ -0,0 +1,198 @@ +# The code in this file was copied from https://github.com/Kitware/CMake/blob/master/Modules/FindProtobuf.cmake + +#[[ +Add custom commands to process ``.proto`` files to C++:: + +protobuf_generate_cpp ( + [DESCRIPTORS ] [EXPORT_MACRO ] [...]) + +``SRCS`` + Variable to define with autogenerated source files +``HDRS`` + Variable to define with autogenerated header files +``DESCRIPTORS`` + Variable to define with autogenerated descriptor files, if requested. +``EXPORT_MACRO`` + is a macro which should expand to ``__declspec(dllexport)`` or + ``__declspec(dllimport)`` depending on what is being compiled. +``ARGN`` + ``.proto`` files +#]] + +function(PROTOBUF_GENERATE_CPP SRCS HDRS) + cmake_parse_arguments(protobuf_generate_cpp "" "EXPORT_MACRO;DESCRIPTORS" "" ${ARGN}) + + set(_proto_files "${protobuf_generate_cpp_UNPARSED_ARGUMENTS}") + if(NOT _proto_files) + message(SEND_ERROR "Error: PROTOBUF_GENERATE_CPP() called without any proto files") + return() + endif() + + if(PROTOBUF_GENERATE_CPP_APPEND_PATH) + set(_append_arg APPEND_PATH) + endif() + + if(protobuf_generate_cpp_DESCRIPTORS) + set(_descriptors DESCRIPTORS) + endif() + + if(DEFINED PROTOBUF_IMPORT_DIRS AND NOT DEFINED Protobuf_IMPORT_DIRS) + set(Protobuf_IMPORT_DIRS "${PROTOBUF_IMPORT_DIRS}") + endif() + + if(DEFINED Protobuf_IMPORT_DIRS) + set(_import_arg IMPORT_DIRS ${Protobuf_IMPORT_DIRS}) + endif() + + set(_outvar) + protobuf_generate(${_append_arg} ${_descriptors} LANGUAGE cpp EXPORT_MACRO ${protobuf_generate_cpp_EXPORT_MACRO} OUT_VAR _outvar ${_import_arg} PROTOS ${_proto_files}) + + set(${SRCS}) + set(${HDRS}) + if(protobuf_generate_cpp_DESCRIPTORS) + set(${protobuf_generate_cpp_DESCRIPTORS}) + endif() + + foreach(_file ${_outvar}) + if(_file MATCHES "cc$") + list(APPEND ${SRCS} ${_file}) + elseif(_file MATCHES "desc$") + list(APPEND ${protobuf_generate_cpp_DESCRIPTORS} ${_file}) + else() + list(APPEND ${HDRS} ${_file}) + endif() + endforeach() + set(${SRCS} ${${SRCS}} PARENT_SCOPE) + set(${HDRS} ${${HDRS}} PARENT_SCOPE) + if(protobuf_generate_cpp_DESCRIPTORS) + set(${protobuf_generate_cpp_DESCRIPTORS} "${${protobuf_generate_cpp_DESCRIPTORS}}" PARENT_SCOPE) + endif() +endfunction() + +# By default have PROTOBUF_GENERATE_CPP macro pass -I to protoc +# for each directory where a proto file is referenced. +if(NOT DEFINED PROTOBUF_GENERATE_CPP_APPEND_PATH) + set(PROTOBUF_GENERATE_CPP_APPEND_PATH TRUE) +endif() + +function(protobuf_generate) + set(_options APPEND_PATH DESCRIPTORS) + set(_singleargs LANGUAGE OUT_VAR EXPORT_MACRO PROTOC_OUT_DIR) + if(COMMAND target_sources) + list(APPEND _singleargs TARGET) + endif() + set(_multiargs PROTOS IMPORT_DIRS GENERATE_EXTENSIONS) + + cmake_parse_arguments(protobuf_generate "${_options}" "${_singleargs}" "${_multiargs}" "${ARGN}") + + if(NOT protobuf_generate_PROTOS AND NOT protobuf_generate_TARGET) + message(SEND_ERROR "Error: protobuf_generate called without any targets or source files") + return() + endif() + + if(NOT protobuf_generate_OUT_VAR AND NOT protobuf_generate_TARGET) + message(SEND_ERROR "Error: protobuf_generate called without a target or output variable") + return() + endif() + + if(NOT protobuf_generate_LANGUAGE) + set(protobuf_generate_LANGUAGE cpp) + endif() + string(TOLOWER ${protobuf_generate_LANGUAGE} protobuf_generate_LANGUAGE) + + if(NOT protobuf_generate_PROTOC_OUT_DIR) + set(protobuf_generate_PROTOC_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}) + endif() + + if(protobuf_generate_EXPORT_MACRO AND protobuf_generate_LANGUAGE STREQUAL cpp) + set(_dll_export_decl "dllexport_decl=${protobuf_generate_EXPORT_MACRO}:") + endif() + + if(NOT protobuf_generate_GENERATE_EXTENSIONS) + if(protobuf_generate_LANGUAGE STREQUAL cpp) + set(protobuf_generate_GENERATE_EXTENSIONS .pb.h .pb.cc) + elseif(protobuf_generate_LANGUAGE STREQUAL python) + set(protobuf_generate_GENERATE_EXTENSIONS _pb2.py) + else() + message(SEND_ERROR "Error: protobuf_generate given unknown Language ${LANGUAGE}, please provide a value for GENERATE_EXTENSIONS") + return() + endif() + endif() + + if(protobuf_generate_TARGET) + get_target_property(_source_list ${protobuf_generate_TARGET} SOURCES) + foreach(_file ${_source_list}) + if(_file MATCHES "proto$") + list(APPEND protobuf_generate_PROTOS ${_file}) + endif() + endforeach() + endif() + + if(NOT protobuf_generate_PROTOS) + message(SEND_ERROR "Error: protobuf_generate could not find any .proto files") + return() + endif() + + if(protobuf_generate_APPEND_PATH) + # Create an include path for each file specified + foreach(_file ${protobuf_generate_PROTOS}) + get_filename_component(_abs_file ${_file} ABSOLUTE) + get_filename_component(_abs_path ${_abs_file} PATH) + list(FIND _protobuf_include_path ${_abs_path} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _protobuf_include_path -I ${_abs_path}) + endif() + endforeach() + else() + set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR}) + endif() + + foreach(DIR ${protobuf_generate_IMPORT_DIRS}) + get_filename_component(ABS_PATH ${DIR} ABSOLUTE) + list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _protobuf_include_path -I ${ABS_PATH}) + endif() + endforeach() + + set(_generated_srcs_all) + foreach(_proto ${protobuf_generate_PROTOS}) + get_filename_component(_abs_file ${_proto} ABSOLUTE) + get_filename_component(_abs_dir ${_abs_file} DIRECTORY) + get_filename_component(_basename ${_proto} NAME_WE) + file(RELATIVE_PATH _rel_dir ${CMAKE_CURRENT_SOURCE_DIR} ${_abs_dir}) + + set(_possible_rel_dir) + if (NOT protobuf_generate_APPEND_PATH) + set(_possible_rel_dir ${_rel_dir}/) + endif() + + set(_generated_srcs) + foreach(_ext ${protobuf_generate_GENERATE_EXTENSIONS}) + list(APPEND _generated_srcs "${protobuf_generate_PROTOC_OUT_DIR}/${_possible_rel_dir}${_basename}${_ext}") + endforeach() + + if(protobuf_generate_DESCRIPTORS AND protobuf_generate_LANGUAGE STREQUAL cpp) + set(_descriptor_file "${CMAKE_CURRENT_BINARY_DIR}/${_basename}.desc") + set(_dll_desc_out "--descriptor_set_out=${_descriptor_file}") + list(APPEND _generated_srcs ${_descriptor_file}) + endif() + list(APPEND _generated_srcs_all ${_generated_srcs}) + + add_custom_command( + OUTPUT ${_generated_srcs} + COMMAND protobuf::protoc + ARGS --${protobuf_generate_LANGUAGE}_out ${_dll_export_decl}${protobuf_generate_PROTOC_OUT_DIR} ${_dll_desc_out} ${_protobuf_include_path} ${_abs_file} + DEPENDS ${_abs_file} protobuf::protoc + COMMENT "Running ${protobuf_generate_LANGUAGE} protocol buffer compiler on ${_proto}" + VERBATIM ) + endforeach() + + set_source_files_properties(${_generated_srcs_all} PROPERTIES GENERATED TRUE) + if(protobuf_generate_OUT_VAR) + set(${protobuf_generate_OUT_VAR} ${_generated_srcs_all} PARENT_SCOPE) + endif() + if(protobuf_generate_TARGET) + target_sources(${protobuf_generate_TARGET} PRIVATE ${_generated_srcs_all}) + endif() +endfunction()