thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-29 21:20:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into consistent_metadata2

Browse Source
This commit is contained in:
alesapin 2020-05-27 12:51:56 +03:00
commit c7cda399c7
125 changed files with 2279 additions and 1038 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
cmake/find/grpc.cmake
View File

@ -1,57 +1,26 @@
set(_PROTOBUF_PROTOC $<TARGET_FILE:protoc>)
set(_GRPC_CPP_PLUGIN_EXECUTABLE $<TARGET_FILE:grpc_cpp_plugin>)
option (ENABLE_GRPC "Use gRPC" ${ENABLE_LIBRARIES})
function(PROTOBUF_GENERATE_GRPC_CPP SRCS HDRS)
if(NOT ARGN)
message(SEND_ERROR "Error: PROTOBUF_GENERATE_GRPC_CPP() called without any proto files")
return()
endif()
if (ENABLE_GRPC)
option (USE_INTERNAL_GRPC_LIBRARY "Set to FALSE to use system gRPC library instead of bundled" ${NOT_UNBUNDLED})
if(PROTOBUF_GENERATE_CPP_APPEND_PATH)
foreach(FIL ${ARGN})
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
get_filename_component(ABS_PATH ${ABS_FIL} PATH)
list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
if(${_contains_already} EQUAL -1)
list(APPEND _protobuf_include_path -I ${ABS_PATH})
endif()
endforeach()
else()
set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})
endif()
if (USE_INTERNAL_GRPC_LIBRARY)
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/grpc/include/grpc++/grpc++.h")
message(WARNING "submodule contrib/grpc is missing. To fix try run: \n git submodule update --init --recursive")
set (USE_INTERNAL_GRPC_LIBRARY OFF)
elif (NOT USE_PROTOBUF)
message(WARNING "gRPC requires protobuf which is disabled")
set (USE_INTERNAL_GRPC_LIBRARY OFF)
else()
set (GRPC_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/grpc/include")
set (GRPC_LIBRARY "libgrpc++")
set (USE_GRPC ON)
endif()
else()
find_package(grpc)
if (GRPC_INCLUDE_DIR AND GRPC_LIBRARY)
set (USE_GRPC ON)
endif()
endif()
endif()
if(DEFINED PROTOBUF_IMPORT_DIRS)
foreach(DIR ${Protobuf_IMPORT_DIRS})
get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
if(${_contains_already} EQUAL -1)
list(APPEND _protobuf_include_path -I ${ABS_PATH})
endif()
endforeach()
endif()
set(${SRCS})
set(${HDRS})
foreach(FIL ${ARGN})
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
get_filename_component(FIL_WE ${FIL} NAME_WE)
list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.cc")
list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.h")
add_custom_command(
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.cc"
"${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.h"
COMMAND ${_PROTOBUF_PROTOC}
ARGS --grpc_out=${CMAKE_CURRENT_BINARY_DIR}
--plugin=protoc-gen-grpc=${_GRPC_CPP_PLUGIN_EXECUTABLE}
${_protobuf_include_path} ${ABS_FIL}
DEPENDS ${ABS_FIL}
COMMENT "Running gRPC C++ protocol buffer compiler on ${FIL}"
VERBATIM)
endforeach()
set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
set(${SRCS} ${${SRCS}} PARENT_SCOPE)
set(${HDRS} ${${HDRS}} PARENT_SCOPE)
endfunction()
message(STATUS "Using gRPC=${USE_GRPC}: ${GRPC_INCLUDE_DIR} : ${GRPC_LIBRARY}")

63
cmake/find/protobuf.cmake
View File

@ -28,68 +28,6 @@ elseif(NOT MISSING_INTERNAL_PROTOBUF_LIBRARY)
set(Protobuf_LITE_LIBRARY libprotobuf-lite)
set(Protobuf_PROTOC_EXECUTABLE "$<TARGET_FILE:protoc>")
if(NOT DEFINED PROTOBUF_GENERATE_CPP_APPEND_PATH)
set(PROTOBUF_GENERATE_CPP_APPEND_PATH TRUE)
endif()
function(PROTOBUF_GENERATE_CPP SRCS HDRS)
if(NOT ARGN)
message(SEND_ERROR "Error: PROTOBUF_GENERATE_CPP() called without any proto files")
return()
endif()
if(PROTOBUF_GENERATE_CPP_APPEND_PATH)
# Create an include path for each file specified
foreach(FIL ${ARGN})
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
get_filename_component(ABS_PATH ${ABS_FIL} PATH)
list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
if(${_contains_already} EQUAL -1)
list(APPEND _protobuf_include_path -I ${ABS_PATH})
endif()
endforeach()
else()
set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})
endif()
if(DEFINED PROTOBUF_IMPORT_DIRS AND NOT DEFINED Protobuf_IMPORT_DIRS)
set(Protobuf_IMPORT_DIRS "${PROTOBUF_IMPORT_DIRS}")
endif()
if(DEFINED Protobuf_IMPORT_DIRS)
foreach(DIR ${Protobuf_IMPORT_DIRS})
get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
if(${_contains_already} EQUAL -1)
list(APPEND _protobuf_include_path -I ${ABS_PATH})
endif()
endforeach()
endif()
set(${SRCS})
set(${HDRS})
foreach(FIL ${ARGN})
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
get_filename_component(FIL_WE ${FIL} NAME_WE)
list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc")
list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h")
add_custom_command(
OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.cc"
"${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h"
COMMAND ${Protobuf_PROTOC_EXECUTABLE}
ARGS --cpp_out ${CMAKE_CURRENT_BINARY_DIR} ${_protobuf_include_path} ${ABS_FIL}
DEPENDS ${ABS_FIL} ${Protobuf_PROTOC_EXECUTABLE}
COMMENT "Running C++ protocol buffer compiler on ${FIL}"
VERBATIM )
endforeach()
set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
set(${SRCS} ${${SRCS}} PARENT_SCOPE)
set(${HDRS} ${${HDRS}} PARENT_SCOPE)
endfunction()
endif()
if(OS_FREEBSD AND SANITIZE STREQUAL "address")
@ -102,6 +40,7 @@ if(OS_FREEBSD AND SANITIZE STREQUAL "address")
endif()
endif()
include (${ClickHouse_SOURCE_DIR}/cmake/protobuf_generate_cpp.cmake)
endif()
message(STATUS "Using protobuf=${USE_PROTOBUF}: ${Protobuf_INCLUDE_DIR} : ${Protobuf_LIBRARY} : ${Protobuf_PROTOC_EXECUTABLE}")

166
cmake/protobuf_generate_cpp.cmake Normal file
View File

@ -0,0 +1,166 @@
# This file declares functions adding custom commands for generating C++ files from *.proto files:
# function (protobuf_generate_cpp SRCS HDRS)
# function (protobuf_generate_grpc_cpp SRCS HDRS)
if (NOT USE_PROTOBUF)
message (WARNING "Could not use protobuf_generate_cpp() without the protobuf library")
return()
endif()
if (NOT DEFINED PROTOBUF_PROTOC_EXECUTABLE)
set (PROTOBUF_PROTOC_EXECUTABLE "$<TARGET_FILE:protoc>")
endif()
if (NOT DEFINED GRPC_CPP_PLUGIN_EXECUTABLE)
set (GRPC_CPP_PLUGIN_EXECUTABLE $<TARGET_FILE:grpc_cpp_plugin>)
endif()
if (NOT DEFINED PROTOBUF_GENERATE_CPP_APPEND_PATH)
set (PROTOBUF_GENERATE_CPP_APPEND_PATH TRUE)
endif()
function(protobuf_generate_cpp_impl SRCS HDRS MODES OUTPUT_FILE_EXTS PLUGIN)
if(NOT ARGN)
message(SEND_ERROR "Error: protobuf_generate_cpp() called without any proto files")
return()
endif()
if(PROTOBUF_GENERATE_CPP_APPEND_PATH)
# Create an include path for each file specified
foreach(FIL ${ARGN})
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
get_filename_component(ABS_PATH ${ABS_FIL} PATH)
list(FIND protobuf_include_path ${ABS_PATH} _contains_already)
if(${_contains_already} EQUAL -1)
list(APPEND protobuf_include_path -I ${ABS_PATH})
endif()
endforeach()
else()
set(protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})
endif()
if(DEFINED PROTOBUF_IMPORT_DIRS AND NOT DEFINED Protobuf_IMPORT_DIRS)
set(Protobuf_IMPORT_DIRS "${PROTOBUF_IMPORT_DIRS}")
endif()
if(DEFINED Protobuf_IMPORT_DIRS)
foreach(DIR ${Protobuf_IMPORT_DIRS})
get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
list(FIND protobuf_include_path ${ABS_PATH} _contains_already)
if(${_contains_already} EQUAL -1)
list(APPEND protobuf_include_path -I ${ABS_PATH})
endif()
endforeach()
endif()
set (intermediate_dir ${CMAKE_CURRENT_BINARY_DIR}/intermediate)
set (protoc_args)
foreach (mode ${MODES})
list (APPEND protoc_args "--${mode}_out" ${intermediate_dir})
endforeach()
if (PLUGIN)
list (APPEND protoc_args "--plugin=${PLUGIN}")
endif()
set(srcs)
set(hdrs)
set(all_intermediate_outputs)
foreach(input_name ${ARGN})
get_filename_component(abs_name ${input_name} ABSOLUTE)
get_filename_component(name ${input_name} NAME_WE)
set (intermediate_outputs)
foreach (ext ${OUTPUT_FILE_EXTS})
set (filename "${name}${ext}")
set (output "${CMAKE_CURRENT_BINARY_DIR}/${filename}")
set (intermediate_output "${intermediate_dir}/${filename}")
list (APPEND intermediate_outputs "${intermediate_output}")
list (APPEND all_intermediate_outputs "${intermediate_output}")
if (${ext} MATCHES ".*\\.h")
list(APPEND hdrs "${output}")
else()
list(APPEND srcs "${output}")
endif()
add_custom_command(
OUTPUT ${output}
COMMAND ${CMAKE_COMMAND} -DPROTOBUF_GENERATE_CPP_SCRIPT_MODE=1 -DUSE_PROTOBUF=1 -DDIR=${CMAKE_CURRENT_BINARY_DIR} -DFILENAME=${filename} -DCOMPILER_ID=${CMAKE_CXX_COMPILER_ID} -P ${ClickHouse_SOURCE_DIR}/cmake/protobuf_generate_cpp.cmake
DEPENDS ${intermediate_output})
endforeach()
add_custom_command(
OUTPUT ${intermediate_outputs}
COMMAND ${Protobuf_PROTOC_EXECUTABLE}
ARGS ${protobuf_include_path} ${protoc_args} ${abs_name}
DEPENDS ${abs_name} ${Protobuf_PROTOC_EXECUTABLE} ${PLUGIN}
COMMENT "Running C++ protocol buffer compiler on ${name}"
VERBATIM )
endforeach()
set_source_files_properties(${srcs} ${hdrs} ${all_intermediate_outputs} PROPERTIES GENERATED TRUE)
set(${SRCS} ${srcs} PARENT_SCOPE)
set(${HDRS} ${hdrs} PARENT_SCOPE)
endfunction()
if (PROTOBUF_GENERATE_CPP_SCRIPT_MODE)
set (output "${DIR}/${FILENAME}")
set (intermediate_dir ${DIR}/intermediate)
set (intermediate_output "${intermediate_dir}/${FILENAME}")
if (COMPILER_ID STREQUAL "GNU")
set (pragma_push "#pragma GCC diagnostic push\n")
set (pragma_pop "#pragma GCC diagnostic pop\n")
set (pragma_disable_warnings "#pragma GCC diagnostic ignored \"-Wall\"\n"
"#pragma GCC diagnostic ignored \"-Wextra\"\n"
"#pragma GCC diagnostic ignored \"-Warray-bounds\"\n")
elseif (COMPILER_ID MATCHES "Clang")
set (pragma_push "#pragma clang diagnostic push\n")
set (pragma_pop "#pragma clang diagnostic pop\n")
set (pragma_disable_warnings "#pragma clang diagnostic ignored \"-Weverything\"\n")
endif()
if (${FILENAME} MATCHES ".*\\.h")
file(WRITE "${output}"
"#pragma once\n"
${pragma_push}
${pragma_disable_warnings}
"#include \"${intermediate_output}\"\n"
${pragma_pop}
)
else()
file(WRITE "${output}"
${pragma_disable_warnings}
"#include \"${intermediate_output}\"\n"
)
endif()
return()
endif()
function(protobuf_generate_cpp SRCS HDRS)
set (modes cpp)
set (output_file_exts ".pb.cc" ".pb.h")
set (plugin)
protobuf_generate_cpp_impl(srcs hdrs "${modes}" "${output_file_exts}" "${plugin}" ${ARGN})
set(${SRCS} ${srcs} PARENT_SCOPE)
set(${HDRS} ${hdrs} PARENT_SCOPE)
endfunction()
function(protobuf_generate_grpc_cpp SRCS HDRS)
set (modes cpp grpc)
set (output_file_exts ".pb.cc" ".pb.h" ".grpc.pb.cc" ".grpc.pb.h")
set (plugin "protoc-gen-grpc=${GRPC_CPP_PLUGIN_EXECUTABLE}")
protobuf_generate_cpp_impl(srcs hdrs "${modes}" "${output_file_exts}" "${plugin}" ${ARGN})
set(${SRCS} ${srcs} PARENT_SCOPE)
set(${HDRS} ${hdrs} PARENT_SCOPE)
endfunction()

14
contrib/CMakeLists.txt vendored
View File

@ -21,7 +21,6 @@ add_subdirectory (consistent-hashing-sumbur)
add_subdirectory (consistent-hashing)
add_subdirectory (croaring)
add_subdirectory (FastMemcpy)
add_subdirectory (grpc-cmake)
add_subdirectory (jemalloc-cmake)
add_subdirectory (libcpuid-cmake)
add_subdirectory (murmurhash)
@ -260,20 +259,17 @@ if (USE_INTERNAL_BROTLI_LIBRARY)
endif ()
if (USE_INTERNAL_PROTOBUF_LIBRARY)
if (MAKE_STATIC_LIBRARIES)
set(protobuf_BUILD_SHARED_LIBS OFF CACHE INTERNAL "" FORCE)
else ()
set(protobuf_BUILD_SHARED_LIBS ON CACHE INTERNAL "" FORCE)
endif ()
set(protobuf_WITH_ZLIB 0 CACHE INTERNAL "" FORCE) # actually will use zlib, but skip find
set(protobuf_BUILD_TESTS OFF CACHE INTERNAL "" FORCE)
add_subdirectory(protobuf/cmake)
add_subdirectory(protobuf-cmake)
endif ()
if (USE_INTERNAL_HDFS3_LIBRARY)
add_subdirectory(libhdfs3-cmake)
endif ()
if (USE_INTERNAL_GRPC_LIBRARY)
add_subdirectory(grpc-cmake)
endif ()
if (USE_INTERNAL_AWS_S3_LIBRARY)
set (save_CMAKE_C_FLAGS ${CMAKE_C_FLAGS})
set (save_CMAKE_REQUIRED_LIBRARIES ${CMAKE_REQUIRED_LIBRARIES})

2
contrib/cppkafka vendored

@ -1 +1 @@
Subproject commit 9b184d881c15cc50784b28688c7c99d3d764db24
Subproject commit f555ee36aaa74d17ca0dab3ce472070a610b2966

2
contrib/grpc vendored

@ -1 +1 @@
Subproject commit c1d176528fd8da9dd4066d16554bcd216d29033f
Subproject commit 8aea4e168e78f3eb9828080740fc8cb73d53bf79

480
contrib/grpc-cmake/CMakeLists.txt
View File

@ -4,6 +4,8 @@ cmake_minimum_required(VERSION 3.5.1)
set(GRPC_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/grpc)
set(GRPC_INCLUDE_DIR ${GRPC_SOURCE_DIR}/include/)
set(GRPC_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/grpc)
if(UNIX)
if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
set(_gRPC_PLATFORM_LINUX ON)
@ -14,72 +16,28 @@ if(UNIX)
endif()
endif()
set(CMAKE_POSITION_INDEPENDENT_CODE TRUE)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_gRPC_C_CXX_FLAGS} -w")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_gRPC_C_CXX_FLAGS} -w")
set(_gRPC_PROTOBUF_LIBRARY_NAME "libprotobuf")
if(gRPC_BACKWARDS_COMPATIBILITY_MODE)
add_definitions(-DGPR_BACKWARDS_COMPATIBILITY_MODE)
if (_gRPC_PLATFORM_MAC)
# some C++11 constructs not supported before OS X 10.9
set(CMAKE_OSX_DEPLOYMENT_TARGET 10.9)
endif()
if(_gRPC_PLATFORM_MAC)
set(_gRPC_ALLTARGETS_LIBRARIES ${CMAKE_DL_LIBS} m pthread)
elseif(UNIX)
set(_gRPC_ALLTARGETS_LIBRARIES ${CMAKE_DL_LIBS} rt m pthread)
endif()
if (_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC)
set(_gRPC_CORE_NOSTDCXX_FLAGS -fno-exceptions -fno-rtti)
else()
set(_gRPC_CORE_NOSTDCXX_FLAGS "")
endif()
# address_sorting.cmake
include(${GRPC_SOURCE_DIR}/cmake/address_sorting.cmake)
set(_gRPC_ADDRESS_SORTING_INCLUDE_DIR "${GRPC_SOURCE_DIR}/third_party/address_sorting/include")
set(_gRPC_ADDRESS_SORTING_LIBRARIES address_sorting)
# cares.cmake
set(CARES_ROOT_DIR ${GRPC_SOURCE_DIR}/third_party/cares/cares)
set(CARES_BINARY_DIR ${GRPC_BINARY_DIR}/third_party/cares/cares)
set(CARES_SHARED OFF CACHE BOOL "disable shared library")
set(CARES_STATIC ON CACHE BOOL "link cares statically")
if(gRPC_BACKWARDS_COMPATIBILITY_MODE)
# See https://github.com/grpc/grpc/issues/17255
set(HAVE_LIBNSL OFF CACHE BOOL "avoid cares dependency on libnsl")
set(CARES_SHARED ${BUILD_SHARED_LIBS} CACHE BOOL "" FORCE)
if(BUILD_SHARED_LIBS)
set(CARES_STATIC OFF CACHE BOOL "" FORCE)
else()
set(CARES_STATIC ON CACHE BOOL "" FORCE)
endif()
set(_gRPC_CARES_LIBRARIES c-ares)
add_subdirectory(${CARES_ROOT_DIR} ${CARES_BINARY_DIR})
if(TARGET c-ares)
set(_gRPC_CARES_LIBRARIES c-ares)
endif()
# protobuf.cmake
set(PROTOBUF_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../protobuf)
set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests")
if(NOT protobuf_WITH_ZLIB)
set(protobuf_WITH_ZLIB OFF CACHE BOOL "Build protobuf with zlib.")
endif()
set(protobuf_MSVC_STATIC_RUNTIME OFF CACHE BOOL "Link static runtime libraries")
set(_gRPC_PROTOBUF_LIBRARIES libprotobuf)
set(_gRPC_PROTOBUF_PROTOC_LIBRARIES libprotoc)
set(_gRPC_PROTOBUF_PROTOC protoc)
set(_gRPC_PROTOBUF_PROTOC_EXECUTABLE $<TARGET_FILE:protoc>)
set(_gRPC_PROTOBUF_INCLUDE_DIR "${PROTOBUF_ROOT_DIR}/src")
if(gRPC_INSTALL)
message(WARNING "gRPC_INSTALL will be forced to FALSE because gRPC_PROTOBUF_PROVIDER is \"module\"")
set(gRPC_INSTALL FALSE)
endif()
# ssl.cmake
set(BORINGSSL_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../ssl)
if(TARGET ssl)
set(_gRPC_SSL_LIBRARIES ssl)
set(_gRPC_SSL_INCLUDE_DIR ${BORINGSSL_ROOT_DIR}/include)
endif()
# upb.cmake
set(UPB_ROOT_DIR ${GRPC_SOURCE_DIR}/third_party/upb)
@ -87,23 +45,20 @@ set(_gRPC_UPB_INCLUDE_DIR "${UPB_ROOT_DIR}")
set(_gRPC_UPB_GRPC_GENERATED_DIR "${GRPC_SOURCE_DIR}/src/core/ext/upb-generated")
set(_gRPC_UPB_LIBRARIES upb)
# protobuf.cmake
set(_gRPC_PROTOBUF_INCLUDE_DIR ${Protobuf_INCLUDE_DIR})
set(_gRPC_PROTOBUF_LIBRARIES ${Protobuf_LIBRARY})
set(_gRPC_PROTOBUF_PROTOC_LIBRARIES ${Protobuf_PROTOC_LIBRARY})
set(_gRPC_PROTOBUF_PROTOC_EXECUTABLE ${Protobuf_PROTOC_EXECUTABLE})
# ssl.cmake
set(_gRPC_SSL_INCLUDE_DIR ${OPENSSL_INCLUDE_DIR})
set(_gRPC_SSL_LIBRARIES ${OPENSSL_SSL_LIBRARY})
# zlib.cmake
set(ZLIB_ROOT_DIR ${GRPC_SOURCE_DIR}/../zlib-ng)
include_directories("${ZLIB_ROOT_DIR}")
## add_subdirectory(${ZLIB_ROOT_DIR} ${ZLIB_ROOT_DIR})
if(TARGET zlibstatic)
set(_gRPC_ZLIB_LIBRARIES zlibstatic)
set(_gRPC_ZLIB_INCLUDE_DIR "${ZLIB_ROOT_DIR}" "${GRPC_SOURCE_DIR}/third_party/zlib")
endif()
set(_gRPC_ZLIB_INCLUDE_DIR ${ZLIB_INCLUDE_DIR})
set(_gRPC_ZLIB_LIBRARIES ${ZLIB_LIBRARIES})
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
if(_gRPC_PLATFORM_MAC)
set(_gRPC_ALLTARGETS_LIBRARIES ${CMAKE_DL_LIBS} m pthread)
elseif(UNIX)
set(_gRPC_ALLTARGETS_LIBRARIES ${CMAKE_DL_LIBS} rt m pthread)
endif()
add_library(address_sorting
${GRPC_SOURCE_DIR}/third_party/address_sorting/address_sorting.c
@ -112,7 +67,6 @@ add_library(address_sorting
)
target_include_directories(address_sorting
PUBLIC $<INSTALL_INTERFACE:${gRPC_INSTALL_INCLUDEDIR}> $<BUILD_INTERFACE:${GRPC_SOURCE_DIR}/include>
PRIVATE ${GRPC_SOURCE_DIR}
PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR}
PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
@ -124,11 +78,11 @@ target_include_directories(address_sorting
PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
)
target_link_libraries(address_sorting
${_gRPC_BASELIB_LIBRARIES}
${_gRPC_PROTOBUF_LIBRARIES}
${_gRPC_ALLTARGETS_LIBRARIES}
)
add_library(gpr
${GRPC_SOURCE_DIR}/src/core/lib/gpr/alloc.cc
${GRPC_SOURCE_DIR}/src/core/lib/gpr/atm.cc
@ -184,6 +138,7 @@ target_include_directories(gpr
PRIVATE ${_gRPC_UPB_INCLUDE_DIR}
PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
)
target_link_libraries(gpr
${_gRPC_ALLTARGETS_LIBRARIES}
${_gRPC_PROTOBUF_LIBRARIES}
@ -569,7 +524,6 @@ add_library(grpc
${GRPC_SOURCE_DIR}/src/core/plugin_registry/grpc_plugin_registry.cc
)
target_compile_options(grpc PUBLIC -fpermissive)
target_include_directories(grpc
PUBLIC ${GRPC_INCLUDE_DIR}
@ -583,8 +537,8 @@ target_include_directories(grpc
PRIVATE ${_gRPC_UPB_INCLUDE_DIR}
PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
)
target_link_libraries(grpc
${_gRPC_BASELIB_LIBRARIES}
${_gRPC_SSL_LIBRARIES}
${_gRPC_ZLIB_LIBRARIES}
${_gRPC_CARES_LIBRARIES}
@ -597,352 +551,6 @@ if (_gRPC_PLATFORM_MAC)
target_link_libraries(grpc "-framework CoreFoundation")
endif()
add_library(grpc_cronet
${GRPC_SOURCE_DIR}/src/core/ext/transport/cronet/plugin_registry/grpc_cronet_plugin_registry.cc
${GRPC_SOURCE_DIR}/src/core/lib/surface/init.cc
${GRPC_SOURCE_DIR}/src/core/lib/avl/avl.cc
${GRPC_SOURCE_DIR}/src/core/lib/backoff/backoff.cc
${GRPC_SOURCE_DIR}/src/core/lib/channel/channel_args.cc
${GRPC_SOURCE_DIR}/src/core/lib/channel/channel_stack.cc
${GRPC_SOURCE_DIR}/src/core/lib/channel/channel_stack_builder.cc
${GRPC_SOURCE_DIR}/src/core/lib/channel/channel_trace.cc
${GRPC_SOURCE_DIR}/src/core/lib/channel/channelz.cc
${GRPC_SOURCE_DIR}/src/core/lib/channel/channelz_registry.cc
${GRPC_SOURCE_DIR}/src/core/lib/channel/connected_channel.cc
${GRPC_SOURCE_DIR}/src/core/lib/channel/handshaker.cc
${GRPC_SOURCE_DIR}/src/core/lib/channel/handshaker_registry.cc
${GRPC_SOURCE_DIR}/src/core/lib/channel/status_util.cc
${GRPC_SOURCE_DIR}/src/core/lib/compression/compression.cc
${GRPC_SOURCE_DIR}/src/core/lib/compression/compression_args.cc
${GRPC_SOURCE_DIR}/src/core/lib/compression/compression_internal.cc
${GRPC_SOURCE_DIR}/src/core/lib/compression/message_compress.cc
${GRPC_SOURCE_DIR}/src/core/lib/compression/stream_compression.cc
${GRPC_SOURCE_DIR}/src/core/lib/compression/stream_compression_gzip.cc
${GRPC_SOURCE_DIR}/src/core/lib/compression/stream_compression_identity.cc
${GRPC_SOURCE_DIR}/src/core/lib/debug/stats.cc
${GRPC_SOURCE_DIR}/src/core/lib/debug/stats_data.cc
${GRPC_SOURCE_DIR}/src/core/lib/http/format_request.cc
${GRPC_SOURCE_DIR}/src/core/lib/http/httpcli.cc
${GRPC_SOURCE_DIR}/src/core/lib/http/parser.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/buffer_list.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/call_combiner.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/cfstream_handle.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/combiner.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_cfstream.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_pair_posix.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_pair_uv.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_pair_windows.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/error.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/error_cfstream.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_epoll1_linux.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_epollex_linux.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_poll_posix.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_posix.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_windows.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/exec_ctx.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/executor.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/executor/mpmcqueue.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/executor/threadpool.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/fork_posix.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/fork_windows.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_fallback.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_host_name_max.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_sysconf.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/grpc_if_nametoindex_posix.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/grpc_if_nametoindex_unsupported.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/internal_errqueue.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iocp_windows.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_custom.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_internal.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_posix.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_posix_cfstream.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_uv.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_windows.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/is_epollexclusive_available.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/load_file.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/lockfree_event.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/polling_entity.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_custom.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set_custom.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set_windows.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_uv.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_windows.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_custom.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_posix.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_windows.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resource_quota.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/sockaddr_utils.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_factory_posix.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_mutator.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_common_posix.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_linux.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_posix.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_uv.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_windows.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_windows.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_cfstream.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_custom.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_posix.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_windows.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_custom.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_posix.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_custom.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_posix.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_common.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_windows.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_uv.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_windows.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/time_averaged_stats.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_custom.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_generic.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_heap.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_manager.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_uv.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/udp_server.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/unix_sockets_posix.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/unix_sockets_posix_noop.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_eventfd.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_nospecial.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_pipe.cc
${GRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_posix.cc
${GRPC_SOURCE_DIR}/src/core/lib/json/json.cc
${GRPC_SOURCE_DIR}/src/core/lib/json/json_reader.cc
${GRPC_SOURCE_DIR}/src/core/lib/json/json_string.cc
${GRPC_SOURCE_DIR}/src/core/lib/json/json_writer.cc
${GRPC_SOURCE_DIR}/src/core/lib/slice/b64.cc
${GRPC_SOURCE_DIR}/src/core/lib/slice/percent_encoding.cc
${GRPC_SOURCE_DIR}/src/core/lib/slice/slice.cc
${GRPC_SOURCE_DIR}/src/core/lib/slice/slice_buffer.cc
${GRPC_SOURCE_DIR}/src/core/lib/slice/slice_intern.cc
${GRPC_SOURCE_DIR}/src/core/lib/slice/slice_string_helpers.cc
${GRPC_SOURCE_DIR}/src/core/lib/surface/api_trace.cc
${GRPC_SOURCE_DIR}/src/core/lib/surface/byte_buffer.cc
${GRPC_SOURCE_DIR}/src/core/lib/surface/byte_buffer_reader.cc
${GRPC_SOURCE_DIR}/src/core/lib/surface/call.cc
${GRPC_SOURCE_DIR}/src/core/lib/surface/call_details.cc
${GRPC_SOURCE_DIR}/src/core/lib/surface/call_log_batch.cc
${GRPC_SOURCE_DIR}/src/core/lib/surface/channel.cc
${GRPC_SOURCE_DIR}/src/core/lib/surface/channel_init.cc
${GRPC_SOURCE_DIR}/src/core/lib/surface/channel_ping.cc
${GRPC_SOURCE_DIR}/src/core/lib/surface/channel_stack_type.cc
${GRPC_SOURCE_DIR}/src/core/lib/surface/completion_queue.cc
${GRPC_SOURCE_DIR}/src/core/lib/surface/completion_queue_factory.cc
${GRPC_SOURCE_DIR}/src/core/lib/surface/event_string.cc
${GRPC_SOURCE_DIR}/src/core/lib/surface/lame_client.cc
${GRPC_SOURCE_DIR}/src/core/lib/surface/metadata_array.cc
${GRPC_SOURCE_DIR}/src/core/lib/surface/server.cc
${GRPC_SOURCE_DIR}/src/core/lib/surface/validate_metadata.cc
${GRPC_SOURCE_DIR}/src/core/lib/surface/version.cc
${GRPC_SOURCE_DIR}/src/core/lib/transport/bdp_estimator.cc
${GRPC_SOURCE_DIR}/src/core/lib/transport/byte_stream.cc
${GRPC_SOURCE_DIR}/src/core/lib/transport/connectivity_state.cc
${GRPC_SOURCE_DIR}/src/core/lib/transport/error_utils.cc
${GRPC_SOURCE_DIR}/src/core/lib/transport/metadata.cc
${GRPC_SOURCE_DIR}/src/core/lib/transport/metadata_batch.cc
${GRPC_SOURCE_DIR}/src/core/lib/transport/pid_controller.cc
${GRPC_SOURCE_DIR}/src/core/lib/transport/static_metadata.cc
${GRPC_SOURCE_DIR}/src/core/lib/transport/status_conversion.cc
${GRPC_SOURCE_DIR}/src/core/lib/transport/status_metadata.cc
${GRPC_SOURCE_DIR}/src/core/lib/transport/timeout_encoding.cc
${GRPC_SOURCE_DIR}/src/core/lib/transport/transport.cc
${GRPC_SOURCE_DIR}/src/core/lib/transport/transport_op_string.cc
${GRPC_SOURCE_DIR}/src/core/lib/uri/uri_parser.cc
${GRPC_SOURCE_DIR}/src/core/lib/debug/trace.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/cronet/client/secure/cronet_channel_create.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/cronet/transport/cronet_api_dummy.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/cronet/transport/cronet_transport.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/secure/secure_channel_create.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_decoder.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_encoder.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/chttp2_plugin.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/chttp2_transport.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/context_list.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/flow_control.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_data.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_goaway.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_ping.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_rst_stream.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_settings.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_window_update.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_encoder.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_parser.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_table.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http2_settings.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/huffsyms.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/incoming_metadata.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/parsing.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/stream_lists.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/stream_map.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/varint.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/writing.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/alpn/alpn.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/http/client/http_client_filter.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/http/http_filters_plugin.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/http/message_compress/message_compress_filter.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/http/server/http_server_filter.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/backend_metric.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/backup_poller.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/channel_connectivity.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_channelz.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_factory.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_plugin.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/connector.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/global_subchannel_pool.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/health/health_check_client.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/http_connect_handshaker.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/http_proxy.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy_registry.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/local_subchannel_pool.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/parse_address.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/proxy_mapper.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/proxy_mapper_registry.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver_registry.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver_result_parsing.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolving_lb_policy.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/retry_throttle.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/server_address.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/service_config.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/subchannel.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/subchannel_pool_interface.cc
${GRPC_SOURCE_DIR}/src/core/ext/filters/deadline/deadline_filter.cc
${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/health/v1/health.upb.c
${GRPC_SOURCE_DIR}/third_party/upb/upb/decode.c
${GRPC_SOURCE_DIR}/third_party/upb/upb/encode.c
${GRPC_SOURCE_DIR}/third_party/upb/upb/msg.c
${GRPC_SOURCE_DIR}/third_party/upb/upb/port.c
${GRPC_SOURCE_DIR}/third_party/upb/upb/table.c
${GRPC_SOURCE_DIR}/third_party/upb/upb/upb.c
${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/udpa/data/orca/v1/orca_load_report.upb.c
${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/gogoproto/gogo.upb.c
${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/validate/validate.upb.c
${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/annotations.upb.c
${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/http.upb.c
${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/any.upb.c
${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/descriptor.upb.c
${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/duration.upb.c
${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/empty.upb.c
${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/struct.upb.c
${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/timestamp.upb.c
${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/wrappers.upb.c
${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/rpc/status.upb.c
${GRPC_SOURCE_DIR}/src/core/lib/http/httpcli_security_connector.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/context/security_context.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/alts_credentials.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/composite/composite_credentials.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/credentials.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/credentials_metadata.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/fake/fake_credentials.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/google_default/credentials_generic.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/google_default/google_default_credentials.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/iam/iam_credentials.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/jwt/json_token.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/jwt/jwt_credentials.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/jwt/jwt_verifier.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/local/local_credentials.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/plugin/plugin_credentials.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/ssl/ssl_credentials.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/spiffe_credentials.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/alts/alts_security_connector.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/fake/fake_security_connector.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/load_system_roots_fallback.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/load_system_roots_linux.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/local/local_security_connector.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/security_connector.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/ssl/ssl_security_connector.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/ssl_utils.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/ssl_utils_config.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/tls/spiffe_security_connector.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/transport/client_auth_filter.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/transport/secure_endpoint.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/transport/security_handshaker.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/transport/server_auth_filter.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/transport/target_authority_table.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/transport/tsi_error.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/util/json_util.cc
${GRPC_SOURCE_DIR}/src/core/lib/surface/init_secure.cc
${GRPC_SOURCE_DIR}/src/core/tsi/alts/crypt/aes_gcm.cc
${GRPC_SOURCE_DIR}/src/core/tsi/alts/crypt/gsec.cc
${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_counter.cc
${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_crypter.cc
${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_frame_protector.cc
${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_record_protocol_crypter_common.cc
${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_seal_privacy_integrity_crypter.cc
${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_unseal_privacy_integrity_crypter.cc
${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/frame_handler.cc
${GRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_handshaker_client.cc
${GRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_shared_resource.cc
${GRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc
${GRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.cc
${GRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.cc
${GRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol_common.cc
${GRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_iovec_record_protocol.cc
${GRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_no_op.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_windows.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_options.cc
${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_server_options.cc
${GRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_tsi_utils.cc
${GRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/transport_security_common_api.cc
${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/altscontext.upb.c
${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/handshaker.upb.c
${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/transport_security_common.upb.c
${GRPC_SOURCE_DIR}/src/core/tsi/transport_security.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/insecure/channel_create.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/authority.cc
${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/chttp2_connector.cc
${GRPC_SOURCE_DIR}/src/core/tsi/fake_transport_security.cc
${GRPC_SOURCE_DIR}/src/core/tsi/local_transport_security.cc
${GRPC_SOURCE_DIR}/src/core/tsi/ssl/session_cache/ssl_session_boringssl.cc
${GRPC_SOURCE_DIR}/src/core/tsi/ssl/session_cache/ssl_session_cache.cc
${GRPC_SOURCE_DIR}/src/core/tsi/ssl/session_cache/ssl_session_openssl.cc
${GRPC_SOURCE_DIR}/src/core/tsi/ssl_transport_security.cc
${GRPC_SOURCE_DIR}/src/core/tsi/transport_security_grpc.cc
)
target_include_directories(grpc_cronet
PUBLIC ${GRPC_INCLUDE_DIR}
PRIVATE ${GRPC_SOURCE_DIR}
PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR}
PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
PRIVATE ${_gRPC_UPB_GENERATED_DIR}
PRIVATE ${_gRPC_UPB_GRPC_GENERATED_DIR}
PRIVATE ${_gRPC_UPB_INCLUDE_DIR}
PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
)
target_link_libraries(grpc_cronet
${_gRPC_BASELIB_LIBRARIES}
${_gRPC_SSL_LIBRARIES}
${_gRPC_ZLIB_LIBRARIES}
${_gRPC_CARES_LIBRARIES}
${_gRPC_ADDRESS_SORTING_LIBRARIES}
${_gRPC_ALLTARGETS_LIBRARIES}
${_gRPC_PROTOBUF_LIBRARIES}
gpr
)
if (_gRPC_PLATFORM_MAC)
target_link_libraries(grpc_cronet "-framework CoreFoundation")
endif()
add_library(grpc_unsecure
${GRPC_SOURCE_DIR}/src/core/lib/surface/init.cc
@ -1249,19 +857,18 @@ add_library(grpc_unsecure
${GRPC_SOURCE_DIR}/src/core/plugin_registry/grpc_unsecure_plugin_registry.cc
)
target_include_directories(grpc_unsecure
PUBLIC ${GRPC_INCLUDE_DIR}
PRIVATE ${GRPC_SOURCE_DIR}
PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR}
PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
PRIVATE ${_gRPC_UPB_GENERATED_DIR}
PRIVATE ${_gRPC_UPB_GRPC_GENERATED_DIR}
PRIVATE ${_gRPC_UPB_INCLUDE_DIR}
PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
)
target_link_libraries(grpc_unsecure
${_gRPC_BASELIB_LIBRARIES}
${_gRPC_ZLIB_LIBRARIES}
@ -1271,10 +878,12 @@ target_link_libraries(grpc_unsecure
${_gRPC_PROTOBUF_LIBRARIES}
gpr
)
if (_gRPC_PLATFORM_MAC)
target_link_libraries(grpc_unsecure "-framework CoreFoundation")
endif()
add_library(grpc++
${GRPC_SOURCE_DIR}/src/cpp/client/insecure_credentials.cc
${GRPC_SOURCE_DIR}/src/cpp/client/secure_credentials.cc
@ -1331,8 +940,6 @@ add_library(grpc++
${GRPC_SOURCE_DIR}/src/cpp/codegen/codegen_init.cc
)
target_compile_options(grpc++ PUBLIC -w)
target_include_directories(grpc++
PUBLIC ${GRPC_INCLUDE_DIR}
PRIVATE ${GRPC_SOURCE_DIR}
@ -1344,10 +951,9 @@ target_include_directories(grpc++
PRIVATE ${_gRPC_UPB_GRPC_GENERATED_DIR}
PRIVATE ${_gRPC_UPB_INCLUDE_DIR}
PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
PRIVATE ${_gRPC_PROTO_GENS_DIR}
)
target_link_libraries(grpc++
${_gRPC_BASELIB_LIBRARIES}
${_gRPC_SSL_LIBRARIES}
${_gRPC_PROTOBUF_LIBRARIES}
${_gRPC_ALLTARGETS_LIBRARIES}
@ -1355,6 +961,7 @@ target_link_libraries(grpc++
gpr
)
add_library(grpc++_unsecure
${GRPC_SOURCE_DIR}/src/cpp/client/insecure_credentials.cc
${GRPC_SOURCE_DIR}/src/cpp/common/insecure_create_auth_context.cc
@ -1404,21 +1011,19 @@ add_library(grpc++_unsecure
${GRPC_SOURCE_DIR}/src/cpp/codegen/codegen_init.cc
)
target_compile_options(grpc++_unsecure PUBLIC -w)
target_include_directories(grpc++_unsecure
PUBLIC ${GRPC_INCLUDE_DIR}
PRIVATE ${GRPC_SOURCE_DIR}
PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR}
PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
PRIVATE ${_gRPC_UPB_GENERATED_DIR}
PRIVATE ${_gRPC_UPB_GRPC_GENERATED_DIR}
PRIVATE ${_gRPC_UPB_INCLUDE_DIR}
PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
PRIVATE ${_gRPC_PROTO_GENS_DIR}
)
target_link_libraries(grpc++_unsecure
${_gRPC_BASELIB_LIBRARIES}
${_gRPC_PROTOBUF_LIBRARIES}
@ -1427,6 +1032,16 @@ target_link_libraries(grpc++_unsecure
grpc_unsecure
)
if (_gRPC_SSL_INCLUDE_DIR AND _gRPC_SSL_LIBRARIES)
add_library(libgrpc ALIAS grpc)
add_library(libgrpc++ ALIAS grpc++)
else()
add_library(libgrpc ALIAS grpc_unsecure)
add_library(libgrpc++ ALIAS grpc++_unsecure)
endif()
add_library(grpc_plugin_support
${GRPC_SOURCE_DIR}/src/compiler/cpp_generator.cc
)
@ -1436,23 +1051,22 @@ target_include_directories(grpc_plugin_support
PUBLIC ${GRPC_INCLUDE_DIR}
PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}
PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR}
PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
PRIVATE ${_gRPC_UPB_GENERATED_DIR}
PRIVATE ${_gRPC_UPB_GRPC_GENERATED_DIR}
PRIVATE ${_gRPC_UPB_INCLUDE_DIR}
PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
PRIVATE ${_gRPC_PROTO_GENS_DIR}
)
target_link_libraries(grpc_plugin_support
${_gRPC_PROTOBUF_PROTOC_LIBRARIES}
${_gRPC_PROTOBUF_LIBRARIES}
${_gRPC_ALLTARGETS_LIBRARIES}
)
add_executable(grpc_cpp_plugin
${GRPC_SOURCE_DIR}/src/compiler/cpp_plugin.cc
)
@ -1461,16 +1075,13 @@ target_include_directories(grpc_cpp_plugin
PRIVATE ${GRPC_SOURCE_DIR}
PUBLIC ${GRPC_INCLUDE_DIR}
PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR}
PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR}
PRIVATE ${_gRPC_CARES_INCLUDE_DIR}
PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR}
PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR}
PRIVATE ${_gRPC_SSL_INCLUDE_DIR}
PRIVATE ${_gRPC_UPB_GENERATED_DIR}
PRIVATE ${_gRPC_UPB_GRPC_GENERATED_DIR}
PRIVATE ${_gRPC_UPB_INCLUDE_DIR}
PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR}
PRIVATE ${_gRPC_PROTO_GENS_DIR}
)
target_link_libraries(grpc_cpp_plugin
@ -1479,4 +1090,3 @@ target_link_libraries(grpc_cpp_plugin
${_gRPC_ALLTARGETS_LIBRARIES}
grpc_plugin_support
)

13
contrib/protobuf-cmake/CMakeLists.txt Normal file
View File

@ -0,0 +1,13 @@
set(protobuf_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/protobuf)
set(protobuf_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/protobuf)
set(protobuf_WITH_ZLIB 0 CACHE INTERNAL "" FORCE) # actually will use zlib, but skip find
set(protobuf_BUILD_TESTS OFF CACHE INTERNAL "" FORCE)
if (MAKE_STATIC_LIBRARIES)
set(protobuf_BUILD_SHARED_LIBS OFF CACHE INTERNAL "" FORCE)
else ()
set(protobuf_BUILD_SHARED_LIBS ON CACHE INTERNAL "" FORCE)
endif ()
add_subdirectory(${protobuf_SOURCE_DIR}/cmake ${protobuf_BINARY_DIR})

2
docker/test/integration/runner/Dockerfile
View File

@ -36,7 +36,7 @@ RUN apt-get update \
ENV TZ=Europe/Moscow
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
RUN pip install urllib3==1.23 pytest docker-compose==1.22.0 docker dicttoxml kazoo PyMySQL psycopg2==2.7.5 pymongo tzlocal kafka-python protobuf redis aerospike pytest-timeout minio rpm-confluent-schemaregistry grpcio grpcio-tools
RUN pip install urllib3==1.23 pytest docker-compose==1.22.0 docker dicttoxml kazoo PyMySQL psycopg2==2.7.5 pymongo tzlocal kafka-python protobuf redis aerospike pytest-timeout minio rpm-confluent-schemaregistry grpcio grpcio-tools cassandra-driver
ENV DOCKER_CHANNEL stable
ENV DOCKER_VERSION 17.09.1-ce

2
docker/test/performance-comparison/perf.py
View File

@ -163,6 +163,8 @@ for query_index, q in enumerate(test_queries):
prewarm_id = f'{query_prefix}.prewarm0'
res = c.execute(q, query_id = prewarm_id)
print(f'prewarm\t{query_index}\t{prewarm_id}\t{conn_index}\t{c.last_query.elapsed}')
except KeyboardInterrupt:
raise
except:
# If prewarm fails for some query -- skip it, and try to test the others.
# This might happen if the new test introduces some function that the

32
docs/en/development/developer-instruction.md
View File

@ -1,19 +1,17 @@
---
toc_priority: 61
toc_title: The Beginner ClickHouse Developer Instruction
toc_title: For Beginners
---
# The Beginner ClickHouse Developer Instruction
Building of ClickHouse is supported on Linux, FreeBSD and Mac OS X.
# If You Use Windows {#if-you-use-windows}
If you use Windows, you need to create a virtual machine with Ubuntu. To start working with a virtual machine please install VirtualBox. You can download Ubuntu from the website: https://www.ubuntu.com/#download. Please create a virtual machine from the downloaded image (you should reserve at least 4GB of RAM for it). To run a command-line terminal in Ubuntu, please locate a program containing the word “terminal” in its name (gnome-terminal, konsole etc.) or just press Ctrl+Alt+T.
# If You Use a 32-bit System {#if-you-use-a-32-bit-system}
ClickHouse cannot work or build on a 32-bit system. You should acquire access to a 64-bit system and you can continue reading.
# Creating a Repository on GitHub {#creating-a-repository-on-github}
## Creating a Repository on GitHub {#creating-a-repository-on-github}
To start working with ClickHouse repository you will need a GitHub account.
@ -33,7 +31,7 @@ To do that in Ubuntu you would run in the command line terminal:
A brief manual on using Git can be found here: https://services.github.com/on-demand/downloads/github-git-cheat-sheet.pdf.
For a detailed manual on Git see https://git-scm.com/book/en/v2.
# Cloning a Repository to Your Development Machine {#cloning-a-repository-to-your-development-machine}
## Cloning a Repository to Your Development Machine {#cloning-a-repository-to-your-development-machine}
Next, you need to download the source files onto your working machine. This is called “to clone a repository” because it creates a local copy of the repository on your working machine.
@ -77,7 +75,7 @@ You can also add original ClickHouse repo’s address to your local repository t
After successfully running this command you will be able to pull updates from the main ClickHouse repo by running `git pull upstream master`.
## Working with Submodules {#working-with-submodules}
### Working with Submodules {#working-with-submodules}
Working with submodules in git could be painful. Next commands will help to manage it:
@ -107,7 +105,7 @@ The next commands would help you to reset all submodules to the initial state (!
git submodule foreach git submodule foreach git reset --hard
git submodule foreach git submodule foreach git clean -xfd
# Build System {#build-system}
## Build System {#build-system}
ClickHouse uses CMake and Ninja for building.
@ -127,11 +125,11 @@ For installing CMake and Ninja on Mac OS X first install Homebrew and then insta
Next, check the version of CMake: `cmake --version`. If it is below 3.3, you should install a newer version from the website: https://cmake.org/download/.
# Optional External Libraries {#optional-external-libraries}
## Optional External Libraries {#optional-external-libraries}
ClickHouse uses several external libraries for building. All of them do not need to be installed separately as they are built together with ClickHouse from the sources located in the submodules. You can check the list in `contrib`.
# C++ Compiler {#c-compiler}
## C++ Compiler {#c-compiler}
Compilers GCC starting from version 9 and Clang version 8 or above are supported for building ClickHouse.
@ -145,7 +143,7 @@ Mac OS X build is supported only for Clang. Just run `brew install llvm`
If you decide to use Clang, you can also install `libc++` and `lld`, if you know what it is. Using `ccache` is also recommended.
# The Building Process {#the-building-process}
## The Building Process {#the-building-process}
Now that you are ready to build ClickHouse we recommend you to create a separate directory `build` inside `ClickHouse` that will contain all of the build artefacts:
@ -202,7 +200,7 @@ Upon successful build you get an executable file `ClickHouse/<build_dir>/program
ls -l programs/clickhouse
# Running the Built Executable of ClickHouse {#running-the-built-executable-of-clickhouse}
## Running the Built Executable of ClickHouse {#running-the-built-executable-of-clickhouse}
To run the server under the current user you need to navigate to `ClickHouse/programs/server/` (located outside of `build`) and run:
@ -229,7 +227,7 @@ You can also run your custom-built ClickHouse binary with the config file from t
sudo service clickhouse-server stop
sudo -u clickhouse ClickHouse/build/programs/clickhouse server --config-file /etc/clickhouse-server/config.xml
# IDE (Integrated Development Environment) {#ide-integrated-development-environment}
## IDE (Integrated Development Environment) {#ide-integrated-development-environment}
If you do not know which IDE to use, we recommend that you use CLion. CLion is commercial software, but it offers 30 days free trial period. It is also free of charge for students. CLion can be used both on Linux and on Mac OS X.
@ -239,7 +237,7 @@ As simple code editors, you can use Sublime Text or Visual Studio Code, or Kate
Just in case, it is worth mentioning that CLion creates `build` path on its own, it also on its own selects `debug` for build type, for configuration it uses a version of CMake that is defined in CLion and not the one installed by you, and finally, CLion will use `make` to run build tasks instead of `ninja`. This is normal behaviour, just keep that in mind to avoid confusion.
# Writing Code {#writing-code}
## Writing Code {#writing-code}
The description of ClickHouse architecture can be found here: https://clickhouse.tech/docs/en/development/architecture/
@ -249,7 +247,7 @@ Writing tests: https://clickhouse.tech/docs/en/development/tests/
List of tasks: https://github.com/ClickHouse/ClickHouse/blob/master/testsructions/easy\_tasks\_sorted\_en.md
# Test Data {#test-data}
## Test Data {#test-data}
Developing ClickHouse often requires loading realistic datasets. It is particularly important for performance testing. We have a specially prepared set of anonymized data from Yandex.Metrica. It requires additionally some 3GB of free disk space. Note that this data is not required to accomplish most of the development tasks.
@ -272,7 +270,7 @@ Developing ClickHouse often requires loading realistic datasets. It is particula
clickhouse-client --max_insert_block_size 100000 --query "INSERT INTO test.hits FORMAT TSV" < hits_v1.tsv
clickhouse-client --max_insert_block_size 100000 --query "INSERT INTO test.visits FORMAT TSV" < visits_v1.tsv
# Creating Pull Request {#creating-pull-request}
## Creating Pull Request {#creating-pull-request}
Navigate to your fork repository in GitHub’s UI. If you have been developing in a branch, you need to select that branch. There will be a “Pull request” button located on the screen. In essence, this means “create a request for accepting my changes into the main repository”.

1
docs/en/interfaces/third-party/integrations.md vendored
View File

@ -25,6 +25,7 @@ toc_title: Integrations
- Message queues
- [Kafka](https://kafka.apache.org)
- [clickhouse\_sinker](https://github.com/housepower/clickhouse_sinker) (uses [Go client](https://github.com/ClickHouse/clickhouse-go/))
- [stream-loader-clickhouse](https://github.com/adform/stream-loader)
- Stream processing
- [Flink](https://flink.apache.org)
- [flink-clickhouse-sink](https://github.com/ivi-ru/flink-clickhouse-sink)

30
docs/en/operations/settings/settings.md
View File

@ -404,6 +404,36 @@ Possible values:
Default value: 0.
## any_join_distinct_right_table_keys {#any_join_distinct_right_table_keys}
Enables legacy ClickHouse server behavior in `ANY INNER|LEFT JOIN` operations.
!!! note "Warning"
Use this setting only for the purpose of backward compatibility if your use cases depend on legacy `JOIN` behavior.
When the legacy behavior enabled:
- Results of `t1 ANY LEFT JOIN t2` and `t2 ANY RIGHT JOIN t1` operations are not equal because ClickHouse uses the logic with many-to-one left-to-right table keys mapping.
- Results of `ANY INNER JOIN` operations contain all rows from the left table like the `SEMI LEFT JOIN` operations do.
When the legacy behavior disabled:
- Results of `t1 ANY LEFT JOIN t2` and `t2 ANY RIGHT JOIN t1` operations are equal because ClickHouse uses the logic which provides one-to-many keys mapping in `ANY RIGHT JOIN` operations.
- Results of `ANY INNER JOIN` operations contain one row per key from both left and right tables.
Possible values:
- 0 — Legacy behavior is disabled.
- 1 — Legacy behavior is enabled.
Default value: 0.
See also:
- [JOIN strictness](../../sql-reference/statements/select/join.md#select-join-strictness)
## max\_block\_size {#setting-max_block_size}
In ClickHouse, data is processed by blocks (sets of column parts). The internal processing cycles for a single block are efficient enough, but there are noticeable expenditures on each block. The `max_block_size` setting is a recommendation for what size of the block (in a count of rows) to load from tables. The block size shouldn’t be too small, so that the expenditures on each block are still noticeable, but not too large so that the query with LIMIT that is completed after the first block is processed quickly. The goal is to avoid consuming too much memory when extracting a large number of columns in multiple threads and to preserve at least some cache locality.

2
docs/en/operations/troubleshooting.md
View File

@ -116,7 +116,7 @@ Check:
Check:
- The [tcp\_port\_secure](server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port_secure) setting.
- Settings for [SSL sertificates](server-configuration-parameters/settings.md#server_configuration_parameters-openssl).
- Settings for [SSL certificates](server-configuration-parameters/settings.md#server_configuration_parameters-openssl).
Use proper parameters while connecting. For example, use the `port_secure` parameter with `clickhouse_client`.

86
docs/en/sql-reference/functions/type-conversion-functions.md
View File

@ -568,4 +568,90 @@ Result:
└───────────────────────┘
```
## toUnixTimestamp64Milli
## toUnixTimestamp64Micro
## toUnixTimestamp64Nano
Converts a `DateTime64` to a `Int64` value with fixed sub-second precision. Input value is scaled up or down appropriately depending on it precision. Please note that output value is a timestamp in UTC, not in timezone of `DateTime64`.
**Syntax**
``` sql
toUnixTimestamp64Milli(value)
```
**Parameters**
- `value` — DateTime64 value with any precision.
**Returned value**
- `value` converted to the `Int64` data type.
**Examples**
Query:
``` sql
WITH toDateTime64('2019-09-16 19:20:12.345678910', 6) AS dt64
SELECT toUnixTimestamp64Milli(dt64)
```
Result:
``` text
┌─toUnixTimestamp64Milli(dt64)─┐
│ 1568650812345 │
└──────────────────────────────┘
```
``` sql
WITH toDateTime64('2019-09-16 19:20:12.345678910', 6) AS dt64
SELECT toUnixTimestamp64Nano(dt64)
```
Result:
``` text
┌─toUnixTimestamp64Nano(dt64)─┐
│ 1568650812345678000 │
└─────────────────────────────┘
```
## fromUnixTimestamp64Milli
## fromUnixTimestamp64Micro
## fromUnixTimestamp64Nano
Converts an `Int64` to a `DateTime64` value with fixed sub-second precision and optional timezone. Input value is scaled up or down appropriately depending on it's precision. Please note that input value is treated as UTC timestamp, not timestamp at given (or implicit) timezone.
**Syntax**
``` sql
fromUnixTimestamp64Milli(value [, ti])
```
**Parameters**
- `value` — `Int64` value with any precision.
- `timezone` — `String` (optional) timezone name of the result.
**Returned value**
- `value` converted to the `DateTime64` data type.
**Examples**
``` sql
WITH CAST(1234567891011, 'Int64') AS i64
SELECT fromUnixTimestamp64Milli(i64, 'UTC')
```
``` text
┌─fromUnixTimestamp64Milli(i64, 'UTC')─┐
│ 2009-02-13 23:31:31.011 │
└──────────────────────────────────────┘
```
[Original article](https://clickhouse.tech/docs/en/query_language/functions/type_conversion_functions/) <!--hide-->

2
docs/en/sql-reference/statements/select/join.md
View File

@ -44,6 +44,8 @@ Modifies how matching by "join keys" is performed
!!! note "Note"
The default strictness value can be overriden using [join\_default\_strictness](../../../operations/settings/settings.md#settings-join_default_strictness) setting.
Also the behavior of ClickHouse server for `ANY JOIN` operations depends on the [any_join_distinct_right_table_keys](../../../operations/settings/settings.md#any_join_distinct_right_table_keys) setting.
### ASOF JOIN Usage

195
docs/zh/operations/access-rights.md
View File

@ -1,101 +1,146 @@
# 访问权限 {#access-rights}
---
toc_priority: 48
toc_title: "访问权限和账户管理"
---
用户和访问权限在用户配置中设置。 这通常是 `users.xml`.
# 访问权限和账户管理 {#access-rights}
ClickHouse支持基于[RBAC](https://en.wikipedia.org/wiki/Role-based_access_control)的访问控制管理。
用户被记录在 `users` 科。 这里是一个片段 `users.xml` 文件:
ClickHouse权限实体包括：
- [用户账户](#user-account-management)
- [角色](#role-management)
- [行策略](#row-policy-management)
- [设置描述](#settings-profiles-management)
- [配额](#quotas-management)
``` xml
<!-- Users and ACL. -->
<users>
<!-- If the user name is not specified, the 'default' user is used. -->
<default>
<!-- Password could be specified in plaintext or in SHA256 (in hex format).
你可以通过如下方式配置权限实体：
If you want to specify password in plaintext (not recommended), place it in 'password' element.
Example: <password>qwerty</password>.
Password could be empty.
- 通过SQL驱动的工作流方式.
If you want to specify SHA256, place it in 'password_sha256_hex' element.
Example: <password_sha256_hex>65e84be33532fb784c48129675f9eff3a682b27168c0ea744b2cf58ee02337c5</password_sha256_hex>
你需要[开启](#enabling-access-control)这个功能.
How to generate decent password:
Execute: PASSWORD=$(base64 < /dev/urandom | head -c8); echo "$PASSWORD"; echo -n "$PASSWORD" | sha256sum | tr -d '-'
In first line will be password and in second - corresponding SHA256.
-->
<password></password>
- 服务端[配置文件](configuration-files.md) `users.xml` 和 `config.xml`.
<!-- A list of networks that access is allowed from.
Each list item has one of the following forms:
<ip> The IP address or subnet mask. For example: 198.51.100.0/24 or 2001:DB8::/32.
<host> Host name. For example: example01. A DNS query is made for verification, and all addresses obtained are compared with the address of the customer.
<host_regexp> Regular expression for host names. For example, ^example\d\d-\d\d-\d\.yandex\.ru$
To check it, a DNS PTR request is made for the client's address and a regular expression is applied to the result.
Then another DNS query is made for the result of the PTR query, and all received address are compared to the client address.
We strongly recommend that the regex ends with \.yandex\.ru$.
我们建议你使用SQL工作流的方式。当然配置的方式也可以同时起作用, 所以如果你正在用服务端配置的方式来管理权限和账户，你可以平滑的切换到SQL驱动的工作流方式。
If you are installing ClickHouse yourself, specify here:
<networks>
<ip>::/0</ip>
</networks>
-->
<networks incl="networks" />
!!! note "警告"
你无法同时使用两个配置的方式来管理同一个权限实体。
<!-- Settings profile for the user. -->
<profile>default</profile>
<!-- Quota for the user. -->
<quota>default</quota>
</default>
## 用法 {#access-control-usage}
<!-- For requests from the Yandex.Metrica user interface via the API for data on specific counters. -->
<web>
<password></password>
<networks incl="networks" />
<profile>web</profile>
<quota>default</quota>
<allow_databases>
<database>test</database>
</allow_databases>
</web>
```
默认ClickHouse提供了一个 `default` 账号，这个账号有所有的权限，但是不能使用SQL驱动方式的访问权限和账户管理。`default`主要用在用户名还未设置的情况，比如从客户端登录或者执行分布式查询。在分布式查询中如果服务端或者集群没有指定[用户名密码](../engines/table-engines/special/distributed.md)那默认的账户就会被使用。
您可以看到两个用户的声明: `default`和`web`. 我们添加了 `web` 用户分开。
如果你刚开始使用ClickHouse，考虑如下场景：
该 `default` 在用户名未通过的情况下选择用户。 该 `default` 如果服务器或群集的配置没有指定分布式查询处理，则user也用于分布式查询处理 `user` 和 `password` （见上的部分 [分布](../engines/table-engines/special/distributed.md) 发动机）。
1. 为 `default` 用户[开启SQL驱动方式的访问权限和账户管理](#enabling-access-control) .
2. 使用 `default` 用户登录并且创建所需要的所有用户。 不要忘记创建管理员账户 (`GRANT ALL ON *.* WITH GRANT OPTION TO admin_user_account`)。
3. [限制](settings/permissions-for-queries.md#permissions_for_queries) `default` 用户的权限并且禁用SQL驱动方式的访问权限和账户管理。
The user that is used for exchanging information between servers combined in a cluster must not have substantial restrictions or quotas – otherwise, distributed queries will fail.
### 当前解决方案的特性 {#access-control-properties}
密码以明文（不推荐）或SHA-256形式指定。 哈希没有腌制。 在这方面，您不应将这些密码视为提供了针对潜在恶意攻击的安全性。 相反，他们是必要的保护员工。
- 你甚至可以在数据库和表不存在的时候授予权限。
- 如果表被删除，和这张表关联的特权不会被删除。这意味着如果你创建一张同名的表，所有的特权仍旧有效。如果想删除这张表关联的特权，你可以执行 `REVOKE ALL PRIVILEGES ON db.table FROM ALL` 查询。
- 特权没有生命周期。
指定允许访问的网络列表。 在此示例中，将从单独的文件加载两个用户的网络列表 (`/etc/metrika.xml`）包含 `networks` 替代。 这里是它的一个片段:
## 用户账户 {#user-account-management}
``` xml
<yandex>
...
<networks>
<ip>::/64</ip>
<ip>203.0.113.0/24</ip>
<ip>2001:DB8::/32</ip>
...
</networks>
</yandex>
```
用户账户是权限实体，用来授权操作ClickHouse，用户账户包含：
您可以直接在以下内容中定义此网络列表 `users.xml`，或在文件中 `users.d` directory (for more information, see the section «[配置文件](configuration-files.md#configuration_files)»).
- 标识符信息。
- [特权](../sql-reference/statements/grant.md#grant-privileges)用来定义用户可以执行的查询的范围。
- 可以连接到ClickHouse的主机。
- 指定或者默认的角色。
- 用户登录的时候默认的限制设置。
- 指定的设置描述。
该配置包括解释如何从任何地方打开访问的注释。
特权可以通过[GRANT](../sql-reference/statements/grant.md)查询授权给用户或者通过[角色](#role-management)授予。如果想撤销特权，可以使用[REVOKE](../sql-reference/statements/revoke.md)查询。查询用户所有的特权，使用[SHOW GRANTS](../sql-reference/statements/show.md#show-grants-statement)语句。
对于在生产中使用，仅指定 `ip` 元素（IP地址及其掩码），因为使用 `host` 和 `hoost_regexp` 可能会导致额外的延迟。
查询管理：
Next the user settings profile is specified (see the section «[设置配置文件](settings/settings-profiles.md)»). You can specify the default profile, `default'`. 配置文件可以有任何名称。 您可以为不同的用户指定相同的配置文件。 您可以在设置配置文件中编写的最重要的事情是 `readonly=1`，这确保只读访问。
Then specify the quota to be used (see the section «[配额](quotas.md#quotas)»). You can specify the default quota: `default`. It is set in the config by default to only count resource usage, without restricting it. The quota can have any name. You can specify the same quota for different users – in this case, resource usage is calculated for each user individually.
- [CREATE USER](../sql-reference/statements/create.md#create-user-statement)
- [ALTER USER](../sql-reference/statements/alter.md#alter-user-statement)
- [DROP USER](../sql-reference/statements/misc.md#drop-user-statement)
- [SHOW CREATE USER](../sql-reference/statements/show.md#show-create-user-statement)
在可选 `<allow_databases>` 您还可以指定用户可以访问的数据库列表。 默认情况下，所有数据库都可供用户使用。 您可以指定 `default` 数据库。 在这种情况下，默认情况下，用户将接收对数据库的访问权限。
### 设置应用规则 {#access-control-settings-applying}
访问 `system` 始终允许数据库（因为此数据库用于处理查询）。
对于一个用户账户来说，设置可以通过多种方式配置：通过角色扮演和设置描述。对于一个登陆的账号来说，如果一个设置对应了多个不同的权限实体，这些设置的应用规则如下（优先权从高到底）：
用户可以通过以下方式获取其中所有数据库和表的列表 `SHOW` 查询或系统表，即使不允许访问单个数据库。
1. 用户账户设置。
2. 用户账号默认的角色设置。如果这个设置配置了多个角色，那设置的应用是没有规定的顺序。
3. 从设置描述分批给用户或者角色的设置。如果这个设置配置了多个角色，那设置的应用是没有规定的顺序。
4. 对所有服务器有效的默认或者[default profile](server-configuration-parameters/settings.md#default-profile)的设置。
数据库访问是不相关的 [只读](settings/permissions-for-queries.md#settings_readonly) 设置。 您不能授予对一个数据库的完全访问权限，并 `readonly` 进入另一个。
[原始文章](https://clickhouse.tech/docs/en/operations/access_rights/) <!--hide-->
## 角色 {#role-management}
角色是权限实体的集合，可以被授予用户账号。
角色包括：
- [特权](../sql-reference/statements/grant.md#grant-privileges)
- 设置和限制
- 分配的角色列表
查询管理:
- [CREATE ROLE](../sql-reference/statements/create.md#create-role-statement)
- [ALTER ROLE](../sql-reference/statements/alter.md#alter-role-statement)
- [DROP ROLE](../sql-reference/statements/misc.md#drop-role-statement)
- [SET ROLE](../sql-reference/statements/misc.md#set-role-statement)
- [SET DEFAULT ROLE](../sql-reference/statements/misc.md#set-default-role-statement)
- [SHOW CREATE ROLE](../sql-reference/statements/show.md#show-create-role-statement)
使用[GRANT](../sql-reference/statements/grant.md) 查询可以把特权授予给角色。用[REVOKE](../sql-reference/statements/revoke.md)来撤回特权。
## 行策略 {#row-policy-management}
行策略是一个过滤器，用来定义哪些行数据可以被账户或者角色访问。对一个特定的表来说，行策略包括过滤器和使用这个策略的账户和角色。
查询管理：
- [CREATE ROW POLICY](../sql-reference/statements/create.md#create-row-policy-statement)
- [ALTER ROW POLICY](../sql-reference/statements/alter.md#alter-row-policy-statement)
- [DROP ROW POLICY](../sql-reference/statements/misc.md#drop-row-policy-statement)
- [SHOW CREATE ROW POLICY](../sql-reference/statements/show.md#show-create-row-policy-statement)
## 设置描述 {#settings-profiles-management}
设置描述是[设置](settings/index.md)的汇总。设置汇总包括设置和限制，当然也包括这些描述的对象：角色和账户。
查询管理:
- [CREATE SETTINGS PROFILE](../sql-reference/statements/create.md#create-settings-profile-statement)
- [ALTER SETTINGS PROFILE](../sql-reference/statements/alter.md#alter-settings-profile-statement)
- [DROP SETTINGS PROFILE](../sql-reference/statements/misc.md#drop-settings-profile-statement)
- [SHOW CREATE SETTINGS PROFILE](../sql-reference/statements/show.md#show-create-settings-profile-statement)
## 配额 {#quotas-management}
配额用来限制资源的使用情况。参考[配额](quotas.md).
配额包括特定时间的限制条件和使用这个配额的账户和角色。
Management queries:
- [CREATE QUOTA](../sql-reference/statements/create.md#create-quota-statement)
- [ALTER QUOTA](../sql-reference/statements/alter.md#alter-quota-statement)
- [DROP QUOTA](../sql-reference/statements/misc.md#drop-quota-statement)
- [SHOW CREATE QUOTA](../sql-reference/statements/show.md#show-create-quota-statement)
## 开启SQL驱动方式的访问权限和账户管理 {#enabling-access-control}
- 为配置的存储设置一个目录.
ClickHouse把访问实体的相关配置存储在[访问控制目录](server-configuration-parameters/settings.md#access_control_path)，而这个目录可以通过服务端进行配置.
- 为至少一个账户开启SQL驱动方式的访问权限和账户管理.
默认情况，SQL驱动方式的访问权限和账户管理对所有用户都是关闭的。你需要在 `users.xml` 中配置至少一个用户，并且把[权限管理](settings/settings-users.md#access_management-user-setting)的值设置为1。
[Original article](https://clickhouse.tech/docs/en/operations/access_rights/) <!--hide-->

10
docs/zh/operations/index.md
View File

@ -1,3 +1,13 @@
---
toc_priority: 43
toc_title: "操作"
---
# 操作 {#operations}
Clickhouse运维手册主要包含下面几部分：
- 安装要求
[原始文章](https://clickhouse.tech/docs/en/operations/) <!--hide-->

13
docs/zh/operations/monitoring.md
View File

@ -1,3 +1,8 @@
---
toc_priority: 45
toc_title: "监控"
---
# 监控 {#jian-kong}
可以监控到：
@ -13,7 +18,7 @@ ClickHouse 本身不会去监控硬件资源的状态。
- 处理器上的负载和温度。
可以使用 [dmesg](https://en.wikipedia.org/wiki/Dmesg), [turbostat](https://www.linux.org/docs/man8/turbostat.html) 或者其他工具。
可以使用[dmesg](https://en.wikipedia.org/wiki/Dmesg), [turbostat](https://www.linux.org/docs/man8/turbostat.html)或者其他工具。
- 磁盘存储，RAM和网络的使用率。
@ -21,17 +26,17 @@ ClickHouse 本身不会去监控硬件资源的状态。
ClickHouse服务本身具有用于自我状态监视指标。
要跟踪服务器事件，请观察服务器日志。 请参阅配置文件的\[logger\]（server\_settings/settings.md\#server\_settings-logger）部分。
要跟踪服务器事件，请观察服务器日志。 请参阅配置文件的 [logger](server-configuration-parameters/settings.md#server_configuration_parameters-logger)部分。
ClickHouse 收集的指标项：
- 服务用于计算的资源占用的各种指标。
- 关于查询处理的常见统计信息。
可以在 [系统。指标](system-tables.md#system_tables-metrics) ，[系统。活动](system-tables.md#system_tables-events) 以及[系统。asynchronous\_metrics](system-tables.md#system_tables-asynchronous_metrics) 等系统表查看所有的指标项。
可以在 [系统指标](system-tables.md#system_tables-metrics) ，[系统事件](system-tables.md#system_tables-events) 以及[系统异步指标](system-tables.md#system_tables-asynchronous_metrics) 等系统表查看所有的指标项。
可以配置ClickHouse 往 [石墨](https://github.com/graphite-project)导入指标。 参考 [石墨部分](server-configuration-parameters/settings.md#server_configuration_parameters-graphite) 配置文件。在配置指标导出之前，需要参考Graphite[官方教程](https://graphite.readthedocs.io/en/latest/install.html)搭建服务。
此外，您可以通过HTTP API监视服务器可用性。 将HTTP GET请求发送到 `/ping`。 如果服务器可用，它将以 `200 OK` 响应。
要监视服务器集群的配置中，应设置[max\_replica\_delay\_for\_distributed\_queries](settings/settings.md#settings-max_replica_delay_for_distributed_queries)参数并使用HTTP资源`/replicas_status`。 如果副本可用，并且不延迟在其他副本之后，则对`/replicas_status`的请求将返回200 OK。 如果副本滞后，请求将返回 `503 HTTP_SERVICE_UNAVAILABLE`，包括有关待办事项大小的信息。
要监视服务器集群的配置，应设置[max\_replica\_delay\_for\_distributed\_queries](settings/settings.md#settings-max_replica_delay_for_distributed_queries)参数并使用HTTP资源`/replicas_status`。 如果副本可用，并且不延迟在其他副本之后，则对`/replicas_status`的请求将返回200 OK。 如果副本滞后，请求将返回 `503 HTTP_SERVICE_UNAVAILABLE`，包括有关待办事项大小的信息。

22
docs/zh/operations/requirements.md
View File

@ -1,8 +1,6 @@
---
machine_translated: true
machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd
toc_priority: 44
toc_title: "\u8981\u6C42"
toc_title: "要求"
---
# 要求 {#requirements}
@ -13,20 +11,20 @@ toc_title: "\u8981\u6C42"
ClickHouse实现并行数据处理并使用所有可用的硬件资源。 在选择处理器时，考虑到ClickHouse在具有大量内核但时钟速率较低的配置中的工作效率要高于具有较少内核和较高时钟速率的配置。 例如，具有2600MHz的16核心优于具有3600MHz的8核心。
建议使用 **涡轮增压** 和 **超线程** 技术。 它显着提高了典型工作负载的性能。
建议使用 **睿频加速** 和 **超线程** 技术。 它显着提高了典型工作负载的性能。
## RAM {#ram}
我们建议使用至少4GB的RAM来执行非平凡的查询。 ClickHouse服务器可以使用少得多的RAM运行，但它需要处理查询的内存。
我们建议使用至少4GB的RAM来执行重要的查询。 ClickHouse服务器可以使用少得多的RAM运行，但它需要处理查询的内存。
RAM所需的体积取决于:
- 查询的复杂性。
- 在查询中处理的数据量。
- 查询中处理的数据量。
要计算所需的RAM体积，您应该估计临时数据的大小 [GROUP BY](../sql-reference/statements/select/group-by.md#select-group-by-clause), [DISTINCT](../sql-reference/statements/select/distinct.md#select-distinct), [JOIN](../sql-reference/statements/select/join.md#select-join) 和您使用的其他操作。
ClickHouse可以使用外部存储器来存储临时数据。 看 [在外部存储器中分组](../sql-reference/statements/select/group-by.md#select-group-by-in-external-memory) 有关详细信息。
ClickHouse可以使用外部存储器来存储临时数据。看 [在外部存储器中分组](../sql-reference/statements/select/group-by.md#select-group-by-in-external-memory) 有关详细信息。
## 交换文件 {#swap-file}
@ -42,20 +40,20 @@ ClickHouse可以使用外部存储器来存储临时数据。 看 [在外部存
您可以采取数据的样本并从中获取行的平均大小。 然后将该值乘以计划存储的行数。
- 的数据压缩系数。
- 数据压缩系数。
要估计数据压缩系数，请将数据的样本加载到ClickHouse中，并将数据的实际大小与存储的表的大小进行比较。 例如，点击流数据通常被压缩6-10次。
要估计数据压缩系数，请将数据的样本加载到ClickHouse中，并将数据的实际大小与存储的表的大小进行比较。 例如，点击流数据通常被压缩6-10倍。
要计算要存储的最终数据量，请将压缩系数应用于估计的数据量。 如果计划将数据存储在多个副本中，则将估计的卷乘以副本数。
要计算要存储的最终数据量，请将压缩系数应用于估计的数据量。 如果计划将数据存储在多个副本中，则将估计的量乘以副本数。
## 网络 {#network}
如果可能的话，使用10G或更高级别的网络。
网络带宽对于处理具有大量中间数据的分布式查询至关重要。 此外，网络速度会影响复制过程。
网络带宽对于处理具有大量中间结果数据的分布式查询至关重要。 此外，网络速度会影响复制过程。
## 软件 {#software}
ClickHouse主要是为Linux系列操作系统开发的。 推荐的Linux发行版是Ubuntu。 该 `tzdata` 软件包应安装在系统中。
ClickHouse主要是为Linux系列操作系统开发的。 推荐的Linux发行版是Ubuntu。 `tzdata` 软件包应安装在系统中。
ClickHouse也可以在其他操作系统系列中工作。 查看详细信息 [开始](../getting-started/index.md) 文档的部分。

34
docs/zh/operations/troubleshooting.md
View File

@ -1,23 +1,21 @@
---
machine_translated: true
machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd
toc_priority: 46
toc_title: "\u7591\u96BE\u89E3\u7B54"
toc_title: "常见问题"
---
# 疑难解答 {#troubleshooting}
# 常见问题 {#troubleshooting}
- [安装方式](#troubleshooting-installation-errors)
- [安装](#troubleshooting-installation-errors)
- [连接到服务器](#troubleshooting-accepts-no-connections)
- [查询处理](#troubleshooting-does-not-process-queries)
- [查询处理效率](#troubleshooting-too-slow)
## 安装方式 {#troubleshooting-installation-errors}
## 安装 {#troubleshooting-installation-errors}
### 您无法使用Apt-get从ClickHouse存储库获取Deb软件包 {#you-cannot-get-deb-packages-from-clickhouse-repository-with-apt-get}
- 检查防火墙设置。
- 如果出于任何原因无法访问存储库，请按照以下文件中的描述下载软件包 [开始](../getting-started/index.md) 文章并使用手动安装它们 `sudo dpkg -i <packages>` 指挥部 您还需要 `tzdata` 包。
- 如果出于任何原因无法访问存储库，请按照[开始](../getting-started/index.md)中的描述下载软件包，并使用命令 `sudo dpkg -i <packages>` 手动安装它们。除此之外你还需要 `tzdata` 包。
## 连接到服务器 {#troubleshooting-accepts-no-connections}
@ -44,7 +42,7 @@ $ sudo service clickhouse-server start
**检查日志**
主日志 `clickhouse-server` 是在 `/var/log/clickhouse-server/clickhouse-server.log` 默认情况下。
主日志 `clickhouse-server` 默认情况是在 `/var/log/clickhouse-server/clickhouse-server.log` 下。
如果服务器成功启动，您应该看到字符串:
@ -57,13 +55,13 @@ $ sudo service clickhouse-server start
2019.01.11 15:23:25.549505 [ 45 ] {} <Error> ExternalDictionaries: Failed reloading 'event2id' external dictionary: Poco::Exception. Code: 1000, e.code() = 111, e.displayText() = Connection refused, e.what() = Connection refused
```
如果在文件末尾没有看到错误，请从字符串开始查看整个文件:
如果在文件末尾没有看到错误，请从如下字符串开始查看整个文件:
``` text
<Information> Application: starting up.
```
如果您尝试启动第二个实例 `clickhouse-server` 在服务器上，您将看到以下日志:
如果您尝试在服务器上启动第二个实例 `clickhouse-server` ，您将看到以下日志:
``` text
2019.01.11 15:25:11.151730 [ 1 ] {} <Information> : Starting ClickHouse 19.1.0 with revision 54413
@ -79,9 +77,9 @@ Revision: 54413
2019.01.11 15:25:11.156716 [ 2 ] {} <Information> BaseDaemon: Stop SignalListener thread
```
**请参阅系统。d日志**
**查看系统日志**
如果你没有找到任何有用的信息 `clickhouse-server` 日志或没有任何日志，您可以查看 `system.d` 使用命令记录:
如果你在 `clickhouse-server` 没有找到任何有用的信息或根本没有任何日志，您可以使用命令查看 `system.d` :
``` bash
$ sudo journalctl -u clickhouse-server
@ -99,9 +97,9 @@ $ sudo -u clickhouse /usr/bin/clickhouse-server --config-file /etc/clickhouse-se
检查:
- 码头工人设置。
- Docker设置。
如果您在IPv6网络中的Docker中运行ClickHouse，请确保 `network=host` 已设置。
如果您在IPv6网络中的Docker中运行ClickHouse，请确保 `network=host` 被设置。
- 端点设置。
@ -117,10 +115,10 @@ $ sudo -u clickhouse /usr/bin/clickhouse-server --config-file /etc/clickhouse-se
检查:
- 该 [tcp\_port\_secure](server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port_secure) 设置。
- 设置 [SSL序列](server-configuration-parameters/settings.md#server_configuration_parameters-openssl).
- [tcp\_port\_secure](server-configuration-parameters/settings.md#server_configuration_parameters-tcp_port_secure) 设置。
- [SSL证书](server-configuration-parameters/settings.md#server_configuration_parameters-openssl) 设置.
连接时使用正确的参数。 例如，使用 `port_secure` 参数 `clickhouse_client`.
连接时使用正确的参数。 例如，使用 `clickhouse_client` 的时候使用 `port_secure` 参数 .
- 用户设置。
@ -135,7 +133,7 @@ $ curl 'http://localhost:8123/' --data-binary "SELECT a"
Code: 47, e.displayText() = DB::Exception: Unknown identifier: a. Note that there are no tables (FROM clause) in your query, context: required_names: 'a' source_tables: table_aliases: private_aliases: column_aliases: public_columns: 'a' masked_columns: array_join_columns: source_columns: , e.what() = DB::Exception
```
如果你开始 `clickhouse-client` 与 `stack-trace` 参数，ClickHouse返回包含错误描述的服务器堆栈跟踪。
如果你使用 `clickhouse-client` 时设置了 `stack-trace` 参数，ClickHouse返回包含错误描述的服务器堆栈跟踪信息。
您可能会看到一条关于连接中断的消息。 在这种情况下，可以重复查询。 如果每次执行查询时连接中断，请检查服务器日志中是否存在错误。

10
docs/zh/operations/update.md
View File

@ -1,11 +1,9 @@
---
machine_translated: true
machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd
toc_priority: 47
toc_title: "\u70B9\u51FB\u66F4\u65B0"
toc_title: "更新"
---
# 点击更新 {#clickhouse-update}
# 更新 {#clickhouse-update}
如果从deb包安装ClickHouse，请在服务器上执行以下命令:
@ -15,6 +13,6 @@ $ sudo apt-get install clickhouse-client clickhouse-server
$ sudo service clickhouse-server restart
```
如果您使用除推荐的deb包之外的其他内容安装ClickHouse，请使用适当的更新方法。
如果您使用除推荐的deb包之外的其他方式安装ClickHouse，请使用适当的更新方法。
ClickHouse不支持分布式更新。 该操作应在每个单独的服务器上连续执行。 不要同时更新群集上的所有服务器，否则群集将在一段时间内不可用。
ClickHouse不支持分布式更新。该操作应在每个单独的服务器上连续执行。不要同时更新群集上的所有服务器，否则群集将在一段时间内不可用。

2
programs/CMakeLists.txt
View File

@ -207,7 +207,7 @@ if (TARGET clickhouse-server AND TARGET copy-headers)
endif ()
if (ENABLE_TESTS AND USE_GTEST)
set (CLICKHOUSE_ALL_TESTS_TARGETS local_date_time_comparison unit_tests_libcommon unit_tests_dbms hashing_write_buffer hashing_read_buffer in_join_subqueries_preprocessor expression_analyzer)
set (CLICKHOUSE_ALL_TESTS_TARGETS local_date_time_comparison unit_tests_libcommon unit_tests_dbms hashing_write_buffer hashing_read_buffer in_join_subqueries_preprocessor)
add_custom_target (clickhouse-tests ALL DEPENDS ${CLICKHOUSE_ALL_TESTS_TARGETS})
add_dependencies(clickhouse-bundle clickhouse-tests)
endif()

8
src/Access/MultipleAccessStorage.cpp
View File

@ -143,6 +143,14 @@ const IAccessStorage & MultipleAccessStorage::getStorage(const UUID & id) const
return const_cast<MultipleAccessStorage *>(this)->getStorage(id);
}
void MultipleAccessStorage::addStorage(std::unique_ptr<Storage> nested_storage)
{
/// Note that IStorage::storage_name is not changed. It is ok as this method
/// is considered as a temporary solution allowing third-party Arcadia applications
/// using CH as a library to register their own access storages. Do not remove
/// this method without providing any alternative :)
nested_storages.emplace_back(std::move(nested_storage));
}
AccessEntityPtr MultipleAccessStorage::readImpl(const UUID & id) const
{

2
src/Access/MultipleAccessStorage.h
View File

@ -25,6 +25,8 @@ public:
const Storage & getStorage(const UUID & id) const;
Storage & getStorage(const UUID & id);
void addStorage(std::unique_ptr<Storage> nested_storage);
Storage & getStorageByIndex(size_t i) { return *(nested_storages[i]); }
const Storage & getStorageByIndex(size_t i) const { return *(nested_storages[i]); }

14
src/Access/UsersConfigAccessStorage.cpp
View File

@ -52,18 +52,20 @@ namespace
String user_config = "users." + user_name;
bool has_password = config.has(user_config + ".password");
bool has_no_password = config.has(user_config + ".no_password");
bool has_password_plaintext = config.has(user_config + ".password");
bool has_password_sha256_hex = config.has(user_config + ".password_sha256_hex");
bool has_password_double_sha1_hex = config.has(user_config + ".password_double_sha1_hex");
if (has_password + has_password_sha256_hex + has_password_double_sha1_hex > 1)
throw Exception("More than one field of 'password', 'password_sha256_hex', 'password_double_sha1_hex' is used to specify password for user " + user_name + ". Must be only one of them.",
size_t num_password_fields = has_no_password + has_password_plaintext + has_password_sha256_hex + has_password_double_sha1_hex;
if (num_password_fields > 1)
throw Exception("More than one field of 'password', 'password_sha256_hex', 'password_double_sha1_hex', 'no_password' are used to specify password for user " + user_name + ". Must be only one of them.",
ErrorCodes::BAD_ARGUMENTS);
if (!has_password && !has_password_sha256_hex && !has_password_double_sha1_hex)
throw Exception("Either 'password' or 'password_sha256_hex' or 'password_double_sha1_hex' must be specified for user " + user_name + ".", ErrorCodes::BAD_ARGUMENTS);
if (num_password_fields < 1)
throw Exception("Either 'password' or 'password_sha256_hex' or 'password_double_sha1_hex' or 'no_password' must be specified for user " + user_name + ".", ErrorCodes::BAD_ARGUMENTS);
if (has_password)
if (has_password_plaintext)
{
user->authentication = Authentication{Authentication::PLAINTEXT_PASSWORD};
user->authentication.setPassword(config.getString(user_config + ".password"));

12
src/AggregateFunctions/AggregateFunctionEntropy.h
View File

@ -23,18 +23,10 @@ struct EntropyData
{
using Weight = UInt64;
using HashingMap = HashMap<
Value, Weight,
HashCRC32<Value>,
HashTableGrower<4>,
HashTableAllocatorWithStackMemory<sizeof(std::pair<Value, Weight>) * (1 << 3)>>;
using HashingMap = HashMapWithStackMemory<Value, Weight, HashCRC32<Value>, 4>;
/// For the case of pre-hashed values.
using TrivialMap = HashMap<
Value, Weight,
UInt128TrivialHash,
HashTableGrower<4>,
HashTableAllocatorWithStackMemory<sizeof(std::pair<Value, Weight>) * (1 << 3)>>;
using TrivialMap = HashMapWithStackMemory<Value, Weight, UInt128TrivialHash, 4>;
using Map = std::conditional_t<std::is_same_v<UInt128, Value>, TrivialMap, HashingMap>;

14
src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
View File

@ -28,12 +28,7 @@ template <typename T>
struct AggregateFunctionGroupUniqArrayData
{
/// When creating, the hash table must be small.
using Set = HashSet<
T,
DefaultHash<T>,
HashTableGrower<4>,
HashTableAllocatorWithStackMemory<sizeof(T) * (1 << 4)>
>;
using Set = HashSetWithStackMemory<T, DefaultHash<T>, 4>;
Set value;
};
@ -126,9 +121,10 @@ public:
/// Generic implementation, it uses serialized representation as object descriptor.
struct AggregateFunctionGroupUniqArrayGenericData
{
static constexpr size_t INIT_ELEMS = 2; /// adjustable
static constexpr size_t ELEM_SIZE = sizeof(HashSetCellWithSavedHash<StringRef, StringRefHash>);
using Set = HashSetWithSavedHash<StringRef, StringRefHash, HashTableGrower<INIT_ELEMS>, HashTableAllocatorWithStackMemory<INIT_ELEMS * ELEM_SIZE>>;
static constexpr size_t INITIAL_SIZE_DEGREE = 3; /// adjustable
using Set = HashSetWithSavedHashWithStackMemory<StringRef, StringRefHash,
INITIAL_SIZE_DEGREE>;
Set value;
};

17
src/AggregateFunctions/AggregateFunctionTopK.h
View File

@ -23,13 +23,8 @@ namespace DB
template <typename T>
struct AggregateFunctionTopKData
{
using Set = SpaceSaving
<
T,
HashCRC32<T>,
HashTableGrower<4>,
HashTableAllocatorWithStackMemory<sizeof(T) * (1 << 4)>
>;
using Set = SpaceSaving<T, HashCRC32<T>>;
Set value;
};
@ -109,13 +104,7 @@ public:
/// Generic implementation, it uses serialized representation as object descriptor.
struct AggregateFunctionTopKGenericData
{
using Set = SpaceSaving
<
StringRef,
StringRefHash,
HashTableGrower<4>,
HashTableAllocatorWithStackMemory<sizeof(StringRef) * (1 << 4)>
>;
using Set = SpaceSaving<StringRef, StringRefHash>;
Set value;
};

7
src/AggregateFunctions/QuantileExactWeighted.h
View File

@ -33,12 +33,7 @@ struct QuantileExactWeighted
using Hasher = std::conditional_t<std::is_same_v<Value, Decimal128>, Int128Hash, HashCRC32<UnderlyingType>>;
/// When creating, the hash table must be small.
using Map = HashMap<
UnderlyingType, Weight,
Hasher,
HashTableGrower<4>,
HashTableAllocatorWithStackMemory<sizeof(std::pair<Value, Weight>) * (1 << 3)>
>;
using Map = HashMapWithStackMemory<UnderlyingType, Weight, Hasher, 4>;
Map map;

2
src/Columns/ColumnAggregateFunction.cpp
View File

@ -549,6 +549,8 @@ void ColumnAggregateFunction::getPermutation(bool /*reverse*/, size_t /*limit*/,
res[i] = i;
}
void ColumnAggregateFunction::updatePermutation(bool, size_t, int, Permutation &, EqualRanges&) const {}
void ColumnAggregateFunction::gather(ColumnGathererStream & gatherer)
{
gatherer.gather(*this);

1
src/Columns/ColumnAggregateFunction.h
View File

@ -193,6 +193,7 @@ public:
}
void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override;
void updatePermutation(bool reverse, size_t limit, int, Permutation & res, EqualRanges & equal_range) const override;
/** More efficient manipulation methods */
Container & getData()

70
src/Columns/ColumnArray.cpp
View File

@ -737,6 +737,76 @@ void ColumnArray::getPermutation(bool reverse, size_t limit, int nan_direction_h
}
}
void ColumnArray::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges & equal_range) const
{
if (limit >= size() || limit >= equal_range.back().second)
limit = 0;
size_t n = equal_range.size();
if (limit)
--n;
EqualRanges new_ranges;
for (size_t i = 0; i < n; ++i)
{
const auto& [first, last] = equal_range[i];
if (reverse)
std::sort(res.begin() + first, res.begin() + last, Less<false>(*this, nan_direction_hint));
else
std::sort(res.begin() + first, res.begin() + last, Less<true>(*this, nan_direction_hint));
auto new_first = first;
for (auto j = first + 1; j < last; ++j)
{
if (compareAt(res[new_first], res[j], *this, nan_direction_hint) != 0)
{
if (j - new_first > 1)
new_ranges.emplace_back(new_first, j);
new_first = j;
}
}
if (last - new_first > 1)
new_ranges.emplace_back(new_first, last);
}
if (limit)
{
const auto& [first, last] = equal_range.back();
if (reverse)
std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, Less<false>(*this, nan_direction_hint));
else
std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, Less<true>(*this, nan_direction_hint));
auto new_first = first;
for (auto j = first + 1; j < limit; ++j)
{
if (compareAt(res[new_first], res[j], *this, nan_direction_hint) != 0)
{
if (j - new_first > 1)
new_ranges.emplace_back(new_first, j);
new_first = j;
}
}
auto new_last = limit;
for (auto j = limit; j < last; ++j)
{
if (compareAt(res[new_first], res[j], *this, nan_direction_hint) == 0)
{
std::swap(res[new_last], res[j]);
++new_last;
}
}
if (new_last - new_first > 1)
{
new_ranges.emplace_back(new_first, new_last);
}
}
equal_range = std::move(new_ranges);
}
ColumnPtr ColumnArray::replicate(const Offsets & replicate_offsets) const
{

1
src/Columns/ColumnArray.h
View File

@ -73,6 +73,7 @@ public:
template <typename Type> ColumnPtr indexImpl(const PaddedPODArray<Type> & indexes, size_t limit) const;
int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override;
void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override;
void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges & equal_range) const override;
void reserve(size_t n) override;
size_t byteSize() const override;
size_t allocatedBytes() const override;

2
src/Columns/ColumnConst.cpp
View File

@ -120,6 +120,8 @@ void ColumnConst::getPermutation(bool /*reverse*/, size_t /*limit*/, int /*nan_d
res[i] = i;
}
void ColumnConst::updatePermutation(bool, size_t, int, Permutation &, EqualRanges &) const {}
void ColumnConst::updateWeakHash32(WeakHash32 & hash) const
{
if (hash.getData().size() != s)

1
src/Columns/ColumnConst.h
View File

@ -170,6 +170,7 @@ public:
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
ColumnPtr index(const IColumn & indexes, size_t limit) const override;
void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override;
void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges & equal_range) const override;
size_t byteSize() const override
{

70
src/Columns/ColumnDecimal.cpp
View File

@ -108,6 +108,76 @@ void ColumnDecimal<T>::getPermutation(bool reverse, size_t limit, int , IColumn:
permutation(reverse, limit, res);
}
template <typename T>
void ColumnDecimal<T>::updatePermutation(bool reverse, size_t limit, int, IColumn::Permutation & res, EqualRanges & equal_range) const
{
if (limit >= data.size() || limit >= equal_range.back().second)
limit = 0;
size_t n = equal_range.size();
if (limit)
--n;
EqualRanges new_ranges;
for (size_t i = 0; i < n; ++i)
{
const auto& [first, last] = equal_range[i];
if (reverse)
std::partial_sort(res.begin() + first, res.begin() + last, res.begin() + last,
[this](size_t a, size_t b) { return data[a] > data[b]; });
else
std::partial_sort(res.begin() + first, res.begin() + last, res.begin() + last,
[this](size_t a, size_t b) { return data[a] < data[b]; });
auto new_first = first;
for (auto j = first + 1; j < last; ++j)
{
if (data[res[new_first]] != data[res[j]])
{
if (j - new_first > 1)
new_ranges.emplace_back(new_first, j);
new_first = j;
}
}
if (last - new_first > 1)
new_ranges.emplace_back(new_first, last);
}
if (limit)
{
const auto& [first, last] = equal_range.back();
if (reverse)
std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last,
[this](size_t a, size_t b) { return data[a] > data[b]; });
else
std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last,
[this](size_t a, size_t b) { return data[a] < data[b]; });
auto new_first = first;
for (auto j = first + 1; j < limit; ++j)
{
if (data[res[new_first]] != data[res[j]])
{
if (j - new_first > 1)
new_ranges.emplace_back(new_first, j);
new_first = j;
}
}
auto new_last = limit;
for (auto j = limit; j < last; ++j)
{
if (data[res[new_first]] == data[res[j]])
{
std::swap(res[new_last], res[j]);
++new_last;
}
}
if (new_last - new_first > 1)
new_ranges.emplace_back(new_first, new_last);
}
equal_range = std::move(new_ranges);
}
template <typename T>
ColumnPtr ColumnDecimal<T>::permute(const IColumn::Permutation & perm, size_t limit) const
{

3
src/Columns/ColumnDecimal.h
View File

@ -108,6 +108,7 @@ public:
void updateWeakHash32(WeakHash32 & hash) const override;
int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override;
void getPermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res) const override;
void updatePermutation(bool reverse, size_t limit, int, IColumn::Permutation & res, EqualRanges& equal_range) const override;
MutableColumnPtr cloneResized(size_t size) const override;
@ -152,6 +153,8 @@ public:
const T & getElement(size_t n) const { return data[n]; }
T & getElement(size_t n) { return data[n]; }
UInt32 getScale() const {return scale;}
protected:
Container data;
UInt32 scale;

65
src/Columns/ColumnFixedString.cpp
View File

@ -162,6 +162,71 @@ void ColumnFixedString::getPermutation(bool reverse, size_t limit, int /*nan_dir
}
}
void ColumnFixedString::updatePermutation(bool reverse, size_t limit, int, Permutation & res, EqualRanges & equal_range) const
{
if (limit >= size() || limit >= equal_range.back().second)
limit = 0;
size_t k = equal_range.size();
if (limit)
--k;
EqualRanges new_ranges;
for (size_t i = 0; i < k; ++i)
{
const auto& [first, last] = equal_range[i];
if (reverse)
std::sort(res.begin() + first, res.begin() + last, less<false>(*this));
else
std::sort(res.begin() + first, res.begin() + last, less<true>(*this));
auto new_first = first;
for (auto j = first + 1; j < last; ++j)
{
if (memcmpSmallAllowOverflow15(chars.data() + j * n, chars.data() + new_first * n, n) != 0)
{
if (j - new_first > 1)
new_ranges.emplace_back(new_first, j);
new_first = j;
}
}
if (last - new_first > 1)
new_ranges.emplace_back(new_first, last);
}
if (limit)
{
const auto& [first, last] = equal_range.back();
if (reverse)
std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less<false>(*this));
else
std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less<true>(*this));
auto new_first = first;
for (auto j = first + 1; j < limit; ++j)
{
if (memcmpSmallAllowOverflow15(chars.data() + j * n, chars.data() + new_first * n, n) != 0)
{
if (j - new_first > 1)
new_ranges.emplace_back(new_first, j);
new_first = j;
}
}
auto new_last = limit;
for (auto j = limit; j < last; ++j)
{
if (memcmpSmallAllowOverflow15(chars.data() + j * n, chars.data() + new_first * n, n) == 0)
{
std::swap(res[new_last], res[j]);
++new_last;
}
}
if (new_last - new_first > 1)
new_ranges.emplace_back(new_first, new_last);
}
equal_range = std::move(new_ranges);
}
void ColumnFixedString::insertRangeFrom(const IColumn & src, size_t start, size_t length)
{
const ColumnFixedString & src_concrete = assert_cast<const ColumnFixedString &>(src);

2
src/Columns/ColumnFixedString.h
View File

@ -118,6 +118,8 @@ public:
void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override;
void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges & equal_range) const override;
void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
ColumnPtr filter(const IColumn::Filter & filt, ssize_t result_size_hint) const override;

5
src/Columns/ColumnFunction.h
View File

@ -121,6 +121,11 @@ public:
throw Exception("getPermutation is not implemented for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
void updatePermutation(bool, size_t, int, Permutation &, EqualRanges &) const override
{
throw Exception("updatePermutation is not implemented for " + getName(), ErrorCodes::NOT_IMPLEMENTED);
}
void gather(ColumnGathererStream &) override
{
throw Exception("Method gather is not supported for " + getName(), ErrorCodes::NOT_IMPLEMENTED);

70
src/Columns/ColumnLowCardinality.cpp
View File

@ -314,6 +314,76 @@ void ColumnLowCardinality::getPermutation(bool reverse, size_t limit, int nan_di
}
}
void ColumnLowCardinality::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const
{
if (limit >= size() || limit >= equal_range.back().second)
limit = 0;
size_t n = equal_range.size();
if (limit)
--n;
EqualRanges new_ranges;
for (size_t i = 0; i < n; ++i)
{
const auto& [first, last] = equal_range[i];
if (reverse)
std::sort(res.begin() + first, res.begin() + last, [this, nan_direction_hint](size_t a, size_t b)
{return getDictionary().compareAt(getIndexes().getUInt(a), getIndexes().getUInt(b), getDictionary(), nan_direction_hint) > 0; });
else
std::sort(res.begin() + first, res.begin() + last, [this, nan_direction_hint](size_t a, size_t b)
{return getDictionary().compareAt(getIndexes().getUInt(a), getIndexes().getUInt(b), getDictionary(), nan_direction_hint) < 0; });
auto new_first = first;
for (auto j = first + 1; j < last; ++j)
{
if (compareAt(new_first, j, *this, nan_direction_hint) != 0)
{
if (j - new_first > 1)
new_ranges.emplace_back(new_first, j);
new_first = j;
}
}
if (last - new_first > 1)
new_ranges.emplace_back(new_first, last);
}
if (limit)
{
const auto& [first, last] = equal_range.back();
if (reverse)
std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this, nan_direction_hint](size_t a, size_t b)
{return getDictionary().compareAt(getIndexes().getUInt(a), getIndexes().getUInt(b), getDictionary(), nan_direction_hint) > 0; });
else
std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, [this, nan_direction_hint](size_t a, size_t b)
{return getDictionary().compareAt(getIndexes().getUInt(a), getIndexes().getUInt(b), getDictionary(), nan_direction_hint) < 0; });
auto new_first = first;
for (auto j = first + 1; j < limit; ++j)
{
if (getDictionary().compareAt(getIndexes().getUInt(new_first), getIndexes().getUInt(j), getDictionary(), nan_direction_hint) != 0)
{
if (j - new_first > 1)
new_ranges.emplace_back(new_first, j);
new_first = j;
}
}
auto new_last = limit;
for (auto j = limit; j < last; ++j)
{
if (getDictionary().compareAt(getIndexes().getUInt(new_first), getIndexes().getUInt(j), getDictionary(), nan_direction_hint) == 0)
{
std::swap(res[new_last], res[j]);
++new_last;
}
}
if (new_last - new_first > 1)
new_ranges.emplace_back(new_first, new_last);
}
equal_range = std::move(new_ranges);
}
std::vector<MutableColumnPtr> ColumnLowCardinality::scatter(ColumnIndex num_columns, const Selector & selector) const
{
auto columns = getIndexes().scatter(num_columns, selector);

2
src/Columns/ColumnLowCardinality.h
View File

@ -111,6 +111,8 @@ public:
void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override;
void updatePermutation(bool reverse, size_t limit, int, IColumn::Permutation & res, EqualRanges & equal_range) const override;
ColumnPtr replicate(const Offsets & offsets) const override
{
return ColumnLowCardinality::create(dictionary.getColumnUniquePtr(), getIndexes().replicate(offsets));

69
src/Columns/ColumnNullable.cpp
View File

@ -321,6 +321,75 @@ void ColumnNullable::getPermutation(bool reverse, size_t limit, int null_directi
}
}
void ColumnNullable::updatePermutation(bool reverse, size_t limit, int null_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const
{
if (limit >= equal_range.back().second || limit >= size())
limit = 0;
EqualRanges new_ranges, temp_ranges;
for (const auto &[first, last] : equal_range)
{
bool direction = ((null_direction_hint > 0) != reverse);
/// Shift all NULL values to the end.
size_t read_idx = first;
size_t write_idx = first;
while (read_idx < last && (isNullAt(res[read_idx])^direction))
{
++read_idx;
++write_idx;
}
++read_idx;
/// Invariants:
/// write_idx < read_idx
/// write_idx points to NULL
/// read_idx will be incremented to position of next not-NULL
/// there are range of NULLs between write_idx and read_idx - 1,
/// We are moving elements from end to begin of this range,
/// so range will "bubble" towards the end.
/// Relative order of NULL elements could be changed,
/// but relative order of non-NULLs is preserved.
while (read_idx < last && write_idx < last)
{
if (isNullAt(res[read_idx])^direction)
{
std::swap(res[read_idx], res[write_idx]);
++write_idx;
}
++read_idx;
}
if (write_idx - first > 1)
{
if (direction)
temp_ranges.emplace_back(first, write_idx);
else
new_ranges.emplace_back(first, write_idx);
}
if (last - write_idx > 1)
{
if (direction)
new_ranges.emplace_back(write_idx, last);
else
temp_ranges.emplace_back(write_idx, last);
}
}
while (!new_ranges.empty() && limit && limit <= new_ranges.back().first)
new_ranges.pop_back();
if (!temp_ranges.empty())
getNestedColumn().updatePermutation(reverse, limit, null_direction_hint, res, temp_ranges);
equal_range.resize(temp_ranges.size() + new_ranges.size());
std::merge(temp_ranges.begin(), temp_ranges.end(), new_ranges.begin(), new_ranges.end(), equal_range.begin());
}
void ColumnNullable::gather(ColumnGathererStream & gatherer)
{
gatherer.gather(*this);

1
src/Columns/ColumnNullable.h
View File

@ -78,6 +78,7 @@ public:
ColumnPtr index(const IColumn & indexes, size_t limit) const override;
int compareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const override;
void getPermutation(bool reverse, size_t limit, int null_direction_hint, Permutation & res) const override;
void updatePermutation(bool reverse, size_t limit, int, Permutation & res, EqualRanges & equal_range) const override;
void reserve(size_t n) override;
size_t byteSize() const override;
size_t allocatedBytes() const override;

142
src/Columns/ColumnString.cpp
View File

@ -302,6 +302,77 @@ void ColumnString::getPermutation(bool reverse, size_t limit, int /*nan_directio
}
}
void ColumnString::updatePermutation(bool reverse, size_t limit, int /*nan_direction_hint*/, Permutation & res, EqualRanges & equal_range) const
{
if (limit >= size() || limit > equal_range.back().second)
limit = 0;
EqualRanges new_ranges;
auto less_true = less<true>(*this);
auto less_false = less<false>(*this);
size_t n = equal_range.size();
if (limit)
--n;
for (size_t i = 0; i < n; ++i)
{
const auto &[first, last] = equal_range[i];
if (reverse)
std::sort(res.begin() + first, res.begin() + last, less_false);
else
std::sort(res.begin() + first, res.begin() + last, less_true);
size_t new_first = first;
for (size_t j = first + 1; j < last; ++j)
{
if (memcmpSmallAllowOverflow15(
chars.data() + offsetAt(res[j]), sizeAt(res[j]) - 1,
chars.data() + offsetAt(res[new_first]), sizeAt(res[new_first]) - 1) != 0)
{
if (j - new_first > 1)
new_ranges.emplace_back(new_first, j);
new_first = j;
}
}
if (last - new_first > 1)
new_ranges.emplace_back(new_first, last);
}
if (limit)
{
const auto &[first, last] = equal_range.back();
if (reverse)
std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less_false);
else
std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less_true);
size_t new_first = first;
for (size_t j = first + 1; j < limit; ++j)
{
if (memcmpSmallAllowOverflow15(
chars.data() + offsetAt(res[j]), sizeAt(res[j]) - 1,
chars.data() + offsetAt(res[new_first]), sizeAt(res[new_first]) - 1) != 0)
{
if (j - new_first > 1)
new_ranges.emplace_back(new_first, j);
new_first = j;
}
}
size_t new_last = limit;
for (size_t j = limit; j < last; ++j)
{
if (memcmpSmallAllowOverflow15(
chars.data() + offsetAt(res[j]), sizeAt(res[j]) - 1,
chars.data() + offsetAt(res[new_first]), sizeAt(res[new_first]) - 1) == 0)
{
std::swap(res[j], res[new_last]);
++new_last;
}
}
if (new_last - new_first > 1)
new_ranges.emplace_back(new_first, new_last);
}
equal_range = std::move(new_ranges);
}
ColumnPtr ColumnString::replicate(const Offsets & replicate_offsets) const
{
@ -440,6 +511,77 @@ void ColumnString::getPermutationWithCollation(const Collator & collator, bool r
}
}
void ColumnString::updatePermutationWithCollation(const Collator & collator, bool reverse, size_t limit, int, Permutation &res, EqualRanges &equal_range) const
{
if (limit >= size() || limit >= equal_range.back().second)
limit = 0;
size_t n = equal_range.size();
if (limit)
--n;
EqualRanges new_ranges;
for (size_t i = 0; i < n; ++i)
{
const auto& [first, last] = equal_range[i];
if (reverse)
std::sort(res.begin() + first, res.begin() + last, lessWithCollation<false>(*this, collator));
else
std::sort(res.begin() + first, res.begin() + last, lessWithCollation<true>(*this, collator));
auto new_first = first;
for (auto j = first + 1; j < last; ++j)
{
if (collator.compare(
reinterpret_cast<const char *>(&chars[offsetAt(res[new_first])]), sizeAt(res[new_first]),
reinterpret_cast<const char *>(&chars[offsetAt(res[j])]), sizeAt(res[j])) != 0)
{
if (j - new_first > 1)
new_ranges.emplace_back(new_first, j);
new_first = j;
}
}
if (last - new_first > 1)
new_ranges.emplace_back(new_first, last);
}
if (limit)
{
const auto& [first, last] = equal_range.back();
if (reverse)
std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, lessWithCollation<false>(*this, collator));
else
std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, lessWithCollation<true>(*this, collator));
auto new_first = first;
for (auto j = first + 1; j < limit; ++j)
{
if (collator.compare(
reinterpret_cast<const char *>(&chars[offsetAt(res[new_first])]), sizeAt(res[new_first]),
reinterpret_cast<const char *>(&chars[offsetAt(res[j])]), sizeAt(res[j])) != 0)
{
if (j - new_first > 1)
new_ranges.emplace_back(new_first, j);
new_first = j;
}
}
auto new_last = limit;
for (auto j = limit; j < last; ++j)
{
if (collator.compare(
reinterpret_cast<const char *>(&chars[offsetAt(res[new_first])]), sizeAt(res[new_first]),
reinterpret_cast<const char *>(&chars[offsetAt(res[j])]), sizeAt(res[j])) == 0)
{
std::swap(res[new_last], res[j]);
++new_last;
}
}
if (new_last - new_first > 1)
new_ranges.emplace_back(new_first, new_last);
}
equal_range = std::move(new_ranges);
}
void ColumnString::protect()
{

4
src/Columns/ColumnString.h
View File

@ -225,9 +225,13 @@ public:
void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override;
void updatePermutation(bool reverse, size_t limit, int, Permutation & res, EqualRanges & equal_range) const override;
/// Sorting with respect of collation.
void getPermutationWithCollation(const Collator & collator, bool reverse, size_t limit, Permutation & res) const;
void updatePermutationWithCollation(const Collator & collator, bool reverse, size_t limit, int, Permutation & res, EqualRanges& equal_range) const;
ColumnPtr replicate(const Offsets & replicate_offsets) const override;
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override

13
src/Columns/ColumnTuple.cpp
View File

@ -329,6 +329,19 @@ void ColumnTuple::getPermutation(bool reverse, size_t limit, int nan_direction_h
}
}
void ColumnTuple::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const
{
for (const auto& column : columns)
{
column->updatePermutation(reverse, limit, nan_direction_hint, res, equal_range);
while (limit && limit <= equal_range.back().first)
equal_range.pop_back();
if (equal_range.empty())
break;
}
}
void ColumnTuple::gather(ColumnGathererStream & gatherer)
{
gatherer.gather(*this);

1
src/Columns/ColumnTuple.h
View File

@ -72,6 +72,7 @@ public:
int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
void getExtremes(Field & min, Field & max) const override;
void getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const override;
void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const override;
void reserve(size_t n) override;
size_t byteSize() const override;
size_t allocatedBytes() const override;

34
src/Columns/ColumnUnique.h
View File

@ -77,6 +77,7 @@ public:
}
int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const override;
void getExtremes(Field & min, Field & max) const override { column_holder->getExtremes(min, max); }
bool valuesHaveFixedSize() const override { return column_holder->valuesHaveFixedSize(); }
@ -374,6 +375,39 @@ int ColumnUnique<ColumnType>::compareAt(size_t n, size_t m, const IColumn & rhs,
return getNestedColumn()->compareAt(n, m, *column_unique.getNestedColumn(), nan_direction_hint);
}
template <typename ColumnType>
void ColumnUnique<ColumnType>::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const
{
bool found_null_value_index = false;
for (size_t i = 0; i < equal_range.size() && !found_null_value_index; ++i)
{
auto& [first, last] = equal_range[i];
for (auto j = first; j < last; ++j)
{
if (res[j] == getNullValueIndex())
{
if ((nan_direction_hint > 0) != reverse)
{
std::swap(res[j], res[last - 1]);
--last;
}
else
{
std::swap(res[j], res[first]);
++first;
}
if (last - first <= 1)
{
equal_range.erase(equal_range.begin() + i);
}
found_null_value_index = true;
break;
}
}
}
getNestedColumn()->updatePermutation(reverse, limit, nan_direction_hint, res, equal_range);
}
template <typename IndexType>
static void checkIndexes(const ColumnVector<IndexType> & indexes, size_t max_dictionary_size)
{

70
src/Columns/ColumnVector.cpp
View File

@ -219,6 +219,76 @@ void ColumnVector<T>::getPermutation(bool reverse, size_t limit, int nan_directi
}
}
template <typename T>
void ColumnVector<T>::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const
{
if (limit >= data.size() || limit >= equal_range.back().second)
limit = 0;
EqualRanges new_ranges;
for (size_t i = 0; i < equal_range.size() - bool(limit); ++i)
{
const auto & [first, last] = equal_range[i];
if (reverse)
pdqsort(res.begin() + first, res.begin() + last, greater(*this, nan_direction_hint));
else
pdqsort(res.begin() + first, res.begin() + last, less(*this, nan_direction_hint));
size_t new_first = first;
for (size_t j = first + 1; j < last; ++j)
{
if (less(*this, nan_direction_hint)(res[j], res[new_first]) || greater(*this, nan_direction_hint)(res[j], res[new_first]))
{
if (j - new_first > 1)
{
new_ranges.emplace_back(new_first, j);
}
new_first = j;
}
}
if (last - new_first > 1)
{
new_ranges.emplace_back(new_first, last);
}
}
if (limit)
{
const auto & [first, last] = equal_range.back();
if (reverse)
std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, greater(*this, nan_direction_hint));
else
std::partial_sort(res.begin() + first, res.begin() + limit, res.begin() + last, less(*this, nan_direction_hint));
size_t new_first = first;
for (size_t j = first + 1; j < limit; ++j)
{
if (less(*this, nan_direction_hint)(res[j], res[new_first]) || greater(*this, nan_direction_hint)(res[j], res[new_first]))
{
if (j - new_first > 1)
{
new_ranges.emplace_back(new_first, j);
}
new_first = j;
}
}
size_t new_last = limit;
for (size_t j = limit; j < last; ++j)
{
if (!less(*this, nan_direction_hint)(res[j], res[new_first]) && !greater(*this, nan_direction_hint)(res[j], res[new_first]))
{
std::swap(res[j], res[new_last]);
++new_last;
}
}
if (new_last - new_first > 1)
{
new_ranges.emplace_back(new_first, new_last);
}
}
equal_range = std::move(new_ranges);
}
template <typename T>
const char * ColumnVector<T>::getFamilyName() const

2
src/Columns/ColumnVector.h
View File

@ -192,6 +192,8 @@ public:
void getSpecialPermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res,
IColumn::SpecialSort) const override;
void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges& equal_range) const override;
void reserve(size_t n) override
{
data.reserve(n);

17
src/Columns/IColumn.h
View File

@ -25,6 +25,13 @@ class ColumnGathererStream;
class Field;
class WeakHash32;
/*
* Represents a set of equal ranges in previous column to perform sorting in current column.
* Used in sorting by tuples.
* */
using EqualRanges = std::vector<std::pair<size_t, size_t> >;
/// Declares interface to store columns in memory.
class IColumn : public COW<IColumn>
{
@ -256,6 +263,16 @@ public:
getPermutation(reverse, limit, nan_direction_hint, res);
}
/*in updatePermutation we pass the current permutation and the intervals at which it should be sorted
* Then for each interval separately (except for the last one, if there is a limit)
* We sort it based on data about the current column, and find all the intervals within this
* interval that had the same values in this column. we can't tell about these values in what order they
* should have been, we form a new array with intervals that need to be sorted
* If there is a limit, then for the last interval we do partial sorting and all that is described above,
* but in addition we still find all the elements equal to the largest sorted, they will also need to be sorted.
*/
virtual void updatePermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res, EqualRanges & equal_ranges) const = 0;
/** Copies each element according offsets parameter.
* (i-th element should be copied offsets[i] - offsets[i - 1] times.)
* It is necessary in ARRAY JOIN operation.

2
src/Columns/IColumnDummy.h
View File

@ -107,6 +107,8 @@ public:
res[i] = i;
}
void updatePermutation(bool, size_t, int, Permutation &, EqualRanges&) const override {}
ColumnPtr replicate(const Offsets & offsets) const override
{
if (s != offsets.size())

28
src/Common/Allocator.h
View File

@ -278,13 +278,15 @@ private:
/** Allocator with optimization to place small memory ranges in automatic memory.
*/
template <typename Base, size_t N, size_t Alignment>
template <typename Base, size_t _initial_bytes, size_t Alignment>
class AllocatorWithStackMemory : private Base
{
private:
alignas(Alignment) char stack_memory[N];
alignas(Alignment) char stack_memory[_initial_bytes];
public:
static constexpr size_t initial_bytes = _initial_bytes;
/// Do not use boost::noncopyable to avoid the warning about direct base
/// being inaccessible due to ambiguity, when derived classes are also
/// noncopiable (-Winaccessible-base).
@ -295,10 +297,10 @@ public:
void * alloc(size_t size)
{
if (size <= N)
if (size <= initial_bytes)
{
if constexpr (Base::clear_memory)
memset(stack_memory, 0, N);
memset(stack_memory, 0, initial_bytes);
return stack_memory;
}
@ -307,18 +309,18 @@ public:
void free(void * buf, size_t size)
{
if (size > N)
if (size > initial_bytes)
Base::free(buf, size);
}
void * realloc(void * buf, size_t old_size, size_t new_size)
{
/// Was in stack_memory, will remain there.
if (new_size <= N)
if (new_size <= initial_bytes)
return buf;
/// Already was big enough to not fit in stack_memory.
if (old_size > N)
if (old_size > initial_bytes)
return Base::realloc(buf, old_size, new_size, Alignment);
/// Was in stack memory, but now will not fit there.
@ -330,10 +332,20 @@ public:
protected:
static constexpr size_t getStackThreshold()
{
return N;
return initial_bytes;
}
};
// A constant that gives the number of initially available bytes in
// the allocator. Used to check that this number is in sync with the
// initial size of array or hash table that uses the allocator.
template<typename TAllocator>
constexpr size_t allocatorInitialBytes = 0;
template<typename Base, size_t initial_bytes, size_t Alignment>
constexpr size_t allocatorInitialBytes<AllocatorWithStackMemory<
Base, initial_bytes, Alignment>> = initial_bytes;
#if !__clang__
#pragma GCC diagnostic pop

11
src/Common/HashTable/ClearableHashMap.h
View File

@ -43,3 +43,14 @@ public:
this->m_size = 0;
}
};
template <typename Key, typename Mapped, typename Hash,
size_t initial_size_degree>
using ClearableHashMapWithStackMemory = ClearableHashMap<
Key,
Mapped,
Hash,
HashTableGrower<initial_size_degree>,
HashTableAllocatorWithStackMemory<
(1ULL << initial_size_degree)
* sizeof(ClearableHashMapCell<Key, Mapped, Hash>)>>;

12
src/Common/HashTable/ClearableHashSet.h
View File

@ -84,3 +84,15 @@ public:
this->m_size = 0;
}
};
template <typename Key, typename Hash, size_t initial_size_degree>
using ClearableHashSetWithStackMemory = ClearableHashSet<
Key,
Hash,
HashTableGrower<initial_size_degree>,
HashTableAllocatorWithStackMemory<
(1ULL << initial_size_degree)
* sizeof(
ClearableHashTableCell<
Key,
HashTableCell<Key, Hash, ClearableHashSetState>>)>>;

11
src/Common/HashTable/HashMap.h
View File

@ -239,3 +239,14 @@ template <
typename Grower = HashTableGrower<>,
typename Allocator = HashTableAllocator>
using HashMapWithSavedHash = HashMapTable<Key, HashMapCellWithSavedHash<Key, Mapped, Hash>, Hash, Grower, Allocator>;
template <typename Key, typename Mapped, typename Hash,
size_t initial_size_degree>
using HashMapWithStackMemory = HashMapTable<
Key,
HashMapCellWithSavedHash<Key, Mapped, Hash>,
Hash,
HashTableGrower<initial_size_degree>,
HashTableAllocatorWithStackMemory<
(1ULL << initial_size_degree)
* sizeof(HashMapCellWithSavedHash<Key, Mapped, Hash>)>>;

17
src/Common/HashTable/HashSet.h
View File

@ -93,6 +93,14 @@ template
>
using HashSet = HashSetTable<Key, HashTableCell<Key, Hash>, Hash, Grower, Allocator>;
template <typename Key, typename Hash, size_t initial_size_degree>
using HashSetWithStackMemory = HashSet<
Key,
Hash,
HashTableGrower<initial_size_degree>,
HashTableAllocatorWithStackMemory<
(1ULL << initial_size_degree)
* sizeof(HashTableCell<Key, Hash>)>>;
template
<
@ -102,3 +110,12 @@ template
typename Allocator = HashTableAllocator
>
using HashSetWithSavedHash = HashSetTable<Key, HashSetCellWithSavedHash<Key, Hash>, Hash, Grower, Allocator>;
template <typename Key, typename Hash, size_t initial_size_degree>
using HashSetWithSavedHashWithStackMemory = HashSetWithSavedHash<
Key,
Hash,
HashTableGrower<initial_size_degree>,
HashTableAllocatorWithStackMemory<
(1ULL << initial_size_degree)
* sizeof(HashSetCellWithSavedHash<Key, Hash>)>>;

11
src/Common/HashTable/HashTable.h
View File

@ -208,6 +208,7 @@ struct HashTableGrower
/// The state of this structure is enough to get the buffer size of the hash table.
UInt8 size_degree = initial_size_degree;
static constexpr auto initial_count = 1ULL << initial_size_degree;
/// The size of the hash table in the cells.
size_t bufSize() const { return 1ULL << size_degree; }
@ -255,6 +256,7 @@ struct HashTableGrower
template <size_t key_bits>
struct HashTableFixedGrower
{
static constexpr auto initial_count = 1ULL << key_bits;
size_t bufSize() const { return 1ULL << key_bits; }
size_t place(size_t x) const { return x; }
/// You could write __builtin_unreachable(), but the compiler does not optimize everything, and it turns out less efficiently.
@ -309,6 +311,7 @@ struct ZeroValueStorage<false, Cell>
};
// The HashTable
template
<
typename Key,
@ -324,6 +327,14 @@ class HashTable :
protected Cell::State,
protected ZeroValueStorage<Cell::need_zero_value_storage, Cell> /// empty base optimization
{
public:
// If we use an allocator with inline memory, check that the initial
// size of the hash table is in sync with the amount of this memory.
static constexpr size_t initial_buffer_bytes
= Grower::initial_count * sizeof(Cell);
static_assert(allocatorInitialBytes<Allocator> == 0
|| allocatorInitialBytes<Allocator> == initial_buffer_bytes);
protected:
friend class const_iterator;
friend class iterator;

4
src/Common/HashTable/HashTableAllocator.h
View File

@ -10,5 +10,5 @@
*/
using HashTableAllocator = Allocator<true /* clear_memory */, true /* mmap_populate */>;
template <size_t N = 64>
using HashTableAllocatorWithStackMemory = AllocatorWithStackMemory<HashTableAllocator, N>;
template <size_t initial_bytes = 64>
using HashTableAllocatorWithStackMemory = AllocatorWithStackMemory<HashTableAllocator, initial_bytes>;

3
src/Common/PODArray.cpp
View File

@ -2,7 +2,8 @@
namespace DB
{
/// Used for left padding of PODArray when empty
const char EmptyPODArray[EmptyPODArraySize]{};
const char empty_pod_array[empty_pod_array_size]{};
}

27
src/Common/PODArray.h
View File

@ -63,8 +63,8 @@ namespace ErrorCodes
* TODO Pass alignment to Allocator.
* TODO Allow greater alignment than alignof(T). Example: array of char aligned to page size.
*/
static constexpr size_t EmptyPODArraySize = 1024;
extern const char EmptyPODArray[EmptyPODArraySize];
static constexpr size_t empty_pod_array_size = 1024;
extern const char empty_pod_array[empty_pod_array_size];
/** Base class that depend only on size of element, not on element itself.
* You can static_cast to this class if you want to insert some data regardless to the actual type T.
@ -81,9 +81,14 @@ protected:
/// pad_left is also rounded up to 16 bytes to maintain alignment of allocated memory.
static constexpr size_t pad_left = integerRoundUp(integerRoundUp(pad_left_, ELEMENT_SIZE), 16);
/// Empty array will point to this static memory as padding.
static constexpr char * null = pad_left ? const_cast<char *>(EmptyPODArray) + EmptyPODArraySize : nullptr;
static constexpr char * null = pad_left ? const_cast<char *>(empty_pod_array) + empty_pod_array_size : nullptr;
static_assert(pad_left <= EmptyPODArraySize && "Left Padding exceeds EmptyPODArraySize. Is the element size too large?");
static_assert(pad_left <= empty_pod_array_size && "Left Padding exceeds empty_pod_array_size. Is the element size too large?");
// If we are using allocator with inline memory, the minimal size of
// array must be in sync with the size of this memory.
static_assert(allocatorInitialBytes<TAllocator> == 0
|| allocatorInitialBytes<TAllocator> == initial_bytes);
char * c_start = null; /// Does not include pad_left.
char * c_end = null;
@ -224,13 +229,19 @@ public:
}
template <typename ... TAllocatorParams>
void push_back_raw(const char * ptr, TAllocatorParams &&... allocator_params)
void push_back_raw(const void * ptr, TAllocatorParams &&... allocator_params)
{
push_back_raw_many(1, ptr, std::forward<TAllocatorParams>(allocator_params)...);
}
template <typename ... TAllocatorParams>
void push_back_raw_many(size_t number_of_items, const void * ptr, TAllocatorParams &&... allocator_params)
{
if (unlikely(c_end == c_end_of_storage))
reserveForNextSize(std::forward<TAllocatorParams>(allocator_params)...);
reserve(number_of_items, std::forward<TAllocatorParams>(allocator_params)...);
memcpy(c_end, ptr, ELEMENT_SIZE);
c_end += byte_size(1);
memcpy(c_end, ptr, ELEMENT_SIZE * number_of_items);
c_end += byte_size(number_of_items);
}
void protect()

6
src/Common/SpaceSaving.h
View File

@ -67,9 +67,7 @@ private:
template
<
typename TKey,
typename Hash = DefaultHash<TKey>,
typename Grower = HashTableGrower<>,
typename Allocator = HashTableAllocator
typename Hash = DefaultHash<TKey>
>
class SpaceSaving
{
@ -380,7 +378,7 @@ private:
counter_map[counter->key] = counter;
}
using CounterMap = HashMap<TKey, Counter *, Hash, Grower, Allocator>;
using CounterMap = HashMapWithStackMemory<TKey, Counter *, Hash, 4>;
CounterMap counter_map;
std::vector<Counter *> counter_list;

2
src/Common/UInt128.h
View File

@ -72,7 +72,7 @@ template <typename T> bool inline operator<= (T a, const UInt128 b) { return UIn
template <typename T> bool inline operator< (T a, const UInt128 b) { return UInt128(a) < b; }
template <> inline constexpr bool IsNumber<UInt128> = true;
template <> struct TypeName<UInt128> { static const char * get() { return "UInt128"; } };
template <> struct TypeName<UInt128> { static constexpr const char * get() { return "UInt128"; } };
template <> struct TypeId<UInt128> { static constexpr const TypeIndex value = TypeIndex::UInt128; };
struct UInt128Hash

1
src/Common/config.h.in
View File

@ -9,4 +9,5 @@
#cmakedefine01 USE_BROTLI
#cmakedefine01 USE_UNWIND
#cmakedefine01 USE_OPENCL
#cmakedefine01 USE_GRPC
#cmakedefine01 CLICKHOUSE_SPLIT_BINARY

18
src/Common/tests/pod_array.cpp
View File

@ -18,9 +18,9 @@ static void test1()
{
using namespace DB;
static constexpr size_t initial_size = 8;
static constexpr size_t stack_threshold = 32;
using Array = PODArray<UInt64, initial_size, AllocatorWithStackMemory<Allocator<false>, stack_threshold>>;
static constexpr size_t initial_bytes = 32;
using Array = PODArray<UInt64, initial_bytes,
AllocatorWithStackMemory<Allocator<false>, initial_bytes>>;
bool res = true;
@ -139,9 +139,9 @@ static void test2()
{
using namespace DB;
static constexpr size_t initial_size = 8;
static constexpr size_t stack_threshold = 32;
using Array = PODArray<UInt64, initial_size, AllocatorWithStackMemory<Allocator<false>, stack_threshold>>;
static constexpr size_t initial_bytes = 32;
using Array = PODArray<UInt64, initial_bytes,
AllocatorWithStackMemory<Allocator<false>, initial_bytes>>;
bool res = true;
@ -389,9 +389,9 @@ static void test3()
{
using namespace DB;
static constexpr size_t initial_size = 8;
static constexpr size_t stack_threshold = 32;
using Array = PODArray<UInt64, initial_size, AllocatorWithStackMemory<Allocator<false>, stack_threshold>>;
static constexpr size_t initial_bytes = 32;
using Array = PODArray<UInt64, initial_bytes,
AllocatorWithStackMemory<Allocator<false>, initial_bytes>>;
bool res = true;

32
src/Core/Types.h
View File

@ -85,17 +85,17 @@ template <> inline constexpr bool IsNumber<Float64> = true;
template <typename T> struct TypeName;
template <> struct TypeName<UInt8> { static const char * get() { return "UInt8"; } };
template <> struct TypeName<UInt16> { static const char * get() { return "UInt16"; } };
template <> struct TypeName<UInt32> { static const char * get() { return "UInt32"; } };
template <> struct TypeName<UInt64> { static const char * get() { return "UInt64"; } };
template <> struct TypeName<Int8> { static const char * get() { return "Int8"; } };
template <> struct TypeName<Int16> { static const char * get() { return "Int16"; } };
template <> struct TypeName<Int32> { static const char * get() { return "Int32"; } };
template <> struct TypeName<Int64> { static const char * get() { return "Int64"; } };
template <> struct TypeName<Float32> { static const char * get() { return "Float32"; } };
template <> struct TypeName<Float64> { static const char * get() { return "Float64"; } };
template <> struct TypeName<String> { static const char * get() { return "String"; } };
template <> struct TypeName<UInt8> { static constexpr const char * get() { return "UInt8"; } };
template <> struct TypeName<UInt16> { static constexpr const char * get() { return "UInt16"; } };
template <> struct TypeName<UInt32> { static constexpr const char * get() { return "UInt32"; } };
template <> struct TypeName<UInt64> { static constexpr const char * get() { return "UInt64"; } };
template <> struct TypeName<Int8> { static constexpr const char * get() { return "Int8"; } };
template <> struct TypeName<Int16> { static constexpr const char * get() { return "Int16"; } };
template <> struct TypeName<Int32> { static constexpr const char * get() { return "Int32"; } };
template <> struct TypeName<Int64> { static constexpr const char * get() { return "Int64"; } };
template <> struct TypeName<Float32> { static constexpr const char * get() { return "Float32"; } };
template <> struct TypeName<Float64> { static constexpr const char * get() { return "Float64"; } };
template <> struct TypeName<String> { static constexpr const char * get() { return "String"; } };
template <typename T> struct TypeId;
template <> struct TypeId<UInt8> { static constexpr const TypeIndex value = TypeIndex::UInt8; };
@ -115,7 +115,7 @@ using Strings = std::vector<String>;
using Int128 = __int128;
template <> inline constexpr bool IsNumber<Int128> = true;
template <> struct TypeName<Int128> { static const char * get() { return "Int128"; } };
template <> struct TypeName<Int128> { static constexpr const char * get() { return "Int128"; } };
template <> struct TypeId<Int128> { static constexpr const TypeIndex value = TypeIndex::Int128; };
/// Own FieldType for Decimal.
@ -161,9 +161,9 @@ using Decimal128 = Decimal<Int128>;
using DateTime64 = Decimal64;
template <> struct TypeName<Decimal32> { static const char * get() { return "Decimal32"; } };
template <> struct TypeName<Decimal64> { static const char * get() { return "Decimal64"; } };
template <> struct TypeName<Decimal128> { static const char * get() { return "Decimal128"; } };
template <> struct TypeName<Decimal32> { static constexpr const char * get() { return "Decimal32"; } };
template <> struct TypeName<Decimal64> { static constexpr const char * get() { return "Decimal64"; } };
template <> struct TypeName<Decimal128> { static constexpr const char * get() { return "Decimal128"; } };
template <> struct TypeId<Decimal32> { static constexpr const TypeIndex value = TypeIndex::Decimal32; };
template <> struct TypeId<Decimal64> { static constexpr const TypeIndex value = TypeIndex::Decimal64; };
@ -183,7 +183,7 @@ template <> inline Int32 Decimal32::getScaleMultiplier(UInt32 scale) { return co
template <> inline Int64 Decimal64::getScaleMultiplier(UInt32 scale) { return common::exp10_i64(scale); }
template <> inline Int128 Decimal128::getScaleMultiplier(UInt32 scale) { return common::exp10_i128(scale); }
inline const char * getTypeName(TypeIndex idx)
inline constexpr const char * getTypeName(TypeIndex idx)
{
switch (idx)
{

4
src/DataTypes/DataTypeDateTime64.h
View File

@ -17,7 +17,9 @@ class DataTypeDateTime64 final : public DataTypeDecimalBase<DateTime64>, public
{
public:
static constexpr UInt8 default_scale = 3;
static constexpr auto family_name = "DateTime64";
static constexpr auto type_id = TypeIndex::DateTime64;
explicit DataTypeDateTime64(UInt32 scale_, const std::string & time_zone_name = "");
@ -26,7 +28,7 @@ public:
const char * getFamilyName() const override { return family_name; }
std::string doGetName() const override;
TypeIndex getTypeId() const override { return TypeIndex::DateTime64; }
TypeIndex getTypeId() const override { return type_id; }
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings &) const override;

7
src/DataTypes/DataTypeNumberBase.h
View File

@ -19,12 +19,15 @@ class DataTypeNumberBase : public DataTypeWithSimpleSerialization
public:
static constexpr bool is_parametric = false;
using FieldType = T;
static constexpr auto type_id = TypeId<T>::value;
static constexpr auto family_name = TypeName<T>::get();
using ColumnType = ColumnVector<T>;
const char * getFamilyName() const override { return TypeName<T>::get(); }
TypeIndex getTypeId() const override { return TypeId<T>::value; }
const char * getFamilyName() const override { return family_name; }
TypeIndex getTypeId() const override { return type_id; }
void serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const override;
void deserializeText(IColumn & column, ReadBuffer & istr, const FormatSettings & settings) const override;

3
src/DataTypes/DataTypeString.h
View File

@ -13,13 +13,14 @@ class DataTypeString final : public IDataType
public:
using FieldType = String;
static constexpr bool is_parametric = false;
static constexpr auto type_id = TypeIndex::String;
const char * getFamilyName() const override
{
return "String";
}
TypeIndex getTypeId() const override { return TypeIndex::String; }
TypeIndex getTypeId() const override { return type_id; }
void serializeBinary(const Field & field, WriteBuffer & ostr) const override;
void deserializeBinary(Field & field, ReadBuffer & istr) const override;

26
src/DataTypes/IDataType.h
View File

@ -529,10 +529,15 @@ struct WhichDataType
/// IDataType helpers (alternative for IDataType virtual methods with single point of truth)
inline bool isDate(const DataTypePtr & data_type) { return WhichDataType(data_type).isDate(); }
inline bool isDateOrDateTime(const DataTypePtr & data_type) { return WhichDataType(data_type).isDateOrDateTime(); }
inline bool isDateTime(const DataTypePtr & data_type) { return WhichDataType(data_type).isDateTime(); }
inline bool isDateTime64(const DataTypePtr & data_type) { return WhichDataType(data_type).isDateTime64(); }
template <typename T>
inline bool isDate(const T & data_type) { return WhichDataType(data_type).isDate(); }
template <typename T>
inline bool isDateOrDateTime(const T & data_type) { return WhichDataType(data_type).isDateOrDateTime(); }
template <typename T>
inline bool isDateTime(const T & data_type) { return WhichDataType(data_type).isDateTime(); }
template <typename T>
inline bool isDateTime64(const T & data_type) { return WhichDataType(data_type).isDateTime64(); }
inline bool isEnum(const DataTypePtr & data_type) { return WhichDataType(data_type).isEnum(); }
inline bool isDecimal(const DataTypePtr & data_type) { return WhichDataType(data_type).isDecimal(); }
inline bool isTuple(const DataTypePtr & data_type) { return WhichDataType(data_type).isTuple(); }
@ -636,6 +641,19 @@ inline bool isCompilableType(const DataTypePtr & data_type)
return data_type->isValueRepresentedByNumber() && !isDecimal(data_type);
}
template <TypeIndex TYPE_IDX, typename DataType>
inline bool isDataType(const DataType & data_type)
{
WhichDataType which(data_type);
return which.idx == TYPE_IDX;
}
template <typename ExpectedDataType, typename DataType>
inline bool isDataType(const DataType & data_type)
{
return isDataType<ExpectedDataType::type_id>(data_type);
}
template <typename DataType> constexpr bool IsDataTypeDecimal = false;
template <typename DataType> constexpr bool IsDataTypeNumber = false;
template <typename DataType> constexpr bool IsDataTypeDateOrDateTime = false;

148
src/Functions/FunctionUnixTimestamp64.h Normal file
View File

@ -0,0 +1,148 @@
#pragma once
#include <Functions/extractTimeZoneFromFunctionArguments.h>
#include <Functions/IFunctionImpl.h>
#include <Functions/FunctionHelpers.h>
#include <DataTypes/DataTypeDateTime64.h>
#include <DataTypes/DataTypesNumber.h>
#include <common/arithmeticOverflow.h>
namespace DB
{
namespace ErrorCodes
{
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int DECIMAL_OVERFLOW;
}
/** Casts DateTim64 to or from Int64 representation narrowed down (or scaled up) to any scale value defined in Impl.
*/
template <typename Impl>
class FunctionUnixTimestamp64 : public IFunction
{
public:
static constexpr auto name = Impl::name;
static constexpr auto target_scale = Impl::target_scale;
using SourceDataType = typename Impl::SourceDataType;
using ResultDataType = typename Impl::ResultDataType;
static constexpr bool is_result_datetime64 = std::is_same_v<ResultDataType, DataTypeDateTime64>;
static_assert(std::is_same_v<SourceDataType, DataTypeDateTime64> || std::is_same_v<ResultDataType, DataTypeDateTime64>);
static auto create(const Context &)
{
return std::make_shared<FunctionUnixTimestamp64<Impl>>();
}
String getName() const override { return name; }
size_t getNumberOfArguments() const override { return is_result_datetime64 ? 2 : 1; }
bool isVariadic() const override { return is_result_datetime64; }
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
if constexpr (is_result_datetime64)
{
validateFunctionArgumentTypes(*this, arguments,
FunctionArgumentDescriptors{{"value", isDataType<SourceDataType>, nullptr, std::string(SourceDataType::family_name).c_str()}},
// optional
FunctionArgumentDescriptors{
// {"precision", isDataType<DataTypeUInt8>, isColumnConst, ("Precision of the result, default is " + std::to_string(target_scale)).c_str()},
{"timezone", isStringOrFixedString, isColumnConst, "Timezone of the result"},
});
const auto timezone = extractTimeZoneNameFromFunctionArguments(arguments, 1, 0);
return std::make_shared<DataTypeDateTime64>(target_scale, timezone);
}
else
{
validateFunctionArgumentTypes(*this, arguments,
FunctionArgumentDescriptors{{"value", isDataType<SourceDataType>, nullptr, std::string(SourceDataType::family_name).c_str()}});
return std::make_shared<DataTypeInt64>();
}
}
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override
{
using SourceColumnType = typename SourceDataType::ColumnType;
using ResultColumnType = typename ResultDataType::ColumnType;
const auto & src = block.getByPosition(arguments[0]);
auto & res = block.getByPosition(result);
const auto & col = *src.column;
const SourceColumnType * source_col_typed = checkAndGetColumn<SourceColumnType>(col);
if (!source_col_typed && !(source_col_typed = checkAndGetColumnConstData<SourceColumnType>(&col)))
throw Exception("Invalid column type" + col.getName() + " expected "
+ std::string(SourceDataType::family_name),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
res.column = res.type->createColumn();
if (input_rows_count == 0)
return;
auto & result_data = assert_cast<ResultColumnType &>(res.column->assumeMutableRef()).getData();
result_data.reserve(source_col_typed->size());
const auto & source_data = source_col_typed->getData();
const auto scale_diff = getScaleDiff(*checkAndGetDataType<SourceDataType>(src.type.get()), *checkAndGetDataType<ResultDataType>(res.type.get()));
if (scale_diff == 0)
{
static_assert(sizeof(typename SourceColumnType::Container::value_type) == sizeof(typename ResultColumnType::Container::value_type));
// no conversion necessary
result_data.push_back_raw_many(source_data.size(), source_data.data());
}
else if (scale_diff < 0)
{
const Int64 scale_multiplier = DecimalUtils::scaleMultiplier<Int64>(std::abs(scale_diff));
for (const auto & v : source_data)
{
Int64 result_value = toDestValue(v);
if (common::mulOverflow(result_value, scale_multiplier, result_value))
throw Exception("Decimal overflow in " + getName(), ErrorCodes::DECIMAL_OVERFLOW);
result_data.push_back(result_value);
}
}
else
{
const Int64 scale_multiplier = DecimalUtils::scaleMultiplier<Int64>(scale_diff);
for (const auto & v : source_data)
result_data.push_back(static_cast<Int64>(toDestValue(v) / scale_multiplier));
}
}
private:
static Int64 getScaleDiff(const SourceDataType & src, const ResultDataType & dst)
{
Int64 src_scale = target_scale;
if constexpr (std::is_same_v<SourceDataType, DataTypeDateTime64>)
{
src_scale = src.getScale();
}
Int64 dst_scale = target_scale;
if constexpr (std::is_same_v<ResultDataType, DataTypeDateTime64>)
{
dst_scale = dst.getScale();
}
return src_scale - dst_scale;
}
static auto toDestValue(const DateTime64 & v)
{
return Int64{v.value};
}
template <typename T>
static auto toDestValue(const T & v)
{
return Int64{v};
}
};
}

2
src/Functions/FunctionsMiscellaneous.h
View File

@ -115,6 +115,7 @@ public:
String getName() const override { return "FunctionCapture"; }
bool useDefaultImplementationForNulls() const override { return false; }
bool useDefaultImplementationForLowCardinalityColumns() const override { return false; }
void execute(Block & block, const ColumnNumbers & arguments, size_t result, size_t input_rows_count) override
{
@ -243,6 +244,7 @@ public:
String getName() const override { return name; }
bool useDefaultImplementationForNulls() const override { return false; }
bool useDefaultImplementationForLowCardinalityColumns() const override { return false; }
DataTypePtr getReturnType(const ColumnsWithTypeAndName &) const override { return return_type; }
size_t getNumberOfArguments() const override { return capture->captured_types.size(); }

16
src/Functions/array/arrayDistinct.cpp
View File

@ -153,10 +153,8 @@ bool FunctionArrayDistinct::executeNumber(
if (nullable_col)
src_null_map = &nullable_col->getNullMapData();
using Set = ClearableHashSet<T,
DefaultHash<T>,
HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(T)>>;
using Set = ClearableHashSetWithStackMemory<T, DefaultHash<T>,
INITIAL_SIZE_DEGREE>;
Set set;
@ -201,10 +199,8 @@ bool FunctionArrayDistinct::executeString(
ColumnString & res_data_column_string = typeid_cast<ColumnString &>(res_data_col);
using Set = ClearableHashSet<StringRef,
StringRefHash,
HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(StringRef)>>;
using Set = ClearableHashSetWithStackMemory<StringRef, StringRefHash,
INITIAL_SIZE_DEGREE>;
const PaddedPODArray<UInt8> * src_null_map = nullptr;
@ -249,8 +245,8 @@ void FunctionArrayDistinct::executeHashed(
ColumnArray::Offsets & res_offsets,
const ColumnNullable * nullable_col)
{
using Set = ClearableHashSet<UInt128, UInt128TrivialHash, HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(UInt128)>>;
using Set = ClearableHashSetWithStackMemory<UInt128, UInt128TrivialHash,
INITIAL_SIZE_DEGREE>;
const PaddedPODArray<UInt8> * src_null_map = nullptr;

25
src/Functions/array/arrayEnumerateExtended.h
View File

@ -64,36 +64,41 @@ private:
template <typename T>
struct MethodOneNumber
{
using Set = ClearableHashMap<T, UInt32, DefaultHash<T>, HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(T)>>;
using Set = ClearableHashMapWithStackMemory<T, UInt32, DefaultHash<T>,
INITIAL_SIZE_DEGREE>;
using Method = ColumnsHashing::HashMethodOneNumber<typename Set::value_type, UInt32, T, false>;
};
struct MethodString
{
using Set = ClearableHashMap<StringRef, UInt32, StringRefHash, HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(StringRef)>>;
using Set = ClearableHashMapWithStackMemory<StringRef, UInt32, StringRefHash,
INITIAL_SIZE_DEGREE>;
using Method = ColumnsHashing::HashMethodString<typename Set::value_type, UInt32, false, false>;
};
struct MethodFixedString
{
using Set = ClearableHashMap<StringRef, UInt32, StringRefHash, HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(StringRef)>>;
using Set = ClearableHashMapWithStackMemory<StringRef, UInt32, StringRefHash,
INITIAL_SIZE_DEGREE>;
using Method = ColumnsHashing::HashMethodFixedString<typename Set::value_type, UInt32, false, false>;
};
struct MethodFixed
{
using Set = ClearableHashMap<UInt128, UInt32, UInt128HashCRC32, HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(UInt128)>>;
using Set = ClearableHashMapWithStackMemory<UInt128, UInt32, UInt128HashCRC32,
INITIAL_SIZE_DEGREE>;
using Method = ColumnsHashing::HashMethodKeysFixed<typename Set::value_type, UInt128, UInt32, false, false, false>;
};
struct MethodHashed
{
using Set = ClearableHashMap<UInt128, UInt32, UInt128TrivialHash, HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(UInt128)>>;
using Set = ClearableHashMapWithStackMemory<UInt128, UInt32, UInt128TrivialHash,
INITIAL_SIZE_DEGREE>;
using Method = ColumnsHashing::HashMethodHashed<typename Set::value_type, UInt32, false>;
};

9
src/Functions/array/arrayEnumerateRanked.h
View File

@ -308,12 +308,9 @@ void FunctionArrayEnumerateRankedExtended<Derived>::executeMethodImpl(
const size_t depth_to_look = arrays_depths.max_array_depth;
const auto & offsets = *offsets_by_depth[depth_to_look - 1];
using Map = ClearableHashMap<
UInt128,
UInt32,
UInt128TrivialHash,
HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(UInt128)>>;
using Map = ClearableHashMapWithStackMemory<UInt128, UInt32,
UInt128TrivialHash, INITIAL_SIZE_DEGREE>;
Map indices;
std::vector<size_t> indices_by_depth(depth_to_look);

23
src/Functions/array/arrayIntersect.cpp
View File

@ -418,16 +418,15 @@ void FunctionArrayIntersect::executeImpl(Block & block, const ColumnNumbers & ar
TypeListNativeNumbers::forEach(NumberExecutor(arrays, not_nullable_nested_return_type, result_column));
TypeListDecimalNumbers::forEach(DecimalExecutor(arrays, not_nullable_nested_return_type, result_column));
using DateMap = ClearableHashMap<DataTypeDate::FieldType, size_t, DefaultHash<DataTypeDate::FieldType>,
HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(DataTypeDate::FieldType)>>;
using DateMap = ClearableHashMapWithStackMemory<DataTypeDate::FieldType,
size_t, DefaultHash<DataTypeDate::FieldType>, INITIAL_SIZE_DEGREE>;
using DateTimeMap = ClearableHashMap<DataTypeDateTime::FieldType, size_t, DefaultHash<DataTypeDateTime::FieldType>,
HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(DataTypeDateTime::FieldType)>>;
using DateTimeMap = ClearableHashMapWithStackMemory<
DataTypeDateTime::FieldType, size_t,
DefaultHash<DataTypeDateTime::FieldType>, INITIAL_SIZE_DEGREE>;
using StringMap = ClearableHashMap<StringRef, size_t, StringRefHash, HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(StringRef)>>;
using StringMap = ClearableHashMapWithStackMemory<StringRef, size_t,
StringRefHash, INITIAL_SIZE_DEGREE>;
if (!result_column)
{
@ -455,8 +454,8 @@ void FunctionArrayIntersect::executeImpl(Block & block, const ColumnNumbers & ar
template <typename T, size_t>
void FunctionArrayIntersect::NumberExecutor::operator()()
{
using Map = ClearableHashMap<T, size_t, DefaultHash<T>, HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(T)>>;
using Map = ClearableHashMapWithStackMemory<T, size_t, DefaultHash<T>,
INITIAL_SIZE_DEGREE>;
if (!result && typeid_cast<const DataTypeNumber<T> *>(data_type.get()))
result = execute<Map, ColumnVector<T>, true>(arrays, ColumnVector<T>::create());
@ -465,8 +464,8 @@ void FunctionArrayIntersect::NumberExecutor::operator()()
template <typename T, size_t>
void FunctionArrayIntersect::DecimalExecutor::operator()()
{
using Map = ClearableHashMap<T, size_t, DefaultHash<T>, HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(T)>>;
using Map = ClearableHashMapWithStackMemory<T, size_t, DefaultHash<T>,
INITIAL_SIZE_DEGREE>;
if (!result)
if (auto * decimal = typeid_cast<const DataTypeDecimal<T> *>(data_type.get()))

25
src/Functions/array/arrayUniq.cpp
View File

@ -66,36 +66,41 @@ private:
template <typename T>
struct MethodOneNumber
{
using Set = ClearableHashSet<T, DefaultHash<T>, HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(T)>>;
using Set = ClearableHashSetWithStackMemory<T, DefaultHash<T>,
INITIAL_SIZE_DEGREE>;
using Method = ColumnsHashing::HashMethodOneNumber<typename Set::value_type, void, T, false>;
};
struct MethodString
{
using Set = ClearableHashSet<StringRef, StringRefHash, HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(StringRef)>>;
using Set = ClearableHashSetWithStackMemory<StringRef, StringRefHash,
INITIAL_SIZE_DEGREE>;
using Method = ColumnsHashing::HashMethodString<typename Set::value_type, void, false, false>;
};
struct MethodFixedString
{
using Set = ClearableHashSet<StringRef, StringRefHash, HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(StringRef)>>;
using Set = ClearableHashSetWithStackMemory<StringRef, StringRefHash,
INITIAL_SIZE_DEGREE>;
using Method = ColumnsHashing::HashMethodFixedString<typename Set::value_type, void, false, false>;
};
struct MethodFixed
{
using Set = ClearableHashSet<UInt128, UInt128HashCRC32, HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(UInt128)>>;
using Set = ClearableHashSetWithStackMemory<UInt128, UInt128HashCRC32,
INITIAL_SIZE_DEGREE>;
using Method = ColumnsHashing::HashMethodKeysFixed<typename Set::value_type, UInt128, void, false, false, false>;
};
struct MethodHashed
{
using Set = ClearableHashSet<UInt128, UInt128TrivialHash, HashTableGrower<INITIAL_SIZE_DEGREE>,
HashTableAllocatorWithStackMemory<(1ULL << INITIAL_SIZE_DEGREE) * sizeof(UInt128)>>;
using Set = ClearableHashSetWithStackMemory<UInt128, UInt128TrivialHash,
INITIAL_SIZE_DEGREE>;
using Method = ColumnsHashing::HashMethodHashed<typename Set::value_type, void, false>;
};

20
src/Functions/fromUnixTimestamp64Micro.cpp Normal file
View File

@ -0,0 +1,20 @@
#include <Functions/FunctionUnixTimestamp64.h>
#include <Functions/FunctionFactory.h>
namespace DB
{
struct TransformFromMicro
{
static constexpr auto name = "fromUnixTimestamp64Micro";
static constexpr auto target_scale = 6;
using SourceDataType = DataTypeInt64;
using ResultDataType = DataTypeDateTime64;
};
void registerFromUnixTimestamp64Micro(FunctionFactory & factory)
{
factory.registerFunction<FunctionUnixTimestamp64<TransformFromMicro>>();
}
}

20
src/Functions/fromUnixTimestamp64Milli.cpp Normal file
View File

@ -0,0 +1,20 @@
#include <Functions/FunctionUnixTimestamp64.h>
#include <Functions/FunctionFactory.h>
namespace DB
{
struct TransformFromMilli
{
static constexpr auto name = "fromUnixTimestamp64Milli";
static constexpr auto target_scale = 3;
using SourceDataType = DataTypeInt64;
using ResultDataType = DataTypeDateTime64;
};
void registerFromUnixTimestamp64Milli(FunctionFactory & factory)
{
factory.registerFunction<FunctionUnixTimestamp64<TransformFromMilli>>();
}
}

20
src/Functions/fromUnixTimestamp64Nano.cpp Normal file
View File

@ -0,0 +1,20 @@
#include <Functions/FunctionUnixTimestamp64.h>
#include <Functions/FunctionFactory.h>
namespace DB
{
struct TransformFromNano
{
static constexpr auto name = "fromUnixTimestamp64Nano";
static constexpr auto target_scale = 9;
using SourceDataType = DataTypeInt64;
using ResultDataType = DataTypeDateTime64;
};
void registerFromUnixTimestamp64Nano(FunctionFactory & factory)
{
factory.registerFunction<FunctionUnixTimestamp64<TransformFromNano>>();
}
}

2
src/Functions/registerFunctions.cpp
View File

@ -37,6 +37,7 @@ void registerFunctionsIntrospection(FunctionFactory &);
void registerFunctionsNull(FunctionFactory &);
void registerFunctionsJSON(FunctionFactory &);
void registerFunctionsConsistentHashing(FunctionFactory & factory);
void registerFunctionsUnixTimestamp64(FunctionFactory & factory);
void registerFunctions()
@ -78,6 +79,7 @@ void registerFunctions()
registerFunctionsJSON(factory);
registerFunctionsIntrospection(factory);
registerFunctionsConsistentHashing(factory);
registerFunctionsUnixTimestamp64(factory);
}
}

25
src/Functions/registerFunctionsUnixTimestamp64.cpp Normal file
View File

@ -0,0 +1,25 @@
namespace DB
{
class FunctionFactory;
void registerToUnixTimestamp64Milli(FunctionFactory &);
void registerToUnixTimestamp64Micro(FunctionFactory &);
void registerToUnixTimestamp64Nano(FunctionFactory &);
void registerFromUnixTimestamp64Milli(FunctionFactory &);
void registerFromUnixTimestamp64Micro(FunctionFactory &);
void registerFromUnixTimestamp64Nano(FunctionFactory &);
void registerFunctionsUnixTimestamp64(FunctionFactory & factory)
{
registerToUnixTimestamp64Milli(factory);
registerToUnixTimestamp64Micro(factory);
registerToUnixTimestamp64Nano(factory);
registerFromUnixTimestamp64Milli(factory);
registerFromUnixTimestamp64Micro(factory);
registerFromUnixTimestamp64Nano(factory);
}
}

20
src/Functions/toUnixTimestamp64Micro.cpp Normal file
View File

@ -0,0 +1,20 @@
#include <Functions/FunctionUnixTimestamp64.h>
#include <Functions/FunctionFactory.h>
namespace DB
{
struct TransformToMicro
{
static constexpr auto name = "toUnixTimestamp64Micro";
static constexpr auto target_scale = 6;
using SourceDataType = DataTypeDateTime64;
using ResultDataType = DataTypeInt64;
};
void registerToUnixTimestamp64Micro(FunctionFactory & factory)
{
factory.registerFunction<FunctionUnixTimestamp64<TransformToMicro>>();
}
}

20
src/Functions/toUnixTimestamp64Milli.cpp Normal file
View File

@ -0,0 +1,20 @@
#include <Functions/FunctionUnixTimestamp64.h>
#include <Functions/FunctionFactory.h>
namespace DB
{
struct TransformToMilli
{
static constexpr auto name = "toUnixTimestamp64Milli";
static constexpr auto target_scale = 3;
using SourceDataType = DataTypeDateTime64;
using ResultDataType = DataTypeInt64;
};
void registerToUnixTimestamp64Milli(FunctionFactory & factory)
{
factory.registerFunction<FunctionUnixTimestamp64<TransformToMilli>>();
}
}

20
src/Functions/toUnixTimestamp64Nano.cpp Normal file
View File

@ -0,0 +1,20 @@
#include <Functions/FunctionUnixTimestamp64.h>
#include <Functions/FunctionFactory.h>
namespace DB
{
struct TransformToNano
{
static constexpr auto name = "toUnixTimestamp64Nano";
static constexpr auto target_scale = 9;
using SourceDataType = DataTypeDateTime64;
using ResultDataType = DataTypeInt64;
};
void registerToUnixTimestamp64Nano(FunctionFactory & factory)
{
factory.registerFunction<FunctionUnixTimestamp64<TransformToNano>>();
}
}

8
src/Functions/ya.make
View File

@ -151,6 +151,9 @@ SRCS(
finalizeAggregation.cpp
formatDateTime.cpp
formatString.cpp
fromUnixTimestamp64Micro.cpp
fromUnixTimestamp64Milli.cpp
fromUnixTimestamp64Nano.cpp
FunctionFactory.cpp
FunctionFQDN.cpp
FunctionHelpers.cpp
@ -168,6 +171,7 @@ SRCS(
FunctionsRound.cpp
FunctionsStringArray.cpp
FunctionsStringSimilarity.cpp
FunctionUnixTimestamp64.h
GatherUtils/concat.cpp
GatherUtils/createArraySink.cpp
GatherUtils/createArraySource.cpp
@ -305,6 +309,7 @@ SRCS(
registerFunctionsStringSearch.cpp
registerFunctionsTuple.cpp
registerFunctionsVisitParam.cpp
registerFunctionsUnixTimestamp64.cpp
reinterpretAsFixedString.cpp
reinterpretAsString.cpp
reinterpretStringAs.cpp
@ -386,6 +391,9 @@ SRCS(
toTimeZone.cpp
toTypeName.cpp
toValidUTF8.cpp
toUnixTimestamp64Micro.cpp
toUnixTimestamp64Milli.cpp
toUnixTimestamp64Nano.cpp
toYear.cpp
toYYYYMM.cpp
toYYYYMMDD.cpp

11
src/Interpreters/CrossToInnerJoinVisitor.cpp
View File

@ -239,6 +239,10 @@ bool getTables(ASTSelectQuery & select, std::vector<JoinedElement> & joined_tabl
size_t num_array_join = 0;
size_t num_using = 0;
// For diagnostic messages.
std::vector<IAST *> tables_with_using;
tables_with_using.reserve(num_tables);
for (const auto & child : tables->children)
{
auto * table_element = child->as<ASTTablesInSelectQueryElement>();
@ -257,6 +261,7 @@ bool getTables(ASTSelectQuery & select, std::vector<JoinedElement> & joined_tabl
if (t.hasUsing())
{
++num_using;
tables_with_using.push_back(table_element);
continue;
}
@ -275,7 +280,11 @@ bool getTables(ASTSelectQuery & select, std::vector<JoinedElement> & joined_tabl
}
if (num_using && (num_tables - num_array_join) > 2)
throw Exception("Multiple CROSS/COMMA JOIN do not support USING", ErrorCodes::NOT_IMPLEMENTED);
{
throw Exception("Multiple CROSS/COMMA JOIN do not support USING (while "
"processing '" + IAST::formatForErrorMessage(tables_with_using) + "')",
ErrorCodes::NOT_IMPLEMENTED);
}
return !(num_array_join || num_using);
}

30
src/Interpreters/JoinedTables.cpp
View File

@ -61,19 +61,6 @@ void replaceJoinedTable(const ASTSelectQuery & select_query)
}
}
template <typename T>
void checkTablesWithColumns(const std::vector<T> & tables_with_columns, const Context & context)
{
const auto & settings = context.getSettingsRef();
if (settings.joined_subquery_requires_alias && tables_with_columns.size() > 1)
{
for (auto & t : tables_with_columns)
if (t.table.table.empty() && t.table.alias.empty())
throw Exception("No alias for subquery or table function in JOIN (set joined_subquery_requires_alias=0 to disable restriction).",
ErrorCodes::ALIAS_REQUIRED);
}
}
class RenameQualifiedIdentifiersMatcher
{
public:
@ -200,7 +187,22 @@ StoragePtr JoinedTables::getLeftTableStorage()
bool JoinedTables::resolveTables()
{
tables_with_columns = getDatabaseAndTablesWithColumns(table_expressions, context);
checkTablesWithColumns(tables_with_columns, context);
assert(tables_with_columns.size() == table_expressions.size());
const auto & settings = context.getSettingsRef();
if (settings.joined_subquery_requires_alias && tables_with_columns.size() > 1)
{
for (size_t i = 0; i < tables_with_columns.size(); ++i)
{
const auto & t = tables_with_columns[i];
if (t.table.table.empty() && t.table.alias.empty())
{
throw Exception("No alias for subquery or table function in JOIN (set joined_subquery_requires_alias=0 to disable restriction). While processing '"
+ table_expressions[i]->formatForErrorMessage() + "'",
ErrorCodes::ALIAS_REQUIRED);
}
}
}
return !tables_with_columns.empty();
}

49
src/Interpreters/sortBlock.cpp
View File

@ -101,6 +101,7 @@ struct PartialSortingLessWithCollation
}
};
void sortBlock(Block & block, const SortDescription & description, UInt64 limit)
{
if (!block)
@ -178,21 +179,47 @@ void sortBlock(Block & block, const SortDescription & description, UInt64 limit)
if (need_collation)
{
PartialSortingLessWithCollation less_with_collation(columns_with_sort_desc);
EqualRanges ranges;
ranges.emplace_back(0, perm.size());
for (const auto& column : columns_with_sort_desc)
{
while (!ranges.empty() && limit && limit <= ranges.back().first)
ranges.pop_back();
if (limit)
std::partial_sort(perm.begin(), perm.begin() + limit, perm.end(), less_with_collation);
else
pdqsort(perm.begin(), perm.end(), less_with_collation);
if (ranges.empty())
break;
if (isCollationRequired(column.description))
{
const ColumnString & column_string = assert_cast<const ColumnString &>(*column.column);
column_string.updatePermutationWithCollation(*column.description.collator, column.description.direction < 0, limit, column.description.nulls_direction, perm, ranges);
}
else
{
column.column->updatePermutation(
column.description.direction < 0, limit, column.description.nulls_direction, perm, ranges);
}
}
}
else
{
PartialSortingLess less(columns_with_sort_desc);
if (limit)
std::partial_sort(perm.begin(), perm.begin() + limit, perm.end(), less);
else
pdqsort(perm.begin(), perm.end(), less);
EqualRanges ranges;
ranges.emplace_back(0, perm.size());
for (const auto& column : columns_with_sort_desc)
{
while (!ranges.empty() && limit && limit <= ranges.back().first)
{
ranges.pop_back();
}
if (ranges.empty())
{
break;
}
column.column->updatePermutation(
column.description.direction < 0, limit, column.description.nulls_direction, perm, ranges);
}
}
size_t columns = block.columns();

4
src/Interpreters/tests/CMakeLists.txt
View File

@ -51,10 +51,6 @@ add_executable (in_join_subqueries_preprocessor in_join_subqueries_preprocessor.
target_link_libraries (in_join_subqueries_preprocessor PRIVATE dbms clickhouse_parsers)
add_check(in_join_subqueries_preprocessor)
add_executable (expression_analyzer expression_analyzer.cpp)
target_link_libraries (expression_analyzer PRIVATE dbms clickhouse_storages_system clickhouse_parsers clickhouse_common_io)
add_check(expression_analyzer)
add_executable (users users.cpp)
target_link_libraries (users PRIVATE dbms clickhouse_common_config)

128
src/Interpreters/tests/expression_analyzer.cpp
View File

@ -1,128 +0,0 @@
#include <DataTypes/DataTypesNumber.h>
#include <Storages/System/StorageSystemOne.h>
#include <Storages/System/StorageSystemNumbers.h>
#include <Databases/DatabaseMemory.h>
#include <Parsers/ParserSelectQuery.h>
#include <Parsers/parseQuery.h>
#include <Interpreters/Context.h>
#include <Interpreters/SyntaxAnalyzer.h>
#include <IO/WriteBufferFromFileDescriptor.h>
#include <IO/ReadBufferFromFileDescriptor.h>
#include <vector>
#include <unordered_map>
#include <iostream>
using namespace DB;
namespace DB
{
namespace ErrorCodes
{
extern const int SYNTAX_ERROR;
}
}
struct TestEntry
{
String query;
std::unordered_map<String, String> expected_aliases; /// alias -> AST.getID()
NamesAndTypesList source_columns = {};
bool check(const Context & context)
{
ASTPtr ast = parse(query);
auto res = SyntaxAnalyzer(context).analyze(ast, source_columns);
return checkAliases(*res);
}
private:
bool checkAliases(const SyntaxAnalyzerResult & res)
{
for (const auto & alias : res.aliases)
{
const String & alias_name = alias.first;
if (expected_aliases.count(alias_name) == 0 ||
expected_aliases[alias_name] != alias.second->getID())
{
std::cout << "unexpected alias: " << alias_name << ' ' << alias.second->getID() << std::endl;
return false;
}
else
expected_aliases.erase(alias_name);
}
if (!expected_aliases.empty())
{
std::cout << "missing aliases: " << expected_aliases.size() << std::endl;
return false;
}
return true;
}
static ASTPtr parse(const std::string & query)
{
ParserSelectQuery parser;
std::string message;
const auto * text = query.data();
if (ASTPtr ast = tryParseQuery(parser, text, text + query.size(), message, false, "", false, 0, 0))
return ast;
throw Exception(message, ErrorCodes::SYNTAX_ERROR);
}
};
int main()
{
std::vector<TestEntry> queries =
{
{
"SELECT number AS n FROM system.numbers LIMIT 0",
{{"n", "Identifier_number"}},
{ NameAndTypePair("number", std::make_shared<DataTypeUInt64>()) }
},
{
"SELECT number AS n FROM system.numbers LIMIT 0",
{{"n", "Identifier_number"}}
}
};
SharedContextHolder shared_context = Context::createShared();
Context context = Context::createGlobal(shared_context.get());
context.makeGlobalContext();
auto system_database = std::make_shared<DatabaseMemory>("system");
DatabaseCatalog::instance().attachDatabase("system", system_database);
//context.setCurrentDatabase("system");
system_database->attachTable("one", StorageSystemOne::create("one"), {});
system_database->attachTable("numbers", StorageSystemNumbers::create(StorageID("system", "numbers"), false), {});
size_t success = 0;
for (auto & entry : queries)
{
try
{
if (entry.check(context))
{
++success;
std::cout << "[OK] " << entry.query << std::endl;
}
else
std::cout << "[Failed] " << entry.query << std::endl;
}
catch (Exception & e)
{
std::cout << "[Error] " << entry.query << std::endl << e.displayText() << std::endl;
}
}
return success != queries.size();
}

3
src/Interpreters/tests/hash_map3.cpp
View File

@ -66,7 +66,8 @@ int main(int, char **)
UInt64,
SimpleHash,
Grower,
HashTableAllocatorWithStackMemory<4 * 24>>;
HashTableAllocatorWithStackMemory<
4 * sizeof(HashMapCell<StringRef, UInt64, SimpleHash>)>>;
Map map;

6
src/Parsers/IAST.cpp
View File

@ -87,6 +87,12 @@ size_t IAST::checkDepthImpl(size_t max_depth, size_t level) const
return res;
}
std::string IAST::formatForErrorMessage() const
{
std::stringstream ss;
format(FormatSettings(ss, true /* one line */));
return ss.str();
}
void IAST::cloneChildren()
{

21
src/Parsers/IAST.h
View File

@ -9,6 +9,7 @@
#include <algorithm>
#include <ostream>
#include <set>
#include <sstream>
class SipHash;
@ -215,6 +216,11 @@ public:
throw Exception("Unknown element in AST: " + getID(), ErrorCodes::UNKNOWN_ELEMENT_IN_AST);
}
// A simple way to add some user-readable context to an error message.
std::string formatForErrorMessage() const;
template <typename AstArray>
static std::string formatForErrorMessage(const AstArray & array);
void cloneChildren();
public:
@ -231,4 +237,19 @@ private:
size_t checkDepthImpl(size_t max_depth, size_t level) const;
};
template <typename AstArray>
std::string IAST::formatForErrorMessage(const AstArray & array)
{
std::stringstream ss;
for (size_t i = 0; i < array.size(); ++i)
{
if (i > 0)
{
ss << ", ";
}
array[i]->format(IAST::FormatSettings(ss, true /* one line */));
}
return ss.str();
}
}

2
src/Processors/Executors/PipelineExecutor.cpp
View File

@ -565,7 +565,7 @@ void PipelineExecutor::executeStepImpl(size_t thread_num, size_t num_threads, st
{
/// First, find any processor to execute.
/// Just travers graph and prepare any processor.
while (!finished)
while (!finished && state == nullptr)
{
{
std::unique_lock lock(task_queue_mutex);

26
src/Processors/ResizeProcessor.cpp
View File

@ -347,11 +347,16 @@ IProcessor::Status StrictResizeProcessor::prepare(const PortNumbers & updated_in
auto & waiting_output = output_ports[input_with_data.waiting_output];
if (waiting_output.status != OutputStatus::NeedData)
throw Exception("Invalid status for associated output.", ErrorCodes::LOGICAL_ERROR);
if (waiting_output.status == OutputStatus::NotActive)
throw Exception("Invalid status NotActive for associated output.", ErrorCodes::LOGICAL_ERROR);
waiting_output.port->pushData(input_with_data.port->pullData(/* set_not_needed = */ true));
waiting_output.status = OutputStatus::NotActive;
if (waiting_output.status != OutputStatus::Finished)
{
waiting_output.port->pushData(input_with_data.port->pullData(/* set_not_needed = */ true));
waiting_output.status = OutputStatus::NotActive;
}
else
abandoned_chunks.emplace_back(input_with_data.port->pullData(/* set_not_needed = */ true));
if (input_with_data.port->isFinished())
{
@ -370,6 +375,18 @@ IProcessor::Status StrictResizeProcessor::prepare(const PortNumbers & updated_in
return Status::Finished;
}
/// Process abandoned chunks if any.
while (!abandoned_chunks.empty() && !waiting_outputs.empty())
{
auto & waiting_output = output_ports[waiting_outputs.front()];
waiting_outputs.pop();
waiting_output.port->pushData(std::move(abandoned_chunks.back()));
abandoned_chunks.pop_back();
waiting_output.status = OutputStatus::NotActive;
}
/// Enable more inputs if needed.
while (!disabled_input_ports.empty() && !waiting_outputs.empty())
{
@ -383,6 +400,7 @@ IProcessor::Status StrictResizeProcessor::prepare(const PortNumbers & updated_in
waiting_outputs.pop();
}
/// Close all other waiting for data outputs (there is no corresponding input for them).
while (!waiting_outputs.empty())
{
auto & output = output_ports[waiting_outputs.front()];

3
src/Processors/ResizeProcessor.h
View File

@ -128,6 +128,9 @@ private:
std::vector<InputPortWithStatus> input_ports;
std::vector<OutputPortWithStatus> output_ports;
/// This field contained chunks which were read for output which had became finished while reading was happening.
/// They will be pushed to any next waiting output.
std::vector<Port::Data> abandoned_chunks;
};
}

24
src/Storages/IStorage.cpp
View File

@ -444,9 +444,14 @@ void IStorage::setPartitionKey(const StorageMetadataKeyField & partition_key_)
partition_key = partition_key_;
}
bool IStorage::isPartitionKeyDefined() const
{
return partition_key.definition_ast != nullptr;
}
bool IStorage::hasPartitionKey() const
{
return partition_key.expression != nullptr;
return !partition_key.column_names.empty();
}
Names IStorage::getColumnsRequiredForPartitionKey() const
@ -466,9 +471,14 @@ void IStorage::setSortingKey(const StorageMetadataKeyField & sorting_key_)
sorting_key = sorting_key_;
}
bool IStorage::isSortingKeyDefined() const
{
return sorting_key.definition_ast != nullptr;
}
bool IStorage::hasSortingKey() const
{
return sorting_key.expression != nullptr;
return !sorting_key.column_names.empty();
}
Names IStorage::getColumnsRequiredForSortingKey() const
@ -502,7 +512,7 @@ bool IStorage::isPrimaryKeyDefined() const
bool IStorage::hasPrimaryKey() const
{
return primary_key.expression != nullptr;
return !primary_key.column_names.empty();
}
Names IStorage::getColumnsRequiredForPrimaryKey() const
@ -529,9 +539,15 @@ void IStorage::setSamplingKey(const StorageMetadataKeyField & sampling_key_)
sampling_key = sampling_key_;
}
bool IStorage::isSamplingKeyDefined() const
{
return sampling_key.definition_ast != nullptr;
}
bool IStorage::hasSamplingKey() const
{
return sampling_key.expression != nullptr;
return !sampling_key.column_names.empty();
}
Names IStorage::getColumnsRequiredForSampling() const

Some files were not shown because too many files have changed in this diff Show More