thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into zk-columns-compact-storage

Browse Source 
Conflicts:
	dbms/src/Storages/StorageReplicatedMergeTree.cpp
This commit is contained in:
Alexey Zatelepin 2019-01-16 17:45:16 +03:00
commit 0f8e5f8522
687 changed files with 9747 additions and 6476 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -251,3 +251,6 @@ website/package-lock.json
# cquery cache
/.cquery-cache
# ccls cache
/.ccls-cache

3
.gitmodules vendored
View File

@ -61,3 +61,6 @@
[submodule "contrib/libgsasl"]
path = contrib/libgsasl
url = https://github.com/ClickHouse-Extras/libgsasl.git
[submodule "contrib/cppkafka"]
path = contrib/cppkafka
url = https://github.com/mfontanini/cppkafka.git

20
CMakeLists.txt
View File

@ -25,11 +25,6 @@ endif ()
# Write compile_commands.json
set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
set (MAX_COMPILER_MEMORY 2000 CACHE INTERNAL "")
set (MAX_LINKER_MEMORY 3500 CACHE INTERNAL "")
include (cmake/limit_jobs.cmake)
include (cmake/find_ccache.cmake)
if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None")
@ -95,8 +90,6 @@ if (GLIBC_COMPATIBILITY)
set (USE_INTERNAL_MEMCPY ON)
endif ()
set (COMPILER_FLAGS "${COMPILER_FLAGS}")
string(REGEX MATCH "-?[0-9]+(.[0-9]+)?$" COMPILER_POSTFIX ${CMAKE_CXX_COMPILER})
find_program (LLD_PATH NAMES "lld${COMPILER_POSTFIX}" "lld")
@ -113,10 +106,15 @@ if (LINKER_NAME)
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}")
endif ()
option (PIPE "-pipe compiler option [less /tmp usage, more ram usage]" ON)
if (PIPE)
set (COMPILER_FLAGS "${COMPILER_FLAGS} -pipe")
endif ()
cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd
if(NOT AVAILABLE_PHYSICAL_MEMORY OR AVAILABLE_PHYSICAL_MEMORY GREATER 8000)
option(COMPILER_PIPE "-pipe compiler option [less /tmp usage, more ram usage]" ON)
endif()
if(COMPILER_PIPE)
set(COMPILER_FLAGS "${COMPILER_FLAGS} -pipe")
else()
message(STATUS "Disabling compiler -pipe option (have only ${AVAILABLE_PHYSICAL_MEMORY} mb of memory)")
endif()
include (cmake/test_cpu.cmake)

2
LICENSE
View File

@ -186,7 +186,7 @@
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2016-2018 Yandex LLC
Copyright 2016-2019 Yandex LLC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.

6
cmake/find_llvm.cmake
View File

@ -79,7 +79,11 @@ endif()
function(llvm_libs_all REQUIRED_LLVM_LIBRARIES)
llvm_map_components_to_libnames (result all)
list (REMOVE_ITEM result "LTO" "LLVM")
if (USE_STATIC_LIBRARIES OR NOT "LLVM" IN_LIST result)
list (REMOVE_ITEM result "LTO" "LLVM")
else()
set (result "LLVM")
endif ()
if (TERMCAP_LIBRARY)
list (APPEND result ${TERMCAP_LIBRARY})
endif ()

5
cmake/find_rdkafka.cmake
View File

@ -20,11 +20,13 @@ if (NOT USE_INTERNAL_RDKAFKA_LIBRARY)
if (USE_STATIC_LIBRARIES AND NOT OS_FREEBSD)
find_library (SASL2_LIBRARY sasl2)
endif ()
set (CPPKAFKA_LIBRARY cppkafka) # TODO: try to use unbundled version.
endif ()
if (RDKAFKA_LIB AND RDKAFKA_INCLUDE_DIR)
set (USE_RDKAFKA 1)
set (RDKAFKA_LIBRARY ${RDKAFKA_LIB} ${OPENSSL_LIBRARIES})
set (CPPKAFKA_LIBRARY cppkafka)
if (SASL2_LIBRARY)
list (APPEND RDKAFKA_LIBRARY ${SASL2_LIBRARY})
endif ()
@ -35,9 +37,10 @@ elseif (NOT MISSING_INTERNAL_RDKAFKA_LIBRARY AND NOT ARCH_ARM)
set (USE_INTERNAL_RDKAFKA_LIBRARY 1)
set (RDKAFKA_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/librdkafka/src")
set (RDKAFKA_LIBRARY rdkafka)
set (CPPKAFKA_LIBRARY cppkafka)
set (USE_RDKAFKA 1)
endif ()
endif ()
message (STATUS "Using librdkafka=${USE_RDKAFKA}: ${RDKAFKA_INCLUDE_DIR} : ${RDKAFKA_LIBRARY}")
message (STATUS "Using librdkafka=${USE_RDKAFKA}: ${RDKAFKA_INCLUDE_DIR} : ${RDKAFKA_LIBRARY} ${CPPKAFKA_LIBRARY}")

2
cmake/find_ssl.cmake
View File

@ -6,7 +6,7 @@ set (OPENSSL_USE_STATIC_LIBS ${USE_STATIC_LIBRARIES})
if (NOT USE_INTERNAL_SSL_LIBRARY)
if (APPLE)
set (OPENSSL_ROOT_DIR "/usr/local/opt/openssl")
set (OPENSSL_ROOT_DIR "/usr/local/opt/openssl" CACHE INTERNAL "")
# https://rt.openssl.org/Ticket/Display.html?user=guest&pass=guest&id=2232
if (USE_STATIC_LIBRARIES)
message(WARNING "Disable USE_STATIC_LIBRARIES if you have linking problems with OpenSSL on MacOS")

13
cmake/find_zlib.cmake
View File

@ -8,23 +8,24 @@ endif ()
if (NOT ZLIB_FOUND)
if (NOT MSVC)
set (INTERNAL_ZLIB_NAME "zlib-ng")
set (INTERNAL_ZLIB_NAME "zlib-ng" CACHE INTERNAL "")
else ()
set (INTERNAL_ZLIB_NAME "zlib")
set (INTERNAL_ZLIB_NAME "zlib" CACHE INTERNAL "")
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}")
message (WARNING "Will use standard zlib, please clone manually:\n git clone https://github.com/madler/zlib.git ${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}")
endif ()
endif ()
set (USE_INTERNAL_ZLIB_LIBRARY 1)
set (ZLIB_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}" "${ClickHouse_BINARY_DIR}/contrib/${INTERNAL_ZLIB_NAME}") # generated zconf.h
set (ZLIB_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/${INTERNAL_ZLIB_NAME}" "${ClickHouse_BINARY_DIR}/contrib/${INTERNAL_ZLIB_NAME}" CACHE INTERNAL "") # generated zconf.h
set (ZLIB_INCLUDE_DIRS ${ZLIB_INCLUDE_DIR}) # for poco
set (ZLIB_INCLUDE_DIRECTORIES ${ZLIB_INCLUDE_DIR}) # for protobuf
set (ZLIB_FOUND 1) # for poco
if (USE_STATIC_LIBRARIES)
set (ZLIB_LIBRARIES zlibstatic)
set (ZLIB_LIBRARIES zlibstatic CACHE INTERNAL "")
else ()
set (ZLIB_LIBRARIES zlib)
set (ZLIB_LIBRARIES zlib CACHE INTERNAL "")
endif ()
endif ()
message (STATUS "Using zlib: ${ZLIB_INCLUDE_DIR} : ${ZLIB_LIBRARIES}")
message (STATUS "Using ${INTERNAL_ZLIB_NAME}: ${ZLIB_INCLUDE_DIR} : ${ZLIB_LIBRARIES}")

30
cmake/limit_jobs.cmake
View File

@ -4,32 +4,34 @@
# include (cmake/limit_jobs.cmake)
cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd
cmake_host_system_information(RESULT NUMBER_OF_LOGICAL_CORES QUERY NUMBER_OF_LOGICAL_CORES)
option(PARALLEL_COMPILE_JOBS "Define the maximum number of concurrent compilation jobs" "")
if (NOT PARALLEL_COMPILE_JOBS AND AVAILABLE_PHYSICAL_MEMORY)
math(EXPR PARALLEL_COMPILE_JOBS ${AVAILABLE_PHYSICAL_MEMORY}/2500) # ~2.5gb max per one compiler
if (NOT PARALLEL_COMPILE_JOBS AND AVAILABLE_PHYSICAL_MEMORY AND MAX_COMPILER_MEMORY)
math(EXPR PARALLEL_COMPILE_JOBS ${AVAILABLE_PHYSICAL_MEMORY}/${MAX_COMPILER_MEMORY})
if (NOT PARALLEL_COMPILE_JOBS)
set (PARALLEL_COMPILE_JOBS 1)
endif ()
endif ()
if (PARALLEL_COMPILE_JOBS)
set_property(GLOBAL APPEND PROPERTY JOB_POOLS compile_job_pool=${PARALLEL_COMPILE_JOBS})
set(CMAKE_JOB_POOL_COMPILE compile_job_pool)
if (PARALLEL_COMPILE_JOBS AND (NOT NUMBER_OF_LOGICAL_CORES OR PARALLEL_COMPILE_JOBS LESS NUMBER_OF_LOGICAL_CORES))
set(CMAKE_JOB_POOL_COMPILE compile_job_pool${CMAKE_CURRENT_SOURCE_DIR})
string (REGEX REPLACE "[^a-zA-Z0-9]+" "_" CMAKE_JOB_POOL_COMPILE ${CMAKE_JOB_POOL_COMPILE})
set_property(GLOBAL APPEND PROPERTY JOB_POOLS ${CMAKE_JOB_POOL_COMPILE}=${PARALLEL_COMPILE_JOBS})
endif ()
option(PARALLEL_LINK_JOBS "Define the maximum number of concurrent link jobs" "")
if (NOT PARALLEL_LINK_JOBS AND AVAILABLE_PHYSICAL_MEMORY)
math(EXPR PARALLEL_LINK_JOBS ${AVAILABLE_PHYSICAL_MEMORY}/4000) # ~4gb max per one linker
if (NOT PARALLEL_LINK_JOBS AND AVAILABLE_PHYSICAL_MEMORY AND MAX_LINKER_MEMORY)
math(EXPR PARALLEL_LINK_JOBS ${AVAILABLE_PHYSICAL_MEMORY}/${MAX_LINKER_MEMORY})
if (NOT PARALLEL_LINK_JOBS)
set (PARALLEL_LINK_JOBS 1)
endif ()
endif ()
if (PARALLEL_LINK_JOBS AND (NOT NUMBER_OF_LOGICAL_CORES OR PARALLEL_COMPILE_JOBS LESS NUMBER_OF_LOGICAL_CORES))
set(CMAKE_JOB_POOL_LINK link_job_pool${CMAKE_CURRENT_SOURCE_DIR})
string (REGEX REPLACE "[^a-zA-Z0-9]+" "_" CMAKE_JOB_POOL_LINK ${CMAKE_JOB_POOL_LINK})
set_property(GLOBAL APPEND PROPERTY JOB_POOLS ${CMAKE_JOB_POOL_LINK}=${PARALLEL_LINK_JOBS})
endif ()
if (PARALLEL_COMPILE_JOBS OR PARALLEL_LINK_JOBS)
message(STATUS "Have ${AVAILABLE_PHYSICAL_MEMORY} megabytes of memory. Limiting concurrent linkers jobs to ${PARALLEL_LINK_JOBS} and compiler jobs to ${PARALLEL_COMPILE_JOBS}")
message(STATUS "${CMAKE_CURRENT_SOURCE_DIR}: Have ${AVAILABLE_PHYSICAL_MEMORY} megabytes of memory. Limiting concurrent linkers jobs to ${PARALLEL_LINK_JOBS} and compiler jobs to ${PARALLEL_COMPILE_JOBS}")
endif ()
if (LLVM_PARALLEL_LINK_JOBS)
set_property(GLOBAL APPEND PROPERTY JOB_POOLS link_job_pool=${PARALLEL_LINK_JOBS})
set(CMAKE_JOB_POOL_LINK link_job_pool)
endif ()

12
contrib/CMakeLists.txt vendored
View File

@ -1,11 +1,11 @@
# Third-party libraries may have substandard code.
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-stringop-overflow")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-stringop-overflow -Wno-implicit-function-declaration -Wno-return-type -Wno-array-bounds -Wno-bool-compare -Wno-int-conversion -Wno-switch")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-implicit-fallthrough -Wno-class-memaccess -Wno-sign-compare -std=c++1z")
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-format -Wno-parentheses-equality")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-format -std=c++1z")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-format -Wno-parentheses-equality -Wno-tautological-constant-compare -Wno-tautological-constant-out-of-range-compare -Wno-implicit-function-declaration -Wno-return-type -Wno-pointer-bool-conversion -Wno-enum-conversion -Wno-int-conversion -Wno-switch")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-format -Wno-inconsistent-missing-override -std=c++1z")
endif ()
if (USE_INTERNAL_BOOST_LIBRARY)
@ -125,6 +125,10 @@ if (USE_INTERNAL_RDKAFKA_LIBRARY)
target_include_directories(rdkafka BEFORE PRIVATE ${OPENSSL_INCLUDE_DIR})
endif ()
if (USE_RDKAFKA)
add_subdirectory (cppkafka-cmake)
endif()
if (ENABLE_ODBC AND USE_INTERNAL_ODBC_LIBRARY)
add_subdirectory (unixodbc-cmake)
endif ()
@ -139,6 +143,7 @@ if (USE_INTERNAL_CAPNP_LIBRARY)
endif ()
if (USE_INTERNAL_POCO_LIBRARY)
set (POCO_VERBOSE_MESSAGES 0 CACHE INTERNAL "")
set (save_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
set (save_CMAKE_C_FLAGS ${CMAKE_C_FLAGS})
set (_save ${ENABLE_TESTS})
@ -206,6 +211,7 @@ if (USE_INTERNAL_HDFS3_LIBRARY)
if (USE_INTERNAL_PROTOBUF_LIBRARY)
set(protobuf_BUILD_TESTS OFF CACHE INTERNAL "" FORCE)
set(protobuf_BUILD_SHARED_LIBS OFF CACHE INTERNAL "" FORCE)
set(protobuf_WITH_ZLIB 0 CACHE INTERNAL "" FORCE) # actually will use zlib, but skip find
add_subdirectory(protobuf/cmake)
endif ()
add_subdirectory(libhdfs3-cmake)

1
contrib/cppkafka vendored Submodule

@ -0,0 +1 @@
Subproject commit 520465510efef7704346cf8d140967c4abb057c1

31
contrib/cppkafka-cmake/CMakeLists.txt Normal file
View File

@ -0,0 +1,31 @@
set(CPPKAFKA_DIR ${CMAKE_SOURCE_DIR}/contrib/cppkafka)
set(SRCS
${CPPKAFKA_DIR}/src/configuration.cpp
${CPPKAFKA_DIR}/src/topic_configuration.cpp
${CPPKAFKA_DIR}/src/configuration_option.cpp
${CPPKAFKA_DIR}/src/exceptions.cpp
${CPPKAFKA_DIR}/src/topic.cpp
${CPPKAFKA_DIR}/src/buffer.cpp
${CPPKAFKA_DIR}/src/queue.cpp
${CPPKAFKA_DIR}/src/message.cpp
${CPPKAFKA_DIR}/src/message_timestamp.cpp
${CPPKAFKA_DIR}/src/message_internal.cpp
${CPPKAFKA_DIR}/src/topic_partition.cpp
${CPPKAFKA_DIR}/src/topic_partition_list.cpp
${CPPKAFKA_DIR}/src/metadata.cpp
${CPPKAFKA_DIR}/src/group_information.cpp
${CPPKAFKA_DIR}/src/error.cpp
${CPPKAFKA_DIR}/src/event.cpp
${CPPKAFKA_DIR}/src/kafka_handle_base.cpp
${CPPKAFKA_DIR}/src/producer.cpp
${CPPKAFKA_DIR}/src/consumer.cpp
)
add_library(cppkafka ${LINK_MODE} ${SRCS})
target_link_libraries(cppkafka PRIVATE ${RDKAFKA_LIBRARY})
target_include_directories(cppkafka PRIVATE ${CPPKAFKA_DIR}/include/cppkafka)
target_include_directories(cppkafka PRIVATE ${Boost_INCLUDE_DIRS})
target_include_directories(cppkafka SYSTEM PUBLIC ${CPPKAFKA_DIR}/include)

2
contrib/jemalloc vendored

@ -1 +1 @@
Subproject commit cd2931ad9bbd78208565716ab102e86d858c2fff
Subproject commit 41b7372eadee941b9164751b8d4963f915d3ceae

48
contrib/libhdfs3-cmake/CMake/CodeCoverage.cmake
View File

@ -1,48 +0,0 @@
# Check prereqs
FIND_PROGRAM(GCOV_PATH gcov)
FIND_PROGRAM(LCOV_PATH lcov)
FIND_PROGRAM(GENHTML_PATH genhtml)
IF(NOT GCOV_PATH)
MESSAGE(FATAL_ERROR "gcov not found! Aborting...")
ENDIF(NOT GCOV_PATH)
IF(NOT CMAKE_BUILD_TYPE STREQUAL Debug)
MESSAGE(WARNING "Code coverage results with an optimised (non-Debug) build may be misleading")
ENDIF(NOT CMAKE_BUILD_TYPE STREQUAL Debug)
#Setup compiler options
ADD_DEFINITIONS(-fprofile-arcs -ftest-coverage)
SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs ")
SET(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fprofile-arcs ")
IF(NOT LCOV_PATH)
MESSAGE(FATAL_ERROR "lcov not found! Aborting...")
ENDIF(NOT LCOV_PATH)
IF(NOT GENHTML_PATH)
MESSAGE(FATAL_ERROR "genhtml not found! Aborting...")
ENDIF(NOT GENHTML_PATH)
#Setup target
ADD_CUSTOM_TARGET(ShowCoverage
#Capturing lcov counters and generating report
COMMAND ${LCOV_PATH} --directory . --capture --output-file CodeCoverage.info
COMMAND ${LCOV_PATH} --remove CodeCoverage.info '${CMAKE_CURRENT_BINARY_DIR}/*' 'test/*' 'mock/*' '/usr/*' '/opt/*' '*ext/rhel5_x86_64*' '*ext/osx*' --output-file CodeCoverage.info.cleaned
COMMAND ${GENHTML_PATH} -o CodeCoverageReport CodeCoverage.info.cleaned
)
ADD_CUSTOM_TARGET(ShowAllCoverage
#Capturing lcov counters and generating report
COMMAND ${LCOV_PATH} -a CodeCoverage.info.cleaned -a CodeCoverage.info.cleaned_withoutHA -o AllCodeCoverage.info
COMMAND sed -e 's|/.*/src|${CMAKE_SOURCE_DIR}/src|' -ig AllCodeCoverage.info
COMMAND ${GENHTML_PATH} -o AllCodeCoverageReport AllCodeCoverage.info
)
ADD_CUSTOM_TARGET(ResetCoverage
#Cleanup lcov
COMMAND ${LCOV_PATH} --directory . --zerocounters
)

1162
contrib/libhdfs3-cmake/CMake/FindBoost.cmake
View File

File diff suppressed because it is too large Load Diff

128
contrib/libhdfs3-cmake/CMake/Options.cmake
View File

@ -1,38 +1,15 @@
OPTION(ENABLE_COVERAGE "enable code coverage" OFF)
OPTION(ENABLE_DEBUG "enable debug build" OFF)
OPTION(ENABLE_SSE "enable SSE4.2 buildin function" ON)
OPTION(ENABLE_FRAME_POINTER "enable frame pointer on 64bit system with flag -fno-omit-frame-pointer, on 32bit system, it is always enabled" ON)
OPTION(ENABLE_LIBCPP "using libc++ instead of libstdc++, only valid for clang compiler" OFF)
OPTION(ENABLE_BOOST "using boost instead of native compiler c++0x support" OFF)
INCLUDE (CheckFunctionExists)
CHECK_FUNCTION_EXISTS(dladdr HAVE_DLADDR)
CHECK_FUNCTION_EXISTS(nanosleep HAVE_NANOSLEEP)
IF(ENABLE_DEBUG STREQUAL ON)
SET(CMAKE_BUILD_TYPE Debug CACHE
STRING "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel." FORCE)
SET(CMAKE_CXX_FLAGS_DEBUG "-g -O0" CACHE STRING "compiler flags for debug" FORCE)
SET(CMAKE_C_FLAGS_DEBUG "-g -O0" CACHE STRING "compiler flags for debug" FORCE)
ELSE(ENABLE_DEBUG STREQUAL ON)
SET(CMAKE_BUILD_TYPE RelWithDebInfo CACHE
STRING "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel." FORCE)
ENDIF(ENABLE_DEBUG STREQUAL ON)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-strict-aliasing")
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-strict-aliasing")
IF(ENABLE_COVERAGE STREQUAL ON)
INCLUDE(CodeCoverage)
ENDIF(ENABLE_COVERAGE STREQUAL ON)
IF(ENABLE_FRAME_POINTER STREQUAL ON)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
ENDIF(ENABLE_FRAME_POINTER STREQUAL ON)
IF(ENABLE_SSE STREQUAL ON)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2")
ENDIF(ENABLE_SSE STREQUAL ON)
ENDIF(ENABLE_SSE STREQUAL ON)
IF(NOT TEST_HDFS_PREFIX)
SET(TEST_HDFS_PREFIX "./" CACHE STRING "default directory prefix used for test." FORCE)
@ -41,76 +18,7 @@ ENDIF(NOT TEST_HDFS_PREFIX)
ADD_DEFINITIONS(-DTEST_HDFS_PREFIX="${TEST_HDFS_PREFIX}")
ADD_DEFINITIONS(-D__STDC_FORMAT_MACROS)
ADD_DEFINITIONS(-D_GNU_SOURCE)
IF(OS_MACOSX AND CMAKE_COMPILER_IS_GNUCXX)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-bind_at_load")
ENDIF(OS_MACOSX AND CMAKE_COMPILER_IS_GNUCXX)
IF(OS_LINUX)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--export-dynamic")
ENDIF(OS_LINUX)
SET(BOOST_ROOT ${CMAKE_PREFIX_PATH})
IF(ENABLE_BOOST STREQUAL ON)
MESSAGE(STATUS "using boost instead of native compiler c++0x support.")
FIND_PACKAGE(Boost 1.50 REQUIRED)
SET(NEED_BOOST true CACHE INTERNAL "boost is required")
ELSE(ENABLE_BOOST STREQUAL ON)
SET(NEED_BOOST false CACHE INTERNAL "boost is required")
ENDIF(ENABLE_BOOST STREQUAL ON)
IF(CMAKE_COMPILER_IS_GNUCXX)
IF(ENABLE_LIBCPP STREQUAL ON)
MESSAGE(FATAL_ERROR "Unsupport using GCC compiler with libc++")
ENDIF(ENABLE_LIBCPP STREQUAL ON)
IF((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR EQUAL 4) AND OS_MACOSX)
SET(NEED_GCCEH true CACHE INTERNAL "Explicitly link with gcc_eh")
MESSAGE(STATUS "link with -lgcc_eh for TLS")
ENDIF((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR EQUAL 4) AND OS_MACOSX)
IF((GCC_COMPILER_VERSION_MAJOR LESS 4) OR ((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 4)))
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
IF(NOT ENABLE_BOOST STREQUAL ON)
MESSAGE(STATUS "gcc version is older than 4.6.0, boost is required.")
FIND_PACKAGE(Boost 1.50 REQUIRED)
SET(NEED_BOOST true CACHE INTERNAL "boost is required")
ENDIF(NOT ENABLE_BOOST STREQUAL ON)
ELSEIF((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 7))
IF(NOT ENABLE_BOOST STREQUAL ON)
MESSAGE(STATUS "gcc version is older than 4.6.0, boost is required.")
FIND_PACKAGE(Boost 1.50 REQUIRED)
SET(NEED_BOOST true CACHE INTERNAL "boost is required")
ENDIF(NOT ENABLE_BOOST STREQUAL ON)
MESSAGE(STATUS "adding c++0x support for gcc compiler")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x")
ELSE((GCC_COMPILER_VERSION_MAJOR LESS 4) OR ((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 4)))
MESSAGE(STATUS "adding c++0x support for gcc compiler")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x")
ENDIF((GCC_COMPILER_VERSION_MAJOR LESS 4) OR ((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 4)))
IF(NEED_BOOST)
IF((Boost_MAJOR_VERSION LESS 1) OR ((Boost_MAJOR_VERSION EQUAL 1) AND (Boost_MINOR_VERSION LESS 50)))
MESSAGE(FATAL_ERROR "boost 1.50+ is required")
ENDIF()
ELSE(NEED_BOOST)
IF(HAVE_NANOSLEEP)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_NANOSLEEP")
ELSE(HAVE_NANOSLEEP)
MESSAGE(FATAL_ERROR "nanosleep() is required")
ENDIF(HAVE_NANOSLEEP)
ENDIF(NEED_BOOST)
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
ELSEIF(CMAKE_COMPILER_IS_CLANG)
MESSAGE(STATUS "adding c++0x support for clang compiler")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x")
SET(CMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LANGUAGE_STANDARD "c++0x")
IF(ENABLE_LIBCPP STREQUAL ON)
MESSAGE(STATUS "using libc++ instead of libstdc++")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
SET(CMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LIBRARY "libc++")
ENDIF(ENABLE_LIBCPP STREQUAL ON)
ENDIF(CMAKE_COMPILER_IS_GNUCXX)
ADD_DEFINITIONS(-D_GLIBCXX_USE_NANOSLEEP)
TRY_COMPILE(STRERROR_R_RETURN_INT
${CMAKE_CURRENT_BINARY_DIR}
@ -138,32 +46,8 @@ TRY_COMPILE(HAVE_NESTED_EXCEPTION
CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'"
OUTPUT_VARIABLE OUTPUT)
FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include <boost/chrono.hpp>")
TRY_COMPILE(HAVE_BOOST_CHRONO
${CMAKE_CURRENT_BINARY_DIR}
${CMAKE_CURRENT_BINARY_DIR}/test.cpp
CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'"
-DINCLUDE_DIRECTORIES=${Boost_INCLUDE_DIR}
OUTPUT_VARIABLE OUTPUT)
SET(HAVE_BOOST_CHRONO 0)
SET(HAVE_BOOST_ATOMIC 0)
FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include <chrono>")
TRY_COMPILE(HAVE_STD_CHRONO
${CMAKE_CURRENT_BINARY_DIR}
${CMAKE_CURRENT_BINARY_DIR}/test.cpp
CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'"
OUTPUT_VARIABLE OUTPUT)
FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include <boost/atomic.hpp>")
TRY_COMPILE(HAVE_BOOST_ATOMIC
${CMAKE_CURRENT_BINARY_DIR}
${CMAKE_CURRENT_BINARY_DIR}/test.cpp
CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'"
-DINCLUDE_DIRECTORIES=${Boost_INCLUDE_DIR}
OUTPUT_VARIABLE OUTPUT)
FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include <atomic>")
TRY_COMPILE(HAVE_STD_ATOMIC
${CMAKE_CURRENT_BINARY_DIR}
${CMAKE_CURRENT_BINARY_DIR}/test.cpp
CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'"
OUTPUT_VARIABLE OUTPUT)
SET(HAVE_STD_CHRONO 1)
SET(HAVE_STD_ATOMIC 1)

2
contrib/librdkafka vendored

@ -1 +1 @@
Subproject commit 7478b5ef16aadd6543fe38bc6a2deb895c70da98
Subproject commit 363dcad5a23dc29381cc626620e68ae418b3af19

107
contrib/librdkafka-cmake/CMakeLists.txt
View File

@ -1,60 +1,63 @@
set(RDKAFKA_SOURCE_DIR ${CMAKE_SOURCE_DIR}/contrib/librdkafka/src)
set(SRCS
${RDKAFKA_SOURCE_DIR}/crc32c.c
${RDKAFKA_SOURCE_DIR}/rdaddr.c
${RDKAFKA_SOURCE_DIR}/rdavl.c
${RDKAFKA_SOURCE_DIR}/rdbuf.c
${RDKAFKA_SOURCE_DIR}/rdcrc32.c
${RDKAFKA_SOURCE_DIR}/rdkafka.c
${RDKAFKA_SOURCE_DIR}/rdkafka_assignor.c
${RDKAFKA_SOURCE_DIR}/rdkafka_broker.c
${RDKAFKA_SOURCE_DIR}/rdkafka_buf.c
${RDKAFKA_SOURCE_DIR}/rdkafka_cgrp.c
${RDKAFKA_SOURCE_DIR}/rdkafka_conf.c
${RDKAFKA_SOURCE_DIR}/rdkafka_event.c
${RDKAFKA_SOURCE_DIR}/rdkafka_feature.c
${RDKAFKA_SOURCE_DIR}/rdkafka_lz4.c
${RDKAFKA_SOURCE_DIR}/rdkafka_metadata.c
${RDKAFKA_SOURCE_DIR}/rdkafka_metadata_cache.c
${RDKAFKA_SOURCE_DIR}/rdkafka_msg.c
${RDKAFKA_SOURCE_DIR}/rdkafka_msgset_reader.c
${RDKAFKA_SOURCE_DIR}/rdkafka_msgset_writer.c
${RDKAFKA_SOURCE_DIR}/rdkafka_offset.c
${RDKAFKA_SOURCE_DIR}/rdkafka_op.c
${RDKAFKA_SOURCE_DIR}/rdkafka_partition.c
${RDKAFKA_SOURCE_DIR}/rdkafka_pattern.c
${RDKAFKA_SOURCE_DIR}/rdkafka_queue.c
${RDKAFKA_SOURCE_DIR}/rdkafka_range_assignor.c
${RDKAFKA_SOURCE_DIR}/rdkafka_request.c
${RDKAFKA_SOURCE_DIR}/rdkafka_roundrobin_assignor.c
${RDKAFKA_SOURCE_DIR}/rdkafka_sasl.c
${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_plain.c
${RDKAFKA_SOURCE_DIR}/rdkafka_subscription.c
${RDKAFKA_SOURCE_DIR}/rdkafka_timer.c
${RDKAFKA_SOURCE_DIR}/rdkafka_topic.c
${RDKAFKA_SOURCE_DIR}/rdkafka_transport.c
${RDKAFKA_SOURCE_DIR}/rdkafka_interceptor.c
${RDKAFKA_SOURCE_DIR}/rdkafka_header.c
${RDKAFKA_SOURCE_DIR}/rdlist.c
${RDKAFKA_SOURCE_DIR}/rdlog.c
${RDKAFKA_SOURCE_DIR}/rdmurmur2.c
${RDKAFKA_SOURCE_DIR}/rdports.c
${RDKAFKA_SOURCE_DIR}/rdrand.c
${RDKAFKA_SOURCE_DIR}/rdregex.c
${RDKAFKA_SOURCE_DIR}/rdstring.c
${RDKAFKA_SOURCE_DIR}/rdunittest.c
${RDKAFKA_SOURCE_DIR}/rdvarint.c
${RDKAFKA_SOURCE_DIR}/snappy.c
${RDKAFKA_SOURCE_DIR}/tinycthread.c
${RDKAFKA_SOURCE_DIR}/xxhash.c
${RDKAFKA_SOURCE_DIR}/lz4.c
${RDKAFKA_SOURCE_DIR}/lz4frame.c
${RDKAFKA_SOURCE_DIR}/lz4hc.c
${RDKAFKA_SOURCE_DIR}/rdgz.c
${RDKAFKA_SOURCE_DIR}/crc32c.c
${RDKAFKA_SOURCE_DIR}/rdaddr.c
${RDKAFKA_SOURCE_DIR}/rdavl.c
${RDKAFKA_SOURCE_DIR}/rdbuf.c
${RDKAFKA_SOURCE_DIR}/rdcrc32.c
${RDKAFKA_SOURCE_DIR}/rdkafka.c
${RDKAFKA_SOURCE_DIR}/rdkafka_assignor.c
${RDKAFKA_SOURCE_DIR}/rdkafka_background.c
${RDKAFKA_SOURCE_DIR}/rdkafka_broker.c
${RDKAFKA_SOURCE_DIR}/rdkafka_buf.c
${RDKAFKA_SOURCE_DIR}/rdkafka_cgrp.c
${RDKAFKA_SOURCE_DIR}/rdkafka_conf.c
${RDKAFKA_SOURCE_DIR}/rdkafka_event.c
${RDKAFKA_SOURCE_DIR}/rdkafka_feature.c
${RDKAFKA_SOURCE_DIR}/rdkafka_idempotence.c
${RDKAFKA_SOURCE_DIR}/rdkafka_lz4.c
${RDKAFKA_SOURCE_DIR}/rdkafka_metadata.c
${RDKAFKA_SOURCE_DIR}/rdkafka_metadata_cache.c
${RDKAFKA_SOURCE_DIR}/rdkafka_msg.c
${RDKAFKA_SOURCE_DIR}/rdkafka_msgset_reader.c
${RDKAFKA_SOURCE_DIR}/rdkafka_msgset_writer.c
${RDKAFKA_SOURCE_DIR}/rdkafka_offset.c
${RDKAFKA_SOURCE_DIR}/rdkafka_op.c
${RDKAFKA_SOURCE_DIR}/rdkafka_partition.c
${RDKAFKA_SOURCE_DIR}/rdkafka_pattern.c
${RDKAFKA_SOURCE_DIR}/rdkafka_queue.c
${RDKAFKA_SOURCE_DIR}/rdkafka_range_assignor.c
${RDKAFKA_SOURCE_DIR}/rdkafka_request.c
${RDKAFKA_SOURCE_DIR}/rdkafka_roundrobin_assignor.c
${RDKAFKA_SOURCE_DIR}/rdkafka_sasl.c
${RDKAFKA_SOURCE_DIR}/rdkafka_sasl_plain.c
${RDKAFKA_SOURCE_DIR}/rdkafka_subscription.c
${RDKAFKA_SOURCE_DIR}/rdkafka_timer.c
${RDKAFKA_SOURCE_DIR}/rdkafka_topic.c
${RDKAFKA_SOURCE_DIR}/rdkafka_transport.c
${RDKAFKA_SOURCE_DIR}/rdkafka_interceptor.c
${RDKAFKA_SOURCE_DIR}/rdkafka_header.c
${RDKAFKA_SOURCE_DIR}/rdlist.c
${RDKAFKA_SOURCE_DIR}/rdlog.c
${RDKAFKA_SOURCE_DIR}/rdmurmur2.c
${RDKAFKA_SOURCE_DIR}/rdports.c
${RDKAFKA_SOURCE_DIR}/rdrand.c
${RDKAFKA_SOURCE_DIR}/rdregex.c
${RDKAFKA_SOURCE_DIR}/rdstring.c
${RDKAFKA_SOURCE_DIR}/rdunittest.c
${RDKAFKA_SOURCE_DIR}/rdvarint.c
${RDKAFKA_SOURCE_DIR}/snappy.c
${RDKAFKA_SOURCE_DIR}/tinycthread.c
${RDKAFKA_SOURCE_DIR}/tinycthread_extra.c
${RDKAFKA_SOURCE_DIR}/xxhash.c
${RDKAFKA_SOURCE_DIR}/lz4.c
${RDKAFKA_SOURCE_DIR}/lz4frame.c
${RDKAFKA_SOURCE_DIR}/lz4hc.c
${RDKAFKA_SOURCE_DIR}/rdgz.c
)
add_library(rdkafka ${LINK_MODE} ${SRCS})
target_include_directories(rdkafka PRIVATE include)
target_include_directories(rdkafka SYSTEM PUBLIC include)
target_include_directories(rdkafka SYSTEM PUBLIC ${RDKAFKA_SOURCE_DIR})
target_link_libraries(rdkafka PUBLIC ${ZLIB_LIBRARIES} ${OPENSSL_SSL_LIBRARY} ${OPENSSL_CRYPTO_LIBRARY})

6
contrib/librdkafka-cmake/config.h
View File

@ -1,4 +1,4 @@
// Automatically generated by ./configure
// Automatically generated by ./configure
#ifndef _CONFIG_H_
#define _CONFIG_H_
#define ARCH "x86_64"
@ -71,4 +71,8 @@
#define HAVE_PTHREAD_SETNAME_GNU 1
// python
//#define HAVE_PYTHON 1
// C11 threads
#if (__STDC_VERSION__ >= 201112L) && !defined(__STDC_NO_THREADS__)
# define WITH_C11THREADS 1
#endif
#endif /* _CONFIG_H_ */

5
contrib/librdkafka-cmake/include/librdkafka/rdkafka.h Normal file
View File

@ -0,0 +1,5 @@
#if __has_include(<rdkafka.h>) // maybe bundled
# include_next <rdkafka.h> // Y_IGNORE
#else // system
# include_next <librdkafka/rdkafka.h>
#endif

6
contrib/murmurhash/include/murmurhash2.h
View File

@ -2,8 +2,7 @@
// MurmurHash2 was written by Austin Appleby, and is placed in the public
// domain. The author hereby disclaims copyright to this source code.
#ifndef _MURMURHASH2_H_
#define _MURMURHASH2_H_
#pragma once
//-----------------------------------------------------------------------------
// Platform-specific functions and macros
@ -30,6 +29,3 @@ uint64_t MurmurHash64B (const void * key, int len, uint64_t seed);
uint32_t MurmurHash2A (const void * key, int len, uint32_t seed);
uint32_t MurmurHashNeutral2 (const void * key, int len, uint32_t seed);
uint32_t MurmurHashAligned2 (const void * key, int len, uint32_t seed);
#endif // _MURMURHASH2_H_

5
contrib/murmurhash/include/murmurhash3.h
View File

@ -2,8 +2,7 @@
// MurmurHash3 was written by Austin Appleby, and is placed in the public
// domain. The author hereby disclaims copyright to this source code.
#ifndef _MURMURHASH3_H_
#define _MURMURHASH3_H_
#pragma once
//-----------------------------------------------------------------------------
// Platform-specific functions and macros
@ -33,5 +32,3 @@ void MurmurHash3_x86_128 ( const void * key, int len, uint32_t seed, void * out
void MurmurHash3_x64_128 ( const void * key, int len, uint32_t seed, void * out );
//-----------------------------------------------------------------------------
#endif // _MURMURHASH3_H_

71
dbms/CMakeLists.txt
View File

@ -2,15 +2,27 @@ if (USE_INCLUDE_WHAT_YOU_USE)
set (CMAKE_CXX_INCLUDE_WHAT_YOU_USE ${IWYU_PATH})
endif ()
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/find_vectorclass.cmake)
if(COMPILER_PIPE)
set(MAX_COMPILER_MEMORY 2500)
else()
set(MAX_COMPILER_MEMORY 1500)
endif()
if(MAKE_STATIC_LIBRARIES)
set(MAX_LINKER_MEMORY 3500)
else()
set(MAX_LINKER_MEMORY 2500)
endif()
include(../cmake/limit_jobs.cmake)
include(cmake/find_vectorclass.cmake)
set (CONFIG_VERSION ${CMAKE_CURRENT_BINARY_DIR}/src/Common/config_version.h)
set (CONFIG_COMMON ${CMAKE_CURRENT_BINARY_DIR}/src/Common/config.h)
include (cmake/version.cmake)
message (STATUS "Will build ${VERSION_FULL}")
configure_file (${CMAKE_CURRENT_SOURCE_DIR}/src/Common/config.h.in ${CONFIG_COMMON})
configure_file (${CMAKE_CURRENT_SOURCE_DIR}/src/Common/config_version.h.in ${CONFIG_VERSION})
message (STATUS "Will build ${VERSION_FULL} revision ${VERSION_REVISION}")
configure_file (src/Common/config.h.in ${CONFIG_COMMON})
configure_file (src/Common/config_version.h.in ${CONFIG_VERSION})
if (NOT MSVC)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra")
@ -21,13 +33,35 @@ if (NOT NO_WERROR)
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror")
endif ()
# Add some warnings that are not available even with -Wall -Wextra.
# Add some warnings that are not available even with -Wall -Wextra -Wpedantic.
option (WEVERYTHING "Enables -Weverything option with some exceptions. This is intended for exploration of new compiler warnings that may be found to be useful. Only makes sense for clang." ON)
if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra-semi -Wcomma -Winconsistent-missing-destructor-override -Wunused-exception-parameter -Wshadow-uncaptured-local")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wpedantic -Wno-vla-extension -Wno-zero-length-array -Wno-gnu-anonymous-struct -Wno-nested-anon-types")
if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wredundant-parens -Wzero-as-null-pointer-constant")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wshadow -Wshadow-uncaptured-local -Wextra-semi -Wcomma -Winconsistent-missing-destructor-override -Wunused-exception-parameter -Wcovered-switch-default -Wold-style-cast -Wrange-loop-analysis -Wunused-member-function -Wunreachable-code -Wunreachable-code-return -Wnewline-eof -Wembedded-directive -Wgnu-case-range -Wunused-macros -Wconditional-uninitialized -Wdeprecated -Wundef -Wreserved-id-macro -Wredundant-parens -Wzero-as-null-pointer-constant")
if (WEVERYTHING)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Weverything -Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-missing-noreturn -Wno-padded -Wno-switch-enum -Wno-shadow-field-in-constructor -Wno-deprecated-dynamic-exception-spec -Wno-float-equal -Wno-weak-vtables -Wno-shift-sign-overflow -Wno-sign-conversion -Wno-conversion -Wno-exit-time-destructors -Wno-undefined-func-template -Wno-documentation-unknown-command -Wno-missing-variable-declarations -Wno-unused-template -Wno-global-constructors -Wno-c99-extensions -Wno-missing-prototypes -Wno-weak-template-vtables -Wno-zero-length-array -Wno-gnu-anonymous-struct -Wno-nested-anon-types -Wno-double-promotion -Wno-disabled-macro-expansion -Wno-used-but-marked-unused -Wno-vla-extension -Wno-vla -Wno-packed")
# TODO Enable conversion, sign-conversion, double-promotion warnings.
endif ()
if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
if (WEVERYTHING)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-return-std-move-in-c++11")
endif ()
endif ()
if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra-semi-stmt -Wshadow-field -Wstring-plus-int")
if (WEVERYTHING)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
endif ()
endif ()
endif ()
@ -53,7 +87,7 @@ add_subdirectory (src)
set(dbms_headers)
set(dbms_sources)
include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
include(../cmake/dbms_glob_sources.cmake)
add_headers_and_sources(clickhouse_common_io src/Common)
add_headers_and_sources(clickhouse_common_io src/Common/HashTable)
@ -142,10 +176,6 @@ if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELW
PROPERTIES COMPILE_FLAGS -g0)
endif ()
if (NOT ARCH_ARM AND CPUID_LIBRARY)
set (LINK_LIBRARIES_ONLY_ON_X86_64 ${CPUID_LIBRARY})
endif()
target_link_libraries (clickhouse_common_io
PUBLIC
common
@ -153,8 +183,6 @@ target_link_libraries (clickhouse_common_io
string_utils
widechar_width
${LINK_LIBRARIES_ONLY_ON_X86_64}
${LZ4_LIBRARY}
${ZSTD_LIBRARY}
${DOUBLE_CONVERSION_LIBRARIES}
pocoext
PUBLIC
@ -175,8 +203,13 @@ target_link_libraries (clickhouse_common_io
${CMAKE_DL_LIBS}
)
if (NOT ARCH_ARM AND CPUID_LIBRARY)
target_link_libraries (clickhouse_common_io PRIVATE ${CPUID_LIBRARY})
endif()
target_link_libraries (dbms
PRIVATE
clickhouse_compression
clickhouse_parsers
clickhouse_common_config
PUBLIC
@ -254,6 +287,7 @@ endif ()
if (USE_RDKAFKA)
target_link_libraries (dbms PRIVATE ${RDKAFKA_LIBRARY})
target_link_libraries (dbms PRIVATE ${CPPKAFKA_LIBRARY})
if (NOT USE_INTERNAL_RDKAFKA_LIBRARY)
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${RDKAFKA_INCLUDE_DIR})
endif ()
@ -269,13 +303,6 @@ if (USE_HDFS)
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${HDFS3_INCLUDE_DIR})
endif()
if (NOT USE_INTERNAL_LZ4_LIBRARY)
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${LZ4_INCLUDE_DIR})
endif ()
if (NOT USE_INTERNAL_ZSTD_LIBRARY)
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${ZSTD_INCLUDE_DIR})
endif ()
if (USE_JEMALLOC)
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR}) # used in Interpreters/AsynchronousMetrics.cpp
endif ()

20
dbms/cmake/version.cmake
View File

@ -1,11 +1,11 @@
# This strings autochanged from release_lib.sh:
set(VERSION_REVISION 54412 CACHE STRING "") # changed manually for tests
set(VERSION_MAJOR 18 CACHE STRING "")
set(VERSION_MINOR 16 CACHE STRING "")
set(VERSION_PATCH 0 CACHE STRING "")
set(VERSION_GITHASH b9b48c646c253358340bd39fd57754e92f88cd8a CACHE STRING "")
set(VERSION_DESCRIBE v18.16.0-testing CACHE STRING "")
set(VERSION_STRING 18.16.0 CACHE STRING "")
set(VERSION_REVISION 54413)
set(VERSION_MAJOR 19)
set(VERSION_MINOR 1)
set(VERSION_PATCH 0)
set(VERSION_GITHASH 014e344a36bc19a58621e0add379984cf62b9067)
set(VERSION_DESCRIBE v19.1.0-testing)
set(VERSION_STRING 19.1.0)
# end of autochange
set(VERSION_EXTRA "" CACHE STRING "")
@ -19,8 +19,8 @@ if (VERSION_EXTRA)
string(CONCAT VERSION_STRING ${VERSION_STRING} "." ${VERSION_EXTRA})
endif ()
set (VERSION_NAME "${PROJECT_NAME}" CACHE STRING "")
set (VERSION_FULL "${VERSION_NAME} ${VERSION_STRING}" CACHE STRING "")
set (VERSION_SO "${VERSION_STRING}" CACHE STRING "")
set (VERSION_NAME "${PROJECT_NAME}")
set (VERSION_FULL "${VERSION_NAME} ${VERSION_STRING}")
set (VERSION_SO "${VERSION_STRING}")
math (EXPR VERSION_INTEGER "${VERSION_PATCH} + ${VERSION_MINOR}*1000 + ${VERSION_MAJOR}*1000000")

10
dbms/programs/benchmark/Benchmark.cpp
View File

@ -229,7 +229,7 @@ private:
report(info_per_interval);
delay_watch.restart();
}
};
}
return true;
}
@ -324,7 +324,7 @@ private:
double seconds = watch.elapsedSeconds();
std::lock_guard<std::mutex> lock(mutex);
std::lock_guard lock(mutex);
info_per_interval.add(seconds, progress.rows, progress.bytes, info.rows, info.bytes);
info_total.add(seconds, progress.rows, progress.bytes, info.rows, info.bytes);
}
@ -332,7 +332,7 @@ private:
void report(Stats & info)
{
std::lock_guard<std::mutex> lock(mutex);
std::lock_guard lock(mutex);
/// Avoid zeros, nans or exceptions
if (0 == info.queries)
@ -369,7 +369,7 @@ private:
{
WriteBufferFromFile json_out(filename);
std::lock_guard<std::mutex> lock(mutex);
std::lock_guard lock(mutex);
auto print_key_value = [&](auto key, auto value, bool with_comma = true)
{
@ -503,6 +503,4 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
std::cerr << getCurrentExceptionMessage(print_stacktrace, true) << std::endl;
return getCurrentExceptionCode();
}
return 0;
}

2
dbms/programs/client/CMakeLists.txt
View File

@ -1,5 +1,5 @@
add_library (clickhouse-client-lib ${LINK_MODE} Client.cpp)
target_link_libraries (clickhouse-client-lib PRIVATE clickhouse_common_io clickhouse_functions clickhouse_aggregate_functions ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-client-lib PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io ${LINE_EDITING_LIBS} ${Boost_PROGRAM_OPTIONS_LIBRARY})
if (READLINE_INCLUDE_DIR)
target_include_directories (clickhouse-client-lib SYSTEM PRIVATE ${READLINE_INCLUDE_DIR})
endif ()

48
dbms/programs/client/Client.cpp
View File

@ -74,8 +74,8 @@
/// http://en.wikipedia.org/wiki/ANSI_escape_code
/// Similar codes \e[s, \e[u don't work in VT100 and Mosh.
#define SAVE_CURSOR_POSITION "\e7"
#define RESTORE_CURSOR_POSITION "\e8"
#define SAVE_CURSOR_POSITION "\033""7"
#define RESTORE_CURSOR_POSITION "\033""8"
#define CLEAR_TO_END_OF_LINE "\033[K"
@ -554,10 +554,10 @@ private:
void loop()
{
String query;
String prev_query;
String input;
String prev_input;
while (char * line_ = readline(query.empty() ? prompt().c_str() : ":-] "))
while (char * line_ = readline(input.empty() ? prompt().c_str() : ":-] "))
{
String line = line_;
free(line_);
@ -577,17 +577,17 @@ private:
if (ends_with_backslash)
line = line.substr(0, ws - 1);
query += line;
input += line;
if (!ends_with_backslash && (ends_with_semicolon || has_vertical_output_suffix || (!config().has("multiline") && !hasDataInSTDIN())))
{
if (query != prev_query)
if (input != prev_input)
{
/// Replace line breaks with spaces to prevent the following problem.
/// Every line of multi-line query is saved to history file as a separate line.
/// If the user restarts the client then after pressing the "up" button
/// every line of the query will be displayed separately.
std::string logged_query = query;
std::string logged_query = input;
std::replace(logged_query.begin(), logged_query.end(), '\n', ' ');
add_history(logged_query.c_str());
@ -596,18 +596,18 @@ private:
throwFromErrno("Cannot append history to file " + history_file, ErrorCodes::CANNOT_APPEND_HISTORY);
#endif
prev_query = query;
prev_input = input;
}
if (has_vertical_output_suffix)
query = query.substr(0, query.length() - 2);
input = input.substr(0, input.length() - 2);
try
{
/// Determine the terminal size.
ioctl(0, TIOCGWINSZ, &terminal_size);
if (!process(query))
if (!process(input))
break;
}
catch (const Exception & e)
@ -633,11 +633,11 @@ private:
connect();
}
query = "";
input = "";
}
else
{
query += '\n';
input += '\n';
}
}
}
@ -666,11 +666,15 @@ private:
const bool test_mode = config().has("testmode");
if (config().has("multiquery"))
{
{ /// disable logs if expects errors
TestHint test_hint(test_mode, text);
if (test_hint.clientError() || test_hint.serverError())
process("SET send_logs_level = 'none'");
}
/// Several queries separated by ';'.
/// INSERT data is ended by the end of line, not ';'.
String query;
const char * begin = text.data();
const char * end = begin + text.size();
@ -702,19 +706,19 @@ private:
insert->end = pos;
}
query = text.substr(begin - text.data(), pos - begin);
String str = text.substr(begin - text.data(), pos - begin);
begin = pos;
while (isWhitespaceASCII(*begin) || *begin == ';')
++begin;
TestHint test_hint(test_mode, query);
TestHint test_hint(test_mode, str);
expected_client_error = test_hint.clientError();
expected_server_error = test_hint.serverError();
try
{
if (!processSingleQuery(query, ast) && !ignore_error)
if (!processSingleQuery(str, ast) && !ignore_error)
return false;
}
catch (...)
@ -722,7 +726,7 @@ private:
last_exception = std::make_unique<Exception>(getCurrentExceptionMessage(true), getCurrentExceptionCode());
actual_client_error = last_exception->code();
if (!ignore_error && (!actual_client_error || actual_client_error != expected_client_error))
std::cerr << "Error on processing query: " << query << std::endl << last_exception->message();
std::cerr << "Error on processing query: " << str << std::endl << last_exception->message();
got_exception = true;
}
@ -898,8 +902,6 @@ private:
ParserQuery parser(end, true);
ASTPtr res;
const auto ignore_error = config().getBool("ignore-error", false);
if (is_interactive || ignore_error)
{
String message;
@ -1610,10 +1612,10 @@ public:
for (size_t i = 0; i < external_tables_arguments.size(); ++i)
{
/// Parse commandline options related to external tables.
po::parsed_options parsed = po::command_line_parser(
po::parsed_options parsed_tables = po::command_line_parser(
external_tables_arguments[i].size(), external_tables_arguments[i].data()).options(external_description).run();
po::variables_map external_options;
po::store(parsed, external_options);
po::store(parsed_tables, external_options);
try
{

2
dbms/programs/client/Suggest.h
View File

@ -56,7 +56,7 @@ private:
{
std::string prefix_str(prefix);
std::tie(pos, end) = std::equal_range(words.begin(), words.end(), prefix_str,
[prefix_length](const std::string & s, const std::string & prefix) { return strncmp(s.c_str(), prefix.c_str(), prefix_length) < 0; });
[prefix_length](const std::string & s, const std::string & prefix_searched) { return strncmp(s.c_str(), prefix_searched.c_str(), prefix_length) < 0; });
}
/// Iterates through matched range.

33
dbms/programs/client/TestHint.h
View File

@ -5,6 +5,7 @@
#include <iostream>
#include <Core/Types.h>
#include <Common/Exception.h>
#include <Parsers/Lexer.h>
namespace DB
@ -27,25 +28,27 @@ public:
if (!enabled_)
return;
/// TODO: This is absolutely wrong. Fragment may be contained inside string literal.
size_t pos = query.find("--");
Lexer lexer(query.data(), query.data() + query.size());
if (pos != String::npos && query.find("--", pos + 2) != String::npos)
return; /// It's not last comment. Hint belongs to commented query. /// TODO Absolutely wrong: there maybe the following comment for the next query.
if (pos != String::npos)
for (Token token = lexer.nextToken(); !token.isEnd(); token = lexer.nextToken())
{
/// TODO: This is also wrong. Comment may already have ended by line break.
pos = query.find('{', pos + 2);
if (pos != String::npos)
if (token.type == TokenType::Comment)
{
String hint = query.substr(pos + 1);
String comment(token.begin, token.begin + token.size());
/// TODO: And this is wrong for the same reason.
pos = hint.find('}');
hint.resize(pos);
parse(hint);
if (!comment.empty())
{
size_t pos_start = comment.find('{', 0);
if (pos_start != String::npos)
{
size_t pos_end = comment.find('}', pos_start);
if (pos_end != String::npos)
{
String hint(comment.begin() + pos_start + 1, comment.begin() + pos_end);
parse(hint);
}
}
}
}
}
}

2
dbms/programs/compressor/CMakeLists.txt
View File

@ -1,5 +1,5 @@
add_library (clickhouse-compressor-lib ${LINK_MODE} Compressor.cpp)
target_link_libraries (clickhouse-compressor-lib PRIVATE clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
target_link_libraries (clickhouse-compressor-lib PRIVATE clickhouse_compression clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
if (CLICKHOUSE_SPLIT_BINARY)
# Also in utils

53
dbms/programs/compressor/Compressor.cpp
View File

@ -1,21 +1,23 @@
#include <iostream>
#include <optional>
#include <boost/program_options.hpp>
#include <Common/Exception.h>
#include <IO/WriteBufferFromFileDescriptor.h>
#include <IO/ReadBufferFromFileDescriptor.h>
#include <IO/CompressedWriteBuffer.h>
#include <IO/CompressedReadBuffer.h>
#include <Compression/CompressedWriteBuffer.h>
#include <Compression/CompressedReadBuffer.h>
#include <IO/WriteHelpers.h>
#include <IO/copyData.h>
#include <Compression/CompressionFactory.h>
namespace DB
{
namespace ErrorCodes
{
extern const int TOO_LARGE_SIZE_COMPRESSED;
extern const int BAD_ARGUMENTS;
}
}
@ -61,7 +63,8 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
("block-size,b", boost::program_options::value<unsigned>()->default_value(DBMS_DEFAULT_BUFFER_SIZE), "compress in blocks of specified size")
("hc", "use LZ4HC instead of LZ4")
("zstd", "use ZSTD instead of LZ4")
("level", boost::program_options::value<int>(), "compression level")
("codec", boost::program_options::value<std::vector<std::string>>()->multitoken(), "use codecs combination instead of LZ4")
("level", boost::program_options::value<std::vector<int>>()->multitoken(), "compression levels for codecs specified via --codec")
("none", "use no compression instead of LZ4")
("stat", "print block statistics of compressed data")
;
@ -84,19 +87,45 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
bool stat_mode = options.count("stat");
bool use_none = options.count("none");
unsigned block_size = options["block-size"].as<unsigned>();
std::vector<std::string> codecs;
if (options.count("codec"))
codecs = options["codec"].as<std::vector<std::string>>();
DB::CompressionMethod method = DB::CompressionMethod::LZ4;
if ((use_lz4hc || use_zstd || use_none) && !codecs.empty())
throw DB::Exception("Wrong options, codec flags like --zstd and --codec options are mutually exclusive", DB::ErrorCodes::BAD_ARGUMENTS);
std::string method_family = "LZ4";
if (use_lz4hc)
method = DB::CompressionMethod::LZ4HC;
method_family = "LZ4HC";
else if (use_zstd)
method = DB::CompressionMethod::ZSTD;
method_family = "ZSTD";
else if (use_none)
method = DB::CompressionMethod::NONE;
method_family = "NONE";
std::vector<int> levels;
if (options.count("level"))
levels = options["level"].as<std::vector<int>>();
DB::CompressionCodecPtr codec;
if (!codecs.empty())
{
if (levels.size() > codecs.size())
throw DB::Exception("Specified more levels than codecs", DB::ErrorCodes::BAD_ARGUMENTS);
std::vector<DB::CodecNameWithLevel> codec_names;
for (size_t i = 0; i < codecs.size(); ++i)
{
if (i < levels.size())
codec_names.emplace_back(codecs[i], levels[i]);
else
codec_names.emplace_back(codecs[i], std::nullopt);
}
codec = DB::CompressionCodecFactory::instance().get(codec_names);
}
else
codec = DB::CompressionCodecFactory::instance().get(method_family, levels.empty() ? std::nullopt : std::optional<int>(levels.back()));
DB::CompressionSettings settings(method, options.count("level")
? options["level"].as<int>()
: DB::CompressionSettings::getDefaultLevel(method));
DB::ReadBufferFromFileDescriptor rb(STDIN_FILENO);
DB::WriteBufferFromFileDescriptor wb(STDOUT_FILENO);
@ -115,7 +144,7 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
else
{
/// Compression
DB::CompressedWriteBuffer to(wb, settings, block_size);
DB::CompressedWriteBuffer to(wb, codec, block_size);
DB::copyData(rb, to);
}
}

27
dbms/programs/compressor/README.md Normal file
View File

@ -0,0 +1,27 @@
## ClickHouse compressor
Simple program for data compression and decompression.
### Examples
Compress data with LZ4:
```
$ ./clickhouse-compressor < input_file > output_file
```
Decompress data from LZ4 format:
```
$ ./clickhouse-compressor --decompress < input_file > output_file
```
Compress data with ZSTD at level 5:
```
$ ./clickhouse-compressor --codec ZSTD --level 5 < input_file > output_file
```
Compress data with ZSTD level 10, LZ4HC level 7 and LZ4.
```
$ ./clickhouse-compressor --codec ZSTD --level 5 --codec LZ4HC --level 7 --codec LZ4 < input_file > output_file
```

62
dbms/programs/copier/ClusterCopier.cpp
View File

@ -243,7 +243,7 @@ struct ClusterPartition
UInt64 rows_copied = 0;
UInt64 blocks_copied = 0;
size_t total_tries = 0;
UInt64 total_tries = 0;
};
@ -340,7 +340,7 @@ struct TaskCluster
String default_local_database;
/// Limits number of simultaneous workers
size_t max_workers = 0;
UInt64 max_workers = 0;
/// Base settings for pull and push
Settings settings_common;
@ -773,11 +773,11 @@ public:
}
template <typename T>
decltype(auto) retry(T && func, size_t max_tries = 100)
decltype(auto) retry(T && func, UInt64 max_tries = 100)
{
std::exception_ptr exception;
for (size_t try_number = 1; try_number <= max_tries; ++try_number)
for (UInt64 try_number = 1; try_number <= max_tries; ++try_number)
{
try
{
@ -880,7 +880,7 @@ public:
}
/// Compute set of partitions, assume set of partitions aren't changed during the processing
void discoverTablePartitions(TaskTable & task_table, size_t num_threads = 0)
void discoverTablePartitions(TaskTable & task_table, UInt64 num_threads = 0)
{
/// Fetch partitions list from a shard
{
@ -985,7 +985,7 @@ public:
/// Retry table processing
bool table_is_done = false;
for (size_t num_table_tries = 0; num_table_tries < max_table_tries; ++num_table_tries)
for (UInt64 num_table_tries = 0; num_table_tries < max_table_tries; ++num_table_tries)
{
if (tryProcessTable(task_table))
{
@ -1044,7 +1044,7 @@ protected:
String workers_path = getWorkersPath();
String current_worker_path = getCurrentWorkerNodePath();
size_t num_bad_version_errors = 0;
UInt64 num_bad_version_errors = 0;
while (true)
{
@ -1055,7 +1055,7 @@ protected:
auto version = stat.version;
zookeeper->get(workers_path, &stat);
if (static_cast<size_t>(stat.numChildren) >= task_cluster->max_workers)
if (static_cast<UInt64>(stat.numChildren) >= task_cluster->max_workers)
{
LOG_DEBUG(log, "Too many workers (" << stat.numChildren << ", maximum " << task_cluster->max_workers << ")"
<< ". Postpone processing " << description);
@ -1163,7 +1163,7 @@ protected:
}
// If all task is finished and zxid is not changed then partition could not become dirty again
for (size_t shard_num = 0; shard_num < status_paths.size(); ++shard_num)
for (UInt64 shard_num = 0; shard_num < status_paths.size(); ++shard_num)
{
if (zxid1[shard_num] != zxid2[shard_num])
{
@ -1280,7 +1280,7 @@ protected:
LOG_DEBUG(log, "Execute distributed DROP PARTITION: " << query);
/// Limit number of max executing replicas to 1
size_t num_shards = executeQueryOnCluster(cluster_push, query, nullptr, &settings_push, PoolMode::GET_ONE, 1);
UInt64 num_shards = executeQueryOnCluster(cluster_push, query, nullptr, &settings_push, PoolMode::GET_ONE, 1);
if (num_shards < cluster_push->getShardCount())
{
@ -1299,8 +1299,8 @@ protected:
}
static constexpr size_t max_table_tries = 1000;
static constexpr size_t max_shard_partition_tries = 600;
static constexpr UInt64 max_table_tries = 1000;
static constexpr UInt64 max_shard_partition_tries = 600;
bool tryProcessTable(TaskTable & task_table)
{
@ -1317,7 +1317,7 @@ protected:
Stopwatch watch;
TasksShard expected_shards;
size_t num_failed_shards = 0;
UInt64 num_failed_shards = 0;
++cluster_partition.total_tries;
@ -1368,7 +1368,7 @@ protected:
bool is_unprioritized_task = !previous_shard_is_instantly_finished && shard->priority.is_remote;
PartitionTaskStatus task_status = PartitionTaskStatus::Error;
bool was_error = false;
for (size_t try_num = 0; try_num < max_shard_partition_tries; ++try_num)
for (UInt64 try_num = 0; try_num < max_shard_partition_tries; ++try_num)
{
task_status = tryProcessPartitionTask(partition, is_unprioritized_task);
@ -1434,8 +1434,8 @@ protected:
}
}
size_t required_partitions = task_table.cluster_partitions.size();
size_t finished_partitions = task_table.finished_cluster_partitions.size();
UInt64 required_partitions = task_table.cluster_partitions.size();
UInt64 finished_partitions = task_table.finished_cluster_partitions.size();
bool table_is_done = finished_partitions >= required_partitions;
if (!table_is_done)
@ -1645,7 +1645,7 @@ protected:
String query = queryToString(create_query_push_ast);
LOG_DEBUG(log, "Create destination tables. Query: " << query);
size_t shards = executeQueryOnCluster(task_table.cluster_push, query, create_query_push_ast, &task_cluster->settings_push,
UInt64 shards = executeQueryOnCluster(task_table.cluster_push, query, create_query_push_ast, &task_cluster->settings_push,
PoolMode::GET_MANY);
LOG_DEBUG(log, "Destination tables " << getDatabaseDotTable(task_table.table_push) << " have been created on " << shards
<< " shards of " << task_table.cluster_push->getShardCount());
@ -1699,7 +1699,7 @@ protected:
std::future<Coordination::ExistsResponse> future_is_dirty_checker;
Stopwatch watch(CLOCK_MONOTONIC_COARSE);
constexpr size_t check_period_milliseconds = 500;
constexpr UInt64 check_period_milliseconds = 500;
/// Will asynchronously check that ZooKeeper connection and is_dirty flag appearing while copy data
auto cancel_check = [&] ()
@ -1917,16 +1917,16 @@ protected:
/** Executes simple query (without output streams, for example DDL queries) on each shard of the cluster
* Returns number of shards for which at least one replica executed query successfully
*/
size_t executeQueryOnCluster(
UInt64 executeQueryOnCluster(
const ClusterPtr & cluster,
const String & query,
const ASTPtr & query_ast_ = nullptr,
const Settings * settings = nullptr,
PoolMode pool_mode = PoolMode::GET_ALL,
size_t max_successful_executions_per_shard = 0) const
UInt64 max_successful_executions_per_shard = 0) const
{
auto num_shards = cluster->getShardsInfo().size();
std::vector<size_t> per_shard_num_successful_replicas(num_shards, 0);
std::vector<UInt64> per_shard_num_successful_replicas(num_shards, 0);
ASTPtr query_ast;
if (query_ast_ == nullptr)
@ -1939,10 +1939,10 @@ protected:
/// We need to execute query on one replica at least
auto do_for_shard = [&] (size_t shard_index)
auto do_for_shard = [&] (UInt64 shard_index)
{
const Cluster::ShardInfo & shard = cluster->getShardsInfo().at(shard_index);
size_t & num_successful_executions = per_shard_num_successful_replicas.at(shard_index);
UInt64 & num_successful_executions = per_shard_num_successful_replicas.at(shard_index);
num_successful_executions = 0;
auto increment_and_check_exit = [&] ()
@ -1951,12 +1951,12 @@ protected:
return max_successful_executions_per_shard && num_successful_executions >= max_successful_executions_per_shard;
};
size_t num_replicas = cluster->getShardsAddresses().at(shard_index).size();
size_t num_local_replicas = shard.getLocalNodeCount();
size_t num_remote_replicas = num_replicas - num_local_replicas;
UInt64 num_replicas = cluster->getShardsAddresses().at(shard_index).size();
UInt64 num_local_replicas = shard.getLocalNodeCount();
UInt64 num_remote_replicas = num_replicas - num_local_replicas;
/// In that case we don't have local replicas, but do it just in case
for (size_t i = 0; i < num_local_replicas; ++i)
for (UInt64 i = 0; i < num_local_replicas; ++i)
{
auto interpreter = InterpreterFactory::get(query_ast, context);
interpreter->execute();
@ -1997,16 +1997,16 @@ protected:
};
{
ThreadPool thread_pool(std::min(num_shards, getNumberOfPhysicalCPUCores()));
ThreadPool thread_pool(std::min<UInt64>(num_shards, getNumberOfPhysicalCPUCores()));
for (size_t shard_index = 0; shard_index < num_shards; ++shard_index)
for (UInt64 shard_index = 0; shard_index < num_shards; ++shard_index)
thread_pool.schedule([=] { do_for_shard(shard_index); });
thread_pool.wait();
}
size_t successful_shards = 0;
for (size_t num_replicas : per_shard_num_successful_replicas)
UInt64 successful_shards = 0;
for (UInt64 num_replicas : per_shard_num_successful_replicas)
successful_shards += (num_replicas > 0);
return successful_shards;

14
dbms/programs/local/LocalServer.cpp
View File

@ -66,11 +66,11 @@ void LocalServer::initialize(Poco::Util::Application & self)
}
}
void LocalServer::applyCmdSettings(Context & context)
void LocalServer::applyCmdSettings()
{
#define EXTRACT_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) \
if (cmd_settings.NAME.changed) \
context.getSettingsRef().NAME = cmd_settings.NAME;
context->getSettingsRef().NAME = cmd_settings.NAME;
APPLY_FOR_SETTINGS(EXTRACT_SETTING)
#undef EXTRACT_SETTING
}
@ -179,7 +179,7 @@ try
std::string default_database = config().getString("default_database", "_local");
context->addDatabase(default_database, std::make_shared<DatabaseMemory>(default_database));
context->setCurrentDatabase(default_database);
applyCmdOptions(*context);
applyCmdOptions();
if (!context->getPath().empty())
{
@ -274,7 +274,7 @@ void LocalServer::processQueries()
context->setUser("default", "", Poco::Net::SocketAddress{}, "");
context->setCurrentQueryId("");
applyCmdSettings(*context);
applyCmdSettings();
/// Use the same query_id (and thread group) for all queries
CurrentThread::QueryScope query_scope_holder(*context);
@ -494,10 +494,10 @@ void LocalServer::init(int argc, char ** argv)
config().setBool("ignore-error", true);
}
void LocalServer::applyCmdOptions(Context & context)
void LocalServer::applyCmdOptions()
{
context.setDefaultFormat(config().getString("output-format", config().getString("format", "TSV")));
applyCmdSettings(context);
context->setDefaultFormat(config().getString("output-format", config().getString("format", "TSV")));
applyCmdSettings();
}
}

4
dbms/programs/local/LocalServer.h
View File

@ -34,8 +34,8 @@ private:
std::string getInitialCreateTableQuery();
void tryInitPath();
void applyCmdOptions(Context & context);
void applyCmdSettings(Context & context);
void applyCmdOptions();
void applyCmdSettings();
void attachSystemTables();
void processQueries();
void setupUsers();

8
dbms/programs/obfuscator/Obfuscator.cpp
View File

@ -123,7 +123,7 @@ UInt64 hash(Ts... xs)
UInt64 maskBits(UInt64 x, size_t num_bits)
{
return x & ((1 << num_bits) - 1);
return x & ((1ULL << num_bits) - 1);
}
@ -149,7 +149,7 @@ UInt64 feistelNetwork(UInt64 x, size_t num_bits, UInt64 seed, size_t num_rounds
UInt64 bits = maskBits(x, num_bits);
for (size_t i = 0; i < num_rounds; ++i)
bits = feistelRound(bits, num_bits, seed, i);
return (x & ~((1 << num_bits) - 1)) ^ bits;
return (x & ~((1ULL << num_bits) - 1)) ^ bits;
}
@ -317,8 +317,8 @@ void transformFixedString(const UInt8 * src, UInt8 * dst, size_t size, UInt64 se
if (size >= 16)
{
char * dst = reinterpret_cast<char *>(std::min(pos, end - 16));
hash.get128(dst);
char * hash_dst = reinterpret_cast<char *>(std::min(pos, end - 16));
hash.get128(hash_dst);
}
else
{

2
dbms/programs/odbc-bridge/CMakeLists.txt
View File

@ -9,7 +9,7 @@ add_library (clickhouse-odbc-bridge-lib ${LINK_MODE}
validateODBCConnectionString.cpp
)
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE clickhouse_common_io daemon dbms)
target_link_libraries (clickhouse-odbc-bridge-lib PRIVATE clickhouse_dictionaries daemon dbms clickhouse_common_io)
target_include_directories (clickhouse-odbc-bridge-lib PUBLIC ${ClickHouse_SOURCE_DIR}/libs/libdaemon/include)
if (USE_POCO_SQLODBC)

2
dbms/programs/odbc-bridge/getIdentifierQuote.h
View File

@ -19,4 +19,4 @@ namespace DB
std::string getIdentifierQuote(SQLHDBC hdbc);
}
#endif
#endif

54
dbms/programs/performance-test/PerformanceTest.cpp
View File

@ -112,7 +112,8 @@ public:
{
return asString(padding);
}
String asString(size_t padding) const
String asString(size_t cur_padding) const
{
String repr = "{";
@ -121,10 +122,10 @@ public:
if (it != content.begin())
repr += ',';
/// construct "key": "value" string with padding
repr += "\n" + pad(padding) + '"' + it->first + '"' + ": " + it->second;
repr += "\n" + pad(cur_padding) + '"' + it->first + '"' + ": " + it->second;
}
repr += "\n" + pad(padding - 1) + '}';
repr += "\n" + pad(cur_padding - 1) + '}';
return repr;
}
};
@ -762,13 +763,13 @@ private:
return true;
}
void processTestsConfigurations(const Paths & input_files)
void processTestsConfigurations(const Paths & paths)
{
tests_configurations.resize(input_files.size());
tests_configurations.resize(paths.size());
for (size_t i = 0; i != input_files.size(); ++i)
for (size_t i = 0; i != paths.size(); ++i)
{
const String path = input_files[i];
const String path = paths[i];
tests_configurations[i] = XMLConfigurationPtr(new XMLConfiguration(path));
}
@ -881,8 +882,6 @@ private:
}
}
Query query;
if (!test_config->has("query") && !test_config->has("query_file"))
{
throw DB::Exception("Missing query fields in test's config: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
@ -907,6 +906,7 @@ private:
bool tsv = fs::path(filename).extension().string() == ".tsv";
ReadBufferFromFile query_file(filename);
Query query;
if (tsv)
{
@ -1024,7 +1024,7 @@ private:
}
if (lite_output)
return minOutput(main_metric);
return minOutput();
else
return constructTotalInfo(metrics);
}
@ -1053,11 +1053,8 @@ private:
void runQueries(const QueriesWithIndexes & queries_with_indexes)
{
for (const std::pair<Query, const size_t> & query_and_index : queries_with_indexes)
for (const auto & [query, run_index] : queries_with_indexes)
{
Query query = query_and_index.first;
const size_t run_index = query_and_index.second;
TestStopConditions & stop_conditions = stop_conditions_by_run[run_index];
Stats & statistics = statistics_by_run[run_index];
@ -1139,7 +1136,7 @@ private:
}
}
void constructSubstitutions(ConfigurationPtr & substitutions_view, StringToVector & substitutions)
void constructSubstitutions(ConfigurationPtr & substitutions_view, StringToVector & out_substitutions)
{
Keys xml_substitutions;
substitutions_view->keys(xml_substitutions);
@ -1157,21 +1154,16 @@ private:
for (size_t j = 0; j != xml_values.size(); ++j)
{
substitutions[name].push_back(xml_substitution->getString("values.value[" + std::to_string(j) + "]"));
out_substitutions[name].push_back(xml_substitution->getString("values.value[" + std::to_string(j) + "]"));
}
}
}
std::vector<String> formatQueries(const String & query, StringToVector substitutions)
std::vector<String> formatQueries(const String & query, StringToVector substitutions_to_generate)
{
std::vector<String> queries;
StringToVector::iterator substitutions_first = substitutions.begin();
StringToVector::iterator substitutions_last = substitutions.end();
runThroughAllOptionsAndPush(substitutions_first, substitutions_last, query, queries);
return queries;
std::vector<String> queries_res;
runThroughAllOptionsAndPush(substitutions_to_generate.begin(), substitutions_to_generate.end(), query, queries_res);
return queries_res;
}
/// Recursive method which goes through all substitution blocks in xml
@ -1179,11 +1171,11 @@ private:
void runThroughAllOptionsAndPush(StringToVector::iterator substitutions_left,
StringToVector::iterator substitutions_right,
const String & template_query,
std::vector<String> & queries)
std::vector<String> & out_queries)
{
if (substitutions_left == substitutions_right)
{
queries.push_back(template_query); /// completely substituted query
out_queries.push_back(template_query); /// completely substituted query
return;
}
@ -1191,7 +1183,7 @@ private:
if (template_query.find(substitution_mask) == String::npos) /// nothing to substitute here
{
runThroughAllOptionsAndPush(std::next(substitutions_left), substitutions_right, template_query, queries);
runThroughAllOptionsAndPush(std::next(substitutions_left), substitutions_right, template_query, out_queries);
return;
}
@ -1209,7 +1201,7 @@ private:
query.replace(substr_pos, substitution_mask.length(), value);
}
runThroughAllOptionsAndPush(std::next(substitutions_left), substitutions_right, query, queries);
runThroughAllOptionsAndPush(std::next(substitutions_left), substitutions_right, query, out_queries);
}
}
@ -1343,7 +1335,7 @@ public:
return json_output.asString();
}
String minOutput(const String & main_metric)
String minOutput()
{
String output;
@ -1465,7 +1457,7 @@ try
input_files = options["input-files"].as<Strings>();
Strings collected_files;
for (const String filename : input_files)
for (const String & filename : input_files)
{
fs::path file(filename);

4
dbms/programs/server/HTTPHandler.cpp
View File

@ -19,8 +19,8 @@
#include <IO/ZlibInflatingReadBuffer.h>
#include <IO/ReadBufferFromString.h>
#include <IO/ConcatReadBuffer.h>
#include <IO/CompressedReadBuffer.h>
#include <IO/CompressedWriteBuffer.h>
#include <Compression/CompressedReadBuffer.h>
#include <Compression/CompressedWriteBuffer.h>
#include <IO/WriteBufferFromString.h>
#include <IO/WriteBufferFromHTTPServerResponse.h>
#include <IO/WriteBufferFromFile.h>

24
dbms/programs/server/HTTPHandlerFactory.h
View File

@ -31,20 +31,16 @@ public:
Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest & request) override
{
LOG_TRACE(log,
"HTTP Request for " << name << ". "
<< "Method: "
<< request.getMethod()
<< ", Address: "
<< request.clientAddress().toString()
<< ", User-Agent: "
<< (request.has("User-Agent") ? request.get("User-Agent") : "none")
<< (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : (""))
#if !NDEBUG
<< ", Content Type: " << request.getContentType()
<< ", Transfer Encoding: " << request.getTransferEncoding()
#endif
);
LOG_TRACE(log, "HTTP Request for " << name << ". "
<< "Method: "
<< request.getMethod()
<< ", Address: "
<< request.clientAddress().toString()
<< ", User-Agent: "
<< (request.has("User-Agent") ? request.get("User-Agent") : "none")
<< (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : (""))
<< ", Content Type: " << request.getContentType()
<< ", Transfer Encoding: " << request.getTransferEncoding());
const auto & uri = request.getURI();

2
dbms/programs/server/InterserverIOHTTPHandler.cpp
View File

@ -6,7 +6,7 @@
#include <Common/HTMLForm.h>
#include <Common/setThreadName.h>
#include <IO/CompressedWriteBuffer.h>
#include <Compression/CompressedWriteBuffer.h>
#include <IO/ReadBufferFromIStream.h>
#include <IO/WriteBufferFromHTTPServerResponse.h>
#include <Interpreters/InterserverIOHandler.h>

4
dbms/programs/server/MetricsTransmitter.cpp
View File

@ -21,7 +21,7 @@ MetricsTransmitter::~MetricsTransmitter()
try
{
{
std::lock_guard<std::mutex> lock{mutex};
std::lock_guard lock{mutex};
quit = true;
}
@ -56,7 +56,7 @@ void MetricsTransmitter::run()
std::vector<ProfileEvents::Count> prev_counters(ProfileEvents::end());
std::unique_lock<std::mutex> lock{mutex};
std::unique_lock lock{mutex};
while (true)
{

2
dbms/programs/server/Server.cpp
View File

@ -411,7 +411,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
global_context->setMarkCache(mark_cache_size);
#if USE_EMBEDDED_COMPILER
size_t compiled_expression_cache_size = config().getUInt64("compiled_expression_cache_size", std::numeric_limits<UInt64>::max());
size_t compiled_expression_cache_size = config().getUInt64("compiled_expression_cache_size", 500);
if (compiled_expression_cache_size)
global_context->setCompiledExpressionCache(compiled_expression_cache_size);
#endif

15
dbms/programs/server/TCPHandler.cpp
View File

@ -12,13 +12,12 @@
#include <Common/setThreadName.h>
#include <Common/config_version.h>
#include <IO/Progress.h>
#include <IO/CompressedReadBuffer.h>
#include <IO/CompressedWriteBuffer.h>
#include <Compression/CompressedReadBuffer.h>
#include <Compression/CompressedWriteBuffer.h>
#include <IO/ReadBufferFromPocoSocket.h>
#include <IO/WriteBufferFromPocoSocket.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <IO/CompressionSettings.h>
#include <IO/copyData.h>
#include <DataStreams/AsynchronousBlockInputStream.h>
#include <DataStreams/NativeBlockInputStream.h>
@ -32,6 +31,7 @@
#include <Core/ExternalTable.h>
#include <Storages/ColumnDefault.h>
#include <DataTypes/DataTypeLowCardinality.h>
#include <Compression/CompressionFactory.h>
#include "TCPHandler.h"
@ -728,7 +728,7 @@ bool TCPHandler::receiveData()
{
NamesAndTypesList columns = block.getNamesAndTypesList();
storage = StorageMemory::create(external_table_name,
ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, ColumnComments{}});
ColumnsDescription{columns, NamesAndTypesList{}, NamesAndTypesList{}, ColumnDefaults{}, ColumnComments{}, ColumnCodecs{}});
storage->startup();
query_context.addExternalTable(external_table_name, storage);
}
@ -772,9 +772,14 @@ void TCPHandler::initBlockOutput(const Block & block)
{
if (!state.maybe_compressed_out)
{
std::string method = query_context.getSettingsRef().network_compression_method;
std::optional<int> level;
if (method == "ZSTD")
level = query_context.getSettingsRef().network_zstd_compression_level;
if (state.compression == Protocol::Compression::Enable)
state.maybe_compressed_out = std::make_shared<CompressedWriteBuffer>(
*out, CompressionSettings(query_context.getSettingsRef()));
*out, CompressionCodecFactory::instance().get(method, level));
else
state.maybe_compressed_out = out;
}

2
dbms/src/AggregateFunctions/AggregateFunctionAvg.h
View File

@ -25,7 +25,7 @@ struct AggregateFunctionAvgData
UInt64 count = 0;
template <typename ResultT>
ResultT result() const
ResultT NO_SANITIZE_UNDEFINED result() const
{
if constexpr (std::is_floating_point_v<ResultT>)
if constexpr (std::numeric_limits<ResultT>::is_iec559)

2
dbms/src/AggregateFunctions/AggregateFunctionBoundingRatio.h
View File

@ -96,7 +96,7 @@ private:
/** Calculates the slope of a line between leftmost and rightmost data points.
* (y2 - y1) / (x2 - x1)
*/
Float64 getBoundingRatio(const AggregateFunctionBoundingRatioData & data) const
Float64 NO_SANITIZE_UNDEFINED getBoundingRatio(const AggregateFunctionBoundingRatioData & data) const
{
if (data.empty)
return std::numeric_limits<Float64>::quiet_NaN();

6
dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp
View File

@ -102,16 +102,14 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl(
{
String name = getAliasToOrName(name_param);
/// Find by exact match.
auto it = aggregate_functions.find(name);
if (it != aggregate_functions.end())
if (auto it = aggregate_functions.find(name); it != aggregate_functions.end())
return it->second(name, argument_types, parameters);
/// Find by case-insensitive name.
/// Combinators cannot apply for case insensitive (SQL-style) aggregate function names. Only for native names.
if (recursion_level == 0)
{
auto it = case_insensitive_aggregate_functions.find(Poco::toLower(name));
if (it != case_insensitive_aggregate_functions.end())
if (auto it = case_insensitive_aggregate_functions.find(Poco::toLower(name)); it != case_insensitive_aggregate_functions.end())
return it->second(name, argument_types, parameters);
}

2
dbms/src/AggregateFunctions/AggregateFunctionForEach.h
View File

@ -231,7 +231,7 @@ public:
nested_state += nested_size_of_data;
}
offsets_to.push_back(offsets_to.empty() ? state.dynamic_array_size : offsets_to.back() + state.dynamic_array_size);
offsets_to.push_back(offsets_to.back() + state.dynamic_array_size);
}
bool allocatesMemoryInArena() const override

13
dbms/src/AggregateFunctions/AggregateFunctionGroupArray.h
View File

@ -77,12 +77,14 @@ public:
if (!limit_num_elems)
{
cur_elems.value.insert(rhs_elems.value.begin(), rhs_elems.value.end(), arena);
if (rhs_elems.value.size())
cur_elems.value.insert(rhs_elems.value.begin(), rhs_elems.value.end(), arena);
}
else
{
UInt64 elems_to_insert = std::min(static_cast<size_t>(max_elems) - cur_elems.value.size(), rhs_elems.value.size());
cur_elems.value.insert(rhs_elems.value.begin(), rhs_elems.value.begin() + elems_to_insert, arena);
if (elems_to_insert)
cur_elems.value.insert(rhs_elems.value.begin(), rhs_elems.value.begin() + elems_to_insert, arena);
}
}
@ -121,8 +123,11 @@ public:
offsets_to.push_back(offsets_to.back() + size);
typename ColumnVector<T>::Container & data_to = static_cast<ColumnVector<T> &>(arr_to.getData()).getData();
data_to.insert(this->data(place).value.begin(), this->data(place).value.end());
if (size)
{
typename ColumnVector<T>::Container & data_to = static_cast<ColumnVector<T> &>(arr_to.getData()).getData();
data_to.insert(this->data(place).value.begin(), this->data(place).value.end());
}
}
bool allocatesMemoryInArena() const override

2
dbms/src/AggregateFunctions/AggregateFunctionGroupArrayInsertAt.h
View File

@ -203,7 +203,7 @@ public:
for (size_t i = arr.size(); i < result_array_size; ++i)
to_data.insert(default_value);
to_offsets.push_back((to_offsets.empty() ? 0 : to_offsets.back()) + result_array_size);
to_offsets.push_back(to_offsets.back() + result_array_size);
}
const char * getHeaderFilePath() const override { return __FILE__; }

2
dbms/src/AggregateFunctions/AggregateFunctionGroupUniqArray.h
View File

@ -195,7 +195,7 @@ public:
for (auto & rhs_elem : rhs_set)
{
cur_set.emplace(rhs_elem, it, inserted);
if (inserted)
if (inserted && it->size)
it->data = arena->insert(it->data, it->size);
}
}

2
dbms/src/AggregateFunctions/AggregateFunctionHistogram.cpp
View File

@ -45,8 +45,6 @@ AggregateFunctionPtr createAggregateFunctionHistogram(const std::string & name,
throw Exception("Illegal type " + arguments[0]->getName() + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
return res;
return nullptr;
}
}

8
dbms/src/AggregateFunctions/AggregateFunctionNull.h
View File

@ -187,8 +187,8 @@ template <bool result_is_nullable>
class AggregateFunctionNullUnary final : public AggregateFunctionNullBase<result_is_nullable, AggregateFunctionNullUnary<result_is_nullable>>
{
public:
AggregateFunctionNullUnary(AggregateFunctionPtr nested_function)
: AggregateFunctionNullBase<result_is_nullable, AggregateFunctionNullUnary<result_is_nullable>>(nested_function)
AggregateFunctionNullUnary(AggregateFunctionPtr nested_function_)
: AggregateFunctionNullBase<result_is_nullable, AggregateFunctionNullUnary<result_is_nullable>>(std::move(nested_function_))
{
}
@ -209,8 +209,8 @@ template <bool result_is_nullable>
class AggregateFunctionNullVariadic final : public AggregateFunctionNullBase<result_is_nullable, AggregateFunctionNullVariadic<result_is_nullable>>
{
public:
AggregateFunctionNullVariadic(AggregateFunctionPtr nested_function, const DataTypes & arguments)
: AggregateFunctionNullBase<result_is_nullable, AggregateFunctionNullVariadic<result_is_nullable>>(nested_function),
AggregateFunctionNullVariadic(AggregateFunctionPtr nested_function_, const DataTypes & arguments)
: AggregateFunctionNullBase<result_is_nullable, AggregateFunctionNullVariadic<result_is_nullable>>(std::move(nested_function_)),
number_of_arguments(arguments.size())
{
if (number_of_arguments == 1)

5
dbms/src/AggregateFunctions/AggregateFunctionQuantile.h
View File

@ -100,9 +100,12 @@ public:
return res;
}
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
void NO_SANITIZE_UNDEFINED add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
{
/// Out of range conversion may occur. This is Ok.
const auto & column = static_cast<const ColVecType &>(*columns[0]);
if constexpr (has_second_arg)
this->data(place).add(
column.getData()[row_num],

10
dbms/src/AggregateFunctions/AggregateFunctionStatisticsSimple.h
View File

@ -68,12 +68,12 @@ struct VarMoments
readPODBinary(*this, buf);
}
T getPopulation() const
T NO_SANITIZE_UNDEFINED getPopulation() const
{
return (m2 - m1 * m1 / m0) / m0;
}
T getSample() const
T NO_SANITIZE_UNDEFINED getSample() const
{
if (m0 == 0)
return std::numeric_limits<T>::quiet_NaN();
@ -177,12 +177,12 @@ struct CovarMoments
readPODBinary(*this, buf);
}
T getPopulation() const
T NO_SANITIZE_UNDEFINED getPopulation() const
{
return (xy - x1 * y1 / m0) / m0;
}
T getSample() const
T NO_SANITIZE_UNDEFINED getSample() const
{
if (m0 == 0)
return std::numeric_limits<T>::quiet_NaN();
@ -232,7 +232,7 @@ struct CorrMoments
readPODBinary(*this, buf);
}
T get() const
T NO_SANITIZE_UNDEFINED get() const
{
return (m0 * xy - x1 * y1) / sqrt((m0 * x2 - x1 * x1) * (m0 * y2 - y1 * y1));
}

14
dbms/src/AggregateFunctions/AggregateFunctionSumMap.h
View File

@ -80,11 +80,11 @@ public:
void add(AggregateDataPtr place, const IColumn ** columns, const size_t row_num, Arena *) const override
{
// Column 0 contains array of keys of known type
const ColumnArray & array_column = static_cast<const ColumnArray &>(*columns[0]);
const IColumn::Offsets & offsets = array_column.getOffsets();
const auto & keys_vec = static_cast<const ColVecType &>(array_column.getData());
const size_t keys_vec_offset = offsets[row_num - 1];
const size_t keys_vec_size = (offsets[row_num] - keys_vec_offset);
const ColumnArray & array_column0 = static_cast<const ColumnArray &>(*columns[0]);
const IColumn::Offsets & offsets0 = array_column0.getOffsets();
const auto & keys_vec = static_cast<const ColVecType &>(array_column0.getData());
const size_t keys_vec_offset = offsets0[row_num - 1];
const size_t keys_vec_size = (offsets0[row_num] - keys_vec_offset);
// Columns 1..n contain arrays of numeric values to sum
auto & merged_maps = this->data(place).merged_maps;
@ -226,14 +226,14 @@ public:
// Advance column offsets
auto & to_keys_offsets = to_keys_arr.getOffsets();
to_keys_offsets.push_back((to_keys_offsets.empty() ? 0 : to_keys_offsets.back()) + size);
to_keys_offsets.push_back(to_keys_offsets.back() + size);
to_keys_col.reserve(size);
for (size_t col = 0; col < values_types.size(); ++col)
{
auto & to_values_arr = static_cast<ColumnArray &>(to_tuple.getColumn(col + 1));
auto & to_values_offsets = to_values_arr.getOffsets();
to_values_offsets.push_back((to_values_offsets.empty() ? 0 : to_values_offsets.back()) + size);
to_values_offsets.push_back(to_values_offsets.back() + size);
to_values_arr.getData().reserve(size);
}

4
dbms/src/AggregateFunctions/CMakeLists.txt
View File

@ -22,3 +22,7 @@ list(REMOVE_ITEM clickhouse_aggregate_functions_headers
add_library(clickhouse_aggregate_functions ${LINK_MODE} ${clickhouse_aggregate_functions_sources})
target_link_libraries(clickhouse_aggregate_functions PRIVATE dbms)
target_include_directories (clickhouse_aggregate_functions BEFORE PRIVATE ${COMMON_INCLUDE_DIR})
if (ENABLE_TESTS)
add_subdirectory (tests)
endif ()

20
dbms/src/AggregateFunctions/Helpers.h
View File

@ -28,8 +28,8 @@ static IAggregateFunction * createWithNumericType(const IDataType & argument_typ
if (which.idx == TypeIndex::TYPE) return new AggregateFunctionTemplate<TYPE>(std::forward<TArgs>(args)...);
FOR_NUMERIC_TYPES(DISPATCH)
#undef DISPATCH
if (which.idx == TypeIndex::Enum8) return new AggregateFunctionTemplate<UInt8>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::Enum16) return new AggregateFunctionTemplate<UInt16>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::Enum8) return new AggregateFunctionTemplate<Int8>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::Enum16) return new AggregateFunctionTemplate<Int16>(std::forward<TArgs>(args)...);
return nullptr;
}
@ -41,8 +41,8 @@ static IAggregateFunction * createWithNumericType(const IDataType & argument_typ
if (which.idx == TypeIndex::TYPE) return new AggregateFunctionTemplate<TYPE, Data>(std::forward<TArgs>(args)...);
FOR_NUMERIC_TYPES(DISPATCH)
#undef DISPATCH
if (which.idx == TypeIndex::Enum8) return new AggregateFunctionTemplate<UInt8, Data>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::Enum16) return new AggregateFunctionTemplate<UInt16, Data>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::Enum8) return new AggregateFunctionTemplate<Int8, Data>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::Enum16) return new AggregateFunctionTemplate<Int16, Data>(std::forward<TArgs>(args)...);
return nullptr;
}
@ -54,8 +54,8 @@ static IAggregateFunction * createWithNumericType(const IDataType & argument_typ
if (which.idx == TypeIndex::TYPE) return new AggregateFunctionTemplate<TYPE, Data<TYPE>>(std::forward<TArgs>(args)...);
FOR_NUMERIC_TYPES(DISPATCH)
#undef DISPATCH
if (which.idx == TypeIndex::Enum8) return new AggregateFunctionTemplate<UInt8, Data<UInt8>>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::Enum16) return new AggregateFunctionTemplate<UInt16, Data<UInt16>>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::Enum8) return new AggregateFunctionTemplate<Int8, Data<Int8>>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::Enum16) return new AggregateFunctionTemplate<Int16, Data<Int16>>(std::forward<TArgs>(args)...);
return nullptr;
}
@ -106,8 +106,8 @@ static IAggregateFunction * createWithTwoNumericTypesSecond(const IDataType & se
if (which.idx == TypeIndex::TYPE) return new AggregateFunctionTemplate<FirstType, TYPE>(std::forward<TArgs>(args)...);
FOR_NUMERIC_TYPES(DISPATCH)
#undef DISPATCH
if (which.idx == TypeIndex::Enum8) return new AggregateFunctionTemplate<FirstType, UInt8>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::Enum16) return new AggregateFunctionTemplate<FirstType, UInt16>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::Enum8) return new AggregateFunctionTemplate<FirstType, Int8>(std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::Enum16) return new AggregateFunctionTemplate<FirstType, Int16>(std::forward<TArgs>(args)...);
return nullptr;
}
@ -121,9 +121,9 @@ static IAggregateFunction * createWithTwoNumericTypes(const IDataType & first_ty
FOR_NUMERIC_TYPES(DISPATCH)
#undef DISPATCH
if (which.idx == TypeIndex::Enum8)
return createWithTwoNumericTypesSecond<UInt8, AggregateFunctionTemplate>(second_type, std::forward<TArgs>(args)...);
return createWithTwoNumericTypesSecond<Int8, AggregateFunctionTemplate>(second_type, std::forward<TArgs>(args)...);
if (which.idx == TypeIndex::Enum16)
return createWithTwoNumericTypesSecond<UInt16, AggregateFunctionTemplate>(second_type, std::forward<TArgs>(args)...);
return createWithTwoNumericTypesSecond<Int16, AggregateFunctionTemplate>(second_type, std::forward<TArgs>(args)...);
return nullptr;
}

4
dbms/src/AggregateFunctions/QuantileTDigest.h
View File

@ -225,6 +225,10 @@ public:
summary.resize(size);
buf.read(reinterpret_cast<char *>(summary.data()), size * sizeof(summary[0]));
count = 0;
for (const auto & c : summary)
count += c.count;
}
/** Calculates the quantile q [0, 1] based on the digest.

8
dbms/src/AggregateFunctions/QuantileTiming.h
View File

@ -382,13 +382,13 @@ namespace detail
if (index == BIG_THRESHOLD)
break;
UInt64 count = 0;
readBinary(count, buf);
UInt64 elem_count = 0;
readBinary(elem_count, buf);
if (index < SMALL_THRESHOLD)
count_small[index] = count;
count_small[index] = elem_count;
else
count_big[index - SMALL_THRESHOLD] = count;
count_big[index - SMALL_THRESHOLD] = elem_count;
}
}
}

2
dbms/src/AggregateFunctions/tests/CMakeLists.txt Normal file
View File

@ -0,0 +1,2 @@
add_executable (quantile-t-digest quantile-t-digest.cpp)
target_link_libraries (quantile-t-digest PRIVATE dbms clickhouse_aggregate_functions)

22
dbms/src/AggregateFunctions/tests/quantile-t-digest.cpp Normal file
View File

@ -0,0 +1,22 @@
#include <AggregateFunctions/QuantileTDigest.h>
#include <IO/WriteBufferFromString.h>
#include <IO/ReadBufferFromString.h>
int main(int, char **)
{
using namespace DB;
QuantileTDigest<float> tdigest;
tdigest.add(1);
tdigest.add(2);
tdigest.add(3);
std::cout << tdigest.get(0.5) << "\n";
WriteBufferFromOwnString wb;
tdigest.serialize(wb);
QuantileTDigest<float> other;
ReadBufferFromString rb{wb.str()};
other.deserialize(rb);
std::cout << other.get(0.5) << "\n";
return 0;
}

1
dbms/src/CMakeLists.txt
View File

@ -13,3 +13,4 @@ add_subdirectory (AggregateFunctions)
add_subdirectory (Client)
add_subdirectory (TableFunctions)
add_subdirectory (Formats)
add_subdirectory (Compression)

27
dbms/src/Client/Connection.cpp
View File

@ -2,8 +2,8 @@
#include <Poco/Net/NetException.h>
#include <Core/Defines.h>
#include <IO/CompressedReadBuffer.h>
#include <IO/CompressedWriteBuffer.h>
#include <Compression/CompressedReadBuffer.h>
#include <Compression/CompressedWriteBuffer.h>
#include <IO/ReadBufferFromPocoSocket.h>
#include <IO/WriteBufferFromPocoSocket.h>
#include <IO/ReadHelpers.h>
@ -21,6 +21,7 @@
#include <Common/StringUtils/StringUtils.h>
#include <Common/config_version.h>
#include <Interpreters/ClientInfo.h>
#include <Compression/CompressionFactory.h>
#include <Common/config.h>
#if USE_POCO_NETSSL
@ -353,7 +354,19 @@ void Connection::sendQuery(
if (!connected)
connect();
compression_settings = settings ? CompressionSettings(*settings) : CompressionSettings(CompressionMethod::LZ4);
if (settings)
{
std::optional<int> level;
std::string method = settings->network_compression_method;
/// Bad custom logic
if (method == "ZSTD")
level = settings->network_zstd_compression_level;
compression_codec = CompressionCodecFactory::instance().get(method, level);
}
else
compression_codec = CompressionCodecFactory::instance().getDefaultCodec();
query_id = query_id_;
@ -426,7 +439,7 @@ void Connection::sendData(const Block & block, const String & name)
if (!block_out)
{
if (compression == Protocol::Compression::Enable)
maybe_compressed_out = std::make_shared<CompressedWriteBuffer>(*out, compression_settings);
maybe_compressed_out = std::make_shared<CompressedWriteBuffer>(*out, compression_codec);
else
maybe_compressed_out = out;
@ -719,10 +732,10 @@ std::unique_ptr<Exception> Connection::receiveException()
std::vector<String> Connection::receiveMultistringMessage(UInt64 msg_type)
{
size_t num = Protocol::Server::stringsInMessage(msg_type);
std::vector<String> out(num);
std::vector<String> strings(num);
for (size_t i = 0; i < num; ++i)
readStringBinary(out[i], *in);
return out;
readStringBinary(strings[i], *in);
return strings;
}

5
dbms/src/Client/Connection.h
View File

@ -18,12 +18,13 @@
#include <DataStreams/IBlockOutputStream.h>
#include <DataStreams/BlockStreamProfileInfo.h>
#include <IO/CompressionSettings.h>
#include <IO/ConnectionTimeouts.h>
#include <Interpreters/Settings.h>
#include <Interpreters/TablesStatus.h>
#include <Compression/ICompressionCodec.h>
#include <atomic>
#include <optional>
@ -205,7 +206,7 @@ private:
Protocol::Secure secure; /// Enable data encryption for communication.
/// What compression settings to use while sending data for INSERT queries and external tables.
CompressionSettings compression_settings;
CompressionCodecPtr compression_codec;
/** If not nullptr, used to limit network traffic.
* Only traffic for transferring blocks is accounted. Other packets don't.

14
dbms/src/Client/MultiplexedConnections.cpp
View File

@ -52,7 +52,7 @@ MultiplexedConnections::MultiplexedConnections(
void MultiplexedConnections::sendExternalTablesData(std::vector<ExternalTablesData> & data)
{
std::lock_guard<std::mutex> lock(cancel_mutex);
std::lock_guard lock(cancel_mutex);
if (!sent_query)
throw Exception("Cannot send external tables data: query not yet sent.", ErrorCodes::LOGICAL_ERROR);
@ -79,7 +79,7 @@ void MultiplexedConnections::sendQuery(
const ClientInfo * client_info,
bool with_pending_data)
{
std::lock_guard<std::mutex> lock(cancel_mutex);
std::lock_guard lock(cancel_mutex);
if (sent_query)
throw Exception("Query already sent.", ErrorCodes::LOGICAL_ERROR);
@ -121,14 +121,14 @@ void MultiplexedConnections::sendQuery(
Connection::Packet MultiplexedConnections::receivePacket()
{
std::lock_guard<std::mutex> lock(cancel_mutex);
std::lock_guard lock(cancel_mutex);
Connection::Packet packet = receivePacketUnlocked();
return packet;
}
void MultiplexedConnections::disconnect()
{
std::lock_guard<std::mutex> lock(cancel_mutex);
std::lock_guard lock(cancel_mutex);
for (ReplicaState & state : replica_states)
{
@ -143,7 +143,7 @@ void MultiplexedConnections::disconnect()
void MultiplexedConnections::sendCancel()
{
std::lock_guard<std::mutex> lock(cancel_mutex);
std::lock_guard lock(cancel_mutex);
if (!sent_query || cancelled)
throw Exception("Cannot cancel. Either no query sent or already cancelled.", ErrorCodes::LOGICAL_ERROR);
@ -160,7 +160,7 @@ void MultiplexedConnections::sendCancel()
Connection::Packet MultiplexedConnections::drain()
{
std::lock_guard<std::mutex> lock(cancel_mutex);
std::lock_guard lock(cancel_mutex);
if (!cancelled)
throw Exception("Cannot drain connections: cancel first.", ErrorCodes::LOGICAL_ERROR);
@ -195,7 +195,7 @@ Connection::Packet MultiplexedConnections::drain()
std::string MultiplexedConnections::dumpAddresses() const
{
std::lock_guard<std::mutex> lock(cancel_mutex);
std::lock_guard lock(cancel_mutex);
return dumpAddressesUnlocked();
}

2
dbms/src/Columns/ColumnAggregateFunction.cpp
View File

@ -378,7 +378,7 @@ const char * ColumnAggregateFunction::deserializeAndInsertFromArena(const char *
* as we cannot legally compare pointers after last element + 1 of some valid memory region.
* Probably this will not work under UBSan.
*/
ReadBufferFromMemory read_buffer(src_arena, std::numeric_limits<char *>::max() - src_arena);
ReadBufferFromMemory read_buffer(src_arena, std::numeric_limits<char *>::max() - src_arena - 1);
func->deserialize(data.back(), read_buffer, &dst_arena);
return read_buffer.position();

13
dbms/src/Columns/ColumnArray.cpp
View File

@ -134,13 +134,13 @@ StringRef ColumnArray::getDataAt(size_t n) const
* since it contains only the data laid in succession, but not the offsets.
*/
size_t array_size = sizeAt(n);
if (array_size == 0)
return StringRef();
size_t offset_of_first_elem = offsetAt(n);
StringRef first = getData().getDataAtWithTerminatingZero(offset_of_first_elem);
size_t array_size = sizeAt(n);
if (array_size == 0)
return StringRef(first.data, 0);
size_t offset_of_last_elem = getOffsets()[n] - 1;
StringRef last = getData().getDataAtWithTerminatingZero(offset_of_last_elem);
@ -233,7 +233,10 @@ void ColumnArray::insertFrom(const IColumn & src_, size_t n)
void ColumnArray::insertDefault()
{
getOffsets().push_back(getOffsets().back());
/// NOTE 1: We can use back() even if the array is empty (due to zero -1th element in PODArray).
/// NOTE 2: We cannot use reference in push_back, because reference get invalidated if array is reallocated.
auto last_offset = getOffsets().back();
getOffsets().push_back(last_offset);
}

4
dbms/src/Columns/ColumnArray.h
View File

@ -124,8 +124,8 @@ private:
ColumnPtr data;
ColumnPtr offsets;
size_t ALWAYS_INLINE offsetAt(size_t i) const { return getOffsets()[i - 1]; }
size_t ALWAYS_INLINE sizeAt(size_t i) const { return getOffsets()[i] - getOffsets()[i - 1]; }
size_t ALWAYS_INLINE offsetAt(ssize_t i) const { return getOffsets()[i - 1]; }
size_t ALWAYS_INLINE sizeAt(ssize_t i) const { return getOffsets()[i] - getOffsets()[i - 1]; }
/// Multiply values if the nested column is ColumnVector<T>.

5
dbms/src/Columns/ColumnDecimal.h
View File

@ -3,6 +3,7 @@
#include <cmath>
#include <Columns/IColumn.h>
#include <Columns/ColumnVectorHelper.h>
namespace DB
@ -53,13 +54,13 @@ private:
/// A ColumnVector for Decimals
template <typename T>
class ColumnDecimal final : public COWPtrHelper<IColumn, ColumnDecimal<T>>
class ColumnDecimal final : public COWPtrHelper<ColumnVectorHelper, ColumnDecimal<T>>
{
static_assert(IsDecimalNumber<T>);
private:
using Self = ColumnDecimal;
friend class COWPtrHelper<IColumn, Self>;
friend class COWPtrHelper<ColumnVectorHelper, Self>;
public:
using Container = DecimalPaddedPODArray<T>;

4
dbms/src/Columns/ColumnFixedString.cpp
View File

@ -9,7 +9,7 @@
#include <IO/WriteHelpers.h>
#if __SSE2__
#ifdef __SSE2__
#include <emmintrin.h>
#endif
@ -169,7 +169,7 @@ ColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result
const UInt8 * filt_end = filt_pos + col_size;
const UInt8 * data_pos = chars.data();
#if __SSE2__
#ifdef __SSE2__
/** A slightly more optimized version.
* Based on the assumption that often pieces of consecutive values
* completely pass or do not pass the filter.

7
dbms/src/Columns/ColumnFixedString.h
View File

@ -1,9 +1,10 @@
#pragma once
#include <string.h> // memcpy
#include <string.h> // memcmp
#include <Common/PODArray.h>
#include <Columns/IColumn.h>
#include <Columns/ColumnVectorHelper.h>
namespace DB
@ -12,10 +13,10 @@ namespace DB
/** A column of values of "fixed-length string" type.
* If you insert a smaller string, it will be padded with zero bytes.
*/
class ColumnFixedString final : public COWPtrHelper<IColumn, ColumnFixedString>
class ColumnFixedString final : public COWPtrHelper<ColumnVectorHelper, ColumnFixedString>
{
public:
friend class COWPtrHelper<IColumn, ColumnFixedString>;
friend class COWPtrHelper<ColumnVectorHelper, ColumnFixedString>;
using Chars = PaddedPODArray<UInt8>;

9
dbms/src/Columns/ColumnLowCardinality.cpp
View File

@ -236,6 +236,9 @@ void ColumnLowCardinality::gather(ColumnGathererStream & gatherer)
MutableColumnPtr ColumnLowCardinality::cloneResized(size_t size) const
{
auto unique_ptr = dictionary.getColumnUniquePtr();
if (size == 0)
unique_ptr = unique_ptr->cloneEmpty();
return ColumnLowCardinality::create((*std::move(unique_ptr)).mutate(), getIndexes().cloneResized(size));
}
@ -632,11 +635,11 @@ void ColumnLowCardinality::Dictionary::checkColumn(const IColumn & column)
throw Exception("ColumnUnique expected as an argument of ColumnLowCardinality.", ErrorCodes::ILLEGAL_COLUMN);
}
void ColumnLowCardinality::Dictionary::setShared(const ColumnPtr & dictionary)
void ColumnLowCardinality::Dictionary::setShared(const ColumnPtr & column_unique_)
{
checkColumn(*dictionary);
checkColumn(*column_unique_);
column_unique = dictionary;
column_unique = column_unique_;
shared = true;
}

2
dbms/src/Columns/ColumnLowCardinality.h
View File

@ -133,7 +133,7 @@ public:
}
bool valuesHaveFixedSize() const override { return getDictionary().valuesHaveFixedSize(); }
bool isFixedAndContiguous() const override { return getDictionary().isFixedAndContiguous(); }
bool isFixedAndContiguous() const override { return false; }
size_t sizeOfValueIfFixed() const override { return getDictionary().sizeOfValueIfFixed(); }
bool isNumeric() const override { return getDictionary().isNumeric(); }
bool lowCardinality() const override { return true; }

2
dbms/src/Columns/ColumnNothing.h
View File

@ -20,7 +20,7 @@ private:
public:
const char * getFamilyName() const override { return "Nothing"; }
MutableColumnPtr cloneDummy(size_t s) const override { return ColumnNothing::create(s); }
MutableColumnPtr cloneDummy(size_t s_) const override { return ColumnNothing::create(s_); }
bool canBeInsideNullable() const override { return true; }
};

7
dbms/src/Columns/ColumnString.h
View File

@ -31,10 +31,10 @@ private:
/// For convenience, every string ends with terminating zero byte. Note that strings could contain zero bytes in the middle.
Chars chars;
size_t ALWAYS_INLINE offsetAt(size_t i) const { return offsets[i - 1]; }
size_t ALWAYS_INLINE offsetAt(ssize_t i) const { return offsets[i - 1]; }
/// Size of i-th element, including terminating zero.
size_t ALWAYS_INLINE sizeAt(size_t i) const { return offsets[i] - offsets[i - 1]; }
size_t ALWAYS_INLINE sizeAt(ssize_t i) const { return offsets[i] - offsets[i - 1]; }
template <bool positive>
struct less;
@ -153,7 +153,8 @@ public:
const size_t new_size = old_size + length + 1;
chars.resize(new_size);
memcpy(&chars[old_size], pos, length);
if (length)
memcpy(&chars[old_size], pos, length);
chars[old_size + length] = 0;
offsets.push_back(new_size);
}

4
dbms/src/Columns/ColumnUnique.h
View File

@ -66,9 +66,9 @@ public:
Int64 getInt(size_t n) const override { return getNestedColumn()->getInt(n); }
bool isNullAt(size_t n) const override { return is_nullable && n == getNullValueIndex(); }
StringRef serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const override;
void updateHashWithValue(size_t n, SipHash & hash) const override
void updateHashWithValue(size_t n, SipHash & hash_func) const override
{
return getNestedColumn()->updateHashWithValue(n, hash);
return getNestedColumn()->updateHashWithValue(n, hash_func);
}
int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;

4
dbms/src/Columns/ColumnVector.cpp
View File

@ -13,7 +13,7 @@
#include <DataStreams/ColumnGathererStream.h>
#include <ext/bit_cast.h>
#if __SSE2__
#ifdef __SSE2__
#include <emmintrin.h>
#include <Columns/ColumnsCommon.h>
@ -162,7 +162,7 @@ ColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_s
const UInt8 * filt_end = filt_pos + size;
const T * data_pos = data.data();
#if __SSE2__
#ifdef __SSE2__
/** A slightly more optimized version.
* Based on the assumption that often pieces of consecutive values
* completely pass or do not pass the filter.

36
dbms/src/Columns/ColumnVector.h
View File

@ -2,6 +2,7 @@
#include <cmath>
#include <Columns/IColumn.h>
#include <Columns/ColumnVectorHelper.h>
#include <common/unaligned.h>
@ -86,47 +87,16 @@ template <> struct CompareHelper<Float32> : public FloatCompareHelper<Float32> {
template <> struct CompareHelper<Float64> : public FloatCompareHelper<Float64> {};
/** To implement `get64` function.
*/
template <typename T>
inline UInt64 unionCastToUInt64(T x) { return x; }
template <> inline UInt64 unionCastToUInt64(Float64 x)
{
union
{
Float64 src;
UInt64 res;
};
src = x;
return res;
}
template <> inline UInt64 unionCastToUInt64(Float32 x)
{
union
{
Float32 src;
UInt64 res;
};
res = 0;
src = x;
return res;
}
/** A template for columns that use a simple array to store.
*/
template <typename T>
class ColumnVector final : public COWPtrHelper<IColumn, ColumnVector<T>>
class ColumnVector final : public COWPtrHelper<ColumnVectorHelper, ColumnVector<T>>
{
static_assert(!IsDecimalNumber<T>);
private:
using Self = ColumnVector;
friend class COWPtrHelper<IColumn, Self>;
friend class COWPtrHelper<ColumnVectorHelper, Self>;
struct less;
struct greater;

39
dbms/src/Columns/ColumnVectorHelper.h Normal file
View File

@ -0,0 +1,39 @@
#pragma once
#include <Columns/IColumn.h>
namespace DB
{
/** Allows to access internal array of ColumnVector or ColumnFixedString without cast to concrete type.
* We will inherit ColumnVector and ColumnFixedString from this class instead of IColumn.
* Assumes data layout of ColumnVector, ColumnFixedString and PODArray.
*
* Why it is needed?
*
* There are some algorithms that specialize on the size of data type but doesn't care about concrete type.
* The same specialization may work for UInt64, Int64, Float64, FixedString(8), if it only does byte moving and hashing.
* To avoid code bloat and compile time increase, we can use single template instantiation for these cases
* and just static_cast pointer to some single column type (e. g. ColumnUInt64) assuming that all types have identical memory layout.
*
* But this static_cast (downcast to unrelated type) is illegal according to the C++ standard and UBSan warns about it.
* To allow functional tests to work under UBSan we have to separate some base class that will present the memory layout in explicit way,
* and we will do static_cast to this class.
*/
class ColumnVectorHelper : public IColumn
{
public:
const char * getRawDataBegin() const
{
return *reinterpret_cast<const char * const *>(reinterpret_cast<const char *>(this) + sizeof(*this));
}
template <size_t ELEMENT_SIZE>
void insertRawData(const char * ptr)
{
return reinterpret_cast<PODArrayBase<ELEMENT_SIZE, 4096, Allocator<false>, 15, 16> *>(reinterpret_cast<char *>(this) + sizeof(*this))->push_back_raw(ptr);
}
};
}

18
dbms/src/Columns/ColumnsCommon.cpp
View File

@ -1,4 +1,4 @@
#if __SSE2__
#ifdef __SSE2__
#include <emmintrin.h>
#endif
@ -24,7 +24,7 @@ size_t countBytesInFilter(const IColumn::Filter & filt)
const Int8 * pos = reinterpret_cast<const Int8 *>(filt.data());
const Int8 * end = pos + filt.size();
#if __SSE2__ && __POPCNT__
#if defined(__SSE2__) && defined(__POPCNT__)
const __m128i zero16 = _mm_setzero_si128();
const Int8 * end64 = pos + filt.size() / 64 * 64;
@ -69,7 +69,7 @@ bool memoryIsZero(const void * data, size_t size)
const Int8 * pos = reinterpret_cast<const Int8 *>(data);
const Int8 * end = pos + size;
#if __SSE2__
#ifdef __SSE2__
const __m128 zero16 = _mm_setzero_ps();
const Int8 * end64 = pos + size / 64 * 64;
@ -205,17 +205,17 @@ namespace
/// copy array ending at *end_offset_ptr
const auto copy_array = [&] (const IColumn::Offset * offset_ptr)
{
const auto offset = offset_ptr == offsets_begin ? 0 : offset_ptr[-1];
const auto size = *offset_ptr - offset;
const auto arr_offset = offset_ptr == offsets_begin ? 0 : offset_ptr[-1];
const auto arr_size = *offset_ptr - arr_offset;
result_offsets_builder.insertOne(size);
result_offsets_builder.insertOne(arr_size);
const auto elems_size_old = res_elems.size();
res_elems.resize(elems_size_old + size);
memcpy(&res_elems[elems_size_old], &src_elems[offset], size * sizeof(T));
res_elems.resize(elems_size_old + arr_size);
memcpy(&res_elems[elems_size_old], &src_elems[arr_offset], arr_size * sizeof(T));
};
#if __SSE2__
#ifdef __SSE2__
const __m128i zero_vec = _mm_setzero_si128();
static constexpr size_t SIMD_BYTES = 16;
const auto filt_end_aligned = filt_pos + size / SIMD_BYTES * SIMD_BYTES;

4
dbms/src/Columns/IColumn.h
View File

@ -330,7 +330,9 @@ public:
virtual bool lowCardinality() const { return false; }
virtual ~IColumn() {}
virtual ~IColumn() = default;
IColumn() = default;
IColumn(const IColumn &) = default;
/** Print column name, size, and recursively print all subcolumns.
*/

8
dbms/src/Columns/ReverseIndex.h
View File

@ -393,10 +393,10 @@ UInt64 ReverseIndex<IndexType, ColumnType>::insert(const StringRef & data)
if constexpr (use_saved_hash)
{
auto & data = saved_hash->getData();
if (data.size() <= num_rows)
data.resize(num_rows + 1);
data[num_rows] = hash;
auto & column_data = saved_hash->getData();
if (column_data.size() <= num_rows)
column_data.resize(num_rows + 1);
column_data[num_rows] = hash;
}
else
column->insertData(data.data, data.size);

1
dbms/src/Columns/tests/column_unique.cpp
View File

@ -10,6 +10,7 @@
#pragma GCC diagnostic ignored "-Wsign-compare"
#ifdef __clang__
#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
#pragma clang diagnostic ignored "-Wundef"
#endif
#include <gtest/gtest.h>

8
dbms/src/Common/Arena.h
View File

@ -36,7 +36,7 @@ private:
static constexpr size_t pad_right = 15;
/// Contiguous chunk of memory and pointer to free space inside it. Member of single-linked list.
struct Chunk : private Allocator<false> /// empty base optimization
struct alignas(16) Chunk : private Allocator<false> /// empty base optimization
{
char * begin;
char * pos;
@ -149,6 +149,12 @@ public:
} while (true);
}
template <typename T>
T * alloc()
{
return reinterpret_cast<T *>(alignedAlloc(sizeof(T), alignof(T)));
}
/** Rollback just performed allocation.
* Must pass size not more that was just allocated.
*/

30
dbms/src/Common/ArrayCache.h
View File

@ -94,7 +94,11 @@ private:
Key key;
Payload payload;
void * ptr;
union
{
void * ptr;
char * char_ptr;
};
size_t size;
size_t refcount = 0;
void * chunk;
@ -231,7 +235,7 @@ public:
~Holder()
{
std::lock_guard<std::mutex> cache_lock(cache.mutex);
std::lock_guard cache_lock(cache.mutex);
if (--region.refcount == 0)
cache.lru_list.push_back(region);
cache.total_size_in_use -= region.size;
@ -301,12 +305,12 @@ private:
if (cleaned_up)
return;
std::lock_guard<std::mutex> token_lock(token->mutex);
std::lock_guard token_lock(token->mutex);
if (token->cleaned_up)
return;
std::lock_guard<std::mutex> cache_lock(token->cache.mutex);
std::lock_guard cache_lock(token->cache.mutex);
--token->refcount;
if (token->refcount == 0)
@ -349,7 +353,7 @@ private:
if (left_it->chunk == region.chunk && left_it->isFree())
{
region.size += left_it->size;
*reinterpret_cast<char **>(&region.ptr) -= left_it->size;
region.char_ptr-= left_it->size;
size_multimap.erase(size_multimap.iterator_to(*left_it));
adjacency_list.erase_and_dispose(left_it, [](RegionMetadata * elem) { elem->destroy(); });
}
@ -479,7 +483,7 @@ private:
size_multimap.erase(size_multimap.iterator_to(free_region));
free_region.size -= size;
*reinterpret_cast<char **>(&free_region.ptr) += size;
free_region.char_ptr += size;
size_multimap.insert(free_region);
adjacency_list.insert(adjacency_list.iterator_to(free_region), *allocated_region);
@ -536,7 +540,7 @@ public:
~ArrayCache()
{
std::lock_guard<std::mutex> cache_lock(mutex);
std::lock_guard cache_lock(mutex);
key_map.clear();
lru_list.clear();
@ -563,7 +567,7 @@ public:
{
InsertTokenHolder token_holder;
{
std::lock_guard<std::mutex> cache_lock(mutex);
std::lock_guard cache_lock(mutex);
auto it = key_map.find(key, RegionCompareByKey());
if (key_map.end() != it)
@ -584,7 +588,7 @@ public:
InsertToken * token = token_holder.token.get();
std::lock_guard<std::mutex> token_lock(token->mutex);
std::lock_guard token_lock(token->mutex);
token_holder.cleaned_up = token->cleaned_up;
@ -605,7 +609,7 @@ public:
RegionMetadata * region;
{
std::lock_guard<std::mutex> cache_lock(mutex);
std::lock_guard cache_lock(mutex);
region = allocate(size);
}
@ -626,14 +630,14 @@ public:
catch (...)
{
{
std::lock_guard<std::mutex> cache_lock(mutex);
std::lock_guard cache_lock(mutex);
freeRegion(*region);
}
throw;
}
}
std::lock_guard<std::mutex> cache_lock(mutex);
std::lock_guard cache_lock(mutex);
try
{
@ -692,7 +696,7 @@ public:
Statistics getStatistics() const
{
std::lock_guard<std::mutex> cache_lock(mutex);
std::lock_guard cache_lock(mutex);
Statistics res;
res.total_chunks_size = total_chunks_size;

43
dbms/src/Common/CompactArray.h
View File

@ -22,7 +22,7 @@ namespace ErrorCodes
* simulates an array of `content_width`-bit values.
*/
template <typename BucketIndex, UInt8 content_width, size_t bucket_count>
class __attribute__ ((packed)) CompactArray final
class CompactArray final
{
public:
class Reader;
@ -55,26 +55,6 @@ public:
return locus;
}
void readText(ReadBuffer & in)
{
for (size_t i = 0; i < BITSET_SIZE; ++i)
{
if (i != 0)
assertChar(',', in);
readIntText(bitset[i], in);
}
}
void writeText(WriteBuffer & out) const
{
for (size_t i = 0; i < BITSET_SIZE; ++i)
{
if (i != 0)
writeCString(",", out);
writeIntText(bitset[i], out);
}
}
private:
/// number of bytes in bitset
static constexpr size_t BITSET_SIZE = (static_cast<size_t>(bucket_count) * content_width + 7) / 8;
@ -165,7 +145,9 @@ private:
bool fits_in_byte;
};
/** The `Locus` structure contains the necessary information to find for each cell
/** TODO This code looks very suboptimal.
*
* The `Locus` structure contains the necessary information to find for each cell
* the corresponding byte and offset, in bits, from the beginning of the cell. Since in general
* case the size of one byte is not divisible by the size of one cell, cases possible
* when one cell overlaps two bytes. Therefore, the `Locus` structure contains two
@ -219,13 +201,20 @@ private:
void ALWAYS_INLINE init(BucketIndex bucket_index)
{
/// offset in bits to the leftmost bit
size_t l = static_cast<size_t>(bucket_index) * content_width;
index_l = l >> 3;
offset_l = l & 7;
size_t r = static_cast<size_t>(bucket_index + 1) * content_width;
index_r = r >> 3;
offset_r = r & 7;
/// offset of byte that contains the leftmost bit
index_l = l / 8;
/// offset in bits to the leftmost bit at that byte
offset_l = l % 8;
/// offset of byte that contains the rightmost bit
index_r = (l + content_width - 1) / 8;
/// offset in bits to the next to the rightmost bit at that byte; or zero if the rightmost bit is the rightmost bit in that byte.
offset_r = (l + content_width) % 8;
}
UInt8 ALWAYS_INLINE read(UInt8 value_l) const

16
dbms/src/Common/Config/ConfigProcessor.cpp
View File

@ -447,6 +447,11 @@ XMLDocumentPtr ConfigProcessor::processConfig(
merge(config, with);
contributing_files.push_back(merge_file);
}
catch (Exception & e)
{
e.addMessage("while merging config '" + path + "' with '" + merge_file + "'");
throw;
}
catch (Poco::Exception & e)
{
throw Poco::Exception("Failed to merge config with '" + merge_file + "': " + e.displayText());
@ -479,6 +484,11 @@ XMLDocumentPtr ConfigProcessor::processConfig(
doIncludesRecursive(config, include_from, getRootNode(config.get()), zk_node_cache, zk_changed_event, contributing_zk_paths);
}
catch (Exception & e)
{
e.addMessage("while preprocessing config '" + path + "'");
throw;
}
catch (Poco::Exception & e)
{
throw Poco::Exception("Failed to preprocess config '" + path + "': " + e.displayText(), e);
@ -590,9 +600,9 @@ void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config,
}
preprocessed_path = preprocessed_dir + new_path;
auto path = Poco::Path(preprocessed_path).makeParent();
if (!path.toString().empty())
Poco::File(path).createDirectories();
auto preprocessed_path_parent = Poco::Path(preprocessed_path).makeParent();
if (!preprocessed_path_parent.toString().empty())
Poco::File(preprocessed_path_parent).createDirectories();
}
try
{

18
dbms/src/Common/Config/ConfigReloader.cpp
View File

@ -78,10 +78,10 @@ void ConfigReloader::run()
void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error, bool fallback_to_preprocessed)
{
std::lock_guard<std::mutex> lock(reload_mutex);
std::lock_guard lock(reload_mutex);
FilesChangesTracker new_files = getNewFileList();
if (force || new_files.isDifferOrNewerThan(files))
if (force || need_reload_from_zk || new_files.isDifferOrNewerThan(files))
{
ConfigProcessor config_processor(path);
ConfigProcessor::LoadedConfig loaded_config;
@ -94,6 +94,17 @@ void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error, bool fallbac
loaded_config = config_processor.loadConfigWithZooKeeperIncludes(
zk_node_cache, zk_changed_event, fallback_to_preprocessed);
}
catch (const Coordination::Exception & e)
{
if (Coordination::isHardwareError(e.code))
need_reload_from_zk = true;
if (throw_on_error)
throw;
tryLogCurrentException(log, "ZooKeeper error when loading config from `" + path + "'");
return;
}
catch (...)
{
if (throw_on_error)
@ -110,7 +121,10 @@ void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error, bool fallbac
* When file has been written (and contain valid data), we don't load new data since modification time remains the same.
*/
if (!loaded_config.loaded_from_preprocessed)
{
files = std::move(new_files);
need_reload_from_zk = false;
}
try
{

1
dbms/src/Common/Config/ConfigReloader.h
View File

@ -75,6 +75,7 @@ private:
std::string preprocessed_dir;
FilesChangesTracker files;
zkutil::ZooKeeperNodeCache zk_node_cache;
bool need_reload_from_zk = false;
zkutil::EventPtr zk_changed_event = std::make_shared<Poco::Event>();
Updater updater;

2
dbms/src/Common/CounterInFile.h
View File

@ -54,7 +54,7 @@ public:
template <typename Callback>
Int64 add(Int64 delta, Callback && locked_callback, bool create_if_need = false)
{
std::lock_guard<std::mutex> lock(mutex);
std::lock_guard lock(mutex);
Int64 res = -1;

3
dbms/src/Common/CurrentThread.cpp
View File

@ -3,7 +3,6 @@
#include "CurrentThread.h"
#include <common/logger_useful.h>
#include <Common/ThreadStatus.h>
#include <Common/ObjectPool.h>
#include <Common/TaskStatsInfoGetter.h>
#include <Interpreters/ProcessList.h>
#include <Interpreters/Context.h>
@ -24,8 +23,6 @@ namespace ErrorCodes
extern const int LOGICAL_ERROR;
}
SimpleObjectPool<TaskStatsInfoGetter> task_stats_info_getter_pool;
// Smoker's implementation to avoid thread_local usage: error: undefined symbol: __cxa_thread_atexit
#if defined(ARCADIA_ROOT)
struct ThreadStatusPtrHolder : ThreadStatusPtr

3
dbms/src/Common/ErrorCodes.cpp
View File

@ -405,6 +405,9 @@ namespace ErrorCodes
extern const int UNKNOWN_LOG_LEVEL = 428;
extern const int FAILED_TO_GETPWUID = 429;
extern const int MISMATCHING_USERS_FOR_PROCESS_AND_DATA = 430;
extern const int ILLEGAL_SYNTAX_FOR_CODEC_TYPE = 431;
extern const int UNKNOWN_CODEC = 432;
extern const int ILLEGAL_CODEC_PARAMETER = 433;
extern const int KEEPER_EXCEPTION = 999;
extern const int POCO_EXCEPTION = 1000;

2
dbms/src/Common/FileChecker.cpp
View File

@ -146,7 +146,7 @@ void FileChecker::load(Map & local_map, const std::string & path)
JSON json(out.str());
JSON files = json["yandex"];
for (const auto & name_value : files)
for (const JSON name_value : files)
local_map[unescapeForFileName(name_value.getName())] = name_value.getValue()["size"].toUInt();
}

8
dbms/src/Common/HashTable/Hash.h
View File

@ -35,20 +35,20 @@ inline DB::UInt64 intHash64(DB::UInt64 x)
* due to high speed (latency 3 + 1 clock cycle, throughput 1 clock cycle).
* Works only with SSE 4.2 support.
*/
#if __SSE4_2__
#ifdef __SSE4_2__
#include <nmmintrin.h>
#endif
#if __aarch64__ && __ARM_FEATURE_CRC32
#if defined(__aarch64__) && defined(__ARM_FEATURE_CRC32)
#include <arm_acle.h>
#include <arm_neon.h>
#endif
inline DB::UInt64 intHashCRC32(DB::UInt64 x)
{
#if __SSE4_2__
#ifdef __SSE4_2__
return _mm_crc32_u64(-1ULL, x);
#elif __aarch64__ && __ARM_FEATURE_CRC32
#elif defined(__aarch64__) && defined(__ARM_FEATURE_CRC32)
return __crc32cd(-1U, x);
#else
/// On other platforms we do not have CRC32. NOTE This can be confusing.

2
dbms/src/Common/HyperLogLogCounter.h
View File

@ -278,7 +278,7 @@ template <
typename BiasEstimator = TrivialBiasEstimator,
HyperLogLogMode mode = HyperLogLogMode::FullFeatured,
DenominatorMode denominator_mode = DenominatorMode::StableIfBig>
class __attribute__ ((packed)) HyperLogLogCounter : private Hash
class HyperLogLogCounter : private Hash
{
private:
/// Number of buckets.

22
dbms/src/Common/LRUCache.h
View File

@ -48,7 +48,7 @@ public:
MappedPtr get(const Key & key)
{
std::lock_guard<std::mutex> lock(mutex);
std::lock_guard lock(mutex);
auto res = getImpl(key, lock);
if (res)
@ -61,7 +61,7 @@ public:
void set(const Key & key, const MappedPtr & mapped)
{
std::lock_guard<std::mutex> lock(mutex);
std::lock_guard lock(mutex);
setImpl(key, mapped, lock);
}
@ -79,7 +79,7 @@ public:
{
InsertTokenHolder token_holder;
{
std::lock_guard<std::mutex> cache_lock(mutex);
std::lock_guard cache_lock(mutex);
auto val = getImpl(key, cache_lock);
if (val)
@ -97,7 +97,7 @@ public:
InsertToken * token = token_holder.token.get();
std::lock_guard<std::mutex> token_lock(token->mutex);
std::lock_guard token_lock(token->mutex);
token_holder.cleaned_up = token->cleaned_up;
@ -111,7 +111,7 @@ public:
++misses;
token->value = load_func();
std::lock_guard<std::mutex> cache_lock(mutex);
std::lock_guard cache_lock(mutex);
/// Insert the new value only if the token is still in present in insert_tokens.
/// (The token may be absent because of a concurrent reset() call).
@ -131,26 +131,26 @@ public:
void getStats(size_t & out_hits, size_t & out_misses) const
{
std::lock_guard<std::mutex> lock(mutex);
std::lock_guard lock(mutex);
out_hits = hits;
out_misses = misses;
}
size_t weight() const
{
std::lock_guard<std::mutex> lock(mutex);
std::lock_guard lock(mutex);
return current_size;
}
size_t count() const
{
std::lock_guard<std::mutex> lock(mutex);
std::lock_guard lock(mutex);
return cells.size();
}
void reset()
{
std::lock_guard<std::mutex> lock(mutex);
std::lock_guard lock(mutex);
queue.clear();
cells.clear();
insert_tokens.clear();
@ -234,12 +234,12 @@ private:
if (cleaned_up)
return;
std::lock_guard<std::mutex> token_lock(token->mutex);
std::lock_guard token_lock(token->mutex);
if (token->cleaned_up)
return;
std::lock_guard<std::mutex> cache_lock(token->cache.mutex);
std::lock_guard cache_lock(token->cache.mutex);
--token->refcount;
if (token->refcount == 0)

6
dbms/src/Common/ObjectPool.h
View File

@ -38,7 +38,7 @@ protected:
void operator()(T * owning_ptr) const
{
std::lock_guard<std::mutex> lock{parent->mutex};
std::lock_guard lock{parent->mutex};
parent->stack.emplace(owning_ptr);
}
};
@ -51,7 +51,7 @@ public:
template <typename Factory>
Pointer get(Factory && f)
{
std::unique_lock<std::mutex> lock(mutex);
std::unique_lock lock(mutex);
if (stack.empty())
{
@ -94,7 +94,7 @@ public:
template <typename Factory>
Pointer get(const Key & key, Factory && f)
{
std::unique_lock<std::mutex> lock(mutex);
std::unique_lock lock(mutex);
auto it = container.find(key);
if (container.end() == it)

366
dbms/src/Common/PODArray.h
View File

@ -61,14 +61,17 @@ inline constexpr size_t integerRoundUp(size_t value, size_t dividend)
static constexpr size_t EmptyPODArraySize = 1024;
extern const char EmptyPODArray[EmptyPODArraySize];
template <typename T, size_t INITIAL_SIZE = 4096, typename TAllocator = Allocator<false>, size_t pad_right_ = 0, size_t pad_left_ = 0>
class PODArray : private boost::noncopyable, private TAllocator /// empty base optimization
/** Base class that depend only on size of element, not on element itself.
* You can static_cast to this class if you want to insert some data regardless to the actual type T.
*/
template <size_t ELEMENT_SIZE, size_t INITIAL_SIZE, typename TAllocator, size_t pad_right_, size_t pad_left_>
class PODArrayBase : private boost::noncopyable, private TAllocator /// empty base optimization
{
protected:
/// Round padding up to an whole number of elements to simplify arithmetic.
static constexpr size_t pad_right = integerRoundUp(pad_right_, sizeof(T));
static constexpr size_t pad_right = integerRoundUp(pad_right_, ELEMENT_SIZE);
/// pad_left is also rounded up to 16 bytes to maintain alignment of allocated memory.
static constexpr size_t pad_left = integerRoundUp(integerRoundUp(pad_left_, sizeof(T)), 16);
static constexpr size_t pad_left = integerRoundUp(integerRoundUp(pad_left_, ELEMENT_SIZE), 16);
/// Empty array will point to this static memory as padding.
static constexpr char * null = pad_left ? const_cast<char *>(EmptyPODArray) + EmptyPODArraySize : nullptr;
@ -78,16 +81,8 @@ protected:
char * c_end = null;
char * c_end_of_storage = null; /// Does not include pad_right.
T * t_start() { return reinterpret_cast<T *>(c_start); }
T * t_end() { return reinterpret_cast<T *>(c_end); }
T * t_end_of_storage() { return reinterpret_cast<T *>(c_end_of_storage); }
const T * t_start() const { return reinterpret_cast<const T *>(c_start); }
const T * t_end() const { return reinterpret_cast<const T *>(c_end); }
const T * t_end_of_storage() const { return reinterpret_cast<const T *>(c_end_of_storage); }
/// The amount of memory occupied by the num_elements of the elements.
static size_t byte_size(size_t num_elements) { return num_elements * sizeof(T); }
static size_t byte_size(size_t num_elements) { return num_elements * ELEMENT_SIZE; }
/// Minimum amount of memory to allocate for num_elements, including padding.
static size_t minimum_memory_for_elements(size_t num_elements) { return byte_size(num_elements) + pad_right + pad_left; }
@ -102,8 +97,9 @@ protected:
{
c_start = c_end = reinterpret_cast<char *>(TAllocator::alloc(bytes, std::forward<TAllocatorParams>(allocator_params)...)) + pad_left;
c_end_of_storage = c_start + bytes - pad_right - pad_left;
if (pad_left)
t_start()[-1] = {};
memset(c_start - ELEMENT_SIZE, 0, ELEMENT_SIZE);
}
void dealloc()
@ -126,12 +122,11 @@ protected:
ptrdiff_t end_diff = c_end - c_start;
c_start = reinterpret_cast<char *>(
TAllocator::realloc(c_start - pad_left, allocated_bytes(), bytes, std::forward<TAllocatorParams>(allocator_params)...))
TAllocator::realloc(c_start - pad_left, allocated_bytes(), bytes, std::forward<TAllocatorParams>(allocator_params)...))
+ pad_left;
c_end = c_start + end_diff;
c_end_of_storage = c_start + bytes - pad_right - pad_left;
if (pad_left)
t_start()[-1] = {};
}
bool isInitialized() const
@ -150,9 +145,9 @@ protected:
{
if (size() == 0)
{
// The allocated memory should be multiplication of sizeof(T) to hold the element, otherwise,
// The allocated memory should be multiplication of ELEMENT_SIZE to hold the element, otherwise,
// memory issue such as corruption could appear in edge case.
realloc(std::max(((INITIAL_SIZE - 1) / sizeof(T) + 1) * sizeof(T), minimum_memory_for_elements(1)),
realloc(std::max(((INITIAL_SIZE - 1) / ELEMENT_SIZE + 1) * ELEMENT_SIZE, minimum_memory_for_elements(1)),
std::forward<TAllocatorParams>(allocator_params)...);
}
else
@ -160,83 +155,13 @@ protected:
}
public:
using value_type = T;
bool empty() const { return c_end == c_start; }
size_t size() const { return (c_end - c_start) / ELEMENT_SIZE; }
size_t capacity() const { return (c_end_of_storage - c_start) / ELEMENT_SIZE; }
size_t allocated_bytes() const { return c_end_of_storage - c_start + pad_right + pad_left; }
/// You can not just use `typedef`, because there is ambiguity for the constructors and `assign` functions.
struct iterator : public boost::iterator_adaptor<iterator, T*>
{
iterator() {}
iterator(T * ptr_) : iterator::iterator_adaptor_(ptr_) {}
};
struct const_iterator : public boost::iterator_adaptor<const_iterator, const T*>
{
const_iterator() {}
const_iterator(const T * ptr_) : const_iterator::iterator_adaptor_(ptr_) {}
};
PODArray() {}
PODArray(size_t n)
{
alloc_for_num_elements(n);
c_end += byte_size(n);
}
PODArray(size_t n, const T & x)
{
alloc_for_num_elements(n);
assign(n, x);
}
PODArray(const_iterator from_begin, const_iterator from_end)
{
alloc_for_num_elements(from_end - from_begin);
insert(from_begin, from_end);
}
PODArray(std::initializer_list<T> il) : PODArray(std::begin(il), std::end(il)) {}
~PODArray()
{
dealloc();
}
PODArray(PODArray && other)
{
this->swap(other);
}
PODArray & operator=(PODArray && other)
{
this->swap(other);
return *this;
}
T * data() { return t_start(); }
const T * data() const { return t_start(); }
size_t size() const { return t_end() - t_start(); }
bool empty() const { return t_end() == t_start(); }
size_t capacity() const { return t_end_of_storage() - t_start(); }
T & operator[] (size_t n) { return t_start()[n]; }
const T & operator[] (size_t n) const { return t_start()[n]; }
T & front() { return t_start()[0]; }
T & back() { return t_end()[-1]; }
const T & front() const { return t_start()[0]; }
const T & back() const { return t_end()[-1]; }
iterator begin() { return t_start(); }
iterator end() { return t_end(); }
const_iterator begin() const { return t_start(); }
const_iterator end() const { return t_end(); }
const_iterator cbegin() const { return t_start(); }
const_iterator cend() const { return t_end(); }
void clear() { c_end = c_start; }
template <typename ... TAllocatorParams>
void reserve(size_t n, TAllocatorParams &&... allocator_params)
@ -257,69 +182,168 @@ public:
c_end = c_start + byte_size(n);
}
/// Same as resize, but zeroes new elements.
void resize_fill(size_t n)
const char * raw_data() const
{
size_t old_size = size();
if (n > old_size)
{
reserve(n);
memset(c_end, 0, byte_size(n - old_size));
}
c_end = c_start + byte_size(n);
}
void resize_fill(size_t n, const T & value)
{
size_t old_size = size();
if (n > old_size)
{
reserve(n);
std::fill(t_end(), t_end() + n - old_size, value);
}
c_end = c_start + byte_size(n);
}
void clear()
{
c_end = c_start;
return c_start;
}
template <typename ... TAllocatorParams>
void push_back(const T & x, TAllocatorParams &&... allocator_params)
void push_back_raw(const char * ptr, TAllocatorParams &&... allocator_params)
{
if (unlikely(c_end == c_end_of_storage))
reserveForNextSize(std::forward<TAllocatorParams>(allocator_params)...);
*t_end() = x;
memcpy(c_end, ptr, ELEMENT_SIZE);
c_end += byte_size(1);
}
~PODArrayBase()
{
dealloc();
}
};
template <typename T, size_t INITIAL_SIZE = 4096, typename TAllocator = Allocator<false>, size_t pad_right_ = 0, size_t pad_left_ = 0>
class PODArray : public PODArrayBase<sizeof(T), INITIAL_SIZE, TAllocator, pad_right_, pad_left_>
{
protected:
using Base = PODArrayBase<sizeof(T), INITIAL_SIZE, TAllocator, pad_right_, pad_left_>;
T * t_start() { return reinterpret_cast<T *>(this->c_start); }
T * t_end() { return reinterpret_cast<T *>(this->c_end); }
T * t_end_of_storage() { return reinterpret_cast<T *>(this->c_end_of_storage); }
const T * t_start() const { return reinterpret_cast<const T *>(this->c_start); }
const T * t_end() const { return reinterpret_cast<const T *>(this->c_end); }
const T * t_end_of_storage() const { return reinterpret_cast<const T *>(this->c_end_of_storage); }
public:
using value_type = T;
/// You can not just use `typedef`, because there is ambiguity for the constructors and `assign` functions.
struct iterator : public boost::iterator_adaptor<iterator, T*>
{
iterator() {}
iterator(T * ptr_) : iterator::iterator_adaptor_(ptr_) {}
};
struct const_iterator : public boost::iterator_adaptor<const_iterator, const T*>
{
const_iterator() {}
const_iterator(const T * ptr_) : const_iterator::iterator_adaptor_(ptr_) {}
};
PODArray() {}
PODArray(size_t n)
{
this->alloc_for_num_elements(n);
this->c_end += this->byte_size(n);
}
PODArray(size_t n, const T & x)
{
this->alloc_for_num_elements(n);
assign(n, x);
}
PODArray(const_iterator from_begin, const_iterator from_end)
{
this->alloc_for_num_elements(from_end - from_begin);
insert(from_begin, from_end);
}
PODArray(std::initializer_list<T> il) : PODArray(std::begin(il), std::end(il)) {}
PODArray(PODArray && other)
{
this->swap(other);
}
PODArray & operator=(PODArray && other)
{
this->swap(other);
return *this;
}
T * data() { return t_start(); }
const T * data() const { return t_start(); }
/// The index is signed to access -1th element without pointer overflow.
T & operator[] (ssize_t n) { return t_start()[n]; }
const T & operator[] (ssize_t n) const { return t_start()[n]; }
T & front() { return t_start()[0]; }
T & back() { return t_end()[-1]; }
const T & front() const { return t_start()[0]; }
const T & back() const { return t_end()[-1]; }
iterator begin() { return t_start(); }
iterator end() { return t_end(); }
const_iterator begin() const { return t_start(); }
const_iterator end() const { return t_end(); }
const_iterator cbegin() const { return t_start(); }
const_iterator cend() const { return t_end(); }
/// Same as resize, but zeroes new elements.
void resize_fill(size_t n)
{
size_t old_size = this->size();
if (n > old_size)
{
this->reserve(n);
memset(this->c_end, 0, this->byte_size(n - old_size));
}
this->c_end = this->c_start + this->byte_size(n);
}
void resize_fill(size_t n, const T & value)
{
size_t old_size = this->size();
if (n > old_size)
{
this->reserve(n);
std::fill(t_end(), t_end() + n - old_size, value);
}
this->c_end = this->c_start + this->byte_size(n);
}
template <typename U, typename ... TAllocatorParams>
void push_back(U && x, TAllocatorParams &&... allocator_params)
{
if (unlikely(this->c_end == this->c_end_of_storage))
this->reserveForNextSize(std::forward<TAllocatorParams>(allocator_params)...);
new (t_end()) T(std::forward<U>(x));
this->c_end += this->byte_size(1);
}
/** This method doesn't allow to pass parameters for Allocator,
* and it couldn't be used if Allocator requires custom parameters.
*/
template <typename... Args>
void emplace_back(Args &&... args)
{
if (unlikely(c_end == c_end_of_storage))
reserveForNextSize();
if (unlikely(this->c_end == this->c_end_of_storage))
this->reserveForNextSize();
new (t_end()) T(std::forward<Args>(args)...);
c_end += byte_size(1);
this->c_end += this->byte_size(1);
}
void pop_back()
{
c_end -= byte_size(1);
this->c_end -= this->byte_size(1);
}
/// Do not insert into the array a piece of itself. Because with the resize, the iterators on themselves can be invalidated.
template <typename It1, typename It2, typename ... TAllocatorParams>
void insertPrepare(It1 from_begin, It2 from_end, TAllocatorParams &&... allocator_params)
{
size_t required_capacity = size() + (from_end - from_begin);
if (required_capacity > capacity())
reserve(roundUpToPowerOfTwoOrZero(required_capacity), std::forward<TAllocatorParams>(allocator_params)...);
size_t required_capacity = this->size() + (from_end - from_begin);
if (required_capacity > this->capacity())
this->reserve(roundUpToPowerOfTwoOrZero(required_capacity), std::forward<TAllocatorParams>(allocator_params)...);
}
/// Do not insert into the array a piece of itself. Because with the resize, the iterators on themselves can be invalidated.
@ -336,9 +360,9 @@ public:
{
static_assert(pad_right_ >= 15);
insertPrepare(from_begin, from_end, std::forward<TAllocatorParams>(allocator_params)...);
size_t bytes_to_copy = byte_size(from_end - from_begin);
memcpySmallAllowReadWriteOverflow15(c_end, reinterpret_cast<const void *>(&*from_begin), bytes_to_copy);
c_end += bytes_to_copy;
size_t bytes_to_copy = this->byte_size(from_end - from_begin);
memcpySmallAllowReadWriteOverflow15(this->c_end, reinterpret_cast<const void *>(&*from_begin), bytes_to_copy);
this->c_end += bytes_to_copy;
}
template <typename It1, typename It2>
@ -346,22 +370,22 @@ public:
{
insertPrepare(from_begin, from_end);
size_t bytes_to_copy = byte_size(from_end - from_begin);
size_t bytes_to_copy = this->byte_size(from_end - from_begin);
size_t bytes_to_move = (end() - it) * sizeof(T);
if (unlikely(bytes_to_move))
memcpy(c_end + bytes_to_copy - bytes_to_move, c_end - bytes_to_move, bytes_to_move);
memcpy(this->c_end + bytes_to_copy - bytes_to_move, this->c_end - bytes_to_move, bytes_to_move);
memcpy(c_end - bytes_to_move, reinterpret_cast<const void *>(&*from_begin), bytes_to_copy);
c_end += bytes_to_copy;
memcpy(this->c_end - bytes_to_move, reinterpret_cast<const void *>(&*from_begin), bytes_to_copy);
this->c_end += bytes_to_copy;
}
template <typename It1, typename It2>
void insert_assume_reserved(It1 from_begin, It2 from_end)
{
size_t bytes_to_copy = byte_size(from_end - from_begin);
memcpy(c_end, reinterpret_cast<const void *>(&*from_begin), bytes_to_copy);
c_end += bytes_to_copy;
size_t bytes_to_copy = this->byte_size(from_end - from_begin);
memcpy(this->c_end, reinterpret_cast<const void *>(&*from_begin), bytes_to_copy);
this->c_end += bytes_to_copy;
}
void swap(PODArray & rhs)
@ -369,7 +393,7 @@ public:
/// Swap two PODArray objects, arr1 and arr2, that satisfy the following conditions:
/// - The elements of arr1 are stored on stack.
/// - The elements of arr2 are stored on heap.
auto swap_stack_heap = [](PODArray & arr1, PODArray & arr2)
auto swap_stack_heap = [this](PODArray & arr1, PODArray & arr2)
{
size_t stack_size = arr1.size();
size_t stack_allocated = arr1.allocated_bytes();
@ -383,27 +407,27 @@ public:
/// arr1 takes ownership of the heap memory of arr2.
arr1.c_start = arr2.c_start;
arr1.c_end_of_storage = arr1.c_start + heap_allocated - arr1.pad_right;
arr1.c_end = arr1.c_start + byte_size(heap_size);
arr1.c_end = arr1.c_start + this->byte_size(heap_size);
/// Allocate stack space for arr2.
arr2.alloc(stack_allocated);
/// Copy the stack content.
memcpy(arr2.c_start, stack_c_start, byte_size(stack_size));
arr2.c_end = arr2.c_start + byte_size(stack_size);
memcpy(arr2.c_start, stack_c_start, this->byte_size(stack_size));
arr2.c_end = arr2.c_start + this->byte_size(stack_size);
};
auto do_move = [](PODArray & src, PODArray & dest)
auto do_move = [this](PODArray & src, PODArray & dest)
{
if (src.isAllocatedFromStack())
{
dest.dealloc();
dest.alloc(src.allocated_bytes());
memcpy(dest.c_start, src.c_start, byte_size(src.size()));
memcpy(dest.c_start, src.c_start, this->byte_size(src.size()));
dest.c_end = dest.c_start + (src.c_end - src.c_start);
src.c_start = null;
src.c_end = null;
src.c_end_of_storage = null;
src.c_start = Base::null;
src.c_end = Base::null;
src.c_end_of_storage = Base::null;
}
else
{
@ -413,28 +437,28 @@ public:
}
};
if (!isInitialized() && !rhs.isInitialized())
if (!this->isInitialized() && !rhs.isInitialized())
return;
else if (!isInitialized() && rhs.isInitialized())
else if (!this->isInitialized() && rhs.isInitialized())
{
do_move(rhs, *this);
return;
}
else if (isInitialized() && !rhs.isInitialized())
else if (this->isInitialized() && !rhs.isInitialized())
{
do_move(*this, rhs);
return;
}
if (isAllocatedFromStack() && rhs.isAllocatedFromStack())
if (this->isAllocatedFromStack() && rhs.isAllocatedFromStack())
{
size_t min_size = std::min(size(), rhs.size());
size_t max_size = std::max(size(), rhs.size());
size_t min_size = std::min(this->size(), rhs.size());
size_t max_size = std::max(this->size(), rhs.size());
for (size_t i = 0; i < min_size; ++i)
std::swap(this->operator[](i), rhs[i]);
if (size() == max_size)
if (this->size() == max_size)
{
for (size_t i = min_size; i < max_size; ++i)
rhs[i] = this->operator[](i);
@ -445,33 +469,33 @@ public:
this->operator[](i) = rhs[i];
}
size_t lhs_size = size();
size_t lhs_allocated = allocated_bytes();
size_t lhs_size = this->size();
size_t lhs_allocated = this->allocated_bytes();
size_t rhs_size = rhs.size();
size_t rhs_allocated = rhs.allocated_bytes();
c_end_of_storage = c_start + rhs_allocated - pad_right;
rhs.c_end_of_storage = rhs.c_start + lhs_allocated - pad_right;
this->c_end_of_storage = this->c_start + rhs_allocated - Base::pad_right;
rhs.c_end_of_storage = rhs.c_start + lhs_allocated - Base::pad_right;
c_end = c_start + byte_size(rhs_size);
rhs.c_end = rhs.c_start + byte_size(lhs_size);
this->c_end = this->c_start + this->byte_size(rhs_size);
rhs.c_end = rhs.c_start + this->byte_size(lhs_size);
}
else if (isAllocatedFromStack() && !rhs.isAllocatedFromStack())
else if (this->isAllocatedFromStack() && !rhs.isAllocatedFromStack())
swap_stack_heap(*this, rhs);
else if (!isAllocatedFromStack() && rhs.isAllocatedFromStack())
else if (!this->isAllocatedFromStack() && rhs.isAllocatedFromStack())
swap_stack_heap(rhs, *this);
else
{
std::swap(c_start, rhs.c_start);
std::swap(c_end, rhs.c_end);
std::swap(c_end_of_storage, rhs.c_end_of_storage);
std::swap(this->c_start, rhs.c_start);
std::swap(this->c_end, rhs.c_end);
std::swap(this->c_end_of_storage, rhs.c_end_of_storage);
}
}
void assign(size_t n, const T & x)
{
resize(n);
this->resize(n);
std::fill(begin(), end(), x);
}
@ -479,12 +503,12 @@ public:
void assign(It1 from_begin, It2 from_end)
{
size_t required_capacity = from_end - from_begin;
if (required_capacity > capacity())
reserve(roundUpToPowerOfTwoOrZero(required_capacity));
if (required_capacity > this->capacity())
this->reserve(roundUpToPowerOfTwoOrZero(required_capacity));
size_t bytes_to_copy = byte_size(required_capacity);
memcpy(c_start, reinterpret_cast<const void *>(&*from_begin), bytes_to_copy);
c_end = c_start + bytes_to_copy;
size_t bytes_to_copy = this->byte_size(required_capacity);
memcpy(this->c_start, reinterpret_cast<const void *>(&*from_begin), bytes_to_copy);
this->c_end = this->c_start + bytes_to_copy;
}
void assign(const PODArray & from)
@ -495,7 +519,7 @@ public:
bool operator== (const PODArray & other) const
{
if (size() != other.size())
if (this->size() != other.size())
return false;
const_iterator this_it = begin();

Some files were not shown because too many files have changed in this diff Show More