mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-10-13 03:50:50 +00:00
Merge branch 'master' of github.com:yandex/ClickHouse
This commit is contained in:
commit
f417854983
2
LICENSE
2
LICENSE
@ -186,7 +186,7 @@
|
|||||||
same "printed page" as the copyright notice for easier
|
same "printed page" as the copyright notice for easier
|
||||||
identification within third-party archives.
|
identification within third-party archives.
|
||||||
|
|
||||||
Copyright 2016-2018 Yandex LLC
|
Copyright 2016-2019 Yandex LLC
|
||||||
|
|
||||||
Licensed under the Apache License, Version 2.0 (the "License");
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
you may not use this file except in compliance with the License.
|
you may not use this file except in compliance with the License.
|
||||||
|
@ -79,7 +79,11 @@ endif()
|
|||||||
|
|
||||||
function(llvm_libs_all REQUIRED_LLVM_LIBRARIES)
|
function(llvm_libs_all REQUIRED_LLVM_LIBRARIES)
|
||||||
llvm_map_components_to_libnames (result all)
|
llvm_map_components_to_libnames (result all)
|
||||||
list (REMOVE_ITEM result "LTO" "LLVM")
|
if (USE_STATIC_LIBRARIES OR NOT "LLVM" IN_LIST result)
|
||||||
|
list (REMOVE_ITEM result "LTO" "LLVM")
|
||||||
|
else()
|
||||||
|
set (result "LLVM")
|
||||||
|
endif ()
|
||||||
if (TERMCAP_LIBRARY)
|
if (TERMCAP_LIBRARY)
|
||||||
list (APPEND result ${TERMCAP_LIBRARY})
|
list (APPEND result ${TERMCAP_LIBRARY})
|
||||||
endif ()
|
endif ()
|
||||||
|
2
contrib/CMakeLists.txt
vendored
2
contrib/CMakeLists.txt
vendored
@ -1,7 +1,7 @@
|
|||||||
# Third-party libraries may have substandard code.
|
# Third-party libraries may have substandard code.
|
||||||
|
|
||||||
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
||||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-stringop-overflow")
|
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-stringop-overflow -Wno-implicit-function-declaration -Wno-return-type -Wno-array-bounds -Wno-bool-compare -Wno-int-conversion -Wno-switch")
|
||||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-implicit-fallthrough -Wno-class-memaccess -Wno-sign-compare -std=c++1z")
|
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-old-style-cast -Wno-unused-function -Wno-unused-variable -Wno-unused-but-set-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-non-virtual-dtor -Wno-maybe-uninitialized -Wno-format -Wno-misleading-indentation -Wno-implicit-fallthrough -Wno-class-memaccess -Wno-sign-compare -std=c++1z")
|
||||||
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
|
elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
|
||||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-format -Wno-parentheses-equality -Wno-tautological-constant-compare -Wno-tautological-constant-out-of-range-compare -Wno-implicit-function-declaration -Wno-return-type -Wno-pointer-bool-conversion -Wno-enum-conversion -Wno-int-conversion -Wno-switch")
|
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-function -Wno-unused-variable -Wno-unused-result -Wno-deprecated-declarations -Wno-format -Wno-parentheses-equality -Wno-tautological-constant-compare -Wno-tautological-constant-out-of-range-compare -Wno-implicit-function-declaration -Wno-return-type -Wno-pointer-bool-conversion -Wno-enum-conversion -Wno-int-conversion -Wno-switch")
|
||||||
|
2
contrib/jemalloc
vendored
2
contrib/jemalloc
vendored
@ -1 +1 @@
|
|||||||
Subproject commit cd2931ad9bbd78208565716ab102e86d858c2fff
|
Subproject commit 41b7372eadee941b9164751b8d4963f915d3ceae
|
@ -1,48 +0,0 @@
|
|||||||
# Check prereqs
|
|
||||||
FIND_PROGRAM(GCOV_PATH gcov)
|
|
||||||
FIND_PROGRAM(LCOV_PATH lcov)
|
|
||||||
FIND_PROGRAM(GENHTML_PATH genhtml)
|
|
||||||
|
|
||||||
IF(NOT GCOV_PATH)
|
|
||||||
MESSAGE(FATAL_ERROR "gcov not found! Aborting...")
|
|
||||||
ENDIF(NOT GCOV_PATH)
|
|
||||||
|
|
||||||
IF(NOT CMAKE_BUILD_TYPE STREQUAL Debug)
|
|
||||||
MESSAGE(WARNING "Code coverage results with an optimised (non-Debug) build may be misleading")
|
|
||||||
ENDIF(NOT CMAKE_BUILD_TYPE STREQUAL Debug)
|
|
||||||
|
|
||||||
#Setup compiler options
|
|
||||||
ADD_DEFINITIONS(-fprofile-arcs -ftest-coverage)
|
|
||||||
|
|
||||||
SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs ")
|
|
||||||
SET(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fprofile-arcs ")
|
|
||||||
|
|
||||||
IF(NOT LCOV_PATH)
|
|
||||||
MESSAGE(FATAL_ERROR "lcov not found! Aborting...")
|
|
||||||
ENDIF(NOT LCOV_PATH)
|
|
||||||
|
|
||||||
IF(NOT GENHTML_PATH)
|
|
||||||
MESSAGE(FATAL_ERROR "genhtml not found! Aborting...")
|
|
||||||
ENDIF(NOT GENHTML_PATH)
|
|
||||||
|
|
||||||
#Setup target
|
|
||||||
ADD_CUSTOM_TARGET(ShowCoverage
|
|
||||||
#Capturing lcov counters and generating report
|
|
||||||
COMMAND ${LCOV_PATH} --directory . --capture --output-file CodeCoverage.info
|
|
||||||
COMMAND ${LCOV_PATH} --remove CodeCoverage.info '${CMAKE_CURRENT_BINARY_DIR}/*' 'test/*' 'mock/*' '/usr/*' '/opt/*' '*ext/rhel5_x86_64*' '*ext/osx*' --output-file CodeCoverage.info.cleaned
|
|
||||||
COMMAND ${GENHTML_PATH} -o CodeCoverageReport CodeCoverage.info.cleaned
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
ADD_CUSTOM_TARGET(ShowAllCoverage
|
|
||||||
#Capturing lcov counters and generating report
|
|
||||||
COMMAND ${LCOV_PATH} -a CodeCoverage.info.cleaned -a CodeCoverage.info.cleaned_withoutHA -o AllCodeCoverage.info
|
|
||||||
COMMAND sed -e 's|/.*/src|${CMAKE_SOURCE_DIR}/src|' -ig AllCodeCoverage.info
|
|
||||||
COMMAND ${GENHTML_PATH} -o AllCodeCoverageReport AllCodeCoverage.info
|
|
||||||
)
|
|
||||||
|
|
||||||
ADD_CUSTOM_TARGET(ResetCoverage
|
|
||||||
#Cleanup lcov
|
|
||||||
COMMAND ${LCOV_PATH} --directory . --zerocounters
|
|
||||||
)
|
|
||||||
|
|
File diff suppressed because it is too large
Load Diff
@ -1,35 +1,12 @@
|
|||||||
OPTION(ENABLE_COVERAGE "enable code coverage" OFF)
|
|
||||||
OPTION(ENABLE_DEBUG "enable debug build" OFF)
|
|
||||||
OPTION(ENABLE_SSE "enable SSE4.2 buildin function" ON)
|
OPTION(ENABLE_SSE "enable SSE4.2 buildin function" ON)
|
||||||
OPTION(ENABLE_FRAME_POINTER "enable frame pointer on 64bit system with flag -fno-omit-frame-pointer, on 32bit system, it is always enabled" ON)
|
|
||||||
OPTION(ENABLE_LIBCPP "using libc++ instead of libstdc++, only valid for clang compiler" OFF)
|
|
||||||
OPTION(ENABLE_BOOST "using boost instead of native compiler c++0x support" OFF)
|
|
||||||
|
|
||||||
INCLUDE (CheckFunctionExists)
|
INCLUDE (CheckFunctionExists)
|
||||||
CHECK_FUNCTION_EXISTS(dladdr HAVE_DLADDR)
|
CHECK_FUNCTION_EXISTS(dladdr HAVE_DLADDR)
|
||||||
CHECK_FUNCTION_EXISTS(nanosleep HAVE_NANOSLEEP)
|
CHECK_FUNCTION_EXISTS(nanosleep HAVE_NANOSLEEP)
|
||||||
|
|
||||||
IF(ENABLE_DEBUG STREQUAL ON)
|
|
||||||
SET(CMAKE_BUILD_TYPE Debug CACHE
|
|
||||||
STRING "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel." FORCE)
|
|
||||||
SET(CMAKE_CXX_FLAGS_DEBUG "-g -O0" CACHE STRING "compiler flags for debug" FORCE)
|
|
||||||
SET(CMAKE_C_FLAGS_DEBUG "-g -O0" CACHE STRING "compiler flags for debug" FORCE)
|
|
||||||
ELSE(ENABLE_DEBUG STREQUAL ON)
|
|
||||||
SET(CMAKE_BUILD_TYPE RelWithDebInfo CACHE
|
|
||||||
STRING "Choose the type of build, options are: None Debug Release RelWithDebInfo MinSizeRel." FORCE)
|
|
||||||
ENDIF(ENABLE_DEBUG STREQUAL ON)
|
|
||||||
|
|
||||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-strict-aliasing")
|
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-strict-aliasing")
|
||||||
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-strict-aliasing")
|
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-strict-aliasing")
|
||||||
|
|
||||||
IF(ENABLE_COVERAGE STREQUAL ON)
|
|
||||||
INCLUDE(CodeCoverage)
|
|
||||||
ENDIF(ENABLE_COVERAGE STREQUAL ON)
|
|
||||||
|
|
||||||
IF(ENABLE_FRAME_POINTER STREQUAL ON)
|
|
||||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
|
|
||||||
ENDIF(ENABLE_FRAME_POINTER STREQUAL ON)
|
|
||||||
|
|
||||||
IF(ENABLE_SSE STREQUAL ON)
|
IF(ENABLE_SSE STREQUAL ON)
|
||||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2")
|
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2")
|
||||||
ENDIF(ENABLE_SSE STREQUAL ON)
|
ENDIF(ENABLE_SSE STREQUAL ON)
|
||||||
@ -41,76 +18,7 @@ ENDIF(NOT TEST_HDFS_PREFIX)
|
|||||||
ADD_DEFINITIONS(-DTEST_HDFS_PREFIX="${TEST_HDFS_PREFIX}")
|
ADD_DEFINITIONS(-DTEST_HDFS_PREFIX="${TEST_HDFS_PREFIX}")
|
||||||
ADD_DEFINITIONS(-D__STDC_FORMAT_MACROS)
|
ADD_DEFINITIONS(-D__STDC_FORMAT_MACROS)
|
||||||
ADD_DEFINITIONS(-D_GNU_SOURCE)
|
ADD_DEFINITIONS(-D_GNU_SOURCE)
|
||||||
|
ADD_DEFINITIONS(-D_GLIBCXX_USE_NANOSLEEP)
|
||||||
IF(OS_MACOSX AND CMAKE_COMPILER_IS_GNUCXX)
|
|
||||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-bind_at_load")
|
|
||||||
ENDIF(OS_MACOSX AND CMAKE_COMPILER_IS_GNUCXX)
|
|
||||||
|
|
||||||
IF(OS_LINUX)
|
|
||||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,--export-dynamic")
|
|
||||||
ENDIF(OS_LINUX)
|
|
||||||
|
|
||||||
SET(BOOST_ROOT ${CMAKE_PREFIX_PATH})
|
|
||||||
IF(ENABLE_BOOST STREQUAL ON)
|
|
||||||
MESSAGE(STATUS "using boost instead of native compiler c++0x support.")
|
|
||||||
FIND_PACKAGE(Boost 1.50 REQUIRED)
|
|
||||||
SET(NEED_BOOST true CACHE INTERNAL "boost is required")
|
|
||||||
ELSE(ENABLE_BOOST STREQUAL ON)
|
|
||||||
SET(NEED_BOOST false CACHE INTERNAL "boost is required")
|
|
||||||
ENDIF(ENABLE_BOOST STREQUAL ON)
|
|
||||||
|
|
||||||
IF(CMAKE_COMPILER_IS_GNUCXX)
|
|
||||||
IF(ENABLE_LIBCPP STREQUAL ON)
|
|
||||||
MESSAGE(FATAL_ERROR "Unsupport using GCC compiler with libc++")
|
|
||||||
ENDIF(ENABLE_LIBCPP STREQUAL ON)
|
|
||||||
|
|
||||||
IF((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR EQUAL 4) AND OS_MACOSX)
|
|
||||||
SET(NEED_GCCEH true CACHE INTERNAL "Explicitly link with gcc_eh")
|
|
||||||
MESSAGE(STATUS "link with -lgcc_eh for TLS")
|
|
||||||
ENDIF((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR EQUAL 4) AND OS_MACOSX)
|
|
||||||
|
|
||||||
IF((GCC_COMPILER_VERSION_MAJOR LESS 4) OR ((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 4)))
|
|
||||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
|
|
||||||
IF(NOT ENABLE_BOOST STREQUAL ON)
|
|
||||||
MESSAGE(STATUS "gcc version is older than 4.6.0, boost is required.")
|
|
||||||
FIND_PACKAGE(Boost 1.50 REQUIRED)
|
|
||||||
SET(NEED_BOOST true CACHE INTERNAL "boost is required")
|
|
||||||
ENDIF(NOT ENABLE_BOOST STREQUAL ON)
|
|
||||||
ELSEIF((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 7))
|
|
||||||
IF(NOT ENABLE_BOOST STREQUAL ON)
|
|
||||||
MESSAGE(STATUS "gcc version is older than 4.6.0, boost is required.")
|
|
||||||
FIND_PACKAGE(Boost 1.50 REQUIRED)
|
|
||||||
SET(NEED_BOOST true CACHE INTERNAL "boost is required")
|
|
||||||
ENDIF(NOT ENABLE_BOOST STREQUAL ON)
|
|
||||||
MESSAGE(STATUS "adding c++0x support for gcc compiler")
|
|
||||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x")
|
|
||||||
ELSE((GCC_COMPILER_VERSION_MAJOR LESS 4) OR ((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 4)))
|
|
||||||
MESSAGE(STATUS "adding c++0x support for gcc compiler")
|
|
||||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x")
|
|
||||||
ENDIF((GCC_COMPILER_VERSION_MAJOR LESS 4) OR ((GCC_COMPILER_VERSION_MAJOR EQUAL 4) AND (GCC_COMPILER_VERSION_MINOR LESS 4)))
|
|
||||||
|
|
||||||
IF(NEED_BOOST)
|
|
||||||
IF((Boost_MAJOR_VERSION LESS 1) OR ((Boost_MAJOR_VERSION EQUAL 1) AND (Boost_MINOR_VERSION LESS 50)))
|
|
||||||
MESSAGE(FATAL_ERROR "boost 1.50+ is required")
|
|
||||||
ENDIF()
|
|
||||||
ELSE(NEED_BOOST)
|
|
||||||
IF(HAVE_NANOSLEEP)
|
|
||||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_NANOSLEEP")
|
|
||||||
ELSE(HAVE_NANOSLEEP)
|
|
||||||
MESSAGE(FATAL_ERROR "nanosleep() is required")
|
|
||||||
ENDIF(HAVE_NANOSLEEP)
|
|
||||||
ENDIF(NEED_BOOST)
|
|
||||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall")
|
|
||||||
ELSEIF(CMAKE_COMPILER_IS_CLANG)
|
|
||||||
MESSAGE(STATUS "adding c++0x support for clang compiler")
|
|
||||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++0x")
|
|
||||||
SET(CMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LANGUAGE_STANDARD "c++0x")
|
|
||||||
IF(ENABLE_LIBCPP STREQUAL ON)
|
|
||||||
MESSAGE(STATUS "using libc++ instead of libstdc++")
|
|
||||||
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
|
|
||||||
SET(CMAKE_XCODE_ATTRIBUTE_CLANG_CXX_LIBRARY "libc++")
|
|
||||||
ENDIF(ENABLE_LIBCPP STREQUAL ON)
|
|
||||||
ENDIF(CMAKE_COMPILER_IS_GNUCXX)
|
|
||||||
|
|
||||||
TRY_COMPILE(STRERROR_R_RETURN_INT
|
TRY_COMPILE(STRERROR_R_RETURN_INT
|
||||||
${CMAKE_CURRENT_BINARY_DIR}
|
${CMAKE_CURRENT_BINARY_DIR}
|
||||||
@ -138,32 +46,8 @@ TRY_COMPILE(HAVE_NESTED_EXCEPTION
|
|||||||
CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'"
|
CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'"
|
||||||
OUTPUT_VARIABLE OUTPUT)
|
OUTPUT_VARIABLE OUTPUT)
|
||||||
|
|
||||||
FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include <boost/chrono.hpp>")
|
SET(HAVE_BOOST_CHRONO 0)
|
||||||
TRY_COMPILE(HAVE_BOOST_CHRONO
|
SET(HAVE_BOOST_ATOMIC 0)
|
||||||
${CMAKE_CURRENT_BINARY_DIR}
|
|
||||||
${CMAKE_CURRENT_BINARY_DIR}/test.cpp
|
|
||||||
CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'"
|
|
||||||
-DINCLUDE_DIRECTORIES=${Boost_INCLUDE_DIR}
|
|
||||||
OUTPUT_VARIABLE OUTPUT)
|
|
||||||
|
|
||||||
FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include <chrono>")
|
SET(HAVE_STD_CHRONO 1)
|
||||||
TRY_COMPILE(HAVE_STD_CHRONO
|
SET(HAVE_STD_ATOMIC 1)
|
||||||
${CMAKE_CURRENT_BINARY_DIR}
|
|
||||||
${CMAKE_CURRENT_BINARY_DIR}/test.cpp
|
|
||||||
CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'"
|
|
||||||
OUTPUT_VARIABLE OUTPUT)
|
|
||||||
|
|
||||||
FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include <boost/atomic.hpp>")
|
|
||||||
TRY_COMPILE(HAVE_BOOST_ATOMIC
|
|
||||||
${CMAKE_CURRENT_BINARY_DIR}
|
|
||||||
${CMAKE_CURRENT_BINARY_DIR}/test.cpp
|
|
||||||
CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'"
|
|
||||||
-DINCLUDE_DIRECTORIES=${Boost_INCLUDE_DIR}
|
|
||||||
OUTPUT_VARIABLE OUTPUT)
|
|
||||||
|
|
||||||
FILE(WRITE ${CMAKE_CURRENT_BINARY_DIR}/test.cpp "#include <atomic>")
|
|
||||||
TRY_COMPILE(HAVE_STD_ATOMIC
|
|
||||||
${CMAKE_CURRENT_BINARY_DIR}
|
|
||||||
${CMAKE_CURRENT_BINARY_DIR}/test.cpp
|
|
||||||
CMAKE_FLAGS "-DCMAKE_CXX_LINK_EXECUTABLE='echo not linking now...'"
|
|
||||||
OUTPUT_VARIABLE OUTPUT)
|
|
||||||
|
@ -2,8 +2,7 @@
|
|||||||
// MurmurHash2 was written by Austin Appleby, and is placed in the public
|
// MurmurHash2 was written by Austin Appleby, and is placed in the public
|
||||||
// domain. The author hereby disclaims copyright to this source code.
|
// domain. The author hereby disclaims copyright to this source code.
|
||||||
|
|
||||||
#ifndef _MURMURHASH2_H_
|
#pragma once
|
||||||
#define _MURMURHASH2_H_
|
|
||||||
|
|
||||||
//-----------------------------------------------------------------------------
|
//-----------------------------------------------------------------------------
|
||||||
// Platform-specific functions and macros
|
// Platform-specific functions and macros
|
||||||
@ -30,6 +29,3 @@ uint64_t MurmurHash64B (const void * key, int len, uint64_t seed);
|
|||||||
uint32_t MurmurHash2A (const void * key, int len, uint32_t seed);
|
uint32_t MurmurHash2A (const void * key, int len, uint32_t seed);
|
||||||
uint32_t MurmurHashNeutral2 (const void * key, int len, uint32_t seed);
|
uint32_t MurmurHashNeutral2 (const void * key, int len, uint32_t seed);
|
||||||
uint32_t MurmurHashAligned2 (const void * key, int len, uint32_t seed);
|
uint32_t MurmurHashAligned2 (const void * key, int len, uint32_t seed);
|
||||||
|
|
||||||
#endif // _MURMURHASH2_H_
|
|
||||||
|
|
||||||
|
@ -2,8 +2,7 @@
|
|||||||
// MurmurHash3 was written by Austin Appleby, and is placed in the public
|
// MurmurHash3 was written by Austin Appleby, and is placed in the public
|
||||||
// domain. The author hereby disclaims copyright to this source code.
|
// domain. The author hereby disclaims copyright to this source code.
|
||||||
|
|
||||||
#ifndef _MURMURHASH3_H_
|
#pragma once
|
||||||
#define _MURMURHASH3_H_
|
|
||||||
|
|
||||||
//-----------------------------------------------------------------------------
|
//-----------------------------------------------------------------------------
|
||||||
// Platform-specific functions and macros
|
// Platform-specific functions and macros
|
||||||
@ -33,5 +32,3 @@ void MurmurHash3_x86_128 ( const void * key, int len, uint32_t seed, void * out
|
|||||||
void MurmurHash3_x64_128 ( const void * key, int len, uint32_t seed, void * out );
|
void MurmurHash3_x64_128 ( const void * key, int len, uint32_t seed, void * out );
|
||||||
|
|
||||||
//-----------------------------------------------------------------------------
|
//-----------------------------------------------------------------------------
|
||||||
|
|
||||||
#endif // _MURMURHASH3_H_
|
|
||||||
|
@ -16,7 +16,7 @@ set (CONFIG_VERSION ${CMAKE_CURRENT_BINARY_DIR}/src/Common/config_version.h)
|
|||||||
set (CONFIG_COMMON ${CMAKE_CURRENT_BINARY_DIR}/src/Common/config.h)
|
set (CONFIG_COMMON ${CMAKE_CURRENT_BINARY_DIR}/src/Common/config.h)
|
||||||
|
|
||||||
include (cmake/version.cmake)
|
include (cmake/version.cmake)
|
||||||
message (STATUS "Will build ${VERSION_FULL}")
|
message (STATUS "Will build ${VERSION_FULL} revision ${VERSION_REVISION}")
|
||||||
configure_file (src/Common/config.h.in ${CONFIG_COMMON})
|
configure_file (src/Common/config.h.in ${CONFIG_COMMON})
|
||||||
configure_file (src/Common/config_version.h.in ${CONFIG_VERSION})
|
configure_file (src/Common/config_version.h.in ${CONFIG_VERSION})
|
||||||
|
|
||||||
@ -29,13 +29,35 @@ if (NOT NO_WERROR)
|
|||||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror")
|
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror")
|
||||||
endif ()
|
endif ()
|
||||||
|
|
||||||
# Add some warnings that are not available even with -Wall -Wextra.
|
# Add some warnings that are not available even with -Wall -Wextra -Wpedantic.
|
||||||
|
|
||||||
|
option (WEVERYTHING "Enables -Weverything option with some exceptions. This is intended for exploration of new compiler warnings that may be found to be useful. Only makes sense for clang." ON)
|
||||||
|
|
||||||
if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
|
if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
|
||||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra-semi -Wcomma -Winconsistent-missing-destructor-override -Wunused-exception-parameter -Wshadow-uncaptured-local")
|
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wpedantic -Wno-vla-extension -Wno-zero-length-array -Wno-gnu-anonymous-struct -Wno-nested-anon-types")
|
||||||
|
|
||||||
if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6)
|
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wshadow -Wshadow-uncaptured-local -Wextra-semi -Wcomma -Winconsistent-missing-destructor-override -Wunused-exception-parameter -Wcovered-switch-default -Wold-style-cast -Wrange-loop-analysis -Wunused-member-function -Wunreachable-code -Wunreachable-code-return -Wnewline-eof -Wembedded-directive -Wgnu-case-range -Wunused-macros -Wconditional-uninitialized -Wdeprecated -Wundef -Wreserved-id-macro -Wredundant-parens -Wzero-as-null-pointer-constant")
|
||||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wredundant-parens -Wzero-as-null-pointer-constant")
|
|
||||||
|
if (WEVERYTHING)
|
||||||
|
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Weverything -Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-missing-noreturn -Wno-padded -Wno-switch-enum -Wno-shadow-field-in-constructor -Wno-deprecated-dynamic-exception-spec -Wno-float-equal -Wno-weak-vtables -Wno-shift-sign-overflow -Wno-sign-conversion -Wno-conversion -Wno-exit-time-destructors -Wno-undefined-func-template -Wno-documentation-unknown-command -Wno-missing-variable-declarations -Wno-unused-template -Wno-global-constructors -Wno-c99-extensions -Wno-missing-prototypes -Wno-weak-template-vtables -Wno-zero-length-array -Wno-gnu-anonymous-struct -Wno-nested-anon-types -Wno-double-promotion -Wno-disabled-macro-expansion -Wno-used-but-marked-unused -Wno-vla-extension -Wno-vla -Wno-packed")
|
||||||
|
|
||||||
|
# TODO Enable conversion, sign-conversion, double-promotion warnings.
|
||||||
|
endif ()
|
||||||
|
|
||||||
|
if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7)
|
||||||
|
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||||
|
|
||||||
|
if (WEVERYTHING)
|
||||||
|
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-return-std-move-in-c++11")
|
||||||
|
endif ()
|
||||||
|
endif ()
|
||||||
|
|
||||||
|
if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8)
|
||||||
|
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wextra-semi-stmt -Wshadow-field -Wstring-plus-int")
|
||||||
|
|
||||||
|
if (WEVERYTHING)
|
||||||
|
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
|
||||||
|
endif ()
|
||||||
endif ()
|
endif ()
|
||||||
endif ()
|
endif ()
|
||||||
|
|
||||||
@ -64,7 +86,6 @@ set(dbms_sources)
|
|||||||
include(../cmake/dbms_glob_sources.cmake)
|
include(../cmake/dbms_glob_sources.cmake)
|
||||||
|
|
||||||
add_headers_and_sources(clickhouse_common_io src/Common)
|
add_headers_and_sources(clickhouse_common_io src/Common)
|
||||||
add_headers_and_sources(clickhouse_common_io src/Compression)
|
|
||||||
add_headers_and_sources(clickhouse_common_io src/Common/HashTable)
|
add_headers_and_sources(clickhouse_common_io src/Common/HashTable)
|
||||||
add_headers_and_sources(clickhouse_common_io src/IO)
|
add_headers_and_sources(clickhouse_common_io src/IO)
|
||||||
|
|
||||||
@ -151,20 +172,13 @@ if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELW
|
|||||||
PROPERTIES COMPILE_FLAGS -g0)
|
PROPERTIES COMPILE_FLAGS -g0)
|
||||||
endif ()
|
endif ()
|
||||||
|
|
||||||
if (NOT ARCH_ARM AND CPUID_LIBRARY)
|
|
||||||
set (LINK_LIBRARIES_ONLY_ON_X86_64 ${CPUID_LIBRARY})
|
|
||||||
endif()
|
|
||||||
|
|
||||||
target_link_libraries (clickhouse_common_io
|
target_link_libraries (clickhouse_common_io
|
||||||
PUBLIC
|
PUBLIC
|
||||||
common
|
common
|
||||||
PRIVATE
|
PRIVATE
|
||||||
clickhouse_parsers
|
|
||||||
string_utils
|
string_utils
|
||||||
widechar_width
|
widechar_width
|
||||||
${LINK_LIBRARIES_ONLY_ON_X86_64}
|
${LINK_LIBRARIES_ONLY_ON_X86_64}
|
||||||
${LZ4_LIBRARY}
|
|
||||||
${ZSTD_LIBRARY}
|
|
||||||
${DOUBLE_CONVERSION_LIBRARIES}
|
${DOUBLE_CONVERSION_LIBRARIES}
|
||||||
pocoext
|
pocoext
|
||||||
PUBLIC
|
PUBLIC
|
||||||
@ -185,8 +199,13 @@ target_link_libraries (clickhouse_common_io
|
|||||||
${CMAKE_DL_LIBS}
|
${CMAKE_DL_LIBS}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if (NOT ARCH_ARM AND CPUID_LIBRARY)
|
||||||
|
target_link_libraries (clickhouse_common_io PRIVATE ${CPUID_LIBRARY})
|
||||||
|
endif()
|
||||||
|
|
||||||
target_link_libraries (dbms
|
target_link_libraries (dbms
|
||||||
PRIVATE
|
PRIVATE
|
||||||
|
clickhouse_compression
|
||||||
clickhouse_parsers
|
clickhouse_parsers
|
||||||
clickhouse_common_config
|
clickhouse_common_config
|
||||||
PUBLIC
|
PUBLIC
|
||||||
@ -279,13 +298,6 @@ if (USE_HDFS)
|
|||||||
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${HDFS3_INCLUDE_DIR})
|
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${HDFS3_INCLUDE_DIR})
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
if (NOT USE_INTERNAL_LZ4_LIBRARY)
|
|
||||||
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${LZ4_INCLUDE_DIR})
|
|
||||||
endif ()
|
|
||||||
if (NOT USE_INTERNAL_ZSTD_LIBRARY)
|
|
||||||
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${ZSTD_INCLUDE_DIR})
|
|
||||||
endif ()
|
|
||||||
|
|
||||||
if (USE_JEMALLOC)
|
if (USE_JEMALLOC)
|
||||||
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR}) # used in Interpreters/AsynchronousMetrics.cpp
|
target_include_directories (dbms SYSTEM BEFORE PRIVATE ${JEMALLOC_INCLUDE_DIR}) # used in Interpreters/AsynchronousMetrics.cpp
|
||||||
endif ()
|
endif ()
|
||||||
|
@ -1,11 +1,11 @@
|
|||||||
# This strings autochanged from release_lib.sh:
|
# This strings autochanged from release_lib.sh:
|
||||||
set(VERSION_REVISION 54412 CACHE STRING "") # changed manually for tests
|
set(VERSION_REVISION 54413)
|
||||||
set(VERSION_MAJOR 18 CACHE STRING "")
|
set(VERSION_MAJOR 19)
|
||||||
set(VERSION_MINOR 16 CACHE STRING "")
|
set(VERSION_MINOR 1)
|
||||||
set(VERSION_PATCH 0 CACHE STRING "")
|
set(VERSION_PATCH 0)
|
||||||
set(VERSION_GITHASH b9b48c646c253358340bd39fd57754e92f88cd8a CACHE STRING "")
|
set(VERSION_GITHASH 014e344a36bc19a58621e0add379984cf62b9067)
|
||||||
set(VERSION_DESCRIBE v18.16.0-testing CACHE STRING "")
|
set(VERSION_DESCRIBE v19.1.0-testing)
|
||||||
set(VERSION_STRING 18.16.0 CACHE STRING "")
|
set(VERSION_STRING 19.1.0)
|
||||||
# end of autochange
|
# end of autochange
|
||||||
|
|
||||||
set(VERSION_EXTRA "" CACHE STRING "")
|
set(VERSION_EXTRA "" CACHE STRING "")
|
||||||
@ -19,8 +19,8 @@ if (VERSION_EXTRA)
|
|||||||
string(CONCAT VERSION_STRING ${VERSION_STRING} "." ${VERSION_EXTRA})
|
string(CONCAT VERSION_STRING ${VERSION_STRING} "." ${VERSION_EXTRA})
|
||||||
endif ()
|
endif ()
|
||||||
|
|
||||||
set (VERSION_NAME "${PROJECT_NAME}" CACHE STRING "")
|
set (VERSION_NAME "${PROJECT_NAME}")
|
||||||
set (VERSION_FULL "${VERSION_NAME} ${VERSION_STRING}" CACHE STRING "")
|
set (VERSION_FULL "${VERSION_NAME} ${VERSION_STRING}")
|
||||||
set (VERSION_SO "${VERSION_STRING}" CACHE STRING "")
|
set (VERSION_SO "${VERSION_STRING}")
|
||||||
|
|
||||||
math (EXPR VERSION_INTEGER "${VERSION_PATCH} + ${VERSION_MINOR}*1000 + ${VERSION_MAJOR}*1000000")
|
math (EXPR VERSION_INTEGER "${VERSION_PATCH} + ${VERSION_MINOR}*1000 + ${VERSION_MAJOR}*1000000")
|
||||||
|
@ -229,7 +229,7 @@ private:
|
|||||||
report(info_per_interval);
|
report(info_per_interval);
|
||||||
delay_watch.restart();
|
delay_watch.restart();
|
||||||
}
|
}
|
||||||
};
|
}
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@ -324,7 +324,7 @@ private:
|
|||||||
|
|
||||||
double seconds = watch.elapsedSeconds();
|
double seconds = watch.elapsedSeconds();
|
||||||
|
|
||||||
std::lock_guard<std::mutex> lock(mutex);
|
std::lock_guard lock(mutex);
|
||||||
info_per_interval.add(seconds, progress.rows, progress.bytes, info.rows, info.bytes);
|
info_per_interval.add(seconds, progress.rows, progress.bytes, info.rows, info.bytes);
|
||||||
info_total.add(seconds, progress.rows, progress.bytes, info.rows, info.bytes);
|
info_total.add(seconds, progress.rows, progress.bytes, info.rows, info.bytes);
|
||||||
}
|
}
|
||||||
@ -332,7 +332,7 @@ private:
|
|||||||
|
|
||||||
void report(Stats & info)
|
void report(Stats & info)
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(mutex);
|
std::lock_guard lock(mutex);
|
||||||
|
|
||||||
/// Avoid zeros, nans or exceptions
|
/// Avoid zeros, nans or exceptions
|
||||||
if (0 == info.queries)
|
if (0 == info.queries)
|
||||||
@ -369,7 +369,7 @@ private:
|
|||||||
{
|
{
|
||||||
WriteBufferFromFile json_out(filename);
|
WriteBufferFromFile json_out(filename);
|
||||||
|
|
||||||
std::lock_guard<std::mutex> lock(mutex);
|
std::lock_guard lock(mutex);
|
||||||
|
|
||||||
auto print_key_value = [&](auto key, auto value, bool with_comma = true)
|
auto print_key_value = [&](auto key, auto value, bool with_comma = true)
|
||||||
{
|
{
|
||||||
@ -503,6 +503,4 @@ int mainEntryClickHouseBenchmark(int argc, char ** argv)
|
|||||||
std::cerr << getCurrentExceptionMessage(print_stacktrace, true) << std::endl;
|
std::cerr << getCurrentExceptionMessage(print_stacktrace, true) << std::endl;
|
||||||
return getCurrentExceptionCode();
|
return getCurrentExceptionCode();
|
||||||
}
|
}
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
}
|
||||||
|
@ -74,8 +74,8 @@
|
|||||||
/// http://en.wikipedia.org/wiki/ANSI_escape_code
|
/// http://en.wikipedia.org/wiki/ANSI_escape_code
|
||||||
|
|
||||||
/// Similar codes \e[s, \e[u don't work in VT100 and Mosh.
|
/// Similar codes \e[s, \e[u don't work in VT100 and Mosh.
|
||||||
#define SAVE_CURSOR_POSITION "\e7"
|
#define SAVE_CURSOR_POSITION "\033""7"
|
||||||
#define RESTORE_CURSOR_POSITION "\e8"
|
#define RESTORE_CURSOR_POSITION "\033""8"
|
||||||
|
|
||||||
#define CLEAR_TO_END_OF_LINE "\033[K"
|
#define CLEAR_TO_END_OF_LINE "\033[K"
|
||||||
|
|
||||||
@ -554,10 +554,10 @@ private:
|
|||||||
|
|
||||||
void loop()
|
void loop()
|
||||||
{
|
{
|
||||||
String query;
|
String input;
|
||||||
String prev_query;
|
String prev_input;
|
||||||
|
|
||||||
while (char * line_ = readline(query.empty() ? prompt().c_str() : ":-] "))
|
while (char * line_ = readline(input.empty() ? prompt().c_str() : ":-] "))
|
||||||
{
|
{
|
||||||
String line = line_;
|
String line = line_;
|
||||||
free(line_);
|
free(line_);
|
||||||
@ -577,17 +577,17 @@ private:
|
|||||||
if (ends_with_backslash)
|
if (ends_with_backslash)
|
||||||
line = line.substr(0, ws - 1);
|
line = line.substr(0, ws - 1);
|
||||||
|
|
||||||
query += line;
|
input += line;
|
||||||
|
|
||||||
if (!ends_with_backslash && (ends_with_semicolon || has_vertical_output_suffix || (!config().has("multiline") && !hasDataInSTDIN())))
|
if (!ends_with_backslash && (ends_with_semicolon || has_vertical_output_suffix || (!config().has("multiline") && !hasDataInSTDIN())))
|
||||||
{
|
{
|
||||||
if (query != prev_query)
|
if (input != prev_input)
|
||||||
{
|
{
|
||||||
/// Replace line breaks with spaces to prevent the following problem.
|
/// Replace line breaks with spaces to prevent the following problem.
|
||||||
/// Every line of multi-line query is saved to history file as a separate line.
|
/// Every line of multi-line query is saved to history file as a separate line.
|
||||||
/// If the user restarts the client then after pressing the "up" button
|
/// If the user restarts the client then after pressing the "up" button
|
||||||
/// every line of the query will be displayed separately.
|
/// every line of the query will be displayed separately.
|
||||||
std::string logged_query = query;
|
std::string logged_query = input;
|
||||||
std::replace(logged_query.begin(), logged_query.end(), '\n', ' ');
|
std::replace(logged_query.begin(), logged_query.end(), '\n', ' ');
|
||||||
add_history(logged_query.c_str());
|
add_history(logged_query.c_str());
|
||||||
|
|
||||||
@ -596,18 +596,18 @@ private:
|
|||||||
throwFromErrno("Cannot append history to file " + history_file, ErrorCodes::CANNOT_APPEND_HISTORY);
|
throwFromErrno("Cannot append history to file " + history_file, ErrorCodes::CANNOT_APPEND_HISTORY);
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
prev_query = query;
|
prev_input = input;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (has_vertical_output_suffix)
|
if (has_vertical_output_suffix)
|
||||||
query = query.substr(0, query.length() - 2);
|
input = input.substr(0, input.length() - 2);
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
/// Determine the terminal size.
|
/// Determine the terminal size.
|
||||||
ioctl(0, TIOCGWINSZ, &terminal_size);
|
ioctl(0, TIOCGWINSZ, &terminal_size);
|
||||||
|
|
||||||
if (!process(query))
|
if (!process(input))
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
catch (const Exception & e)
|
catch (const Exception & e)
|
||||||
@ -633,11 +633,11 @@ private:
|
|||||||
connect();
|
connect();
|
||||||
}
|
}
|
||||||
|
|
||||||
query = "";
|
input = "";
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
query += '\n';
|
input += '\n';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -675,8 +675,6 @@ private:
|
|||||||
/// Several queries separated by ';'.
|
/// Several queries separated by ';'.
|
||||||
/// INSERT data is ended by the end of line, not ';'.
|
/// INSERT data is ended by the end of line, not ';'.
|
||||||
|
|
||||||
String query;
|
|
||||||
|
|
||||||
const char * begin = text.data();
|
const char * begin = text.data();
|
||||||
const char * end = begin + text.size();
|
const char * end = begin + text.size();
|
||||||
|
|
||||||
@ -708,19 +706,19 @@ private:
|
|||||||
insert->end = pos;
|
insert->end = pos;
|
||||||
}
|
}
|
||||||
|
|
||||||
query = text.substr(begin - text.data(), pos - begin);
|
String str = text.substr(begin - text.data(), pos - begin);
|
||||||
|
|
||||||
begin = pos;
|
begin = pos;
|
||||||
while (isWhitespaceASCII(*begin) || *begin == ';')
|
while (isWhitespaceASCII(*begin) || *begin == ';')
|
||||||
++begin;
|
++begin;
|
||||||
|
|
||||||
TestHint test_hint(test_mode, query);
|
TestHint test_hint(test_mode, str);
|
||||||
expected_client_error = test_hint.clientError();
|
expected_client_error = test_hint.clientError();
|
||||||
expected_server_error = test_hint.serverError();
|
expected_server_error = test_hint.serverError();
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
if (!processSingleQuery(query, ast) && !ignore_error)
|
if (!processSingleQuery(str, ast) && !ignore_error)
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
catch (...)
|
catch (...)
|
||||||
@ -728,7 +726,7 @@ private:
|
|||||||
last_exception = std::make_unique<Exception>(getCurrentExceptionMessage(true), getCurrentExceptionCode());
|
last_exception = std::make_unique<Exception>(getCurrentExceptionMessage(true), getCurrentExceptionCode());
|
||||||
actual_client_error = last_exception->code();
|
actual_client_error = last_exception->code();
|
||||||
if (!ignore_error && (!actual_client_error || actual_client_error != expected_client_error))
|
if (!ignore_error && (!actual_client_error || actual_client_error != expected_client_error))
|
||||||
std::cerr << "Error on processing query: " << query << std::endl << last_exception->message();
|
std::cerr << "Error on processing query: " << str << std::endl << last_exception->message();
|
||||||
got_exception = true;
|
got_exception = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -904,8 +902,6 @@ private:
|
|||||||
ParserQuery parser(end, true);
|
ParserQuery parser(end, true);
|
||||||
ASTPtr res;
|
ASTPtr res;
|
||||||
|
|
||||||
const auto ignore_error = config().getBool("ignore-error", false);
|
|
||||||
|
|
||||||
if (is_interactive || ignore_error)
|
if (is_interactive || ignore_error)
|
||||||
{
|
{
|
||||||
String message;
|
String message;
|
||||||
@ -1616,10 +1612,10 @@ public:
|
|||||||
for (size_t i = 0; i < external_tables_arguments.size(); ++i)
|
for (size_t i = 0; i < external_tables_arguments.size(); ++i)
|
||||||
{
|
{
|
||||||
/// Parse commandline options related to external tables.
|
/// Parse commandline options related to external tables.
|
||||||
po::parsed_options parsed = po::command_line_parser(
|
po::parsed_options parsed_tables = po::command_line_parser(
|
||||||
external_tables_arguments[i].size(), external_tables_arguments[i].data()).options(external_description).run();
|
external_tables_arguments[i].size(), external_tables_arguments[i].data()).options(external_description).run();
|
||||||
po::variables_map external_options;
|
po::variables_map external_options;
|
||||||
po::store(parsed, external_options);
|
po::store(parsed_tables, external_options);
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
|
@ -56,7 +56,7 @@ private:
|
|||||||
{
|
{
|
||||||
std::string prefix_str(prefix);
|
std::string prefix_str(prefix);
|
||||||
std::tie(pos, end) = std::equal_range(words.begin(), words.end(), prefix_str,
|
std::tie(pos, end) = std::equal_range(words.begin(), words.end(), prefix_str,
|
||||||
[prefix_length](const std::string & s, const std::string & prefix) { return strncmp(s.c_str(), prefix.c_str(), prefix_length) < 0; });
|
[prefix_length](const std::string & s, const std::string & prefix_searched) { return strncmp(s.c_str(), prefix_searched.c_str(), prefix_length) < 0; });
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Iterates through matched range.
|
/// Iterates through matched range.
|
||||||
|
@ -28,25 +28,26 @@ public:
|
|||||||
if (!enabled_)
|
if (!enabled_)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
String full_comment;
|
|
||||||
Lexer lexer(query.data(), query.data() + query.size());
|
Lexer lexer(query.data(), query.data() + query.size());
|
||||||
|
|
||||||
for (Token token = lexer.nextToken(); !token.isEnd(); token = lexer.nextToken())
|
for (Token token = lexer.nextToken(); !token.isEnd(); token = lexer.nextToken())
|
||||||
{
|
{
|
||||||
if (token.type == TokenType::Comment)
|
if (token.type == TokenType::Comment)
|
||||||
full_comment += String(token.begin, token.begin + token.size()) + ' ';
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!full_comment.empty())
|
|
||||||
{
|
|
||||||
size_t pos_start = full_comment.find('{', 0);
|
|
||||||
if (pos_start != String::npos)
|
|
||||||
{
|
{
|
||||||
size_t pos_end = full_comment.find('}', pos_start);
|
String comment(token.begin, token.begin + token.size());
|
||||||
if (pos_end != String::npos)
|
|
||||||
|
if (!comment.empty())
|
||||||
{
|
{
|
||||||
String hint(full_comment.begin() + pos_start + 1, full_comment.begin() + pos_end);
|
size_t pos_start = comment.find('{', 0);
|
||||||
parse(hint);
|
if (pos_start != String::npos)
|
||||||
|
{
|
||||||
|
size_t pos_end = comment.find('}', pos_start);
|
||||||
|
if (pos_end != String::npos)
|
||||||
|
{
|
||||||
|
String hint(comment.begin() + pos_start + 1, comment.begin() + pos_end);
|
||||||
|
parse(hint);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
add_library (clickhouse-compressor-lib ${LINK_MODE} Compressor.cpp)
|
add_library (clickhouse-compressor-lib ${LINK_MODE} Compressor.cpp)
|
||||||
target_link_libraries (clickhouse-compressor-lib PRIVATE clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
target_link_libraries (clickhouse-compressor-lib PRIVATE clickhouse_compression clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
|
||||||
|
|
||||||
if (CLICKHOUSE_SPLIT_BINARY)
|
if (CLICKHOUSE_SPLIT_BINARY)
|
||||||
# Also in utils
|
# Also in utils
|
||||||
|
@ -5,8 +5,8 @@
|
|||||||
#include <Common/Exception.h>
|
#include <Common/Exception.h>
|
||||||
#include <IO/WriteBufferFromFileDescriptor.h>
|
#include <IO/WriteBufferFromFileDescriptor.h>
|
||||||
#include <IO/ReadBufferFromFileDescriptor.h>
|
#include <IO/ReadBufferFromFileDescriptor.h>
|
||||||
#include <IO/CompressedWriteBuffer.h>
|
#include <Compression/CompressedWriteBuffer.h>
|
||||||
#include <IO/CompressedReadBuffer.h>
|
#include <Compression/CompressedReadBuffer.h>
|
||||||
#include <IO/WriteHelpers.h>
|
#include <IO/WriteHelpers.h>
|
||||||
#include <IO/copyData.h>
|
#include <IO/copyData.h>
|
||||||
|
|
||||||
|
@ -243,7 +243,7 @@ struct ClusterPartition
|
|||||||
UInt64 rows_copied = 0;
|
UInt64 rows_copied = 0;
|
||||||
UInt64 blocks_copied = 0;
|
UInt64 blocks_copied = 0;
|
||||||
|
|
||||||
size_t total_tries = 0;
|
UInt64 total_tries = 0;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
@ -340,7 +340,7 @@ struct TaskCluster
|
|||||||
String default_local_database;
|
String default_local_database;
|
||||||
|
|
||||||
/// Limits number of simultaneous workers
|
/// Limits number of simultaneous workers
|
||||||
size_t max_workers = 0;
|
UInt64 max_workers = 0;
|
||||||
|
|
||||||
/// Base settings for pull and push
|
/// Base settings for pull and push
|
||||||
Settings settings_common;
|
Settings settings_common;
|
||||||
@ -773,11 +773,11 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
template <typename T>
|
template <typename T>
|
||||||
decltype(auto) retry(T && func, size_t max_tries = 100)
|
decltype(auto) retry(T && func, UInt64 max_tries = 100)
|
||||||
{
|
{
|
||||||
std::exception_ptr exception;
|
std::exception_ptr exception;
|
||||||
|
|
||||||
for (size_t try_number = 1; try_number <= max_tries; ++try_number)
|
for (UInt64 try_number = 1; try_number <= max_tries; ++try_number)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
@ -880,7 +880,7 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Compute set of partitions, assume set of partitions aren't changed during the processing
|
/// Compute set of partitions, assume set of partitions aren't changed during the processing
|
||||||
void discoverTablePartitions(TaskTable & task_table, size_t num_threads = 0)
|
void discoverTablePartitions(TaskTable & task_table, UInt64 num_threads = 0)
|
||||||
{
|
{
|
||||||
/// Fetch partitions list from a shard
|
/// Fetch partitions list from a shard
|
||||||
{
|
{
|
||||||
@ -985,7 +985,7 @@ public:
|
|||||||
|
|
||||||
/// Retry table processing
|
/// Retry table processing
|
||||||
bool table_is_done = false;
|
bool table_is_done = false;
|
||||||
for (size_t num_table_tries = 0; num_table_tries < max_table_tries; ++num_table_tries)
|
for (UInt64 num_table_tries = 0; num_table_tries < max_table_tries; ++num_table_tries)
|
||||||
{
|
{
|
||||||
if (tryProcessTable(task_table))
|
if (tryProcessTable(task_table))
|
||||||
{
|
{
|
||||||
@ -1044,7 +1044,7 @@ protected:
|
|||||||
String workers_path = getWorkersPath();
|
String workers_path = getWorkersPath();
|
||||||
String current_worker_path = getCurrentWorkerNodePath();
|
String current_worker_path = getCurrentWorkerNodePath();
|
||||||
|
|
||||||
size_t num_bad_version_errors = 0;
|
UInt64 num_bad_version_errors = 0;
|
||||||
|
|
||||||
while (true)
|
while (true)
|
||||||
{
|
{
|
||||||
@ -1055,7 +1055,7 @@ protected:
|
|||||||
auto version = stat.version;
|
auto version = stat.version;
|
||||||
zookeeper->get(workers_path, &stat);
|
zookeeper->get(workers_path, &stat);
|
||||||
|
|
||||||
if (static_cast<size_t>(stat.numChildren) >= task_cluster->max_workers)
|
if (static_cast<UInt64>(stat.numChildren) >= task_cluster->max_workers)
|
||||||
{
|
{
|
||||||
LOG_DEBUG(log, "Too many workers (" << stat.numChildren << ", maximum " << task_cluster->max_workers << ")"
|
LOG_DEBUG(log, "Too many workers (" << stat.numChildren << ", maximum " << task_cluster->max_workers << ")"
|
||||||
<< ". Postpone processing " << description);
|
<< ". Postpone processing " << description);
|
||||||
@ -1163,7 +1163,7 @@ protected:
|
|||||||
}
|
}
|
||||||
|
|
||||||
// If all task is finished and zxid is not changed then partition could not become dirty again
|
// If all task is finished and zxid is not changed then partition could not become dirty again
|
||||||
for (size_t shard_num = 0; shard_num < status_paths.size(); ++shard_num)
|
for (UInt64 shard_num = 0; shard_num < status_paths.size(); ++shard_num)
|
||||||
{
|
{
|
||||||
if (zxid1[shard_num] != zxid2[shard_num])
|
if (zxid1[shard_num] != zxid2[shard_num])
|
||||||
{
|
{
|
||||||
@ -1280,7 +1280,7 @@ protected:
|
|||||||
|
|
||||||
LOG_DEBUG(log, "Execute distributed DROP PARTITION: " << query);
|
LOG_DEBUG(log, "Execute distributed DROP PARTITION: " << query);
|
||||||
/// Limit number of max executing replicas to 1
|
/// Limit number of max executing replicas to 1
|
||||||
size_t num_shards = executeQueryOnCluster(cluster_push, query, nullptr, &settings_push, PoolMode::GET_ONE, 1);
|
UInt64 num_shards = executeQueryOnCluster(cluster_push, query, nullptr, &settings_push, PoolMode::GET_ONE, 1);
|
||||||
|
|
||||||
if (num_shards < cluster_push->getShardCount())
|
if (num_shards < cluster_push->getShardCount())
|
||||||
{
|
{
|
||||||
@ -1299,8 +1299,8 @@ protected:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
static constexpr size_t max_table_tries = 1000;
|
static constexpr UInt64 max_table_tries = 1000;
|
||||||
static constexpr size_t max_shard_partition_tries = 600;
|
static constexpr UInt64 max_shard_partition_tries = 600;
|
||||||
|
|
||||||
bool tryProcessTable(TaskTable & task_table)
|
bool tryProcessTable(TaskTable & task_table)
|
||||||
{
|
{
|
||||||
@ -1317,7 +1317,7 @@ protected:
|
|||||||
|
|
||||||
Stopwatch watch;
|
Stopwatch watch;
|
||||||
TasksShard expected_shards;
|
TasksShard expected_shards;
|
||||||
size_t num_failed_shards = 0;
|
UInt64 num_failed_shards = 0;
|
||||||
|
|
||||||
++cluster_partition.total_tries;
|
++cluster_partition.total_tries;
|
||||||
|
|
||||||
@ -1368,7 +1368,7 @@ protected:
|
|||||||
bool is_unprioritized_task = !previous_shard_is_instantly_finished && shard->priority.is_remote;
|
bool is_unprioritized_task = !previous_shard_is_instantly_finished && shard->priority.is_remote;
|
||||||
PartitionTaskStatus task_status = PartitionTaskStatus::Error;
|
PartitionTaskStatus task_status = PartitionTaskStatus::Error;
|
||||||
bool was_error = false;
|
bool was_error = false;
|
||||||
for (size_t try_num = 0; try_num < max_shard_partition_tries; ++try_num)
|
for (UInt64 try_num = 0; try_num < max_shard_partition_tries; ++try_num)
|
||||||
{
|
{
|
||||||
task_status = tryProcessPartitionTask(partition, is_unprioritized_task);
|
task_status = tryProcessPartitionTask(partition, is_unprioritized_task);
|
||||||
|
|
||||||
@ -1434,8 +1434,8 @@ protected:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t required_partitions = task_table.cluster_partitions.size();
|
UInt64 required_partitions = task_table.cluster_partitions.size();
|
||||||
size_t finished_partitions = task_table.finished_cluster_partitions.size();
|
UInt64 finished_partitions = task_table.finished_cluster_partitions.size();
|
||||||
bool table_is_done = finished_partitions >= required_partitions;
|
bool table_is_done = finished_partitions >= required_partitions;
|
||||||
|
|
||||||
if (!table_is_done)
|
if (!table_is_done)
|
||||||
@ -1645,7 +1645,7 @@ protected:
|
|||||||
String query = queryToString(create_query_push_ast);
|
String query = queryToString(create_query_push_ast);
|
||||||
|
|
||||||
LOG_DEBUG(log, "Create destination tables. Query: " << query);
|
LOG_DEBUG(log, "Create destination tables. Query: " << query);
|
||||||
size_t shards = executeQueryOnCluster(task_table.cluster_push, query, create_query_push_ast, &task_cluster->settings_push,
|
UInt64 shards = executeQueryOnCluster(task_table.cluster_push, query, create_query_push_ast, &task_cluster->settings_push,
|
||||||
PoolMode::GET_MANY);
|
PoolMode::GET_MANY);
|
||||||
LOG_DEBUG(log, "Destination tables " << getDatabaseDotTable(task_table.table_push) << " have been created on " << shards
|
LOG_DEBUG(log, "Destination tables " << getDatabaseDotTable(task_table.table_push) << " have been created on " << shards
|
||||||
<< " shards of " << task_table.cluster_push->getShardCount());
|
<< " shards of " << task_table.cluster_push->getShardCount());
|
||||||
@ -1699,7 +1699,7 @@ protected:
|
|||||||
std::future<Coordination::ExistsResponse> future_is_dirty_checker;
|
std::future<Coordination::ExistsResponse> future_is_dirty_checker;
|
||||||
|
|
||||||
Stopwatch watch(CLOCK_MONOTONIC_COARSE);
|
Stopwatch watch(CLOCK_MONOTONIC_COARSE);
|
||||||
constexpr size_t check_period_milliseconds = 500;
|
constexpr UInt64 check_period_milliseconds = 500;
|
||||||
|
|
||||||
/// Will asynchronously check that ZooKeeper connection and is_dirty flag appearing while copy data
|
/// Will asynchronously check that ZooKeeper connection and is_dirty flag appearing while copy data
|
||||||
auto cancel_check = [&] ()
|
auto cancel_check = [&] ()
|
||||||
@ -1917,16 +1917,16 @@ protected:
|
|||||||
/** Executes simple query (without output streams, for example DDL queries) on each shard of the cluster
|
/** Executes simple query (without output streams, for example DDL queries) on each shard of the cluster
|
||||||
* Returns number of shards for which at least one replica executed query successfully
|
* Returns number of shards for which at least one replica executed query successfully
|
||||||
*/
|
*/
|
||||||
size_t executeQueryOnCluster(
|
UInt64 executeQueryOnCluster(
|
||||||
const ClusterPtr & cluster,
|
const ClusterPtr & cluster,
|
||||||
const String & query,
|
const String & query,
|
||||||
const ASTPtr & query_ast_ = nullptr,
|
const ASTPtr & query_ast_ = nullptr,
|
||||||
const Settings * settings = nullptr,
|
const Settings * settings = nullptr,
|
||||||
PoolMode pool_mode = PoolMode::GET_ALL,
|
PoolMode pool_mode = PoolMode::GET_ALL,
|
||||||
size_t max_successful_executions_per_shard = 0) const
|
UInt64 max_successful_executions_per_shard = 0) const
|
||||||
{
|
{
|
||||||
auto num_shards = cluster->getShardsInfo().size();
|
auto num_shards = cluster->getShardsInfo().size();
|
||||||
std::vector<size_t> per_shard_num_successful_replicas(num_shards, 0);
|
std::vector<UInt64> per_shard_num_successful_replicas(num_shards, 0);
|
||||||
|
|
||||||
ASTPtr query_ast;
|
ASTPtr query_ast;
|
||||||
if (query_ast_ == nullptr)
|
if (query_ast_ == nullptr)
|
||||||
@ -1939,10 +1939,10 @@ protected:
|
|||||||
|
|
||||||
|
|
||||||
/// We need to execute query on one replica at least
|
/// We need to execute query on one replica at least
|
||||||
auto do_for_shard = [&] (size_t shard_index)
|
auto do_for_shard = [&] (UInt64 shard_index)
|
||||||
{
|
{
|
||||||
const Cluster::ShardInfo & shard = cluster->getShardsInfo().at(shard_index);
|
const Cluster::ShardInfo & shard = cluster->getShardsInfo().at(shard_index);
|
||||||
size_t & num_successful_executions = per_shard_num_successful_replicas.at(shard_index);
|
UInt64 & num_successful_executions = per_shard_num_successful_replicas.at(shard_index);
|
||||||
num_successful_executions = 0;
|
num_successful_executions = 0;
|
||||||
|
|
||||||
auto increment_and_check_exit = [&] ()
|
auto increment_and_check_exit = [&] ()
|
||||||
@ -1951,12 +1951,12 @@ protected:
|
|||||||
return max_successful_executions_per_shard && num_successful_executions >= max_successful_executions_per_shard;
|
return max_successful_executions_per_shard && num_successful_executions >= max_successful_executions_per_shard;
|
||||||
};
|
};
|
||||||
|
|
||||||
size_t num_replicas = cluster->getShardsAddresses().at(shard_index).size();
|
UInt64 num_replicas = cluster->getShardsAddresses().at(shard_index).size();
|
||||||
size_t num_local_replicas = shard.getLocalNodeCount();
|
UInt64 num_local_replicas = shard.getLocalNodeCount();
|
||||||
size_t num_remote_replicas = num_replicas - num_local_replicas;
|
UInt64 num_remote_replicas = num_replicas - num_local_replicas;
|
||||||
|
|
||||||
/// In that case we don't have local replicas, but do it just in case
|
/// In that case we don't have local replicas, but do it just in case
|
||||||
for (size_t i = 0; i < num_local_replicas; ++i)
|
for (UInt64 i = 0; i < num_local_replicas; ++i)
|
||||||
{
|
{
|
||||||
auto interpreter = InterpreterFactory::get(query_ast, context);
|
auto interpreter = InterpreterFactory::get(query_ast, context);
|
||||||
interpreter->execute();
|
interpreter->execute();
|
||||||
@ -1997,16 +1997,16 @@ protected:
|
|||||||
};
|
};
|
||||||
|
|
||||||
{
|
{
|
||||||
ThreadPool thread_pool(std::min(num_shards, getNumberOfPhysicalCPUCores()));
|
ThreadPool thread_pool(std::min(num_shards, UInt64(getNumberOfPhysicalCPUCores())));
|
||||||
|
|
||||||
for (size_t shard_index = 0; shard_index < num_shards; ++shard_index)
|
for (UInt64 shard_index = 0; shard_index < num_shards; ++shard_index)
|
||||||
thread_pool.schedule([=] { do_for_shard(shard_index); });
|
thread_pool.schedule([=] { do_for_shard(shard_index); });
|
||||||
|
|
||||||
thread_pool.wait();
|
thread_pool.wait();
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t successful_shards = 0;
|
UInt64 successful_shards = 0;
|
||||||
for (size_t num_replicas : per_shard_num_successful_replicas)
|
for (UInt64 num_replicas : per_shard_num_successful_replicas)
|
||||||
successful_shards += (num_replicas > 0);
|
successful_shards += (num_replicas > 0);
|
||||||
|
|
||||||
return successful_shards;
|
return successful_shards;
|
||||||
|
@ -66,11 +66,11 @@ void LocalServer::initialize(Poco::Util::Application & self)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void LocalServer::applyCmdSettings(Context & context)
|
void LocalServer::applyCmdSettings()
|
||||||
{
|
{
|
||||||
#define EXTRACT_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) \
|
#define EXTRACT_SETTING(TYPE, NAME, DEFAULT, DESCRIPTION) \
|
||||||
if (cmd_settings.NAME.changed) \
|
if (cmd_settings.NAME.changed) \
|
||||||
context.getSettingsRef().NAME = cmd_settings.NAME;
|
context->getSettingsRef().NAME = cmd_settings.NAME;
|
||||||
APPLY_FOR_SETTINGS(EXTRACT_SETTING)
|
APPLY_FOR_SETTINGS(EXTRACT_SETTING)
|
||||||
#undef EXTRACT_SETTING
|
#undef EXTRACT_SETTING
|
||||||
}
|
}
|
||||||
@ -179,7 +179,7 @@ try
|
|||||||
std::string default_database = config().getString("default_database", "_local");
|
std::string default_database = config().getString("default_database", "_local");
|
||||||
context->addDatabase(default_database, std::make_shared<DatabaseMemory>(default_database));
|
context->addDatabase(default_database, std::make_shared<DatabaseMemory>(default_database));
|
||||||
context->setCurrentDatabase(default_database);
|
context->setCurrentDatabase(default_database);
|
||||||
applyCmdOptions(*context);
|
applyCmdOptions();
|
||||||
|
|
||||||
if (!context->getPath().empty())
|
if (!context->getPath().empty())
|
||||||
{
|
{
|
||||||
@ -274,7 +274,7 @@ void LocalServer::processQueries()
|
|||||||
|
|
||||||
context->setUser("default", "", Poco::Net::SocketAddress{}, "");
|
context->setUser("default", "", Poco::Net::SocketAddress{}, "");
|
||||||
context->setCurrentQueryId("");
|
context->setCurrentQueryId("");
|
||||||
applyCmdSettings(*context);
|
applyCmdSettings();
|
||||||
|
|
||||||
/// Use the same query_id (and thread group) for all queries
|
/// Use the same query_id (and thread group) for all queries
|
||||||
CurrentThread::QueryScope query_scope_holder(*context);
|
CurrentThread::QueryScope query_scope_holder(*context);
|
||||||
@ -494,10 +494,10 @@ void LocalServer::init(int argc, char ** argv)
|
|||||||
config().setBool("ignore-error", true);
|
config().setBool("ignore-error", true);
|
||||||
}
|
}
|
||||||
|
|
||||||
void LocalServer::applyCmdOptions(Context & context)
|
void LocalServer::applyCmdOptions()
|
||||||
{
|
{
|
||||||
context.setDefaultFormat(config().getString("output-format", config().getString("format", "TSV")));
|
context->setDefaultFormat(config().getString("output-format", config().getString("format", "TSV")));
|
||||||
applyCmdSettings(context);
|
applyCmdSettings();
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -34,8 +34,8 @@ private:
|
|||||||
std::string getInitialCreateTableQuery();
|
std::string getInitialCreateTableQuery();
|
||||||
|
|
||||||
void tryInitPath();
|
void tryInitPath();
|
||||||
void applyCmdOptions(Context & context);
|
void applyCmdOptions();
|
||||||
void applyCmdSettings(Context & context);
|
void applyCmdSettings();
|
||||||
void attachSystemTables();
|
void attachSystemTables();
|
||||||
void processQueries();
|
void processQueries();
|
||||||
void setupUsers();
|
void setupUsers();
|
||||||
|
@ -123,7 +123,7 @@ UInt64 hash(Ts... xs)
|
|||||||
|
|
||||||
UInt64 maskBits(UInt64 x, size_t num_bits)
|
UInt64 maskBits(UInt64 x, size_t num_bits)
|
||||||
{
|
{
|
||||||
return x & ((1 << num_bits) - 1);
|
return x & ((1ULL << num_bits) - 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -149,7 +149,7 @@ UInt64 feistelNetwork(UInt64 x, size_t num_bits, UInt64 seed, size_t num_rounds
|
|||||||
UInt64 bits = maskBits(x, num_bits);
|
UInt64 bits = maskBits(x, num_bits);
|
||||||
for (size_t i = 0; i < num_rounds; ++i)
|
for (size_t i = 0; i < num_rounds; ++i)
|
||||||
bits = feistelRound(bits, num_bits, seed, i);
|
bits = feistelRound(bits, num_bits, seed, i);
|
||||||
return (x & ~((1 << num_bits) - 1)) ^ bits;
|
return (x & ~((1ULL << num_bits) - 1)) ^ bits;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -317,8 +317,8 @@ void transformFixedString(const UInt8 * src, UInt8 * dst, size_t size, UInt64 se
|
|||||||
|
|
||||||
if (size >= 16)
|
if (size >= 16)
|
||||||
{
|
{
|
||||||
char * dst = reinterpret_cast<char *>(std::min(pos, end - 16));
|
char * hash_dst = reinterpret_cast<char *>(std::min(pos, end - 16));
|
||||||
hash.get128(dst);
|
hash.get128(hash_dst);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
|
@ -112,7 +112,8 @@ public:
|
|||||||
{
|
{
|
||||||
return asString(padding);
|
return asString(padding);
|
||||||
}
|
}
|
||||||
String asString(size_t padding) const
|
|
||||||
|
String asString(size_t cur_padding) const
|
||||||
{
|
{
|
||||||
String repr = "{";
|
String repr = "{";
|
||||||
|
|
||||||
@ -121,10 +122,10 @@ public:
|
|||||||
if (it != content.begin())
|
if (it != content.begin())
|
||||||
repr += ',';
|
repr += ',';
|
||||||
/// construct "key": "value" string with padding
|
/// construct "key": "value" string with padding
|
||||||
repr += "\n" + pad(padding) + '"' + it->first + '"' + ": " + it->second;
|
repr += "\n" + pad(cur_padding) + '"' + it->first + '"' + ": " + it->second;
|
||||||
}
|
}
|
||||||
|
|
||||||
repr += "\n" + pad(padding - 1) + '}';
|
repr += "\n" + pad(cur_padding - 1) + '}';
|
||||||
return repr;
|
return repr;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@ -762,13 +763,13 @@ private:
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
void processTestsConfigurations(const Paths & input_files)
|
void processTestsConfigurations(const Paths & paths)
|
||||||
{
|
{
|
||||||
tests_configurations.resize(input_files.size());
|
tests_configurations.resize(paths.size());
|
||||||
|
|
||||||
for (size_t i = 0; i != input_files.size(); ++i)
|
for (size_t i = 0; i != paths.size(); ++i)
|
||||||
{
|
{
|
||||||
const String path = input_files[i];
|
const String path = paths[i];
|
||||||
tests_configurations[i] = XMLConfigurationPtr(new XMLConfiguration(path));
|
tests_configurations[i] = XMLConfigurationPtr(new XMLConfiguration(path));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -881,8 +882,6 @@ private:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Query query;
|
|
||||||
|
|
||||||
if (!test_config->has("query") && !test_config->has("query_file"))
|
if (!test_config->has("query") && !test_config->has("query_file"))
|
||||||
{
|
{
|
||||||
throw DB::Exception("Missing query fields in test's config: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
|
throw DB::Exception("Missing query fields in test's config: " + test_name, DB::ErrorCodes::BAD_ARGUMENTS);
|
||||||
@ -907,6 +906,7 @@ private:
|
|||||||
bool tsv = fs::path(filename).extension().string() == ".tsv";
|
bool tsv = fs::path(filename).extension().string() == ".tsv";
|
||||||
|
|
||||||
ReadBufferFromFile query_file(filename);
|
ReadBufferFromFile query_file(filename);
|
||||||
|
Query query;
|
||||||
|
|
||||||
if (tsv)
|
if (tsv)
|
||||||
{
|
{
|
||||||
@ -1024,7 +1024,7 @@ private:
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (lite_output)
|
if (lite_output)
|
||||||
return minOutput(main_metric);
|
return minOutput();
|
||||||
else
|
else
|
||||||
return constructTotalInfo(metrics);
|
return constructTotalInfo(metrics);
|
||||||
}
|
}
|
||||||
@ -1053,11 +1053,8 @@ private:
|
|||||||
|
|
||||||
void runQueries(const QueriesWithIndexes & queries_with_indexes)
|
void runQueries(const QueriesWithIndexes & queries_with_indexes)
|
||||||
{
|
{
|
||||||
for (const std::pair<Query, const size_t> & query_and_index : queries_with_indexes)
|
for (const auto & [query, run_index] : queries_with_indexes)
|
||||||
{
|
{
|
||||||
Query query = query_and_index.first;
|
|
||||||
const size_t run_index = query_and_index.second;
|
|
||||||
|
|
||||||
TestStopConditions & stop_conditions = stop_conditions_by_run[run_index];
|
TestStopConditions & stop_conditions = stop_conditions_by_run[run_index];
|
||||||
Stats & statistics = statistics_by_run[run_index];
|
Stats & statistics = statistics_by_run[run_index];
|
||||||
|
|
||||||
@ -1139,7 +1136,7 @@ private:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void constructSubstitutions(ConfigurationPtr & substitutions_view, StringToVector & substitutions)
|
void constructSubstitutions(ConfigurationPtr & substitutions_view, StringToVector & out_substitutions)
|
||||||
{
|
{
|
||||||
Keys xml_substitutions;
|
Keys xml_substitutions;
|
||||||
substitutions_view->keys(xml_substitutions);
|
substitutions_view->keys(xml_substitutions);
|
||||||
@ -1157,21 +1154,16 @@ private:
|
|||||||
|
|
||||||
for (size_t j = 0; j != xml_values.size(); ++j)
|
for (size_t j = 0; j != xml_values.size(); ++j)
|
||||||
{
|
{
|
||||||
substitutions[name].push_back(xml_substitution->getString("values.value[" + std::to_string(j) + "]"));
|
out_substitutions[name].push_back(xml_substitution->getString("values.value[" + std::to_string(j) + "]"));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
std::vector<String> formatQueries(const String & query, StringToVector substitutions)
|
std::vector<String> formatQueries(const String & query, StringToVector substitutions_to_generate)
|
||||||
{
|
{
|
||||||
std::vector<String> queries;
|
std::vector<String> queries_res;
|
||||||
|
runThroughAllOptionsAndPush(substitutions_to_generate.begin(), substitutions_to_generate.end(), query, queries_res);
|
||||||
StringToVector::iterator substitutions_first = substitutions.begin();
|
return queries_res;
|
||||||
StringToVector::iterator substitutions_last = substitutions.end();
|
|
||||||
|
|
||||||
runThroughAllOptionsAndPush(substitutions_first, substitutions_last, query, queries);
|
|
||||||
|
|
||||||
return queries;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Recursive method which goes through all substitution blocks in xml
|
/// Recursive method which goes through all substitution blocks in xml
|
||||||
@ -1179,11 +1171,11 @@ private:
|
|||||||
void runThroughAllOptionsAndPush(StringToVector::iterator substitutions_left,
|
void runThroughAllOptionsAndPush(StringToVector::iterator substitutions_left,
|
||||||
StringToVector::iterator substitutions_right,
|
StringToVector::iterator substitutions_right,
|
||||||
const String & template_query,
|
const String & template_query,
|
||||||
std::vector<String> & queries)
|
std::vector<String> & out_queries)
|
||||||
{
|
{
|
||||||
if (substitutions_left == substitutions_right)
|
if (substitutions_left == substitutions_right)
|
||||||
{
|
{
|
||||||
queries.push_back(template_query); /// completely substituted query
|
out_queries.push_back(template_query); /// completely substituted query
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1191,7 +1183,7 @@ private:
|
|||||||
|
|
||||||
if (template_query.find(substitution_mask) == String::npos) /// nothing to substitute here
|
if (template_query.find(substitution_mask) == String::npos) /// nothing to substitute here
|
||||||
{
|
{
|
||||||
runThroughAllOptionsAndPush(std::next(substitutions_left), substitutions_right, template_query, queries);
|
runThroughAllOptionsAndPush(std::next(substitutions_left), substitutions_right, template_query, out_queries);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1209,7 +1201,7 @@ private:
|
|||||||
query.replace(substr_pos, substitution_mask.length(), value);
|
query.replace(substr_pos, substitution_mask.length(), value);
|
||||||
}
|
}
|
||||||
|
|
||||||
runThroughAllOptionsAndPush(std::next(substitutions_left), substitutions_right, query, queries);
|
runThroughAllOptionsAndPush(std::next(substitutions_left), substitutions_right, query, out_queries);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1343,7 +1335,7 @@ public:
|
|||||||
return json_output.asString();
|
return json_output.asString();
|
||||||
}
|
}
|
||||||
|
|
||||||
String minOutput(const String & main_metric)
|
String minOutput()
|
||||||
{
|
{
|
||||||
String output;
|
String output;
|
||||||
|
|
||||||
@ -1465,7 +1457,7 @@ try
|
|||||||
input_files = options["input-files"].as<Strings>();
|
input_files = options["input-files"].as<Strings>();
|
||||||
Strings collected_files;
|
Strings collected_files;
|
||||||
|
|
||||||
for (const String filename : input_files)
|
for (const String & filename : input_files)
|
||||||
{
|
{
|
||||||
fs::path file(filename);
|
fs::path file(filename);
|
||||||
|
|
||||||
|
@ -19,8 +19,8 @@
|
|||||||
#include <IO/ZlibInflatingReadBuffer.h>
|
#include <IO/ZlibInflatingReadBuffer.h>
|
||||||
#include <IO/ReadBufferFromString.h>
|
#include <IO/ReadBufferFromString.h>
|
||||||
#include <IO/ConcatReadBuffer.h>
|
#include <IO/ConcatReadBuffer.h>
|
||||||
#include <IO/CompressedReadBuffer.h>
|
#include <Compression/CompressedReadBuffer.h>
|
||||||
#include <IO/CompressedWriteBuffer.h>
|
#include <Compression/CompressedWriteBuffer.h>
|
||||||
#include <IO/WriteBufferFromString.h>
|
#include <IO/WriteBufferFromString.h>
|
||||||
#include <IO/WriteBufferFromHTTPServerResponse.h>
|
#include <IO/WriteBufferFromHTTPServerResponse.h>
|
||||||
#include <IO/WriteBufferFromFile.h>
|
#include <IO/WriteBufferFromFile.h>
|
||||||
|
@ -31,20 +31,16 @@ public:
|
|||||||
|
|
||||||
Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest & request) override
|
Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest & request) override
|
||||||
{
|
{
|
||||||
LOG_TRACE(log,
|
LOG_TRACE(log, "HTTP Request for " << name << ". "
|
||||||
"HTTP Request for " << name << ". "
|
<< "Method: "
|
||||||
<< "Method: "
|
<< request.getMethod()
|
||||||
<< request.getMethod()
|
<< ", Address: "
|
||||||
<< ", Address: "
|
<< request.clientAddress().toString()
|
||||||
<< request.clientAddress().toString()
|
<< ", User-Agent: "
|
||||||
<< ", User-Agent: "
|
<< (request.has("User-Agent") ? request.get("User-Agent") : "none")
|
||||||
<< (request.has("User-Agent") ? request.get("User-Agent") : "none")
|
<< (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : (""))
|
||||||
<< (request.hasContentLength() ? (", Length: " + std::to_string(request.getContentLength())) : (""))
|
<< ", Content Type: " << request.getContentType()
|
||||||
#if !NDEBUG
|
<< ", Transfer Encoding: " << request.getTransferEncoding());
|
||||||
<< ", Content Type: " << request.getContentType()
|
|
||||||
<< ", Transfer Encoding: " << request.getTransferEncoding()
|
|
||||||
#endif
|
|
||||||
);
|
|
||||||
|
|
||||||
const auto & uri = request.getURI();
|
const auto & uri = request.getURI();
|
||||||
|
|
||||||
|
@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
#include <Common/HTMLForm.h>
|
#include <Common/HTMLForm.h>
|
||||||
#include <Common/setThreadName.h>
|
#include <Common/setThreadName.h>
|
||||||
#include <IO/CompressedWriteBuffer.h>
|
#include <Compression/CompressedWriteBuffer.h>
|
||||||
#include <IO/ReadBufferFromIStream.h>
|
#include <IO/ReadBufferFromIStream.h>
|
||||||
#include <IO/WriteBufferFromHTTPServerResponse.h>
|
#include <IO/WriteBufferFromHTTPServerResponse.h>
|
||||||
#include <Interpreters/InterserverIOHandler.h>
|
#include <Interpreters/InterserverIOHandler.h>
|
||||||
|
@ -21,7 +21,7 @@ MetricsTransmitter::~MetricsTransmitter()
|
|||||||
try
|
try
|
||||||
{
|
{
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock{mutex};
|
std::lock_guard lock{mutex};
|
||||||
quit = true;
|
quit = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -56,7 +56,7 @@ void MetricsTransmitter::run()
|
|||||||
|
|
||||||
std::vector<ProfileEvents::Count> prev_counters(ProfileEvents::end());
|
std::vector<ProfileEvents::Count> prev_counters(ProfileEvents::end());
|
||||||
|
|
||||||
std::unique_lock<std::mutex> lock{mutex};
|
std::unique_lock lock{mutex};
|
||||||
|
|
||||||
while (true)
|
while (true)
|
||||||
{
|
{
|
||||||
|
@ -12,8 +12,8 @@
|
|||||||
#include <Common/setThreadName.h>
|
#include <Common/setThreadName.h>
|
||||||
#include <Common/config_version.h>
|
#include <Common/config_version.h>
|
||||||
#include <IO/Progress.h>
|
#include <IO/Progress.h>
|
||||||
#include <IO/CompressedReadBuffer.h>
|
#include <Compression/CompressedReadBuffer.h>
|
||||||
#include <IO/CompressedWriteBuffer.h>
|
#include <Compression/CompressedWriteBuffer.h>
|
||||||
#include <IO/ReadBufferFromPocoSocket.h>
|
#include <IO/ReadBufferFromPocoSocket.h>
|
||||||
#include <IO/WriteBufferFromPocoSocket.h>
|
#include <IO/WriteBufferFromPocoSocket.h>
|
||||||
#include <IO/ReadHelpers.h>
|
#include <IO/ReadHelpers.h>
|
||||||
|
@ -102,16 +102,14 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl(
|
|||||||
{
|
{
|
||||||
String name = getAliasToOrName(name_param);
|
String name = getAliasToOrName(name_param);
|
||||||
/// Find by exact match.
|
/// Find by exact match.
|
||||||
auto it = aggregate_functions.find(name);
|
if (auto it = aggregate_functions.find(name); it != aggregate_functions.end())
|
||||||
if (it != aggregate_functions.end())
|
|
||||||
return it->second(name, argument_types, parameters);
|
return it->second(name, argument_types, parameters);
|
||||||
|
|
||||||
/// Find by case-insensitive name.
|
/// Find by case-insensitive name.
|
||||||
/// Combinators cannot apply for case insensitive (SQL-style) aggregate function names. Only for native names.
|
/// Combinators cannot apply for case insensitive (SQL-style) aggregate function names. Only for native names.
|
||||||
if (recursion_level == 0)
|
if (recursion_level == 0)
|
||||||
{
|
{
|
||||||
auto it = case_insensitive_aggregate_functions.find(Poco::toLower(name));
|
if (auto it = case_insensitive_aggregate_functions.find(Poco::toLower(name)); it != case_insensitive_aggregate_functions.end())
|
||||||
if (it != case_insensitive_aggregate_functions.end())
|
|
||||||
return it->second(name, argument_types, parameters);
|
return it->second(name, argument_types, parameters);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -231,7 +231,7 @@ public:
|
|||||||
nested_state += nested_size_of_data;
|
nested_state += nested_size_of_data;
|
||||||
}
|
}
|
||||||
|
|
||||||
offsets_to.push_back(offsets_to.empty() ? state.dynamic_array_size : offsets_to.back() + state.dynamic_array_size);
|
offsets_to.push_back(offsets_to.back() + state.dynamic_array_size);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool allocatesMemoryInArena() const override
|
bool allocatesMemoryInArena() const override
|
||||||
|
@ -203,7 +203,7 @@ public:
|
|||||||
for (size_t i = arr.size(); i < result_array_size; ++i)
|
for (size_t i = arr.size(); i < result_array_size; ++i)
|
||||||
to_data.insert(default_value);
|
to_data.insert(default_value);
|
||||||
|
|
||||||
to_offsets.push_back((to_offsets.empty() ? 0 : to_offsets.back()) + result_array_size);
|
to_offsets.push_back(to_offsets.back() + result_array_size);
|
||||||
}
|
}
|
||||||
|
|
||||||
const char * getHeaderFilePath() const override { return __FILE__; }
|
const char * getHeaderFilePath() const override { return __FILE__; }
|
||||||
|
@ -45,8 +45,6 @@ AggregateFunctionPtr createAggregateFunctionHistogram(const std::string & name,
|
|||||||
throw Exception("Illegal type " + arguments[0]->getName() + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
throw Exception("Illegal type " + arguments[0]->getName() + " of argument for aggregate function " + name, ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||||
|
|
||||||
return res;
|
return res;
|
||||||
|
|
||||||
return nullptr;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -187,8 +187,8 @@ template <bool result_is_nullable>
|
|||||||
class AggregateFunctionNullUnary final : public AggregateFunctionNullBase<result_is_nullable, AggregateFunctionNullUnary<result_is_nullable>>
|
class AggregateFunctionNullUnary final : public AggregateFunctionNullBase<result_is_nullable, AggregateFunctionNullUnary<result_is_nullable>>
|
||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
AggregateFunctionNullUnary(AggregateFunctionPtr nested_function)
|
AggregateFunctionNullUnary(AggregateFunctionPtr nested_function_)
|
||||||
: AggregateFunctionNullBase<result_is_nullable, AggregateFunctionNullUnary<result_is_nullable>>(nested_function)
|
: AggregateFunctionNullBase<result_is_nullable, AggregateFunctionNullUnary<result_is_nullable>>(std::move(nested_function_))
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -209,8 +209,8 @@ template <bool result_is_nullable>
|
|||||||
class AggregateFunctionNullVariadic final : public AggregateFunctionNullBase<result_is_nullable, AggregateFunctionNullVariadic<result_is_nullable>>
|
class AggregateFunctionNullVariadic final : public AggregateFunctionNullBase<result_is_nullable, AggregateFunctionNullVariadic<result_is_nullable>>
|
||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
AggregateFunctionNullVariadic(AggregateFunctionPtr nested_function, const DataTypes & arguments)
|
AggregateFunctionNullVariadic(AggregateFunctionPtr nested_function_, const DataTypes & arguments)
|
||||||
: AggregateFunctionNullBase<result_is_nullable, AggregateFunctionNullVariadic<result_is_nullable>>(nested_function),
|
: AggregateFunctionNullBase<result_is_nullable, AggregateFunctionNullVariadic<result_is_nullable>>(std::move(nested_function_)),
|
||||||
number_of_arguments(arguments.size())
|
number_of_arguments(arguments.size())
|
||||||
{
|
{
|
||||||
if (number_of_arguments == 1)
|
if (number_of_arguments == 1)
|
||||||
|
@ -100,9 +100,12 @@ public:
|
|||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
void add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
|
void NO_SANITIZE_UNDEFINED add(AggregateDataPtr place, const IColumn ** columns, size_t row_num, Arena *) const override
|
||||||
{
|
{
|
||||||
|
/// Out of range conversion may occur. This is Ok.
|
||||||
|
|
||||||
const auto & column = static_cast<const ColVecType &>(*columns[0]);
|
const auto & column = static_cast<const ColVecType &>(*columns[0]);
|
||||||
|
|
||||||
if constexpr (has_second_arg)
|
if constexpr (has_second_arg)
|
||||||
this->data(place).add(
|
this->data(place).add(
|
||||||
column.getData()[row_num],
|
column.getData()[row_num],
|
||||||
|
@ -80,11 +80,11 @@ public:
|
|||||||
void add(AggregateDataPtr place, const IColumn ** columns, const size_t row_num, Arena *) const override
|
void add(AggregateDataPtr place, const IColumn ** columns, const size_t row_num, Arena *) const override
|
||||||
{
|
{
|
||||||
// Column 0 contains array of keys of known type
|
// Column 0 contains array of keys of known type
|
||||||
const ColumnArray & array_column = static_cast<const ColumnArray &>(*columns[0]);
|
const ColumnArray & array_column0 = static_cast<const ColumnArray &>(*columns[0]);
|
||||||
const IColumn::Offsets & offsets = array_column.getOffsets();
|
const IColumn::Offsets & offsets0 = array_column0.getOffsets();
|
||||||
const auto & keys_vec = static_cast<const ColVecType &>(array_column.getData());
|
const auto & keys_vec = static_cast<const ColVecType &>(array_column0.getData());
|
||||||
const size_t keys_vec_offset = offsets[row_num - 1];
|
const size_t keys_vec_offset = offsets0[row_num - 1];
|
||||||
const size_t keys_vec_size = (offsets[row_num] - keys_vec_offset);
|
const size_t keys_vec_size = (offsets0[row_num] - keys_vec_offset);
|
||||||
|
|
||||||
// Columns 1..n contain arrays of numeric values to sum
|
// Columns 1..n contain arrays of numeric values to sum
|
||||||
auto & merged_maps = this->data(place).merged_maps;
|
auto & merged_maps = this->data(place).merged_maps;
|
||||||
@ -226,14 +226,14 @@ public:
|
|||||||
|
|
||||||
// Advance column offsets
|
// Advance column offsets
|
||||||
auto & to_keys_offsets = to_keys_arr.getOffsets();
|
auto & to_keys_offsets = to_keys_arr.getOffsets();
|
||||||
to_keys_offsets.push_back((to_keys_offsets.empty() ? 0 : to_keys_offsets.back()) + size);
|
to_keys_offsets.push_back(to_keys_offsets.back() + size);
|
||||||
to_keys_col.reserve(size);
|
to_keys_col.reserve(size);
|
||||||
|
|
||||||
for (size_t col = 0; col < values_types.size(); ++col)
|
for (size_t col = 0; col < values_types.size(); ++col)
|
||||||
{
|
{
|
||||||
auto & to_values_arr = static_cast<ColumnArray &>(to_tuple.getColumn(col + 1));
|
auto & to_values_arr = static_cast<ColumnArray &>(to_tuple.getColumn(col + 1));
|
||||||
auto & to_values_offsets = to_values_arr.getOffsets();
|
auto & to_values_offsets = to_values_arr.getOffsets();
|
||||||
to_values_offsets.push_back((to_values_offsets.empty() ? 0 : to_values_offsets.back()) + size);
|
to_values_offsets.push_back(to_values_offsets.back() + size);
|
||||||
to_values_arr.getData().reserve(size);
|
to_values_arr.getData().reserve(size);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -382,13 +382,13 @@ namespace detail
|
|||||||
if (index == BIG_THRESHOLD)
|
if (index == BIG_THRESHOLD)
|
||||||
break;
|
break;
|
||||||
|
|
||||||
UInt64 count = 0;
|
UInt64 elem_count = 0;
|
||||||
readBinary(count, buf);
|
readBinary(elem_count, buf);
|
||||||
|
|
||||||
if (index < SMALL_THRESHOLD)
|
if (index < SMALL_THRESHOLD)
|
||||||
count_small[index] = count;
|
count_small[index] = elem_count;
|
||||||
else
|
else
|
||||||
count_big[index - SMALL_THRESHOLD] = count;
|
count_big[index - SMALL_THRESHOLD] = elem_count;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -2,8 +2,8 @@
|
|||||||
|
|
||||||
#include <Poco/Net/NetException.h>
|
#include <Poco/Net/NetException.h>
|
||||||
#include <Core/Defines.h>
|
#include <Core/Defines.h>
|
||||||
#include <IO/CompressedReadBuffer.h>
|
#include <Compression/CompressedReadBuffer.h>
|
||||||
#include <IO/CompressedWriteBuffer.h>
|
#include <Compression/CompressedWriteBuffer.h>
|
||||||
#include <IO/ReadBufferFromPocoSocket.h>
|
#include <IO/ReadBufferFromPocoSocket.h>
|
||||||
#include <IO/WriteBufferFromPocoSocket.h>
|
#include <IO/WriteBufferFromPocoSocket.h>
|
||||||
#include <IO/ReadHelpers.h>
|
#include <IO/ReadHelpers.h>
|
||||||
@ -732,10 +732,10 @@ std::unique_ptr<Exception> Connection::receiveException()
|
|||||||
std::vector<String> Connection::receiveMultistringMessage(UInt64 msg_type)
|
std::vector<String> Connection::receiveMultistringMessage(UInt64 msg_type)
|
||||||
{
|
{
|
||||||
size_t num = Protocol::Server::stringsInMessage(msg_type);
|
size_t num = Protocol::Server::stringsInMessage(msg_type);
|
||||||
std::vector<String> out(num);
|
std::vector<String> strings(num);
|
||||||
for (size_t i = 0; i < num; ++i)
|
for (size_t i = 0; i < num; ++i)
|
||||||
readStringBinary(out[i], *in);
|
readStringBinary(strings[i], *in);
|
||||||
return out;
|
return strings;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -52,7 +52,7 @@ MultiplexedConnections::MultiplexedConnections(
|
|||||||
|
|
||||||
void MultiplexedConnections::sendExternalTablesData(std::vector<ExternalTablesData> & data)
|
void MultiplexedConnections::sendExternalTablesData(std::vector<ExternalTablesData> & data)
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(cancel_mutex);
|
std::lock_guard lock(cancel_mutex);
|
||||||
|
|
||||||
if (!sent_query)
|
if (!sent_query)
|
||||||
throw Exception("Cannot send external tables data: query not yet sent.", ErrorCodes::LOGICAL_ERROR);
|
throw Exception("Cannot send external tables data: query not yet sent.", ErrorCodes::LOGICAL_ERROR);
|
||||||
@ -79,7 +79,7 @@ void MultiplexedConnections::sendQuery(
|
|||||||
const ClientInfo * client_info,
|
const ClientInfo * client_info,
|
||||||
bool with_pending_data)
|
bool with_pending_data)
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(cancel_mutex);
|
std::lock_guard lock(cancel_mutex);
|
||||||
|
|
||||||
if (sent_query)
|
if (sent_query)
|
||||||
throw Exception("Query already sent.", ErrorCodes::LOGICAL_ERROR);
|
throw Exception("Query already sent.", ErrorCodes::LOGICAL_ERROR);
|
||||||
@ -121,14 +121,14 @@ void MultiplexedConnections::sendQuery(
|
|||||||
|
|
||||||
Connection::Packet MultiplexedConnections::receivePacket()
|
Connection::Packet MultiplexedConnections::receivePacket()
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(cancel_mutex);
|
std::lock_guard lock(cancel_mutex);
|
||||||
Connection::Packet packet = receivePacketUnlocked();
|
Connection::Packet packet = receivePacketUnlocked();
|
||||||
return packet;
|
return packet;
|
||||||
}
|
}
|
||||||
|
|
||||||
void MultiplexedConnections::disconnect()
|
void MultiplexedConnections::disconnect()
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(cancel_mutex);
|
std::lock_guard lock(cancel_mutex);
|
||||||
|
|
||||||
for (ReplicaState & state : replica_states)
|
for (ReplicaState & state : replica_states)
|
||||||
{
|
{
|
||||||
@ -143,7 +143,7 @@ void MultiplexedConnections::disconnect()
|
|||||||
|
|
||||||
void MultiplexedConnections::sendCancel()
|
void MultiplexedConnections::sendCancel()
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(cancel_mutex);
|
std::lock_guard lock(cancel_mutex);
|
||||||
|
|
||||||
if (!sent_query || cancelled)
|
if (!sent_query || cancelled)
|
||||||
throw Exception("Cannot cancel. Either no query sent or already cancelled.", ErrorCodes::LOGICAL_ERROR);
|
throw Exception("Cannot cancel. Either no query sent or already cancelled.", ErrorCodes::LOGICAL_ERROR);
|
||||||
@ -160,7 +160,7 @@ void MultiplexedConnections::sendCancel()
|
|||||||
|
|
||||||
Connection::Packet MultiplexedConnections::drain()
|
Connection::Packet MultiplexedConnections::drain()
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(cancel_mutex);
|
std::lock_guard lock(cancel_mutex);
|
||||||
|
|
||||||
if (!cancelled)
|
if (!cancelled)
|
||||||
throw Exception("Cannot drain connections: cancel first.", ErrorCodes::LOGICAL_ERROR);
|
throw Exception("Cannot drain connections: cancel first.", ErrorCodes::LOGICAL_ERROR);
|
||||||
@ -195,7 +195,7 @@ Connection::Packet MultiplexedConnections::drain()
|
|||||||
|
|
||||||
std::string MultiplexedConnections::dumpAddresses() const
|
std::string MultiplexedConnections::dumpAddresses() const
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(cancel_mutex);
|
std::lock_guard lock(cancel_mutex);
|
||||||
return dumpAddressesUnlocked();
|
return dumpAddressesUnlocked();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -233,7 +233,10 @@ void ColumnArray::insertFrom(const IColumn & src_, size_t n)
|
|||||||
|
|
||||||
void ColumnArray::insertDefault()
|
void ColumnArray::insertDefault()
|
||||||
{
|
{
|
||||||
getOffsets().push_back(getOffsets().back());
|
/// NOTE 1: We can use back() even if the array is empty (due to zero -1th element in PODArray).
|
||||||
|
/// NOTE 2: We cannot use reference in push_back, because reference get invalidated if array is reallocated.
|
||||||
|
auto last_offset = getOffsets().back();
|
||||||
|
getOffsets().push_back(last_offset);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -9,7 +9,7 @@
|
|||||||
|
|
||||||
#include <IO/WriteHelpers.h>
|
#include <IO/WriteHelpers.h>
|
||||||
|
|
||||||
#if __SSE2__
|
#ifdef __SSE2__
|
||||||
#include <emmintrin.h>
|
#include <emmintrin.h>
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
@ -169,7 +169,7 @@ ColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result
|
|||||||
const UInt8 * filt_end = filt_pos + col_size;
|
const UInt8 * filt_end = filt_pos + col_size;
|
||||||
const UInt8 * data_pos = chars.data();
|
const UInt8 * data_pos = chars.data();
|
||||||
|
|
||||||
#if __SSE2__
|
#ifdef __SSE2__
|
||||||
/** A slightly more optimized version.
|
/** A slightly more optimized version.
|
||||||
* Based on the assumption that often pieces of consecutive values
|
* Based on the assumption that often pieces of consecutive values
|
||||||
* completely pass or do not pass the filter.
|
* completely pass or do not pass the filter.
|
||||||
|
@ -236,6 +236,9 @@ void ColumnLowCardinality::gather(ColumnGathererStream & gatherer)
|
|||||||
MutableColumnPtr ColumnLowCardinality::cloneResized(size_t size) const
|
MutableColumnPtr ColumnLowCardinality::cloneResized(size_t size) const
|
||||||
{
|
{
|
||||||
auto unique_ptr = dictionary.getColumnUniquePtr();
|
auto unique_ptr = dictionary.getColumnUniquePtr();
|
||||||
|
if (size == 0)
|
||||||
|
unique_ptr = unique_ptr->cloneEmpty();
|
||||||
|
|
||||||
return ColumnLowCardinality::create((*std::move(unique_ptr)).mutate(), getIndexes().cloneResized(size));
|
return ColumnLowCardinality::create((*std::move(unique_ptr)).mutate(), getIndexes().cloneResized(size));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -632,11 +635,11 @@ void ColumnLowCardinality::Dictionary::checkColumn(const IColumn & column)
|
|||||||
throw Exception("ColumnUnique expected as an argument of ColumnLowCardinality.", ErrorCodes::ILLEGAL_COLUMN);
|
throw Exception("ColumnUnique expected as an argument of ColumnLowCardinality.", ErrorCodes::ILLEGAL_COLUMN);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ColumnLowCardinality::Dictionary::setShared(const ColumnPtr & dictionary)
|
void ColumnLowCardinality::Dictionary::setShared(const ColumnPtr & column_unique_)
|
||||||
{
|
{
|
||||||
checkColumn(*dictionary);
|
checkColumn(*column_unique_);
|
||||||
|
|
||||||
column_unique = dictionary;
|
column_unique = column_unique_;
|
||||||
shared = true;
|
shared = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -133,7 +133,7 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
bool valuesHaveFixedSize() const override { return getDictionary().valuesHaveFixedSize(); }
|
bool valuesHaveFixedSize() const override { return getDictionary().valuesHaveFixedSize(); }
|
||||||
bool isFixedAndContiguous() const override { return getDictionary().isFixedAndContiguous(); }
|
bool isFixedAndContiguous() const override { return false; }
|
||||||
size_t sizeOfValueIfFixed() const override { return getDictionary().sizeOfValueIfFixed(); }
|
size_t sizeOfValueIfFixed() const override { return getDictionary().sizeOfValueIfFixed(); }
|
||||||
bool isNumeric() const override { return getDictionary().isNumeric(); }
|
bool isNumeric() const override { return getDictionary().isNumeric(); }
|
||||||
bool lowCardinality() const override { return true; }
|
bool lowCardinality() const override { return true; }
|
||||||
|
@ -20,7 +20,7 @@ private:
|
|||||||
|
|
||||||
public:
|
public:
|
||||||
const char * getFamilyName() const override { return "Nothing"; }
|
const char * getFamilyName() const override { return "Nothing"; }
|
||||||
MutableColumnPtr cloneDummy(size_t s) const override { return ColumnNothing::create(s); }
|
MutableColumnPtr cloneDummy(size_t s_) const override { return ColumnNothing::create(s_); }
|
||||||
|
|
||||||
bool canBeInsideNullable() const override { return true; }
|
bool canBeInsideNullable() const override { return true; }
|
||||||
};
|
};
|
||||||
|
@ -66,9 +66,9 @@ public:
|
|||||||
Int64 getInt(size_t n) const override { return getNestedColumn()->getInt(n); }
|
Int64 getInt(size_t n) const override { return getNestedColumn()->getInt(n); }
|
||||||
bool isNullAt(size_t n) const override { return is_nullable && n == getNullValueIndex(); }
|
bool isNullAt(size_t n) const override { return is_nullable && n == getNullValueIndex(); }
|
||||||
StringRef serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const override;
|
StringRef serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const override;
|
||||||
void updateHashWithValue(size_t n, SipHash & hash) const override
|
void updateHashWithValue(size_t n, SipHash & hash_func) const override
|
||||||
{
|
{
|
||||||
return getNestedColumn()->updateHashWithValue(n, hash);
|
return getNestedColumn()->updateHashWithValue(n, hash_func);
|
||||||
}
|
}
|
||||||
|
|
||||||
int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
|
int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
|
||||||
|
@ -13,7 +13,7 @@
|
|||||||
#include <DataStreams/ColumnGathererStream.h>
|
#include <DataStreams/ColumnGathererStream.h>
|
||||||
#include <ext/bit_cast.h>
|
#include <ext/bit_cast.h>
|
||||||
|
|
||||||
#if __SSE2__
|
#ifdef __SSE2__
|
||||||
#include <emmintrin.h>
|
#include <emmintrin.h>
|
||||||
#include <Columns/ColumnsCommon.h>
|
#include <Columns/ColumnsCommon.h>
|
||||||
|
|
||||||
@ -162,7 +162,7 @@ ColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_s
|
|||||||
const UInt8 * filt_end = filt_pos + size;
|
const UInt8 * filt_end = filt_pos + size;
|
||||||
const T * data_pos = data.data();
|
const T * data_pos = data.data();
|
||||||
|
|
||||||
#if __SSE2__
|
#ifdef __SSE2__
|
||||||
/** A slightly more optimized version.
|
/** A slightly more optimized version.
|
||||||
* Based on the assumption that often pieces of consecutive values
|
* Based on the assumption that often pieces of consecutive values
|
||||||
* completely pass or do not pass the filter.
|
* completely pass or do not pass the filter.
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
#if __SSE2__
|
#ifdef __SSE2__
|
||||||
#include <emmintrin.h>
|
#include <emmintrin.h>
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
@ -24,7 +24,7 @@ size_t countBytesInFilter(const IColumn::Filter & filt)
|
|||||||
const Int8 * pos = reinterpret_cast<const Int8 *>(filt.data());
|
const Int8 * pos = reinterpret_cast<const Int8 *>(filt.data());
|
||||||
const Int8 * end = pos + filt.size();
|
const Int8 * end = pos + filt.size();
|
||||||
|
|
||||||
#if __SSE2__ && __POPCNT__
|
#if defined(__SSE2__) && defined(__POPCNT__)
|
||||||
const __m128i zero16 = _mm_setzero_si128();
|
const __m128i zero16 = _mm_setzero_si128();
|
||||||
const Int8 * end64 = pos + filt.size() / 64 * 64;
|
const Int8 * end64 = pos + filt.size() / 64 * 64;
|
||||||
|
|
||||||
@ -69,7 +69,7 @@ bool memoryIsZero(const void * data, size_t size)
|
|||||||
const Int8 * pos = reinterpret_cast<const Int8 *>(data);
|
const Int8 * pos = reinterpret_cast<const Int8 *>(data);
|
||||||
const Int8 * end = pos + size;
|
const Int8 * end = pos + size;
|
||||||
|
|
||||||
#if __SSE2__
|
#ifdef __SSE2__
|
||||||
const __m128 zero16 = _mm_setzero_ps();
|
const __m128 zero16 = _mm_setzero_ps();
|
||||||
const Int8 * end64 = pos + size / 64 * 64;
|
const Int8 * end64 = pos + size / 64 * 64;
|
||||||
|
|
||||||
@ -205,17 +205,17 @@ namespace
|
|||||||
/// copy array ending at *end_offset_ptr
|
/// copy array ending at *end_offset_ptr
|
||||||
const auto copy_array = [&] (const IColumn::Offset * offset_ptr)
|
const auto copy_array = [&] (const IColumn::Offset * offset_ptr)
|
||||||
{
|
{
|
||||||
const auto offset = offset_ptr == offsets_begin ? 0 : offset_ptr[-1];
|
const auto arr_offset = offset_ptr == offsets_begin ? 0 : offset_ptr[-1];
|
||||||
const auto size = *offset_ptr - offset;
|
const auto arr_size = *offset_ptr - arr_offset;
|
||||||
|
|
||||||
result_offsets_builder.insertOne(size);
|
result_offsets_builder.insertOne(arr_size);
|
||||||
|
|
||||||
const auto elems_size_old = res_elems.size();
|
const auto elems_size_old = res_elems.size();
|
||||||
res_elems.resize(elems_size_old + size);
|
res_elems.resize(elems_size_old + arr_size);
|
||||||
memcpy(&res_elems[elems_size_old], &src_elems[offset], size * sizeof(T));
|
memcpy(&res_elems[elems_size_old], &src_elems[arr_offset], arr_size * sizeof(T));
|
||||||
};
|
};
|
||||||
|
|
||||||
#if __SSE2__
|
#ifdef __SSE2__
|
||||||
const __m128i zero_vec = _mm_setzero_si128();
|
const __m128i zero_vec = _mm_setzero_si128();
|
||||||
static constexpr size_t SIMD_BYTES = 16;
|
static constexpr size_t SIMD_BYTES = 16;
|
||||||
const auto filt_end_aligned = filt_pos + size / SIMD_BYTES * SIMD_BYTES;
|
const auto filt_end_aligned = filt_pos + size / SIMD_BYTES * SIMD_BYTES;
|
||||||
|
@ -330,7 +330,9 @@ public:
|
|||||||
virtual bool lowCardinality() const { return false; }
|
virtual bool lowCardinality() const { return false; }
|
||||||
|
|
||||||
|
|
||||||
virtual ~IColumn() {}
|
virtual ~IColumn() = default;
|
||||||
|
IColumn() = default;
|
||||||
|
IColumn(const IColumn &) = default;
|
||||||
|
|
||||||
/** Print column name, size, and recursively print all subcolumns.
|
/** Print column name, size, and recursively print all subcolumns.
|
||||||
*/
|
*/
|
||||||
|
@ -393,10 +393,10 @@ UInt64 ReverseIndex<IndexType, ColumnType>::insert(const StringRef & data)
|
|||||||
|
|
||||||
if constexpr (use_saved_hash)
|
if constexpr (use_saved_hash)
|
||||||
{
|
{
|
||||||
auto & data = saved_hash->getData();
|
auto & column_data = saved_hash->getData();
|
||||||
if (data.size() <= num_rows)
|
if (column_data.size() <= num_rows)
|
||||||
data.resize(num_rows + 1);
|
column_data.resize(num_rows + 1);
|
||||||
data[num_rows] = hash;
|
column_data[num_rows] = hash;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
column->insertData(data.data, data.size);
|
column->insertData(data.data, data.size);
|
||||||
|
@ -10,6 +10,7 @@
|
|||||||
#pragma GCC diagnostic ignored "-Wsign-compare"
|
#pragma GCC diagnostic ignored "-Wsign-compare"
|
||||||
#ifdef __clang__
|
#ifdef __clang__
|
||||||
#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
|
#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
|
||||||
|
#pragma clang diagnostic ignored "-Wundef"
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#include <gtest/gtest.h>
|
#include <gtest/gtest.h>
|
||||||
|
@ -94,7 +94,11 @@ private:
|
|||||||
Key key;
|
Key key;
|
||||||
Payload payload;
|
Payload payload;
|
||||||
|
|
||||||
void * ptr;
|
union
|
||||||
|
{
|
||||||
|
void * ptr;
|
||||||
|
char * char_ptr;
|
||||||
|
};
|
||||||
size_t size;
|
size_t size;
|
||||||
size_t refcount = 0;
|
size_t refcount = 0;
|
||||||
void * chunk;
|
void * chunk;
|
||||||
@ -231,7 +235,7 @@ public:
|
|||||||
|
|
||||||
~Holder()
|
~Holder()
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> cache_lock(cache.mutex);
|
std::lock_guard cache_lock(cache.mutex);
|
||||||
if (--region.refcount == 0)
|
if (--region.refcount == 0)
|
||||||
cache.lru_list.push_back(region);
|
cache.lru_list.push_back(region);
|
||||||
cache.total_size_in_use -= region.size;
|
cache.total_size_in_use -= region.size;
|
||||||
@ -301,12 +305,12 @@ private:
|
|||||||
if (cleaned_up)
|
if (cleaned_up)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
std::lock_guard<std::mutex> token_lock(token->mutex);
|
std::lock_guard token_lock(token->mutex);
|
||||||
|
|
||||||
if (token->cleaned_up)
|
if (token->cleaned_up)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
std::lock_guard<std::mutex> cache_lock(token->cache.mutex);
|
std::lock_guard cache_lock(token->cache.mutex);
|
||||||
|
|
||||||
--token->refcount;
|
--token->refcount;
|
||||||
if (token->refcount == 0)
|
if (token->refcount == 0)
|
||||||
@ -349,7 +353,7 @@ private:
|
|||||||
if (left_it->chunk == region.chunk && left_it->isFree())
|
if (left_it->chunk == region.chunk && left_it->isFree())
|
||||||
{
|
{
|
||||||
region.size += left_it->size;
|
region.size += left_it->size;
|
||||||
*reinterpret_cast<char **>(®ion.ptr) -= left_it->size;
|
region.char_ptr-= left_it->size;
|
||||||
size_multimap.erase(size_multimap.iterator_to(*left_it));
|
size_multimap.erase(size_multimap.iterator_to(*left_it));
|
||||||
adjacency_list.erase_and_dispose(left_it, [](RegionMetadata * elem) { elem->destroy(); });
|
adjacency_list.erase_and_dispose(left_it, [](RegionMetadata * elem) { elem->destroy(); });
|
||||||
}
|
}
|
||||||
@ -479,7 +483,7 @@ private:
|
|||||||
|
|
||||||
size_multimap.erase(size_multimap.iterator_to(free_region));
|
size_multimap.erase(size_multimap.iterator_to(free_region));
|
||||||
free_region.size -= size;
|
free_region.size -= size;
|
||||||
*reinterpret_cast<char **>(&free_region.ptr) += size;
|
free_region.char_ptr += size;
|
||||||
size_multimap.insert(free_region);
|
size_multimap.insert(free_region);
|
||||||
|
|
||||||
adjacency_list.insert(adjacency_list.iterator_to(free_region), *allocated_region);
|
adjacency_list.insert(adjacency_list.iterator_to(free_region), *allocated_region);
|
||||||
@ -536,7 +540,7 @@ public:
|
|||||||
|
|
||||||
~ArrayCache()
|
~ArrayCache()
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> cache_lock(mutex);
|
std::lock_guard cache_lock(mutex);
|
||||||
|
|
||||||
key_map.clear();
|
key_map.clear();
|
||||||
lru_list.clear();
|
lru_list.clear();
|
||||||
@ -563,7 +567,7 @@ public:
|
|||||||
{
|
{
|
||||||
InsertTokenHolder token_holder;
|
InsertTokenHolder token_holder;
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> cache_lock(mutex);
|
std::lock_guard cache_lock(mutex);
|
||||||
|
|
||||||
auto it = key_map.find(key, RegionCompareByKey());
|
auto it = key_map.find(key, RegionCompareByKey());
|
||||||
if (key_map.end() != it)
|
if (key_map.end() != it)
|
||||||
@ -584,7 +588,7 @@ public:
|
|||||||
|
|
||||||
InsertToken * token = token_holder.token.get();
|
InsertToken * token = token_holder.token.get();
|
||||||
|
|
||||||
std::lock_guard<std::mutex> token_lock(token->mutex);
|
std::lock_guard token_lock(token->mutex);
|
||||||
|
|
||||||
token_holder.cleaned_up = token->cleaned_up;
|
token_holder.cleaned_up = token->cleaned_up;
|
||||||
|
|
||||||
@ -605,7 +609,7 @@ public:
|
|||||||
|
|
||||||
RegionMetadata * region;
|
RegionMetadata * region;
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> cache_lock(mutex);
|
std::lock_guard cache_lock(mutex);
|
||||||
region = allocate(size);
|
region = allocate(size);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -626,14 +630,14 @@ public:
|
|||||||
catch (...)
|
catch (...)
|
||||||
{
|
{
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> cache_lock(mutex);
|
std::lock_guard cache_lock(mutex);
|
||||||
freeRegion(*region);
|
freeRegion(*region);
|
||||||
}
|
}
|
||||||
throw;
|
throw;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
std::lock_guard<std::mutex> cache_lock(mutex);
|
std::lock_guard cache_lock(mutex);
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
@ -692,7 +696,7 @@ public:
|
|||||||
|
|
||||||
Statistics getStatistics() const
|
Statistics getStatistics() const
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> cache_lock(mutex);
|
std::lock_guard cache_lock(mutex);
|
||||||
Statistics res;
|
Statistics res;
|
||||||
|
|
||||||
res.total_chunks_size = total_chunks_size;
|
res.total_chunks_size = total_chunks_size;
|
||||||
|
@ -22,7 +22,7 @@ namespace ErrorCodes
|
|||||||
* simulates an array of `content_width`-bit values.
|
* simulates an array of `content_width`-bit values.
|
||||||
*/
|
*/
|
||||||
template <typename BucketIndex, UInt8 content_width, size_t bucket_count>
|
template <typename BucketIndex, UInt8 content_width, size_t bucket_count>
|
||||||
class __attribute__ ((packed)) CompactArray final
|
class CompactArray final
|
||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
class Reader;
|
class Reader;
|
||||||
|
@ -600,9 +600,9 @@ void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config,
|
|||||||
}
|
}
|
||||||
|
|
||||||
preprocessed_path = preprocessed_dir + new_path;
|
preprocessed_path = preprocessed_dir + new_path;
|
||||||
auto path = Poco::Path(preprocessed_path).makeParent();
|
auto preprocessed_path_parent = Poco::Path(preprocessed_path).makeParent();
|
||||||
if (!path.toString().empty())
|
if (!preprocessed_path_parent.toString().empty())
|
||||||
Poco::File(path).createDirectories();
|
Poco::File(preprocessed_path_parent).createDirectories();
|
||||||
}
|
}
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
|
@ -78,7 +78,7 @@ void ConfigReloader::run()
|
|||||||
|
|
||||||
void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error, bool fallback_to_preprocessed)
|
void ConfigReloader::reloadIfNewer(bool force, bool throw_on_error, bool fallback_to_preprocessed)
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(reload_mutex);
|
std::lock_guard lock(reload_mutex);
|
||||||
|
|
||||||
FilesChangesTracker new_files = getNewFileList();
|
FilesChangesTracker new_files = getNewFileList();
|
||||||
if (force || need_reload_from_zk || new_files.isDifferOrNewerThan(files))
|
if (force || need_reload_from_zk || new_files.isDifferOrNewerThan(files))
|
||||||
|
@ -54,7 +54,7 @@ public:
|
|||||||
template <typename Callback>
|
template <typename Callback>
|
||||||
Int64 add(Int64 delta, Callback && locked_callback, bool create_if_need = false)
|
Int64 add(Int64 delta, Callback && locked_callback, bool create_if_need = false)
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(mutex);
|
std::lock_guard lock(mutex);
|
||||||
|
|
||||||
Int64 res = -1;
|
Int64 res = -1;
|
||||||
|
|
||||||
|
@ -3,7 +3,6 @@
|
|||||||
#include "CurrentThread.h"
|
#include "CurrentThread.h"
|
||||||
#include <common/logger_useful.h>
|
#include <common/logger_useful.h>
|
||||||
#include <Common/ThreadStatus.h>
|
#include <Common/ThreadStatus.h>
|
||||||
#include <Common/ObjectPool.h>
|
|
||||||
#include <Common/TaskStatsInfoGetter.h>
|
#include <Common/TaskStatsInfoGetter.h>
|
||||||
#include <Interpreters/ProcessList.h>
|
#include <Interpreters/ProcessList.h>
|
||||||
#include <Interpreters/Context.h>
|
#include <Interpreters/Context.h>
|
||||||
@ -24,8 +23,6 @@ namespace ErrorCodes
|
|||||||
extern const int LOGICAL_ERROR;
|
extern const int LOGICAL_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
SimpleObjectPool<TaskStatsInfoGetter> task_stats_info_getter_pool;
|
|
||||||
|
|
||||||
// Smoker's implementation to avoid thread_local usage: error: undefined symbol: __cxa_thread_atexit
|
// Smoker's implementation to avoid thread_local usage: error: undefined symbol: __cxa_thread_atexit
|
||||||
#if defined(ARCADIA_ROOT)
|
#if defined(ARCADIA_ROOT)
|
||||||
struct ThreadStatusPtrHolder : ThreadStatusPtr
|
struct ThreadStatusPtrHolder : ThreadStatusPtr
|
||||||
|
@ -146,7 +146,7 @@ void FileChecker::load(Map & local_map, const std::string & path)
|
|||||||
JSON json(out.str());
|
JSON json(out.str());
|
||||||
|
|
||||||
JSON files = json["yandex"];
|
JSON files = json["yandex"];
|
||||||
for (const auto & name_value : files)
|
for (const JSON name_value : files)
|
||||||
local_map[unescapeForFileName(name_value.getName())] = name_value.getValue()["size"].toUInt();
|
local_map[unescapeForFileName(name_value.getName())] = name_value.getValue()["size"].toUInt();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -35,20 +35,20 @@ inline DB::UInt64 intHash64(DB::UInt64 x)
|
|||||||
* due to high speed (latency 3 + 1 clock cycle, throughput 1 clock cycle).
|
* due to high speed (latency 3 + 1 clock cycle, throughput 1 clock cycle).
|
||||||
* Works only with SSE 4.2 support.
|
* Works only with SSE 4.2 support.
|
||||||
*/
|
*/
|
||||||
#if __SSE4_2__
|
#ifdef __SSE4_2__
|
||||||
#include <nmmintrin.h>
|
#include <nmmintrin.h>
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#if __aarch64__ && __ARM_FEATURE_CRC32
|
#if defined(__aarch64__) && defined(__ARM_FEATURE_CRC32)
|
||||||
#include <arm_acle.h>
|
#include <arm_acle.h>
|
||||||
#include <arm_neon.h>
|
#include <arm_neon.h>
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
inline DB::UInt64 intHashCRC32(DB::UInt64 x)
|
inline DB::UInt64 intHashCRC32(DB::UInt64 x)
|
||||||
{
|
{
|
||||||
#if __SSE4_2__
|
#ifdef __SSE4_2__
|
||||||
return _mm_crc32_u64(-1ULL, x);
|
return _mm_crc32_u64(-1ULL, x);
|
||||||
#elif __aarch64__ && __ARM_FEATURE_CRC32
|
#elif defined(__aarch64__) && defined(__ARM_FEATURE_CRC32)
|
||||||
return __crc32cd(-1U, x);
|
return __crc32cd(-1U, x);
|
||||||
#else
|
#else
|
||||||
/// On other platforms we do not have CRC32. NOTE This can be confusing.
|
/// On other platforms we do not have CRC32. NOTE This can be confusing.
|
||||||
|
@ -278,7 +278,7 @@ template <
|
|||||||
typename BiasEstimator = TrivialBiasEstimator,
|
typename BiasEstimator = TrivialBiasEstimator,
|
||||||
HyperLogLogMode mode = HyperLogLogMode::FullFeatured,
|
HyperLogLogMode mode = HyperLogLogMode::FullFeatured,
|
||||||
DenominatorMode denominator_mode = DenominatorMode::StableIfBig>
|
DenominatorMode denominator_mode = DenominatorMode::StableIfBig>
|
||||||
class __attribute__ ((packed)) HyperLogLogCounter : private Hash
|
class HyperLogLogCounter : private Hash
|
||||||
{
|
{
|
||||||
private:
|
private:
|
||||||
/// Number of buckets.
|
/// Number of buckets.
|
||||||
|
@ -48,7 +48,7 @@ public:
|
|||||||
|
|
||||||
MappedPtr get(const Key & key)
|
MappedPtr get(const Key & key)
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(mutex);
|
std::lock_guard lock(mutex);
|
||||||
|
|
||||||
auto res = getImpl(key, lock);
|
auto res = getImpl(key, lock);
|
||||||
if (res)
|
if (res)
|
||||||
@ -61,7 +61,7 @@ public:
|
|||||||
|
|
||||||
void set(const Key & key, const MappedPtr & mapped)
|
void set(const Key & key, const MappedPtr & mapped)
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(mutex);
|
std::lock_guard lock(mutex);
|
||||||
|
|
||||||
setImpl(key, mapped, lock);
|
setImpl(key, mapped, lock);
|
||||||
}
|
}
|
||||||
@ -79,7 +79,7 @@ public:
|
|||||||
{
|
{
|
||||||
InsertTokenHolder token_holder;
|
InsertTokenHolder token_holder;
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> cache_lock(mutex);
|
std::lock_guard cache_lock(mutex);
|
||||||
|
|
||||||
auto val = getImpl(key, cache_lock);
|
auto val = getImpl(key, cache_lock);
|
||||||
if (val)
|
if (val)
|
||||||
@ -97,7 +97,7 @@ public:
|
|||||||
|
|
||||||
InsertToken * token = token_holder.token.get();
|
InsertToken * token = token_holder.token.get();
|
||||||
|
|
||||||
std::lock_guard<std::mutex> token_lock(token->mutex);
|
std::lock_guard token_lock(token->mutex);
|
||||||
|
|
||||||
token_holder.cleaned_up = token->cleaned_up;
|
token_holder.cleaned_up = token->cleaned_up;
|
||||||
|
|
||||||
@ -111,7 +111,7 @@ public:
|
|||||||
++misses;
|
++misses;
|
||||||
token->value = load_func();
|
token->value = load_func();
|
||||||
|
|
||||||
std::lock_guard<std::mutex> cache_lock(mutex);
|
std::lock_guard cache_lock(mutex);
|
||||||
|
|
||||||
/// Insert the new value only if the token is still in present in insert_tokens.
|
/// Insert the new value only if the token is still in present in insert_tokens.
|
||||||
/// (The token may be absent because of a concurrent reset() call).
|
/// (The token may be absent because of a concurrent reset() call).
|
||||||
@ -131,26 +131,26 @@ public:
|
|||||||
|
|
||||||
void getStats(size_t & out_hits, size_t & out_misses) const
|
void getStats(size_t & out_hits, size_t & out_misses) const
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(mutex);
|
std::lock_guard lock(mutex);
|
||||||
out_hits = hits;
|
out_hits = hits;
|
||||||
out_misses = misses;
|
out_misses = misses;
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t weight() const
|
size_t weight() const
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(mutex);
|
std::lock_guard lock(mutex);
|
||||||
return current_size;
|
return current_size;
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t count() const
|
size_t count() const
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(mutex);
|
std::lock_guard lock(mutex);
|
||||||
return cells.size();
|
return cells.size();
|
||||||
}
|
}
|
||||||
|
|
||||||
void reset()
|
void reset()
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(mutex);
|
std::lock_guard lock(mutex);
|
||||||
queue.clear();
|
queue.clear();
|
||||||
cells.clear();
|
cells.clear();
|
||||||
insert_tokens.clear();
|
insert_tokens.clear();
|
||||||
@ -234,12 +234,12 @@ private:
|
|||||||
if (cleaned_up)
|
if (cleaned_up)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
std::lock_guard<std::mutex> token_lock(token->mutex);
|
std::lock_guard token_lock(token->mutex);
|
||||||
|
|
||||||
if (token->cleaned_up)
|
if (token->cleaned_up)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
std::lock_guard<std::mutex> cache_lock(token->cache.mutex);
|
std::lock_guard cache_lock(token->cache.mutex);
|
||||||
|
|
||||||
--token->refcount;
|
--token->refcount;
|
||||||
if (token->refcount == 0)
|
if (token->refcount == 0)
|
||||||
|
@ -38,7 +38,7 @@ protected:
|
|||||||
|
|
||||||
void operator()(T * owning_ptr) const
|
void operator()(T * owning_ptr) const
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock{parent->mutex};
|
std::lock_guard lock{parent->mutex};
|
||||||
parent->stack.emplace(owning_ptr);
|
parent->stack.emplace(owning_ptr);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
@ -51,7 +51,7 @@ public:
|
|||||||
template <typename Factory>
|
template <typename Factory>
|
||||||
Pointer get(Factory && f)
|
Pointer get(Factory && f)
|
||||||
{
|
{
|
||||||
std::unique_lock<std::mutex> lock(mutex);
|
std::unique_lock lock(mutex);
|
||||||
|
|
||||||
if (stack.empty())
|
if (stack.empty())
|
||||||
{
|
{
|
||||||
@ -94,7 +94,7 @@ public:
|
|||||||
template <typename Factory>
|
template <typename Factory>
|
||||||
Pointer get(const Key & key, Factory && f)
|
Pointer get(const Key & key, Factory && f)
|
||||||
{
|
{
|
||||||
std::unique_lock<std::mutex> lock(mutex);
|
std::unique_lock lock(mutex);
|
||||||
|
|
||||||
auto it = container.find(key);
|
auto it = container.find(key);
|
||||||
if (container.end() == it)
|
if (container.end() == it)
|
||||||
|
@ -127,9 +127,6 @@ protected:
|
|||||||
|
|
||||||
c_end = c_start + end_diff;
|
c_end = c_start + end_diff;
|
||||||
c_end_of_storage = c_start + bytes - pad_right - pad_left;
|
c_end_of_storage = c_start + bytes - pad_right - pad_left;
|
||||||
|
|
||||||
if (pad_left)
|
|
||||||
memset(c_start - ELEMENT_SIZE, 0, ELEMENT_SIZE);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
bool isInitialized() const
|
bool isInitialized() const
|
||||||
@ -312,13 +309,13 @@ public:
|
|||||||
this->c_end = this->c_start + this->byte_size(n);
|
this->c_end = this->c_start + this->byte_size(n);
|
||||||
}
|
}
|
||||||
|
|
||||||
template <typename ... TAllocatorParams>
|
template <typename U, typename ... TAllocatorParams>
|
||||||
void push_back(const T & x, TAllocatorParams &&... allocator_params)
|
void push_back(U && x, TAllocatorParams &&... allocator_params)
|
||||||
{
|
{
|
||||||
if (unlikely(this->c_end == this->c_end_of_storage))
|
if (unlikely(this->c_end == this->c_end_of_storage))
|
||||||
this->reserveForNextSize(std::forward<TAllocatorParams>(allocator_params)...);
|
this->reserveForNextSize(std::forward<TAllocatorParams>(allocator_params)...);
|
||||||
|
|
||||||
*t_end() = x;
|
new (t_end()) T(std::forward<U>(x));
|
||||||
this->c_end += this->byte_size(1);
|
this->c_end += this->byte_size(1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -54,7 +54,7 @@ private:
|
|||||||
PoolEntryHelper(PooledObject & data_) : data(data_) { data.in_use = true; }
|
PoolEntryHelper(PooledObject & data_) : data(data_) { data.in_use = true; }
|
||||||
~PoolEntryHelper()
|
~PoolEntryHelper()
|
||||||
{
|
{
|
||||||
std::unique_lock<std::mutex> lock(data.pool.mutex);
|
std::unique_lock lock(data.pool.mutex);
|
||||||
data.in_use = false;
|
data.in_use = false;
|
||||||
data.pool.available.notify_one();
|
data.pool.available.notify_one();
|
||||||
}
|
}
|
||||||
@ -107,7 +107,7 @@ public:
|
|||||||
/** Allocates the object. Wait for free object in pool for 'timeout'. With 'timeout' < 0, the timeout is infinite. */
|
/** Allocates the object. Wait for free object in pool for 'timeout'. With 'timeout' < 0, the timeout is infinite. */
|
||||||
Entry get(Poco::Timespan::TimeDiff timeout)
|
Entry get(Poco::Timespan::TimeDiff timeout)
|
||||||
{
|
{
|
||||||
std::unique_lock<std::mutex> lock(mutex);
|
std::unique_lock lock(mutex);
|
||||||
|
|
||||||
while (true)
|
while (true)
|
||||||
{
|
{
|
||||||
@ -133,7 +133,7 @@ public:
|
|||||||
|
|
||||||
void reserve(size_t count)
|
void reserve(size_t count)
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(mutex);
|
std::lock_guard lock(mutex);
|
||||||
|
|
||||||
while (items.size() < count)
|
while (items.size() < count)
|
||||||
items.emplace_back(std::make_shared<PooledObject>(allocObject(), *this));
|
items.emplace_back(std::make_shared<PooledObject>(allocObject(), *this));
|
||||||
|
@ -195,7 +195,7 @@ PoolWithFailoverBase<TNestedPool>::getMany(
|
|||||||
/// At exit update shared error counts with error counts occured during this call.
|
/// At exit update shared error counts with error counts occured during this call.
|
||||||
SCOPE_EXIT(
|
SCOPE_EXIT(
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(pool_states_mutex);
|
std::lock_guard lock(pool_states_mutex);
|
||||||
for (const ShuffledPool & pool: shuffled_pools)
|
for (const ShuffledPool & pool: shuffled_pools)
|
||||||
shared_pool_states[pool.index].error_count += pool.error_count;
|
shared_pool_states[pool.index].error_count += pool.error_count;
|
||||||
});
|
});
|
||||||
@ -300,7 +300,7 @@ void PoolWithFailoverBase<TNestedPool>::reportError(const Entry & entry)
|
|||||||
{
|
{
|
||||||
if (nested_pools[i]->contains(entry))
|
if (nested_pools[i]->contains(entry))
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(pool_states_mutex);
|
std::lock_guard lock(pool_states_mutex);
|
||||||
++shared_pool_states[i].error_count;
|
++shared_pool_states[i].error_count;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@ -338,7 +338,7 @@ PoolWithFailoverBase<TNestedPool>::updatePoolStates()
|
|||||||
result.reserve(nested_pools.size());
|
result.reserve(nested_pools.size());
|
||||||
|
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(pool_states_mutex);
|
std::lock_guard lock(pool_states_mutex);
|
||||||
|
|
||||||
for (auto & state : shared_pool_states)
|
for (auto & state : shared_pool_states)
|
||||||
state.randomize();
|
state.randomize();
|
||||||
|
@ -70,7 +70,7 @@ RWLockImpl::LockHandler RWLockImpl::getLock(RWLockImpl::Type type)
|
|||||||
GroupsContainer::iterator it_group;
|
GroupsContainer::iterator it_group;
|
||||||
ClientsContainer::iterator it_client;
|
ClientsContainer::iterator it_client;
|
||||||
|
|
||||||
std::unique_lock<std::mutex> lock(mutex);
|
std::unique_lock lock(mutex);
|
||||||
|
|
||||||
/// Check if the same thread is acquiring previously acquired lock
|
/// Check if the same thread is acquiring previously acquired lock
|
||||||
auto it_handler = thread_to_handler.find(this_thread_id);
|
auto it_handler = thread_to_handler.find(this_thread_id);
|
||||||
@ -139,7 +139,7 @@ RWLockImpl::LockHandler RWLockImpl::getLock(RWLockImpl::Type type)
|
|||||||
|
|
||||||
RWLockImpl::LockHandlerImpl::~LockHandlerImpl()
|
RWLockImpl::LockHandlerImpl::~LockHandlerImpl()
|
||||||
{
|
{
|
||||||
std::unique_lock<std::mutex> lock(parent->mutex);
|
std::unique_lock lock(parent->mutex);
|
||||||
|
|
||||||
/// Remove weak_ptr to the handler, since there are no owners of the current lock
|
/// Remove weak_ptr to the handler, since there are no owners of the current lock
|
||||||
parent->thread_to_handler.erase(it_handler);
|
parent->thread_to_handler.erase(it_handler);
|
||||||
|
@ -28,37 +28,29 @@ namespace
|
|||||||
{
|
{
|
||||||
struct Pipe
|
struct Pipe
|
||||||
{
|
{
|
||||||
union
|
int fds_rw[2];
|
||||||
{
|
|
||||||
int fds[2];
|
|
||||||
struct
|
|
||||||
{
|
|
||||||
int read_fd;
|
|
||||||
int write_fd;
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
Pipe()
|
Pipe()
|
||||||
{
|
{
|
||||||
#ifndef __APPLE__
|
#ifndef __APPLE__
|
||||||
if (0 != pipe2(fds, O_CLOEXEC))
|
if (0 != pipe2(fds_rw, O_CLOEXEC))
|
||||||
DB::throwFromErrno("Cannot create pipe", DB::ErrorCodes::CANNOT_PIPE);
|
DB::throwFromErrno("Cannot create pipe", DB::ErrorCodes::CANNOT_PIPE);
|
||||||
#else
|
#else
|
||||||
if (0 != pipe(fds))
|
if (0 != pipe(fds_rw))
|
||||||
DB::throwFromErrno("Cannot create pipe", DB::ErrorCodes::CANNOT_PIPE);
|
DB::throwFromErrno("Cannot create pipe", DB::ErrorCodes::CANNOT_PIPE);
|
||||||
if (0 != fcntl(fds[0], F_SETFD, FD_CLOEXEC))
|
if (0 != fcntl(fds_rw[0], F_SETFD, FD_CLOEXEC))
|
||||||
DB::throwFromErrno("Cannot create pipe", DB::ErrorCodes::CANNOT_PIPE);
|
DB::throwFromErrno("Cannot create pipe", DB::ErrorCodes::CANNOT_PIPE);
|
||||||
if (0 != fcntl(fds[1], F_SETFD, FD_CLOEXEC))
|
if (0 != fcntl(fds_rw[1], F_SETFD, FD_CLOEXEC))
|
||||||
DB::throwFromErrno("Cannot create pipe", DB::ErrorCodes::CANNOT_PIPE);
|
DB::throwFromErrno("Cannot create pipe", DB::ErrorCodes::CANNOT_PIPE);
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
~Pipe()
|
~Pipe()
|
||||||
{
|
{
|
||||||
if (read_fd >= 0)
|
if (fds_rw[0] >= 0)
|
||||||
close(read_fd);
|
close(fds_rw[0]);
|
||||||
if (write_fd >= 0)
|
if (fds_rw[1] >= 0)
|
||||||
close(write_fd);
|
close(fds_rw[1]);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -125,15 +117,15 @@ std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, c
|
|||||||
/// And there is a lot of garbage (including, for example, mutex is blocked). And this can not be done after `vfork` - deadlock happens.
|
/// And there is a lot of garbage (including, for example, mutex is blocked). And this can not be done after `vfork` - deadlock happens.
|
||||||
|
|
||||||
/// Replace the file descriptors with the ends of our pipes.
|
/// Replace the file descriptors with the ends of our pipes.
|
||||||
if (STDIN_FILENO != dup2(pipe_stdin.read_fd, STDIN_FILENO))
|
if (STDIN_FILENO != dup2(pipe_stdin.fds_rw[0], STDIN_FILENO))
|
||||||
_exit(int(ReturnCodes::CANNOT_DUP_STDIN));
|
_exit(int(ReturnCodes::CANNOT_DUP_STDIN));
|
||||||
|
|
||||||
if (!pipe_stdin_only)
|
if (!pipe_stdin_only)
|
||||||
{
|
{
|
||||||
if (STDOUT_FILENO != dup2(pipe_stdout.write_fd, STDOUT_FILENO))
|
if (STDOUT_FILENO != dup2(pipe_stdout.fds_rw[1], STDOUT_FILENO))
|
||||||
_exit(int(ReturnCodes::CANNOT_DUP_STDOUT));
|
_exit(int(ReturnCodes::CANNOT_DUP_STDOUT));
|
||||||
|
|
||||||
if (STDERR_FILENO != dup2(pipe_stderr.write_fd, STDERR_FILENO))
|
if (STDERR_FILENO != dup2(pipe_stderr.fds_rw[1], STDERR_FILENO))
|
||||||
_exit(int(ReturnCodes::CANNOT_DUP_STDERR));
|
_exit(int(ReturnCodes::CANNOT_DUP_STDERR));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -143,12 +135,12 @@ std::unique_ptr<ShellCommand> ShellCommand::executeImpl(const char * filename, c
|
|||||||
_exit(int(ReturnCodes::CANNOT_EXEC));
|
_exit(int(ReturnCodes::CANNOT_EXEC));
|
||||||
}
|
}
|
||||||
|
|
||||||
std::unique_ptr<ShellCommand> res(new ShellCommand(pid, pipe_stdin.write_fd, pipe_stdout.read_fd, pipe_stderr.read_fd, terminate_in_destructor));
|
std::unique_ptr<ShellCommand> res(new ShellCommand(pid, pipe_stdin.fds_rw[1], pipe_stdout.fds_rw[0], pipe_stderr.fds_rw[0], terminate_in_destructor));
|
||||||
|
|
||||||
/// Now the ownership of the file descriptors is passed to the result.
|
/// Now the ownership of the file descriptors is passed to the result.
|
||||||
pipe_stdin.write_fd = -1;
|
pipe_stdin.fds_rw[1] = -1;
|
||||||
pipe_stdout.read_fd = -1;
|
pipe_stdout.fds_rw[0] = -1;
|
||||||
pipe_stderr.read_fd = -1;
|
pipe_stderr.fds_rw[0] = -1;
|
||||||
|
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
@ -158,8 +150,8 @@ std::unique_ptr<ShellCommand> ShellCommand::execute(const std::string & command,
|
|||||||
{
|
{
|
||||||
/// Arguments in non-constant chunks of memory (as required for `execv`).
|
/// Arguments in non-constant chunks of memory (as required for `execv`).
|
||||||
/// Moreover, their copying must be done before calling `vfork`, so after `vfork` do a minimum of things.
|
/// Moreover, their copying must be done before calling `vfork`, so after `vfork` do a minimum of things.
|
||||||
std::vector<char> argv0("sh", "sh" + strlen("sh") + 1);
|
std::vector<char> argv0("sh", &("sh"[3]));
|
||||||
std::vector<char> argv1("-c", "-c" + strlen("-c") + 1);
|
std::vector<char> argv1("-c", &("-c"[3]));
|
||||||
std::vector<char> argv2(command.data(), command.data() + command.size() + 1);
|
std::vector<char> argv2(command.data(), command.data() + command.size() + 1);
|
||||||
|
|
||||||
char * const argv[] = { argv0.data(), argv1.data(), argv2.data(), nullptr };
|
char * const argv[] = { argv0.data(), argv1.data(), argv2.data(), nullptr };
|
||||||
|
@ -32,7 +32,7 @@ public:
|
|||||||
Result operator() (Args &&... args)
|
Result operator() (Args &&... args)
|
||||||
{
|
{
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(mutex);
|
std::lock_guard lock(mutex);
|
||||||
|
|
||||||
Key key{std::forward<Args>(args)...};
|
Key key{std::forward<Args>(args)...};
|
||||||
auto it = cache.find(key);
|
auto it = cache.find(key);
|
||||||
@ -45,7 +45,7 @@ public:
|
|||||||
Result res = f(std::forward<Args>(args)...);
|
Result res = f(std::forward<Args>(args)...);
|
||||||
|
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(mutex);
|
std::lock_guard lock(mutex);
|
||||||
|
|
||||||
cache.emplace(std::forward_as_tuple(args...), res);
|
cache.emplace(std::forward_as_tuple(args...), res);
|
||||||
}
|
}
|
||||||
@ -55,7 +55,7 @@ public:
|
|||||||
|
|
||||||
void drop()
|
void drop()
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(mutex);
|
std::lock_guard lock(mutex);
|
||||||
cache.clear();
|
cache.clear();
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
@ -32,11 +32,10 @@ StackTrace::StackTrace()
|
|||||||
std::string StackTrace::toStringImpl(const Frames & frames, size_t frames_size)
|
std::string StackTrace::toStringImpl(const Frames & frames, size_t frames_size)
|
||||||
{
|
{
|
||||||
char ** symbols = backtrace_symbols(frames.data(), frames_size);
|
char ** symbols = backtrace_symbols(frames.data(), frames_size);
|
||||||
std::stringstream res;
|
|
||||||
|
|
||||||
if (!symbols)
|
if (!symbols)
|
||||||
return "Cannot get symbols for stack trace.\n";
|
return "Cannot get symbols for stack trace.\n";
|
||||||
|
|
||||||
|
std::stringstream res;
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
for (size_t i = 0, size = frames_size; i < size; ++i)
|
for (size_t i = 0, size = frames_size; i < size; ++i)
|
||||||
|
@ -7,11 +7,11 @@
|
|||||||
#include <stdint.h>
|
#include <stdint.h>
|
||||||
#include <string.h>
|
#include <string.h>
|
||||||
|
|
||||||
#if __SSE2__
|
#ifdef __SSE2__
|
||||||
#include <emmintrin.h>
|
#include <emmintrin.h>
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
#if __SSE4_1__
|
#ifdef __SSE4_1__
|
||||||
#include <smmintrin.h>
|
#include <smmintrin.h>
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
@ -32,7 +32,7 @@ namespace ErrorCodes
|
|||||||
|
|
||||||
struct StringSearcherBase
|
struct StringSearcherBase
|
||||||
{
|
{
|
||||||
#if __SSE2__
|
#ifdef __SSE2__
|
||||||
static constexpr auto n = sizeof(__m128i);
|
static constexpr auto n = sizeof(__m128i);
|
||||||
const int page_size = getpagesize();
|
const int page_size = getpagesize();
|
||||||
|
|
||||||
@ -63,7 +63,7 @@ private:
|
|||||||
UInt8 l{};
|
UInt8 l{};
|
||||||
UInt8 u{};
|
UInt8 u{};
|
||||||
|
|
||||||
#if __SSE4_1__
|
#ifdef __SSE4_1__
|
||||||
/// vectors filled with `l` and `u`, for determining leftmost position of the first symbol
|
/// vectors filled with `l` and `u`, for determining leftmost position of the first symbol
|
||||||
__m128i patl, patu;
|
__m128i patl, patu;
|
||||||
/// lower and uppercase vectors of first 16 characters of `needle`
|
/// lower and uppercase vectors of first 16 characters of `needle`
|
||||||
@ -102,7 +102,7 @@ public:
|
|||||||
u = u_seq[0];
|
u = u_seq[0];
|
||||||
}
|
}
|
||||||
|
|
||||||
#if __SSE4_1__
|
#ifdef __SSE4_1__
|
||||||
/// for detecting leftmost position of the first symbol
|
/// for detecting leftmost position of the first symbol
|
||||||
patl = _mm_set1_epi8(l);
|
patl = _mm_set1_epi8(l);
|
||||||
patu = _mm_set1_epi8(u);
|
patu = _mm_set1_epi8(u);
|
||||||
@ -160,7 +160,7 @@ public:
|
|||||||
{
|
{
|
||||||
static const Poco::UTF8Encoding utf8;
|
static const Poco::UTF8Encoding utf8;
|
||||||
|
|
||||||
#if __SSE4_1__
|
#ifdef __SSE4_1__
|
||||||
if (pageSafe(pos))
|
if (pageSafe(pos))
|
||||||
{
|
{
|
||||||
const auto v_haystack = _mm_loadu_si128(reinterpret_cast<const __m128i *>(pos));
|
const auto v_haystack = _mm_loadu_si128(reinterpret_cast<const __m128i *>(pos));
|
||||||
@ -227,7 +227,7 @@ public:
|
|||||||
|
|
||||||
while (haystack < haystack_end)
|
while (haystack < haystack_end)
|
||||||
{
|
{
|
||||||
#if __SSE4_1__
|
#ifdef __SSE4_1__
|
||||||
if (haystack + n <= haystack_end && pageSafe(haystack))
|
if (haystack + n <= haystack_end && pageSafe(haystack))
|
||||||
{
|
{
|
||||||
const auto v_haystack = _mm_loadu_si128(reinterpret_cast<const __m128i *>(haystack));
|
const auto v_haystack = _mm_loadu_si128(reinterpret_cast<const __m128i *>(haystack));
|
||||||
@ -249,15 +249,15 @@ public:
|
|||||||
|
|
||||||
if (haystack < haystack_end && haystack + n <= haystack_end && pageSafe(haystack))
|
if (haystack < haystack_end && haystack + n <= haystack_end && pageSafe(haystack))
|
||||||
{
|
{
|
||||||
const auto v_haystack = _mm_loadu_si128(reinterpret_cast<const __m128i *>(haystack));
|
const auto v_haystack_offset = _mm_loadu_si128(reinterpret_cast<const __m128i *>(haystack));
|
||||||
const auto v_against_l = _mm_cmpeq_epi8(v_haystack, cachel);
|
const auto v_against_l_offset = _mm_cmpeq_epi8(v_haystack_offset, cachel);
|
||||||
const auto v_against_u = _mm_cmpeq_epi8(v_haystack, cacheu);
|
const auto v_against_u_offset = _mm_cmpeq_epi8(v_haystack_offset, cacheu);
|
||||||
const auto v_against_l_or_u = _mm_or_si128(v_against_l, v_against_u);
|
const auto v_against_l_or_u_offset = _mm_or_si128(v_against_l_offset, v_against_u_offset);
|
||||||
const auto mask = _mm_movemask_epi8(v_against_l_or_u);
|
const auto mask_offset = _mm_movemask_epi8(v_against_l_or_u_offset);
|
||||||
|
|
||||||
if (0xffff == cachemask)
|
if (0xffff == cachemask)
|
||||||
{
|
{
|
||||||
if (mask == cachemask)
|
if (mask_offset == cachemask)
|
||||||
{
|
{
|
||||||
auto haystack_pos = haystack + cache_valid_len;
|
auto haystack_pos = haystack + cache_valid_len;
|
||||||
auto needle_pos = needle + cache_valid_len;
|
auto needle_pos = needle + cache_valid_len;
|
||||||
@ -276,7 +276,7 @@ public:
|
|||||||
return haystack;
|
return haystack;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else if ((mask & cachemask) == cachemask)
|
else if ((mask_offset & cachemask) == cachemask)
|
||||||
return haystack;
|
return haystack;
|
||||||
|
|
||||||
/// first octet was ok, but not the first 16, move to start of next sequence and reapply
|
/// first octet was ok, but not the first 16, move to start of next sequence and reapply
|
||||||
@ -334,7 +334,7 @@ private:
|
|||||||
UInt8 l{};
|
UInt8 l{};
|
||||||
UInt8 u{};
|
UInt8 u{};
|
||||||
|
|
||||||
#if __SSE4_1__
|
#ifdef __SSE4_1__
|
||||||
/// vectors filled with `l` and `u`, for determining leftmost position of the first symbol
|
/// vectors filled with `l` and `u`, for determining leftmost position of the first symbol
|
||||||
__m128i patl, patu;
|
__m128i patl, patu;
|
||||||
/// lower and uppercase vectors of first 16 characters of `needle`
|
/// lower and uppercase vectors of first 16 characters of `needle`
|
||||||
@ -352,7 +352,7 @@ public:
|
|||||||
l = static_cast<UInt8>(std::tolower(*needle));
|
l = static_cast<UInt8>(std::tolower(*needle));
|
||||||
u = static_cast<UInt8>(std::toupper(*needle));
|
u = static_cast<UInt8>(std::toupper(*needle));
|
||||||
|
|
||||||
#if __SSE4_1__
|
#ifdef __SSE4_1__
|
||||||
patl = _mm_set1_epi8(l);
|
patl = _mm_set1_epi8(l);
|
||||||
patu = _mm_set1_epi8(u);
|
patu = _mm_set1_epi8(u);
|
||||||
|
|
||||||
@ -376,7 +376,7 @@ public:
|
|||||||
|
|
||||||
bool compare(const UInt8 * pos) const
|
bool compare(const UInt8 * pos) const
|
||||||
{
|
{
|
||||||
#if __SSE4_1__
|
#ifdef __SSE4_1__
|
||||||
if (pageSafe(pos))
|
if (pageSafe(pos))
|
||||||
{
|
{
|
||||||
const auto v_haystack = _mm_loadu_si128(reinterpret_cast<const __m128i *>(pos));
|
const auto v_haystack = _mm_loadu_si128(reinterpret_cast<const __m128i *>(pos));
|
||||||
@ -434,7 +434,7 @@ public:
|
|||||||
|
|
||||||
while (haystack < haystack_end)
|
while (haystack < haystack_end)
|
||||||
{
|
{
|
||||||
#if __SSE4_1__
|
#ifdef __SSE4_1__
|
||||||
if (haystack + n <= haystack_end && pageSafe(haystack))
|
if (haystack + n <= haystack_end && pageSafe(haystack))
|
||||||
{
|
{
|
||||||
const auto v_haystack = _mm_loadu_si128(reinterpret_cast<const __m128i *>(haystack));
|
const auto v_haystack = _mm_loadu_si128(reinterpret_cast<const __m128i *>(haystack));
|
||||||
@ -455,15 +455,15 @@ public:
|
|||||||
|
|
||||||
if (haystack < haystack_end && haystack + n <= haystack_end && pageSafe(haystack))
|
if (haystack < haystack_end && haystack + n <= haystack_end && pageSafe(haystack))
|
||||||
{
|
{
|
||||||
const auto v_haystack = _mm_loadu_si128(reinterpret_cast<const __m128i *>(haystack));
|
const auto v_haystack_offset = _mm_loadu_si128(reinterpret_cast<const __m128i *>(haystack));
|
||||||
const auto v_against_l = _mm_cmpeq_epi8(v_haystack, cachel);
|
const auto v_against_l_offset = _mm_cmpeq_epi8(v_haystack_offset, cachel);
|
||||||
const auto v_against_u = _mm_cmpeq_epi8(v_haystack, cacheu);
|
const auto v_against_u_offset = _mm_cmpeq_epi8(v_haystack_offset, cacheu);
|
||||||
const auto v_against_l_or_u = _mm_or_si128(v_against_l, v_against_u);
|
const auto v_against_l_or_u_offset = _mm_or_si128(v_against_l_offset, v_against_u_offset);
|
||||||
const auto mask = _mm_movemask_epi8(v_against_l_or_u);
|
const auto mask_offset = _mm_movemask_epi8(v_against_l_or_u_offset);
|
||||||
|
|
||||||
if (0xffff == cachemask)
|
if (0xffff == cachemask)
|
||||||
{
|
{
|
||||||
if (mask == cachemask)
|
if (mask_offset == cachemask)
|
||||||
{
|
{
|
||||||
auto haystack_pos = haystack + n;
|
auto haystack_pos = haystack + n;
|
||||||
auto needle_pos = needle + n;
|
auto needle_pos = needle + n;
|
||||||
@ -479,7 +479,7 @@ public:
|
|||||||
return haystack;
|
return haystack;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else if ((mask & cachemask) == cachemask)
|
else if ((mask_offset & cachemask) == cachemask)
|
||||||
return haystack;
|
return haystack;
|
||||||
|
|
||||||
++haystack;
|
++haystack;
|
||||||
@ -532,7 +532,7 @@ private:
|
|||||||
/// first character in `needle`
|
/// first character in `needle`
|
||||||
UInt8 first{};
|
UInt8 first{};
|
||||||
|
|
||||||
#if __SSE4_1__
|
#ifdef __SSE4_1__
|
||||||
/// vector filled `first` for determining leftmost position of the first symbol
|
/// vector filled `first` for determining leftmost position of the first symbol
|
||||||
__m128i pattern;
|
__m128i pattern;
|
||||||
/// vector of first 16 characters of `needle`
|
/// vector of first 16 characters of `needle`
|
||||||
@ -549,7 +549,7 @@ public:
|
|||||||
|
|
||||||
first = *needle;
|
first = *needle;
|
||||||
|
|
||||||
#if __SSE4_1__
|
#ifdef __SSE4_1__
|
||||||
pattern = _mm_set1_epi8(first);
|
pattern = _mm_set1_epi8(first);
|
||||||
|
|
||||||
auto needle_pos = needle;
|
auto needle_pos = needle;
|
||||||
@ -570,7 +570,7 @@ public:
|
|||||||
|
|
||||||
bool compare(const UInt8 * pos) const
|
bool compare(const UInt8 * pos) const
|
||||||
{
|
{
|
||||||
#if __SSE4_1__
|
#ifdef __SSE4_1__
|
||||||
if (pageSafe(pos))
|
if (pageSafe(pos))
|
||||||
{
|
{
|
||||||
const auto v_haystack = _mm_loadu_si128(reinterpret_cast<const __m128i *>(pos));
|
const auto v_haystack = _mm_loadu_si128(reinterpret_cast<const __m128i *>(pos));
|
||||||
@ -620,7 +620,7 @@ public:
|
|||||||
|
|
||||||
while (haystack < haystack_end)
|
while (haystack < haystack_end)
|
||||||
{
|
{
|
||||||
#if __SSE4_1__
|
#ifdef __SSE4_1__
|
||||||
if (haystack + n <= haystack_end && pageSafe(haystack))
|
if (haystack + n <= haystack_end && pageSafe(haystack))
|
||||||
{
|
{
|
||||||
/// find first character
|
/// find first character
|
||||||
@ -642,13 +642,13 @@ public:
|
|||||||
if (haystack < haystack_end && haystack + n <= haystack_end && pageSafe(haystack))
|
if (haystack < haystack_end && haystack + n <= haystack_end && pageSafe(haystack))
|
||||||
{
|
{
|
||||||
/// check for first 16 octets
|
/// check for first 16 octets
|
||||||
const auto v_haystack = _mm_loadu_si128(reinterpret_cast<const __m128i *>(haystack));
|
const auto v_haystack_offset = _mm_loadu_si128(reinterpret_cast<const __m128i *>(haystack));
|
||||||
const auto v_against_cache = _mm_cmpeq_epi8(v_haystack, cache);
|
const auto v_against_cache = _mm_cmpeq_epi8(v_haystack_offset, cache);
|
||||||
const auto mask = _mm_movemask_epi8(v_against_cache);
|
const auto mask_offset = _mm_movemask_epi8(v_against_cache);
|
||||||
|
|
||||||
if (0xffff == cachemask)
|
if (0xffff == cachemask)
|
||||||
{
|
{
|
||||||
if (mask == cachemask)
|
if (mask_offset == cachemask)
|
||||||
{
|
{
|
||||||
auto haystack_pos = haystack + n;
|
auto haystack_pos = haystack + n;
|
||||||
auto needle_pos = needle + n;
|
auto needle_pos = needle + n;
|
||||||
@ -661,7 +661,7 @@ public:
|
|||||||
return haystack;
|
return haystack;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else if ((mask & cachemask) == cachemask)
|
else if ((mask_offset & cachemask) == cachemask)
|
||||||
return haystack;
|
return haystack;
|
||||||
|
|
||||||
++haystack;
|
++haystack;
|
||||||
|
@ -56,7 +56,7 @@ std::string getOrdinalSuffix(T n)
|
|||||||
case 2: return "nd";
|
case 2: return "nd";
|
||||||
case 3: return "rd";
|
case 3: return "rd";
|
||||||
default: return "th";
|
default: return "th";
|
||||||
};
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// More efficient than libc, because doesn't respect locale. But for some functions table implementation could be better.
|
/// More efficient than libc, because doesn't respect locale. But for some functions table implementation could be better.
|
||||||
|
@ -80,11 +80,6 @@ struct NetlinkMessage
|
|||||||
::nlmsgerr error;
|
::nlmsgerr error;
|
||||||
};
|
};
|
||||||
|
|
||||||
size_t payload_size() const
|
|
||||||
{
|
|
||||||
return header.nlmsg_len - sizeof(header) - sizeof(generic_header);
|
|
||||||
}
|
|
||||||
|
|
||||||
const Attribute * end() const
|
const Attribute * end() const
|
||||||
{
|
{
|
||||||
return reinterpret_cast<const Attribute *>(reinterpret_cast<const char *>(this) + header.nlmsg_len);
|
return reinterpret_cast<const Attribute *>(reinterpret_cast<const char *>(this) + header.nlmsg_len);
|
||||||
|
@ -21,9 +21,6 @@ namespace ErrorCodes
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
extern SimpleObjectPool<TaskStatsInfoGetter> task_stats_info_getter_pool;
|
|
||||||
|
|
||||||
|
|
||||||
TasksStatsCounters TasksStatsCounters::current()
|
TasksStatsCounters TasksStatsCounters::current()
|
||||||
{
|
{
|
||||||
TasksStatsCounters res;
|
TasksStatsCounters res;
|
||||||
@ -74,7 +71,7 @@ void ThreadStatus::initPerformanceCounters()
|
|||||||
if (TaskStatsInfoGetter::checkPermissions())
|
if (TaskStatsInfoGetter::checkPermissions())
|
||||||
{
|
{
|
||||||
if (!taskstats_getter)
|
if (!taskstats_getter)
|
||||||
taskstats_getter = task_stats_info_getter_pool.getDefault();
|
taskstats_getter = std::make_unique<TaskStatsInfoGetter>();
|
||||||
|
|
||||||
*last_taskstats = TasksStatsCounters::current();
|
*last_taskstats = TasksStatsCounters::current();
|
||||||
}
|
}
|
||||||
|
@ -2,7 +2,6 @@
|
|||||||
|
|
||||||
#include <Common/ProfileEvents.h>
|
#include <Common/ProfileEvents.h>
|
||||||
#include <Common/MemoryTracker.h>
|
#include <Common/MemoryTracker.h>
|
||||||
#include <Common/ObjectPool.h>
|
|
||||||
|
|
||||||
#include <IO/Progress.h>
|
#include <IO/Progress.h>
|
||||||
|
|
||||||
@ -175,8 +174,7 @@ protected:
|
|||||||
std::unique_ptr<TasksStatsCounters> last_taskstats;
|
std::unique_ptr<TasksStatsCounters> last_taskstats;
|
||||||
|
|
||||||
/// Set to non-nullptr only if we have enough capabilities.
|
/// Set to non-nullptr only if we have enough capabilities.
|
||||||
/// We use pool because creation and destruction of TaskStatsInfoGetter objects are expensive.
|
std::unique_ptr<TaskStatsInfoGetter> taskstats_getter;
|
||||||
SimpleObjectPool<TaskStatsInfoGetter>::Pointer taskstats_getter;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
#include <Core/Types.h>
|
#include <Core/Types.h>
|
||||||
|
|
||||||
#if __SSE4_2__
|
#ifdef __SSE4_2__
|
||||||
#include <nmmintrin.h>
|
#include <nmmintrin.h>
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
@ -75,7 +75,7 @@ struct UInt128Hash
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
#if __SSE4_2__
|
#ifdef __SSE4_2__
|
||||||
|
|
||||||
struct UInt128HashCRC32
|
struct UInt128HashCRC32
|
||||||
{
|
{
|
||||||
@ -153,7 +153,7 @@ struct UInt256Hash
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
#if __SSE4_2__
|
#ifdef __SSE4_2__
|
||||||
|
|
||||||
struct UInt256HashCRC32
|
struct UInt256HashCRC32
|
||||||
{
|
{
|
||||||
|
@ -3,7 +3,7 @@
|
|||||||
#include <Core/Types.h>
|
#include <Core/Types.h>
|
||||||
#include <Common/BitHelpers.h>
|
#include <Common/BitHelpers.h>
|
||||||
|
|
||||||
#if __SSE2__
|
#ifdef __SSE2__
|
||||||
#include <emmintrin.h>
|
#include <emmintrin.h>
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
@ -55,7 +55,7 @@ inline size_t countCodePoints(const UInt8 * data, size_t size)
|
|||||||
size_t res = 0;
|
size_t res = 0;
|
||||||
const auto end = data + size;
|
const auto end = data + size;
|
||||||
|
|
||||||
#if __SSE2__
|
#ifdef __SSE2__
|
||||||
constexpr auto bytes_sse = sizeof(__m128i);
|
constexpr auto bytes_sse = sizeof(__m128i);
|
||||||
const auto src_end_sse = data + size / bytes_sse * bytes_sse;
|
const auto src_end_sse = data + size / bytes_sse * bytes_sse;
|
||||||
|
|
||||||
|
@ -194,9 +194,9 @@ template <bool CaseSensitive, bool ASCII> struct VolnitskyImpl;
|
|||||||
/// Case sensitive comparison
|
/// Case sensitive comparison
|
||||||
template <bool ASCII> struct VolnitskyImpl<true, ASCII> : VolnitskyBase<VolnitskyImpl<true, ASCII>>
|
template <bool ASCII> struct VolnitskyImpl<true, ASCII> : VolnitskyBase<VolnitskyImpl<true, ASCII>>
|
||||||
{
|
{
|
||||||
VolnitskyImpl(const char * const needle, const size_t needle_size, const size_t haystack_size_hint = 0)
|
VolnitskyImpl(const char * const needle_, const size_t needle_size_, const size_t haystack_size_hint = 0)
|
||||||
: VolnitskyBase<VolnitskyImpl<true, ASCII>>{needle, needle_size, haystack_size_hint},
|
: VolnitskyBase<VolnitskyImpl<true, ASCII>>{needle_, needle_size_, haystack_size_hint},
|
||||||
fallback_searcher{needle, needle_size}
|
fallback_searcher{needle_, needle_size_}
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -222,8 +222,8 @@ template <bool ASCII> struct VolnitskyImpl<true, ASCII> : VolnitskyBase<Volnitsk
|
|||||||
/// Case-insensitive ASCII
|
/// Case-insensitive ASCII
|
||||||
template <> struct VolnitskyImpl<false, true> : VolnitskyBase<VolnitskyImpl<false, true>>
|
template <> struct VolnitskyImpl<false, true> : VolnitskyBase<VolnitskyImpl<false, true>>
|
||||||
{
|
{
|
||||||
VolnitskyImpl(const char * const needle, const size_t needle_size, const size_t haystack_size_hint = 0)
|
VolnitskyImpl(const char * const needle_, const size_t needle_size_, const size_t haystack_size_hint = 0)
|
||||||
: VolnitskyBase{needle, needle_size, haystack_size_hint}, fallback_searcher{needle, needle_size}
|
: VolnitskyBase{needle_, needle_size_, haystack_size_hint}, fallback_searcher{needle_, needle_size_}
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -248,8 +248,8 @@ template <> struct VolnitskyImpl<false, true> : VolnitskyBase<VolnitskyImpl<fals
|
|||||||
/// Case-sensitive UTF-8
|
/// Case-sensitive UTF-8
|
||||||
template <> struct VolnitskyImpl<false, false> : VolnitskyBase<VolnitskyImpl<false, false>>
|
template <> struct VolnitskyImpl<false, false> : VolnitskyBase<VolnitskyImpl<false, false>>
|
||||||
{
|
{
|
||||||
VolnitskyImpl(const char * const needle, const size_t needle_size, const size_t haystack_size_hint = 0)
|
VolnitskyImpl(const char * const needle_, const size_t needle_size_, const size_t haystack_size_hint = 0)
|
||||||
: VolnitskyBase{needle, needle_size, haystack_size_hint}, fallback_searcher{needle, needle_size}
|
: VolnitskyBase{needle_, needle_size_, haystack_size_hint}, fallback_searcher{needle_, needle_size_}
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -59,7 +59,9 @@ using Requests = std::vector<RequestPtr>;
|
|||||||
|
|
||||||
struct Request
|
struct Request
|
||||||
{
|
{
|
||||||
virtual ~Request() {}
|
Request() = default;
|
||||||
|
Request(const Request &) = default;
|
||||||
|
virtual ~Request() = default;
|
||||||
virtual String getPath() const = 0;
|
virtual String getPath() const = 0;
|
||||||
virtual void addRootPath(const String & /* root_path */) {}
|
virtual void addRootPath(const String & /* root_path */) {}
|
||||||
};
|
};
|
||||||
@ -72,7 +74,9 @@ using ResponseCallback = std::function<void(const Response &)>;
|
|||||||
struct Response
|
struct Response
|
||||||
{
|
{
|
||||||
int32_t error = 0;
|
int32_t error = 0;
|
||||||
virtual ~Response() {}
|
Response() = default;
|
||||||
|
Response(const Response &) = default;
|
||||||
|
virtual ~Response() = default;
|
||||||
virtual void removeRootPath(const String & /* root_path */) {}
|
virtual void removeRootPath(const String & /* root_path */) {}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -17,7 +17,6 @@
|
|||||||
|
|
||||||
|
|
||||||
#define ZOOKEEPER_CONNECTION_TIMEOUT_MS 1000
|
#define ZOOKEEPER_CONNECTION_TIMEOUT_MS 1000
|
||||||
#define ZOOKEEPER_OPERATION_TIMEOUT_MS 10000
|
|
||||||
|
|
||||||
|
|
||||||
namespace DB
|
namespace DB
|
||||||
@ -840,7 +839,7 @@ int32_t ZooKeeper::tryMultiNoThrow(const Coordination::Requests & requests, Coor
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
size_t KeeperMultiException::getFailedOpIndex(int32_t code, const Coordination::Responses & responses)
|
size_t KeeperMultiException::getFailedOpIndex(int32_t exception_code, const Coordination::Responses & responses)
|
||||||
{
|
{
|
||||||
if (responses.empty())
|
if (responses.empty())
|
||||||
throw DB::Exception("Responses for multi transaction is empty", DB::ErrorCodes::LOGICAL_ERROR);
|
throw DB::Exception("Responses for multi transaction is empty", DB::ErrorCodes::LOGICAL_ERROR);
|
||||||
@ -849,17 +848,17 @@ size_t KeeperMultiException::getFailedOpIndex(int32_t code, const Coordination::
|
|||||||
if (responses[index]->error)
|
if (responses[index]->error)
|
||||||
return index;
|
return index;
|
||||||
|
|
||||||
if (!Coordination::isUserError(code))
|
if (!Coordination::isUserError(exception_code))
|
||||||
throw DB::Exception("There are no failed OPs because '" + ZooKeeper::error2string(code) + "' is not valid response code for that",
|
throw DB::Exception("There are no failed OPs because '" + ZooKeeper::error2string(exception_code) + "' is not valid response code for that",
|
||||||
DB::ErrorCodes::LOGICAL_ERROR);
|
DB::ErrorCodes::LOGICAL_ERROR);
|
||||||
|
|
||||||
throw DB::Exception("There is no failed OpResult", DB::ErrorCodes::LOGICAL_ERROR);
|
throw DB::Exception("There is no failed OpResult", DB::ErrorCodes::LOGICAL_ERROR);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
KeeperMultiException::KeeperMultiException(int32_t code, const Coordination::Requests & requests, const Coordination::Responses & responses)
|
KeeperMultiException::KeeperMultiException(int32_t exception_code, const Coordination::Requests & requests, const Coordination::Responses & responses)
|
||||||
: KeeperException("Transaction failed", code),
|
: KeeperException("Transaction failed", exception_code),
|
||||||
requests(requests), responses(responses), failed_op_index(getFailedOpIndex(code, responses))
|
requests(requests), responses(responses), failed_op_index(getFailedOpIndex(exception_code, responses))
|
||||||
{
|
{
|
||||||
addMessage("Op #" + std::to_string(failed_op_index) + ", path: " + getPathForFirstFailedOp());
|
addMessage("Op #" + std::to_string(failed_op_index) + ", path: " + getPathForFirstFailedOp());
|
||||||
}
|
}
|
||||||
@ -870,15 +869,15 @@ std::string KeeperMultiException::getPathForFirstFailedOp() const
|
|||||||
return requests[failed_op_index]->getPath();
|
return requests[failed_op_index]->getPath();
|
||||||
}
|
}
|
||||||
|
|
||||||
void KeeperMultiException::check(int32_t code, const Coordination::Requests & requests, const Coordination::Responses & responses)
|
void KeeperMultiException::check(int32_t exception_code, const Coordination::Requests & requests, const Coordination::Responses & responses)
|
||||||
{
|
{
|
||||||
if (!code)
|
if (!exception_code)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
if (Coordination::isUserError(code))
|
if (Coordination::isUserError(exception_code))
|
||||||
throw KeeperMultiException(code, requests, responses);
|
throw KeeperMultiException(exception_code, requests, responses);
|
||||||
else
|
else
|
||||||
throw KeeperException(code);
|
throw KeeperException(exception_code);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -14,7 +14,7 @@ using namespace zkutil;
|
|||||||
|
|
||||||
ZooKeeperHolder::UnstorableZookeeperHandler ZooKeeperHolder::getZooKeeper()
|
ZooKeeperHolder::UnstorableZookeeperHandler ZooKeeperHolder::getZooKeeper()
|
||||||
{
|
{
|
||||||
std::unique_lock<std::mutex> lock(mutex);
|
std::unique_lock lock(mutex);
|
||||||
return UnstorableZookeeperHandler(ptr);
|
return UnstorableZookeeperHandler(ptr);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -25,7 +25,7 @@ void ZooKeeperHolder::initFromInstance(const ZooKeeper::Ptr & zookeeper_ptr)
|
|||||||
|
|
||||||
bool ZooKeeperHolder::replaceZooKeeperSessionToNewOne()
|
bool ZooKeeperHolder::replaceZooKeeperSessionToNewOne()
|
||||||
{
|
{
|
||||||
std::unique_lock<std::mutex> lock(mutex);
|
std::unique_lock lock(mutex);
|
||||||
|
|
||||||
if (ptr.unique())
|
if (ptr.unique())
|
||||||
{
|
{
|
||||||
|
@ -1039,8 +1039,8 @@ void ZooKeeper::sendThread()
|
|||||||
{
|
{
|
||||||
/// Wait for the next request in queue. No more than operation timeout. No more than until next heartbeat time.
|
/// Wait for the next request in queue. No more than operation timeout. No more than until next heartbeat time.
|
||||||
UInt64 max_wait = std::min(
|
UInt64 max_wait = std::min(
|
||||||
std::chrono::duration_cast<std::chrono::milliseconds>(next_heartbeat_time - now).count(),
|
UInt64(std::chrono::duration_cast<std::chrono::milliseconds>(next_heartbeat_time - now).count()),
|
||||||
operation_timeout.totalMilliseconds());
|
UInt64(operation_timeout.totalMilliseconds()));
|
||||||
|
|
||||||
RequestInfo info;
|
RequestInfo info;
|
||||||
if (requests_queue.tryPop(info, max_wait))
|
if (requests_queue.tryPop(info, max_wait))
|
||||||
@ -1181,9 +1181,9 @@ void ZooKeeper::receiveEvent()
|
|||||||
ProfileEvents::increment(ProfileEvents::ZooKeeperWatchResponse);
|
ProfileEvents::increment(ProfileEvents::ZooKeeperWatchResponse);
|
||||||
response = std::make_shared<ZooKeeperWatchResponse>();
|
response = std::make_shared<ZooKeeperWatchResponse>();
|
||||||
|
|
||||||
request_info.callback = [this](const Response & response)
|
request_info.callback = [this](const Response & response_)
|
||||||
{
|
{
|
||||||
const WatchResponse & watch_response = dynamic_cast<const WatchResponse &>(response);
|
const WatchResponse & watch_response = dynamic_cast<const WatchResponse &>(response_);
|
||||||
|
|
||||||
std::lock_guard lock(watches_mutex);
|
std::lock_guard lock(watches_mutex);
|
||||||
|
|
||||||
|
@ -252,7 +252,9 @@ struct ZooKeeperRequest : virtual Request
|
|||||||
/// If the request was sent and we didn't get the response and the error happens, then we cannot be sure was it processed or not.
|
/// If the request was sent and we didn't get the response and the error happens, then we cannot be sure was it processed or not.
|
||||||
bool probably_sent = false;
|
bool probably_sent = false;
|
||||||
|
|
||||||
virtual ~ZooKeeperRequest() {}
|
ZooKeeperRequest() = default;
|
||||||
|
ZooKeeperRequest(const ZooKeeperRequest &) = default;
|
||||||
|
virtual ~ZooKeeperRequest() = default;
|
||||||
|
|
||||||
virtual ZooKeeper::OpNum getOpNum() const = 0;
|
virtual ZooKeeper::OpNum getOpNum() const = 0;
|
||||||
|
|
||||||
|
@ -22,7 +22,7 @@ ZooKeeperNodeCache::ZNode ZooKeeperNodeCache::get(const std::string & path, Coor
|
|||||||
{
|
{
|
||||||
std::unordered_set<std::string> invalidated_paths;
|
std::unordered_set<std::string> invalidated_paths;
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(context->mutex);
|
std::lock_guard lock(context->mutex);
|
||||||
|
|
||||||
if (context->all_paths_invalidated)
|
if (context->all_paths_invalidated)
|
||||||
{
|
{
|
||||||
@ -57,7 +57,7 @@ ZooKeeperNodeCache::ZNode ZooKeeperNodeCache::get(const std::string & path, Coor
|
|||||||
|
|
||||||
bool changed = false;
|
bool changed = false;
|
||||||
{
|
{
|
||||||
std::lock_guard<std::mutex> lock(owned_context->mutex);
|
std::lock_guard lock(owned_context->mutex);
|
||||||
|
|
||||||
if (response.type != Coordination::SESSION)
|
if (response.type != Coordination::SESSION)
|
||||||
changed = owned_context->invalidated_paths.emplace(response.path).second;
|
changed = owned_context->invalidated_paths.emplace(response.path).second;
|
||||||
|
@ -8,6 +8,7 @@
|
|||||||
#pragma GCC diagnostic ignored "-Wsign-compare"
|
#pragma GCC diagnostic ignored "-Wsign-compare"
|
||||||
#ifdef __clang__
|
#ifdef __clang__
|
||||||
#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
|
#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
|
||||||
|
#pragma clang diagnostic ignored "-Wundef"
|
||||||
#endif
|
#endif
|
||||||
#include <gtest/gtest.h>
|
#include <gtest/gtest.h>
|
||||||
|
|
||||||
|
@ -63,6 +63,4 @@ int main(int argc, char ** argv)
|
|||||||
std::cerr << "Some exception" << std::endl;
|
std::cerr << "Some exception" << std::endl;
|
||||||
return 2;
|
return 2;
|
||||||
}
|
}
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
}
|
||||||
|
@ -67,6 +67,4 @@ int main(int argc, char ** argv)
|
|||||||
std::cerr << "Some exception: " << DB::getCurrentExceptionMessage(true) << std::endl;
|
std::cerr << "Some exception: " << DB::getCurrentExceptionMessage(true) << std::endl;
|
||||||
return 2;
|
return 2;
|
||||||
}
|
}
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
}
|
||||||
|
@ -16,6 +16,7 @@
|
|||||||
#cmakedefine01 USE_BASE64
|
#cmakedefine01 USE_BASE64
|
||||||
#cmakedefine01 USE_HDFS
|
#cmakedefine01 USE_HDFS
|
||||||
#cmakedefine01 USE_XXHASH
|
#cmakedefine01 USE_XXHASH
|
||||||
|
#cmakedefine01 USE_INTERNAL_LLVM_LIBRARY
|
||||||
|
|
||||||
#cmakedefine01 CLICKHOUSE_SPLIT_BINARY
|
#cmakedefine01 CLICKHOUSE_SPLIT_BINARY
|
||||||
#cmakedefine01 LLVM_HAS_RTTI
|
#cmakedefine01 LLVM_HAS_RTTI
|
||||||
|
@ -3,7 +3,7 @@
|
|||||||
#include <string.h>
|
#include <string.h>
|
||||||
#include <Core/Defines.h>
|
#include <Core/Defines.h>
|
||||||
|
|
||||||
#if __SSE2__
|
#ifdef __SSE2__
|
||||||
#include <emmintrin.h>
|
#include <emmintrin.h>
|
||||||
|
|
||||||
|
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
add_executable (hashes_test hashes_test.cpp)
|
add_executable (hashes_test hashes_test.cpp)
|
||||||
target_link_libraries (hashes_test PRIVATE dbms ${OPENSSL_CRYPTO_LIBRARY})
|
target_link_libraries (hashes_test PRIVATE clickhouse_common_io ${OPENSSL_CRYPTO_LIBRARY} ${CITYHASH_LIBRARIES})
|
||||||
|
|
||||||
add_executable (sip_hash sip_hash.cpp)
|
add_executable (sip_hash sip_hash.cpp)
|
||||||
target_link_libraries (sip_hash PRIVATE clickhouse_common_io)
|
target_link_libraries (sip_hash PRIVATE clickhouse_common_io)
|
||||||
@ -20,10 +20,10 @@ add_executable (small_table small_table.cpp)
|
|||||||
target_link_libraries (small_table PRIVATE clickhouse_common_io)
|
target_link_libraries (small_table PRIVATE clickhouse_common_io)
|
||||||
|
|
||||||
add_executable (parallel_aggregation parallel_aggregation.cpp)
|
add_executable (parallel_aggregation parallel_aggregation.cpp)
|
||||||
target_link_libraries (parallel_aggregation PRIVATE clickhouse_common_io)
|
target_link_libraries (parallel_aggregation PRIVATE clickhouse_compression clickhouse_common_io)
|
||||||
|
|
||||||
add_executable (parallel_aggregation2 parallel_aggregation2.cpp)
|
add_executable (parallel_aggregation2 parallel_aggregation2.cpp)
|
||||||
target_link_libraries (parallel_aggregation2 PRIVATE clickhouse_common_io)
|
target_link_libraries (parallel_aggregation2 PRIVATE clickhouse_compression clickhouse_common_io)
|
||||||
|
|
||||||
add_executable (int_hashes_perf int_hashes_perf.cpp AvalancheTest.cpp Random.cpp)
|
add_executable (int_hashes_perf int_hashes_perf.cpp AvalancheTest.cpp Random.cpp)
|
||||||
target_link_libraries (int_hashes_perf PRIVATE clickhouse_common_io)
|
target_link_libraries (int_hashes_perf PRIVATE clickhouse_common_io)
|
||||||
@ -42,7 +42,7 @@ add_executable (shell_command_test shell_command_test.cpp)
|
|||||||
target_link_libraries (shell_command_test PRIVATE clickhouse_common_io)
|
target_link_libraries (shell_command_test PRIVATE clickhouse_common_io)
|
||||||
|
|
||||||
add_executable (arena_with_free_lists arena_with_free_lists.cpp)
|
add_executable (arena_with_free_lists arena_with_free_lists.cpp)
|
||||||
target_link_libraries (arena_with_free_lists PRIVATE clickhouse_common_io)
|
target_link_libraries (arena_with_free_lists PRIVATE clickhouse_compression clickhouse_common_io)
|
||||||
|
|
||||||
add_executable (pod_array pod_array.cpp)
|
add_executable (pod_array pod_array.cpp)
|
||||||
target_link_libraries (pod_array PRIVATE clickhouse_common_io)
|
target_link_libraries (pod_array PRIVATE clickhouse_common_io)
|
||||||
@ -61,7 +61,7 @@ target_link_libraries (space_saving PRIVATE clickhouse_common_io)
|
|||||||
|
|
||||||
add_executable (integer_hash_tables_and_hashes integer_hash_tables_and_hashes.cpp)
|
add_executable (integer_hash_tables_and_hashes integer_hash_tables_and_hashes.cpp)
|
||||||
target_include_directories (integer_hash_tables_and_hashes SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
|
target_include_directories (integer_hash_tables_and_hashes SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
|
||||||
target_link_libraries (integer_hash_tables_and_hashes PRIVATE clickhouse_common_io)
|
target_link_libraries (integer_hash_tables_and_hashes PRIVATE clickhouse_compression clickhouse_common_io)
|
||||||
|
|
||||||
add_executable (allocator allocator.cpp)
|
add_executable (allocator allocator.cpp)
|
||||||
target_link_libraries (allocator PRIVATE clickhouse_common_io)
|
target_link_libraries (allocator PRIVATE clickhouse_common_io)
|
||||||
|
@ -20,7 +20,7 @@
|
|||||||
#include <Core/Field.h>
|
#include <Core/Field.h>
|
||||||
#include <Common/Stopwatch.h>
|
#include <Common/Stopwatch.h>
|
||||||
#include <IO/ReadBufferFromFileDescriptor.h>
|
#include <IO/ReadBufferFromFileDescriptor.h>
|
||||||
#include <IO/CompressedReadBuffer.h>
|
#include <Compression/CompressedReadBuffer.h>
|
||||||
#include <IO/ReadHelpers.h>
|
#include <IO/ReadHelpers.h>
|
||||||
|
|
||||||
using namespace DB;
|
using namespace DB;
|
||||||
|
@ -60,7 +60,7 @@ int main(int argc, char ** argv)
|
|||||||
{
|
{
|
||||||
pcg64 generator(randomSeed());
|
pcg64 generator(randomSeed());
|
||||||
|
|
||||||
for (size_t i = 0; i < num_iterations; ++i)
|
for (size_t j = 0; j < num_iterations; ++j)
|
||||||
{
|
{
|
||||||
size_t size = std::uniform_int_distribution<size_t>(1, region_max_size)(generator);
|
size_t size = std::uniform_int_distribution<size_t>(1, region_max_size)(generator);
|
||||||
int key = std::uniform_int_distribution<int>(1, max_key)(generator);
|
int key = std::uniform_int_distribution<int>(1, max_key)(generator);
|
||||||
@ -70,8 +70,8 @@ int main(int argc, char ** argv)
|
|||||||
[=]{ return size; },
|
[=]{ return size; },
|
||||||
[=](void * /*ptr*/, int & payload)
|
[=](void * /*ptr*/, int & payload)
|
||||||
{
|
{
|
||||||
payload = i;
|
payload = j;
|
||||||
// memset(ptr, i, size);
|
// memset(ptr, j, size);
|
||||||
},
|
},
|
||||||
nullptr);
|
nullptr);
|
||||||
|
|
||||||
|
@ -9,7 +9,9 @@ private:
|
|||||||
virtual MutablePtr clone() const = 0;
|
virtual MutablePtr clone() const = 0;
|
||||||
|
|
||||||
public:
|
public:
|
||||||
virtual ~IColumn() {}
|
IColumn() = default;
|
||||||
|
IColumn(const IColumn &) = default;
|
||||||
|
virtual ~IColumn() = default;
|
||||||
|
|
||||||
virtual int get() const = 0;
|
virtual int get() const = 0;
|
||||||
virtual void set(int value) = 0;
|
virtual void set(int value) = 0;
|
||||||
|
@ -1,6 +1,7 @@
|
|||||||
#pragma GCC diagnostic ignored "-Wsign-compare"
|
#pragma GCC diagnostic ignored "-Wsign-compare"
|
||||||
#ifdef __clang__
|
#ifdef __clang__
|
||||||
#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
|
#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
|
||||||
|
#pragma clang diagnostic ignored "-Wundef"
|
||||||
#endif
|
#endif
|
||||||
#include <gtest/gtest.h>
|
#include <gtest/gtest.h>
|
||||||
|
|
||||||
|
@ -3,6 +3,7 @@
|
|||||||
#pragma GCC diagnostic ignored "-Wsign-compare"
|
#pragma GCC diagnostic ignored "-Wsign-compare"
|
||||||
#ifdef __clang__
|
#ifdef __clang__
|
||||||
#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
|
#pragma clang diagnostic ignored "-Wzero-as-null-pointer-constant"
|
||||||
|
#pragma clang diagnostic ignored "-Wundef"
|
||||||
#endif
|
#endif
|
||||||
#include <gtest/gtest.h>
|
#include <gtest/gtest.h>
|
||||||
|
|
||||||
|
@ -14,7 +14,7 @@
|
|||||||
|
|
||||||
#include <Core/Types.h>
|
#include <Core/Types.h>
|
||||||
#include <IO/ReadBufferFromFile.h>
|
#include <IO/ReadBufferFromFile.h>
|
||||||
#include <IO/CompressedReadBuffer.h>
|
#include <Compression/CompressedReadBuffer.h>
|
||||||
#include <Common/HashTable/HashMap.h>
|
#include <Common/HashTable/HashMap.h>
|
||||||
#include <Common/SipHash.h>
|
#include <Common/SipHash.h>
|
||||||
|
|
||||||
@ -171,7 +171,7 @@ namespace Hashes
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
#if __SSE4_2__
|
#ifdef __SSE4_2__
|
||||||
#include <nmmintrin.h>
|
#include <nmmintrin.h>
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
@ -179,7 +179,7 @@ namespace Hashes
|
|||||||
{
|
{
|
||||||
size_t operator()(Key x) const
|
size_t operator()(Key x) const
|
||||||
{
|
{
|
||||||
#if __SSE4_2__
|
#ifdef __SSE4_2__
|
||||||
return _mm_crc32_u64(-1ULL, x);
|
return _mm_crc32_u64(-1ULL, x);
|
||||||
#else
|
#else
|
||||||
/// On other platforms we do not have CRC32. NOTE This can be confusing.
|
/// On other platforms we do not have CRC32. NOTE This can be confusing.
|
||||||
|
@ -13,7 +13,7 @@
|
|||||||
//#include <Common/HashTable/HashTableMerge.h>
|
//#include <Common/HashTable/HashTableMerge.h>
|
||||||
|
|
||||||
#include <IO/ReadBufferFromFile.h>
|
#include <IO/ReadBufferFromFile.h>
|
||||||
#include <IO/CompressedReadBuffer.h>
|
#include <Compression/CompressedReadBuffer.h>
|
||||||
|
|
||||||
#include <Common/Stopwatch.h>
|
#include <Common/Stopwatch.h>
|
||||||
#include <common/ThreadPool.h>
|
#include <common/ThreadPool.h>
|
||||||
|
@ -13,7 +13,7 @@
|
|||||||
//#include <Common/HashTable/HashTableMerge.h>
|
//#include <Common/HashTable/HashTableMerge.h>
|
||||||
|
|
||||||
#include <IO/ReadBufferFromFile.h>
|
#include <IO/ReadBufferFromFile.h>
|
||||||
#include <IO/CompressedReadBuffer.h>
|
#include <Compression/CompressedReadBuffer.h>
|
||||||
|
|
||||||
#include <Common/Stopwatch.h>
|
#include <Common/Stopwatch.h>
|
||||||
#include <common/ThreadPool.h>
|
#include <common/ThreadPool.h>
|
||||||
@ -186,8 +186,8 @@ struct MergeParallelForTwoLevelTable
|
|||||||
for (size_t i = 0; i < num_maps; ++i)
|
for (size_t i = 0; i < num_maps; ++i)
|
||||||
section[i] = &source_maps[i]->impls[bucket];
|
section[i] = &source_maps[i]->impls[bucket];
|
||||||
|
|
||||||
typename Map::Impl * result_map;
|
typename Map::Impl * res;
|
||||||
ImplMerge::execute(section.data(), num_maps, result_map, merger, pool);
|
ImplMerge::execute(section.data(), num_maps, res, merger, pool);
|
||||||
});
|
});
|
||||||
|
|
||||||
pool.wait();
|
pool.wait();
|
||||||
|
@ -0,0 +1,16 @@
|
|||||||
|
include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
|
||||||
|
add_headers_and_sources(clickhouse_compression .)
|
||||||
|
add_library(clickhouse_compression ${LINK_MODE} ${clickhouse_compression_headers} ${clickhouse_compression_sources})
|
||||||
|
target_link_libraries(clickhouse_compression PRIVATE clickhouse_parsers clickhouse_common_io ${ZSTD_LIBRARY} ${LZ4_LIBRARY})
|
||||||
|
target_include_directories(clickhouse_compression PUBLIC ${DBMS_INCLUDE_DIR})
|
||||||
|
|
||||||
|
if (NOT USE_INTERNAL_LZ4_LIBRARY)
|
||||||
|
target_include_directories(clickhouse_compression SYSTEM BEFORE PRIVATE ${LZ4_INCLUDE_DIR})
|
||||||
|
endif ()
|
||||||
|
if (NOT USE_INTERNAL_ZSTD_LIBRARY)
|
||||||
|
target_include_directories(clickhouse_compression SYSTEM BEFORE PRIVATE ${ZSTD_INCLUDE_DIR})
|
||||||
|
endif ()
|
||||||
|
|
||||||
|
if(ENABLE_TESTS)
|
||||||
|
add_subdirectory(tests)
|
||||||
|
endif()
|
@ -1,9 +1,9 @@
|
|||||||
|
#include "CachedCompressedReadBuffer.h"
|
||||||
|
|
||||||
#include <IO/createReadBufferFromFileBase.h>
|
#include <IO/createReadBufferFromFileBase.h>
|
||||||
#include <IO/CachedCompressedReadBuffer.h>
|
|
||||||
#include <IO/WriteHelpers.h>
|
#include <IO/WriteHelpers.h>
|
||||||
#include <Compression/CompressionInfo.h>
|
#include <Compression/CompressionInfo.h>
|
||||||
#include <IO/LZ4_decompress_faster.h>
|
#include <Compression/LZ4_decompress_faster.h>
|
||||||
#include "CachedCompressedReadBuffer.h"
|
|
||||||
|
|
||||||
|
|
||||||
namespace DB
|
namespace DB
|
@ -3,7 +3,7 @@
|
|||||||
#include <memory>
|
#include <memory>
|
||||||
#include <time.h>
|
#include <time.h>
|
||||||
#include <IO/ReadBufferFromFileBase.h>
|
#include <IO/ReadBufferFromFileBase.h>
|
||||||
#include <IO/CompressedReadBufferBase.h>
|
#include "CompressedReadBufferBase.h"
|
||||||
#include <IO/UncompressedCache.h>
|
#include <IO/UncompressedCache.h>
|
||||||
#include <port/clock.h>
|
#include <port/clock.h>
|
||||||
|
|
@ -1,6 +1,6 @@
|
|||||||
#include <IO/CompressedReadBuffer.h>
|
#include "CompressedReadBuffer.h"
|
||||||
#include <Compression/CompressionInfo.h>
|
#include <Compression/CompressionInfo.h>
|
||||||
#include <IO/LZ4_decompress_faster.h>
|
#include <Compression/LZ4_decompress_faster.h>
|
||||||
|
|
||||||
|
|
||||||
namespace DB
|
namespace DB
|
@ -1,6 +1,6 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include <IO/CompressedReadBufferBase.h>
|
#include "CompressedReadBufferBase.h"
|
||||||
#include <IO/BufferWithOwnMemory.h>
|
#include <IO/BufferWithOwnMemory.h>
|
||||||
#include <IO/ReadBuffer.h>
|
#include <IO/ReadBuffer.h>
|
||||||
|
|
@ -1,4 +1,4 @@
|
|||||||
#include <IO/CompressedReadBufferBase.h>
|
#include "CompressedReadBufferBase.h"
|
||||||
|
|
||||||
#include <vector>
|
#include <vector>
|
||||||
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user