thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 08:40:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into watchdog

Browse Source
This commit is contained in:
Alexey Milovidov 2020-12-22 03:10:15 +03:00
commit 932ade2cb8
6262 changed files with 525937 additions and 96341 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.github/ISSUE_TEMPLATE/documentation-issue.md vendored
View File

@ -2,8 +2,7 @@
name: Documentation issue
about: Report something incorrect or missing in documentation
title: ''
labels: documentation
assignees: BayoNet
labels: comp-documentation
---

3
.github/PULL_REQUEST_TEMPLATE.md vendored
View File

@ -24,3 +24,6 @@ Detailed description / Documentation draft:
By adding documentation, you'll allow users to try your new feature immediately, not when someone else will have time to document it later. Documentation is necessary for all features that affect user experience in any way. You can add brief documentation draft above, or add documentation right into your patch as Markdown files in [docs](https://github.com/ClickHouse/ClickHouse/tree/master/docs) folder.
If you are doing this for the first time, it's recommended to read the lightweight [Contributing to ClickHouse Documentation](https://github.com/ClickHouse/ClickHouse/tree/master/docs/README.md) guide first.
Information about CI checks: https://clickhouse.tech/docs/en/development/continuous-integration/

17
.github/codecov.yml vendored Normal file
View File

@ -0,0 +1,17 @@
codecov:
max_report_age: off
strict_yaml_branch: "master"
ignore:
- "contrib"
- "docs"
- "benchmark"
- "tests"
- "docker"
- "debian"
- "cmake"
comment: false
github_checks:
annotations: false

18
.github/workflows/anchore-analysis.yml vendored
View File

@ -1,8 +1,8 @@
# This workflow checks out code, performs an Anchore container image
# vulnerability and compliance scan, and integrates the results with
# GitHub Advanced Security code scanning feature. For more information on
# GitHub Advanced Security code scanning feature. For more information on
# the Anchore scan action usage and parameters, see
# https://github.com/anchore/scan-action. For more information on
# https://github.com/anchore/scan-action. For more information on
# Anchore container image scanning in general, see
# https://docs.anchore.com.
@ -28,18 +28,12 @@ jobs:
perl -pi -e 's|=\$version||g' Dockerfile
docker build . --file Dockerfile --tag localbuild/testimage:latest
- name: Run the local Anchore scan action itself with GitHub Advanced Security code scanning integration enabled
uses: anchore/scan-action@master
uses: anchore/scan-action@v2
id: scan
with:
image-reference: "localbuild/testimage:latest"
dockerfile-path: "docker/server/Dockerfile"
image: "localbuild/testimage:latest"
acs-report-enable: true
fail-build: true
- name: Upload artifact
uses: actions/upload-artifact@v1.0.0
with:
name: AnchoreReports
path: ./anchore-reports/
- name: Upload Anchore Scan Report
uses: github/codeql-action/upload-sarif@v1
with:
sarif_file: results.sarif
sarif_file: ${{ steps.scan.outputs.sarif }}

9
.github/workflows/codeql-analysis.yml vendored
View File

@ -1,3 +1,5 @@
# See the example here: https://github.com/github/codeql-action
name: "CodeQL Scanning"
on:
@ -16,17 +18,14 @@ jobs:
fetch-depth: 2
submodules: 'recursive'
- run: git checkout HEAD^2
if: ${{ github.event_name == 'pull_request' }}
- name: Initialize CodeQL
uses: github/codeql-action/init@v1
with:
languages: cpp
- run: sudo apt-get update && sudo apt-get install -y git cmake python ninja-build gcc-9 g++-9 && mkdir build
- run: cd build && CC=gcc-9 CXX=g++-9 cmake ..
- run: sudo apt-get update && sudo apt-get install -y git cmake python ninja-build gcc-10 g++-10 && mkdir build
- run: cd build && CC=gcc-10 CXX=g++-10 cmake ..
- run: cd build && ninja
- name: Perform CodeQL Analysis

14
.gitignore vendored
View File

@ -118,8 +118,22 @@ website/package-lock.json
# clangd cache
/.clangd
/.cache
/compile_commands.json
# Toolchains
/cmake/toolchain/*
# ANTLR extension cache
.antlr
# ANTLR generated files
/src/Parsers/New/*.interp
/src/Parsers/New/*.tokens
/src/Parsers/New/ClickHouseParserBaseVisitor.*
# pytest-profiling
/prof
*.iml

51
.gitmodules vendored
View File

@ -37,13 +37,14 @@
url = https://github.com/ClickHouse-Extras/mariadb-connector-c.git
[submodule "contrib/jemalloc"]
path = contrib/jemalloc
url = https://github.com/jemalloc/jemalloc.git
url = https://github.com/ClickHouse-Extras/jemalloc.git
[submodule "contrib/unixodbc"]
path = contrib/unixodbc
url = https://github.com/ClickHouse-Extras/UnixODBC.git
[submodule "contrib/protobuf"]
path = contrib/protobuf
url = https://github.com/ClickHouse-Extras/protobuf.git
branch = v3.13.0.1
[submodule "contrib/boost"]
path = contrib/boost
url = https://github.com/ClickHouse-Extras/boost.git
@ -107,7 +108,7 @@
[submodule "contrib/grpc"]
path = contrib/grpc
url = https://github.com/ClickHouse-Extras/grpc.git
branch = v1.25.0
branch = v1.33.2
[submodule "contrib/aws"]
path = contrib/aws
url = https://github.com/ClickHouse-Extras/aws-sdk-cpp.git
@ -123,9 +124,6 @@
[submodule "contrib/curl"]
path = contrib/curl
url = https://github.com/curl/curl.git
[submodule "contrib/openssl"]
path = contrib/openssl
url = https://github.com/ClickHouse-Extras/openssl.git
[submodule "contrib/icudata"]
path = contrib/icudata
url = https://github.com/ClickHouse-Extras/icudata.git
@ -141,9 +139,6 @@
[submodule "contrib/replxx"]
path = contrib/replxx
url = https://github.com/ClickHouse-Extras/replxx.git
[submodule "contrib/ryu"]
path = contrib/ryu
url = https://github.com/ClickHouse-Extras/ryu.git
[submodule "contrib/avro"]
path = contrib/avro
url = https://github.com/ClickHouse-Extras/avro.git
@ -156,10 +151,10 @@
url = https://github.com/ClickHouse-Extras/libcpuid.git
[submodule "contrib/openldap"]
path = contrib/openldap
url = https://github.com/openldap/openldap.git
url = https://github.com/ClickHouse-Extras/openldap.git
[submodule "contrib/AMQP-CPP"]
path = contrib/AMQP-CPP
url = https://github.com/CopernicaMarketingSoftware/AMQP-CPP.git
url = https://github.com/ClickHouse-Extras/AMQP-CPP.git
[submodule "contrib/cassandra"]
path = contrib/cassandra
url = https://github.com/ClickHouse-Extras/cpp-driver.git
@ -171,6 +166,9 @@
[submodule "contrib/fmtlib"]
path = contrib/fmtlib
url = https://github.com/fmtlib/fmt.git
[submodule "contrib/antlr4-runtime"]
path = contrib/antlr4-runtime
url = https://github.com/ClickHouse-Extras/antlr4-runtime.git
[submodule "contrib/sentry-native"]
path = contrib/sentry-native
url = https://github.com/ClickHouse-Extras/sentry-native.git
@ -180,3 +178,36 @@
[submodule "contrib/stats"]
path = contrib/stats
url = https://github.com/kthohr/stats.git
[submodule "contrib/krb5"]
path = contrib/krb5
url = https://github.com/ClickHouse-Extras/krb5
[submodule "contrib/cyrus-sasl"]
path = contrib/cyrus-sasl
url = https://github.com/cyrusimap/cyrus-sasl
branch = cyrus-sasl-2.1
[submodule "contrib/croaring"]
path = contrib/croaring
url = https://github.com/RoaringBitmap/CRoaring
branch = v0.2.66
[submodule "contrib/miniselect"]
path = contrib/miniselect
url = https://github.com/danlark1/miniselect
[submodule "contrib/rocksdb"]
path = contrib/rocksdb
url = https://github.com/ClickHouse-Extras/rocksdb.git
[submodule "contrib/xz"]
path = contrib/xz
url = https://github.com/xz-mirror/xz
[submodule "contrib/abseil-cpp"]
path = contrib/abseil-cpp
url = https://github.com/ClickHouse-Extras/abseil-cpp.git
branch = lts_2020_02_25
[submodule "contrib/dragonbox"]
path = contrib/dragonbox
url = https://github.com/ClickHouse-Extras/dragonbox.git
[submodule "contrib/fast_float"]
path = contrib/fast_float
url = https://github.com/fastfloat/fast_float
[submodule "contrib/boringssl"]
path = contrib/boringssl
url = https://github.com/ClickHouse-Extras/boringssl.git

1215
CHANGELOG.md
View File

File diff suppressed because it is too large Load Diff

215
CMakeLists.txt
View File

@ -28,6 +28,17 @@ endforeach()
project(ClickHouse)
# If turned off: e.g. when ENABLE_FOO is ON, but FOO tool was not found, the CMake will continue.
option(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION
"Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF)
but is not possible to satisfy" ON)
if(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION)
set(RECONFIGURE_MESSAGE_LEVEL FATAL_ERROR)
else()
set(RECONFIGURE_MESSAGE_LEVEL STATUS)
endif()
include (cmake/arch.cmake)
include (cmake/target.cmake)
include (cmake/tools.cmake)
@ -48,21 +59,6 @@ set(CMAKE_DEBUG_POSTFIX "d" CACHE STRING "Generate debug library name with a pos
# For more info see https://cmake.org/cmake/help/latest/prop_gbl/USE_FOLDERS.html
set_property(GLOBAL PROPERTY USE_FOLDERS ON)
option(ENABLE_IPO "Enable full link time optimization (it's usually impractical; see also ENABLE_THINLTO)" OFF) # need cmake 3.9+
if(ENABLE_IPO)
cmake_policy(SET CMP0069 NEW)
include(CheckIPOSupported)
check_ipo_supported(RESULT IPO_SUPPORTED OUTPUT IPO_NOT_SUPPORTED)
if(IPO_SUPPORTED)
message(STATUS "IPO/LTO is supported, enabling")
set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE)
else()
message(STATUS "IPO/LTO is not supported: <${IPO_NOT_SUPPORTED}>")
endif()
else()
message(STATUS "IPO/LTO not enabled.")
endif()
# Check that submodules are present only if source was downloaded with git
if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git" AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/boost/boost")
message (FATAL_ERROR "Submodules are not initialized. Run\n\tgit submodule update --init --recursive")
@ -70,6 +66,11 @@ endif ()
include (cmake/find/ccache.cmake)
option(ENABLE_CHECK_HEAVY_BUILDS "Don't allow C++ translation units to compile too long or to take too much memory while compiling" OFF)
if (ENABLE_CHECK_HEAVY_BUILDS)
set (CMAKE_CXX_COMPILER_LAUNCHER prlimit --rss=10000000 --cpu=600)
endif ()
if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "None")
set (CMAKE_BUILD_TYPE "RelWithDebInfo")
message (STATUS "CMAKE_BUILD_TYPE is not set, set to default = ${CMAKE_BUILD_TYPE}")
@ -78,11 +79,16 @@ message (STATUS "CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC)
option (USE_STATIC_LIBRARIES "Set to FALSE to use shared libraries" ON)
option (MAKE_STATIC_LIBRARIES "Set to FALSE to make shared libraries" ${USE_STATIC_LIBRARIES})
option(USE_STATIC_LIBRARIES "Disable to use shared libraries" ON)
option(MAKE_STATIC_LIBRARIES "Disable to make shared libraries" ${USE_STATIC_LIBRARIES})
if (NOT MAKE_STATIC_LIBRARIES)
option (SPLIT_SHARED_LIBRARIES "DEV ONLY. Keep all internal libs as separate .so for faster linking" OFF)
option (CLICKHOUSE_SPLIT_BINARY "Make several binaries instead one bundled (clickhouse-server, clickhouse-client, ... )" OFF)
# DEVELOPER ONLY.
# Faster linking if turned on.
option(SPLIT_SHARED_LIBRARIES "Keep all internal libraries as separate .so files")
option(CLICKHOUSE_SPLIT_BINARY
"Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled")
endif ()
if (MAKE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES)
@ -97,7 +103,8 @@ if (USE_STATIC_LIBRARIES)
list(REVERSE CMAKE_FIND_LIBRARY_SUFFIXES)
endif ()
option (ENABLE_FUZZING "Enables fuzzing instrumentation" OFF)
# Implies ${WITH_COVERAGE}
option (ENABLE_FUZZING "Fuzzy testing using libfuzzer" OFF)
if (ENABLE_FUZZING)
message (STATUS "Fuzzing instrumentation enabled")
@ -105,6 +112,12 @@ if (ENABLE_FUZZING)
set (FUZZER "libfuzzer")
endif()
# Global libraries
# See:
# - default_libs.cmake
# - sanitize.cmake
add_library(global-libs INTERFACE)
include (cmake/fuzzer.cmake)
include (cmake/sanitize.cmake)
@ -129,10 +142,15 @@ if (COMPILER_CLANG)
endif ()
endif ()
option (ENABLE_TESTS "Enables tests" ON)
# If turned `ON`, assumes the user has either the system GTest library or the bundled one.
option(ENABLE_TESTS "Provide unit_test_dbms target with Google.Test unit tests" ON)
if (OS_LINUX AND NOT UNBUNDLED AND MAKE_STATIC_LIBRARIES AND NOT SPLIT_SHARED_LIBRARIES AND CMAKE_VERSION VERSION_GREATER "3.9.0")
option (GLIBC_COMPATIBILITY "Set to TRUE to enable compatibility with older glibc libraries. Only for x86_64, Linux. Implies ENABLE_FASTMEMCPY." ON)
# Only for Linux, x86_64.
# Implies ${ENABLE_FASTMEMCPY}
option(GLIBC_COMPATIBILITY "Enable compatibility with older glibc libraries." ON)
elseif(GLIBC_COMPATIBILITY)
message (${RECONFIGURE_MESSAGE_LEVEL} "Glibc compatibility cannot be enabled in current configuration")
endif ()
if (NOT CMAKE_VERSION VERSION_GREATER "3.9.0")
@ -142,17 +160,19 @@ endif ()
# Make sure the final executable has symbols exported
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -rdynamic")
find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-10" "llvm-objcopy-9" "llvm-objcopy-8" "objcopy")
if (OBJCOPY_PATH)
message(STATUS "Using objcopy: ${OBJCOPY_PATH}.")
if (OS_LINUX)
find_program (OBJCOPY_PATH NAMES "llvm-objcopy" "llvm-objcopy-11" "llvm-objcopy-10" "llvm-objcopy-9" "llvm-objcopy-8" "objcopy")
if (OBJCOPY_PATH)
message(STATUS "Using objcopy: ${OBJCOPY_PATH}.")
if (ARCH_AMD64)
set(OBJCOPY_ARCH_OPTIONS -O elf64-x86-64 -B i386)
elseif (ARCH_AARCH64)
set(OBJCOPY_ARCH_OPTIONS -O elf64-aarch64 -B aarch64)
if (ARCH_AMD64)
set(OBJCOPY_ARCH_OPTIONS -O elf64-x86-64 -B i386)
elseif (ARCH_AARCH64)
set(OBJCOPY_ARCH_OPTIONS -O elf64-aarch64 -B aarch64)
endif ()
else ()
message(FATAL_ERROR "Cannot find objcopy.")
endif ()
else ()
message(FATAL_ERROR "Cannot find objcopy.")
endif ()
if (OS_DARWIN)
@ -163,7 +183,9 @@ else ()
set(NO_WHOLE_ARCHIVE --no-whole-archive)
endif ()
option (ADD_GDB_INDEX_FOR_GOLD "Set to add .gdb-index to resulting binaries for gold linker. NOOP if lld is used." 0)
# Ignored if `lld` is used
option(ADD_GDB_INDEX_FOR_GOLD "Add .gdb-index to resulting binaries for gold linker.")
if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE")
if (LINKER_NAME STREQUAL "lld")
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gdb-index")
@ -184,9 +206,13 @@ if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE")
endif()
cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd
if(NOT AVAILABLE_PHYSICAL_MEMORY OR AVAILABLE_PHYSICAL_MEMORY GREATER 8000)
option(COMPILER_PIPE "-pipe compiler option [less /tmp usage, more ram usage]" ON)
# Less `/tmp` usage, more RAM usage.
option(COMPILER_PIPE "-pipe compiler option" ON)
endif()
if(COMPILER_PIPE)
set(COMPILER_FLAGS "${COMPILER_FLAGS} -pipe")
else()
@ -197,14 +223,23 @@ if(NOT DISABLE_CPU_OPTIMIZE)
include(cmake/cpu_features.cmake)
endif()
option(ARCH_NATIVE "Enable -march=native compiler flag" 0)
option(ARCH_NATIVE "Add -march=native compiler flag")
if (ARCH_NATIVE)
set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=native")
endif ()
# cmake < 3.12 doesn't supoprt 20. We'll set CMAKE_CXX_FLAGS for now
if (COMPILER_GCC OR COMPILER_CLANG)
# to make numeric_limits<__int128> works with GCC
set (_CXX_STANDARD "gnu++2a")
else()
set (_CXX_STANDARD "c++2a")
endif()
# cmake < 3.12 doesn't support 20. We'll set CMAKE_CXX_FLAGS for now
# set (CMAKE_CXX_STANDARD 20)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++2a")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=${_CXX_STANDARD}")
set (CMAKE_CXX_EXTENSIONS 0) # https://cmake.org/cmake/help/latest/prop_tgt/CXX_EXTENSIONS.html#prop_tgt:CXX_EXTENSIONS
set (CMAKE_CXX_STANDARD_REQUIRED ON)
@ -213,7 +248,8 @@ if (COMPILER_GCC OR COMPILER_CLANG)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsized-deallocation")
endif ()
option(WITH_COVERAGE "Build with coverage." 0)
# Compiler-specific coverage flags e.g. -fcoverage-mapping for gcc
option(WITH_COVERAGE "Profile the resulting binary/binaries" OFF)
if (WITH_COVERAGE AND COMPILER_CLANG)
set(COMPILER_FLAGS "${COMPILER_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
@ -227,6 +263,8 @@ if (WITH_COVERAGE AND COMPILER_GCC)
set(WITHOUT_COVERAGE "-fno-profile-arcs -fno-test-coverage")
endif()
set(COMPILER_FLAGS "${COMPILER_FLAGS}")
set (CMAKE_BUILD_COLOR_MAKEFILE ON)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS}")
set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_CXX_FLAGS_ADD}")
@ -246,9 +284,19 @@ if (COMPILER_CLANG)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-absolute-paths")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-absolute-paths")
option(ENABLE_THINLTO "Enable Thin LTO. Only applicable for clang. It's also suppressed when building with tests or sanitizers." ON)
if (NOT ENABLE_TESTS AND NOT SANITIZE)
# https://clang.llvm.org/docs/ThinLTO.html
# Applies to clang only.
# Disabled when building with tests or sanitizers.
option(ENABLE_THINLTO "Clang-specific link time optimization" ON)
endif()
# We cannot afford to use LTO when compiling unitests, and it's not enough
# Set new experimental pass manager, it's a performance, build time and binary size win.
# Can be removed after https://reviews.llvm.org/D66490 merged and released to at least two versions of clang.
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fexperimental-new-pass-manager")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fexperimental-new-pass-manager")
# We cannot afford to use LTO when compiling unit tests, and it's not enough
# to only supply -fno-lto at the final linking stage. So we disable it
# completely.
if (ENABLE_THINLTO AND NOT ENABLE_TESTS AND NOT SANITIZE)
@ -256,11 +304,13 @@ if (COMPILER_CLANG)
set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -flto=thin")
set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -flto=thin")
set (CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -flto=thin")
elseif (ENABLE_THINLTO)
message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot enable ThinLTO")
endif ()
# Always prefer llvm tools when using clang. For instance, we cannot use GNU ar when llvm LTO is enabled
find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-11" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
find_program (LLVM_AR_PATH NAMES "llvm-ar" "llvm-ar-10" "llvm-ar-9" "llvm-ar-8")
if (LLVM_AR_PATH)
message(STATUS "Using llvm-ar: ${LLVM_AR_PATH}.")
set (CMAKE_AR ${LLVM_AR_PATH})
@ -268,29 +318,39 @@ if (COMPILER_CLANG)
message(WARNING "Cannot find llvm-ar. System ar will be used instead. It does not work with ThinLTO.")
endif ()
find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-10" "llvm-ranlib-9" "llvm-ranlib-8")
find_program (LLVM_RANLIB_PATH NAMES "llvm-ranlib" "llvm-ranlib-11" "llvm-ranlib-10" "llvm-ranlib-9" "llvm-ranlib-8")
if (LLVM_RANLIB_PATH)
message(STATUS "Using llvm-ranlib: ${LLVM_RANLIB_PATH}.")
set (CMAKE_RANLIB ${LLVM_RANLIB_PATH})
else ()
message(WARNING "Cannot find llvm-ranlib. System ranlib will be used instead. It does not work with ThinLTO.")
endif ()
elseif (ENABLE_THINLTO)
message (${RECONFIGURE_MESSAGE_LEVEL} "ThinLTO is only available with CLang")
endif ()
option (ENABLE_LIBRARIES "Enable all libraries (Global default switch)" ON)
# Turns on all external libs like s3, kafka, ODBC, ...
option(ENABLE_LIBRARIES "Enable all external libraries by default" ON)
# We recommend avoiding this mode for production builds because we can't guarantee all needed libraries exist in your
# system.
# This mode exists for enthusiastic developers who are searching for trouble.
# Useful for maintainers of OS packages.
option (UNBUNDLED "Use system libraries instead of ones in contrib/" OFF)
option (UNBUNDLED "Try find all libraries in system. We recommend to avoid this mode for production builds, because we cannot guarantee exact versions and variants of libraries your system has installed. This mode exists for enthusiastic developers who search for trouble. Also it is useful for maintainers of OS packages." OFF)
if (UNBUNDLED)
set(NOT_UNBUNDLED 0)
set(NOT_UNBUNDLED OFF)
else ()
set(NOT_UNBUNDLED 1)
set(NOT_UNBUNDLED ON)
endif ()
if (UNBUNDLED OR NOT (OS_LINUX OR OS_DARWIN))
# Using system libs can cause a lot of warnings in includes (on macro expansion).
option (WERROR "Enable -Werror compiler option" OFF)
option(WERROR "Enable -Werror compiler option" OFF)
else ()
option (WERROR "Enable -Werror compiler option" ON)
option(WERROR "Enable -Werror compiler option" ON)
endif ()
if (WERROR)
@ -332,8 +392,9 @@ else ()
set (CMAKE_POSITION_INDEPENDENT_CODE ON)
endif ()
# Using "include-what-you-use" tool.
option (USE_INCLUDE_WHAT_YOU_USE "Use 'include-what-you-use' tool" OFF)
# https://github.com/include-what-you-use/include-what-you-use
option (USE_INCLUDE_WHAT_YOU_USE "Automatically reduce unneeded includes in source code (external tool)" OFF)
if (USE_INCLUDE_WHAT_YOU_USE)
find_program(IWYU_PATH NAMES include-what-you-use iwyu)
if (NOT IWYU_PATH)
@ -345,8 +406,11 @@ if (USE_INCLUDE_WHAT_YOU_USE)
endif ()
if (ENABLE_TESTS)
message (STATUS "Tests are enabled")
message (STATUS "Unit tests are enabled")
else()
message(STATUS "Unit tests are disabled")
endif ()
enable_testing() # Enable for tests without binary
# when installing to /usr - place configs to /etc but for /usr/local place to /usr/local/etc
@ -356,7 +420,13 @@ else ()
set (CLICKHOUSE_ETC_DIR "${CMAKE_INSTALL_PREFIX}/etc")
endif ()
message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ; USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES} MAKE_STATIC_LIBRARIES=${MAKE_STATIC_LIBRARIES} SPLIT_SHARED=${SPLIT_SHARED_LIBRARIES} UNBUNDLED=${UNBUNDLED} CCACHE=${CCACHE_FOUND} ${CCACHE_VERSION}")
message (STATUS
"Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ;
USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES}
MAKE_STATIC_LIBRARIES=${MAKE_STATIC_LIBRARIES}
SPLIT_SHARED=${SPLIT_SHARED_LIBRARIES}
UNBUNDLED=${UNBUNDLED}
CCACHE=${CCACHE_FOUND} ${CCACHE_VERSION}")
include (GNUInstallDirs)
include (cmake/contrib_finder.cmake)
@ -371,35 +441,50 @@ include (cmake/find/ltdl.cmake) # for odbc
# openssl, zlib before poco
include (cmake/find/sparsehash.cmake)
include (cmake/find/re2.cmake)
include (cmake/find/krb5.cmake)
include (cmake/find/libgsasl.cmake)
include (cmake/find/cyrus-sasl.cmake)
include (cmake/find/rdkafka.cmake)
include (cmake/find/amqpcpp.cmake)
include (cmake/find/capnp.cmake)
include (cmake/find/llvm.cmake)
include (cmake/find/opencl.cmake)
include (cmake/find/termcap.cmake) # for external static llvm
include (cmake/find/h3.cmake)
include (cmake/find/libxml2.cmake)
include (cmake/find/brotli.cmake)
include (cmake/find/protobuf.cmake)
include (cmake/find/grpc.cmake)
include (cmake/find/pdqsort.cmake)
include (cmake/find/miniselect.cmake)
include (cmake/find/hdfs3.cmake) # uses protobuf
include (cmake/find/poco.cmake)
include (cmake/find/curl.cmake)
include (cmake/find/s3.cmake)
include (cmake/find/base64.cmake)
include (cmake/find/parquet.cmake)
include (cmake/find/simdjson.cmake)
include (cmake/find/fast_float.cmake)
include (cmake/find/rapidjson.cmake)
include (cmake/find/fastops.cmake)
include (cmake/find/odbc.cmake)
include (cmake/find/rocksdb.cmake)
if(NOT USE_INTERNAL_PARQUET_LIBRARY)
set (ENABLE_ORC OFF CACHE INTERNAL "")
endif()
include (cmake/find/orc.cmake)
include (cmake/find/avro.cmake)
include (cmake/find/msgpack.cmake)
include (cmake/find/cassandra.cmake)
include (cmake/find/sentry.cmake)
include (cmake/find/stats.cmake)
set (USE_INTERNAL_CITYHASH_LIBRARY ON CACHE INTERNAL "")
find_contrib_lib(cityhash)
find_contrib_lib(farmhash)
find_contrib_lib(btrie)
if (ENABLE_TESTS)
include (cmake/find/gtest.cmake)
@ -410,13 +495,6 @@ include (cmake/find/mysqlclient.cmake)
# When testing for memory leaks with Valgrind, don't link tcmalloc or jemalloc.
if (USE_OPENCL)
if (OS_DARWIN)
set(OPENCL_LINKER_FLAGS "-framework OpenCL")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OPENCL_LINKER_FLAGS}")
endif ()
endif ()
include (cmake/print_flags.cmake)
if (TARGET global-group)
@ -432,11 +510,20 @@ endif ()
macro (add_executable target)
# invoke built-in add_executable
# explicitly acquire and interpose malloc symbols by clickhouse_malloc
_add_executable (${ARGV} $<TARGET_OBJECTS:clickhouse_malloc>)
# if GLIBC_COMPATIBILITY is ON and ENABLE_THINLTO is on than provide memcpy symbol explicitly to neutrialize thinlto's libcall generation.
if (GLIBC_COMPATIBILITY AND ENABLE_THINLTO)
_add_executable (${ARGV} $<TARGET_OBJECTS:clickhouse_malloc> $<TARGET_OBJECTS:clickhouse_memcpy>)
else ()
_add_executable (${ARGV} $<TARGET_OBJECTS:clickhouse_malloc>)
endif ()
get_target_property (type ${target} TYPE)
if (${type} STREQUAL EXECUTABLE)
# operator::new/delete for executables (MemoryTracker stuff)
target_link_libraries (${target} PRIVATE clickhouse_new_delete ${MALLOC_LIBRARIES})
# disabled for TSAN and gcc since libtsan.a provides overrides too
if (TARGET clickhouse_new_delete)
# operator::new/delete for executables (MemoryTracker stuff)
target_link_libraries (${target} PRIVATE clickhouse_new_delete ${MALLOC_LIBRARIES})
endif()
endif()
endmacro()
@ -447,8 +534,8 @@ include_directories(${ConfigIncludePath})
include (cmake/warnings.cmake)
add_subdirectory (base)
add_subdirectory (programs)
add_subdirectory (src)
add_subdirectory (programs)
add_subdirectory (tests)
add_subdirectory (utils)

7
README.md
View File

@ -1,6 +1,6 @@
[![ClickHouse — open source distributed column-oriented DBMS](https://github.com/ClickHouse/ClickHouse/raw/master/website/images/logo-400x240.png)](https://clickhouse.tech)
ClickHouse is an open-source column-oriented database management system that allows generating analytical data reports in real time.
ClickHouse® is an open-source column-oriented database management system that allows generating analytical data reports in real time.
## Useful Links
@ -15,6 +15,5 @@ ClickHouse is an open-source column-oriented database management system that all
* [Contacts](https://clickhouse.tech/#contacts) can help to get your questions answered if there are any.
* You can also [fill this form](https://clickhouse.tech/#meet) to meet Yandex ClickHouse team in person.
## Upcoming Events
* [ClickHouse at ByteDance (in Chinese)](https://mp.weixin.qq.com/s/Em-HjPylO8D7WPui4RREAQ) on August 14, 2020.
## Upcoming Events
* [SF Bay Area ClickHouse Virtual Office Hours (online)](https://www.meetup.com/San-Francisco-Bay-Area-ClickHouse-Meetup/events/274273549/) on 20 January 2020.

9
SECURITY.md
View File

@ -10,11 +10,14 @@ currently being supported with security updates:
| 1.x | :x: |
| 18.x | :x: |
| 19.x | :x: |
| 19.14 | :white_check_mark: |
| 20.1 | :x: |
| 20.3 | :white_check_mark: |
| 20.4 | :white_check_mark: |
| 20.5 | :white_check_mark: |
| 20.4 | :x: |
| 20.5 | :x: |
| 20.6 | :x: |
| 20.7 | :white_check_mark: |
| 20.8 | :white_check_mark: |
| 20.9 | :white_check_mark: |
## Reporting a Vulnerability

2
base/common/CMakeLists.txt
View File

@ -6,6 +6,7 @@ set (SRCS
demangle.cpp
getFQDNOrHostName.cpp
getMemoryAmount.cpp
getPageSize.cpp
getThreadId.cpp
JSON.cpp
LineReader.cpp
@ -18,6 +19,7 @@ set (SRCS
terminalColors.cpp
errnoToString.cpp
getResource.cpp
StringRef.cpp
)
if (ENABLE_REPLXX)

6
base/common/LineReader.cpp
View File

@ -38,10 +38,10 @@ bool hasInputData()
}
LineReader::Suggest::WordsRange LineReader::Suggest::getCompletions(const String & prefix, size_t prefix_length) const
std::optional<LineReader::Suggest::WordsRange> LineReader::Suggest::getCompletions(const String & prefix, size_t prefix_length) const
{
if (!ready)
return std::make_pair(words.end(), words.end());
return std::nullopt;
std::string_view last_word;
@ -127,7 +127,7 @@ String LineReader::readLine(const String & first_prompt, const String & second_p
}
#endif
line += (line.empty() ? "" : " ") + input;
line += (line.empty() ? "" : "\n") + input;
if (!need_next_line)
break;

3
base/common/LineReader.h
View File

@ -4,6 +4,7 @@
#include <atomic>
#include <vector>
#include <optional>
class LineReader
{
@ -18,7 +19,7 @@ public:
std::atomic<bool> ready{false};
/// Get iterators for the matched range of words if any.
WordsRange getCompletions(const String & prefix, size_t prefix_length) const;
std::optional<WordsRange> getCompletions(const String & prefix, size_t prefix_length) const;
};
using Patterns = std::vector<const char *>;

17
base/common/ReadlineLineReader.cpp
View File

@ -1,4 +1,5 @@
#include <common/ReadlineLineReader.h>
#include <common/errnoToString.h>
#include <ext/scope_guard.h>
#include <errno.h>
@ -6,6 +7,8 @@
#include <string.h>
#include <unistd.h>
#include <iostream>
namespace
{
@ -28,7 +31,8 @@ static LineReader::Suggest::Words::const_iterator end;
static void findRange(const char * prefix, size_t prefix_length)
{
std::string prefix_str(prefix);
std::tie(pos, end) = suggest->getCompletions(prefix_str, prefix_length);
if (auto completions = suggest->getCompletions(prefix_str, prefix_length))
std::tie(pos, end) = *completions;
}
/// Iterates through matched range.
@ -66,7 +70,7 @@ ReadlineLineReader::ReadlineLineReader(
{
int res = read_history(history_file_path.c_str());
if (res)
std::cerr << "Cannot read history from file " + history_file_path + ": "+ strerror(errno) << std::endl;
std::cerr << "Cannot read history from file " + history_file_path + ": "+ errnoToString(errno) << std::endl;
}
/// Added '.' to the default list. Because it is used to separate database and table.
@ -104,9 +108,11 @@ ReadlineLineReader::ReadlineLineReader(
};
if (signal(SIGINT, clear_prompt_or_exit) == SIG_ERR)
throw std::runtime_error(std::string("Cannot set signal handler for readline: ") + strerror(errno));
throw std::runtime_error(std::string("Cannot set signal handler for readline: ") + errnoToString(errno));
rl_variable_bind("completion-ignore-case", "on");
// TODO: it doesn't work
// history_write_timestamps = 1;
}
ReadlineLineReader::~ReadlineLineReader()
@ -129,6 +135,11 @@ LineReader::InputStatus ReadlineLineReader::readOneLine(const String & prompt)
void ReadlineLineReader::addToHistory(const String & line)
{
add_history(line.c_str());
// Flush changes to the disk
// NOTE readline builds a buffer of all the lines to write, and write them in one syscall.
// Thus there is no need to lock the history file here.
write_history(history_file_path.c_str());
}
#if RL_VERSION_MAJOR >= 7

34
base/common/ReplxxLineReader.cpp
View File

@ -16,19 +16,6 @@ void trim(String & s)
s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); }).base(), s.end());
}
// Uses separate replxx::Replxx instance to avoid loading them again in the
// current context (replxx::Replxx::history_load() will re-load the history
// from the file), since then they will overlaps with history from the current
// session (this will make behavior compatible with other interpreters, i.e.
// bash).
void history_save(const String & history_file_path, const String & line)
{
replxx::Replxx rx_no_overlap;
rx_no_overlap.history_load(history_file_path);
rx_no_overlap.history_add(line);
rx_no_overlap.history_save(history_file_path);
}
}
ReplxxLineReader::ReplxxLineReader(
@ -58,7 +45,10 @@ ReplxxLineReader::ReplxxLineReader(
}
else
{
rx.history_load(history_file_path);
if (!rx.history_load(history_file_path))
{
rx.print("Loading history failed: %s\n", errnoToString(errno).c_str());
}
if (flock(history_file_fd, LOCK_UN))
{
@ -68,10 +58,13 @@ ReplxxLineReader::ReplxxLineReader(
}
}
rx.install_window_change_handler();
auto callback = [&suggest] (const String & context, size_t context_size)
{
auto range = suggest.getCompletions(context, context_size);
return Replxx::completions_t(range.first, range.second);
if (auto range = suggest.getCompletions(context, context_size))
return Replxx::completions_t(range->first, range->second);
return Replxx::completions_t();
};
rx.set_completion_callback(callback);
@ -95,7 +88,7 @@ ReplxxLineReader::ReplxxLineReader(
ReplxxLineReader::~ReplxxLineReader()
{
if (close(history_file_fd))
rx.print("Close of history file failed: %s\n", strerror(errno));
rx.print("Close of history file failed: %s\n", errnoToString(errno).c_str());
}
LineReader::InputStatus ReplxxLineReader::readOneLine(const String & prompt)
@ -120,17 +113,18 @@ void ReplxxLineReader::addToHistory(const String & line)
// and that is why flock() is added here.
bool locked = false;
if (flock(history_file_fd, LOCK_EX))
rx.print("Lock of history file failed: %s\n", strerror(errno));
rx.print("Lock of history file failed: %s\n", errnoToString(errno).c_str());
else
locked = true;
rx.history_add(line);
// flush changes to the disk
history_save(history_file_path, line);
if (!rx.history_save(history_file_path))
rx.print("Saving history failed: %s\n", errnoToString(errno).c_str());
if (locked && 0 != flock(history_file_fd, LOCK_UN))
rx.print("Unlock of history file failed: %s\n", strerror(errno));
rx.print("Unlock of history file failed: %s\n", errnoToString(errno).c_str());
}
void ReplxxLineReader::enableBracketedPaste()

13
base/common/StringRef.cpp Normal file
View File

@ -0,0 +1,13 @@
#include <ostream>
#include "StringRef.h"
std::ostream & operator<<(std::ostream & os, const StringRef & str)
{
if (str.data)
os.write(str.data, str.size);
return os;
}

37
base/common/StringRef.h
View File

@ -1,10 +1,11 @@
#pragma once
#include <cassert>
#include <stdexcept> // for std::logic_error
#include <string>
#include <vector>
#include <functional>
#include <ostream>
#include <iosfwd>
#include <common/types.h>
#include <common/unaligned.h>
@ -21,21 +22,26 @@
#endif
/// The thing to avoid creating strings to find substrings in the hash table.
/**
* The std::string_view-like container to avoid creating strings to find substrings in the hash table.
*/
struct StringRef
{
const char * data = nullptr;
size_t size = 0;
/// Non-constexpr due to reinterpret_cast.
template <typename CharT, typename = std::enable_if_t<sizeof(CharT) == 1>>
constexpr StringRef(const CharT * data_, size_t size_) : data(reinterpret_cast<const char *>(data_)), size(size_)
StringRef(const CharT * data_, size_t size_) : data(reinterpret_cast<const char *>(data_)), size(size_)
{
/// Sanity check for overflowed values.
assert(size < 0x8000000000000000ULL);
}
constexpr StringRef(const char * data_, size_t size_) : data(data_), size(size_) {}
StringRef(const std::string & s) : data(s.data()), size(s.size()) {}
constexpr explicit StringRef(const std::string_view & s) : data(s.data()), size(s.size()) {}
constexpr explicit StringRef(std::string_view s) : data(s.data()), size(s.size()) {}
constexpr StringRef(const char * data_) : StringRef(std::string_view{data_}) {}
constexpr StringRef() = default;
@ -45,6 +51,12 @@ struct StringRef
constexpr explicit operator std::string_view() const { return {data, size}; }
};
/// Here constexpr doesn't implicate inline, see https://www.viva64.com/en/w/v1043/
/// nullptr can't be used because the StringRef values are used in SipHash's pointer arithmetic
/// and the UBSan thinks that something like nullptr + 8 is UB.
constexpr const inline char empty_string_ref_addr{};
constexpr const inline StringRef EMPTY_STRING_REF{&empty_string_ref_addr, 0};
using StringRefs = std::vector<StringRef>;
@ -302,19 +314,4 @@ namespace ZeroTraits
}
inline bool operator==(StringRef lhs, const char * rhs)
{
for (size_t pos = 0; pos < lhs.size; ++pos)
if (!rhs[pos] || lhs.data[pos] != rhs[pos])
return false;
return true;
}
inline std::ostream & operator<<(std::ostream & os, const StringRef & str)
{
if (str.data)
os.write(str.data, str.size);
return os;
}
std::ostream & operator<<(std::ostream & os, const StringRef & str);

61
base/common/arithmeticOverflow.h
View File

@ -1,5 +1,7 @@
#pragma once
#include <common/extended_types.h>
namespace common
{
template <typename T>
@ -29,12 +31,27 @@ namespace common
template <>
inline bool addOverflow(__int128 x, __int128 y, __int128 & res)
{
static constexpr __int128 min_int128 = __int128(0x8000000000000000ll) << 64;
static constexpr __int128 max_int128 = (__int128(0x7fffffffffffffffll) << 64) + 0xffffffffffffffffll;
static constexpr __int128 min_int128 = minInt128();
static constexpr __int128 max_int128 = maxInt128();
res = x + y;
return (y > 0 && x > max_int128 - y) || (y < 0 && x < min_int128 - y);
}
template <>
inline bool addOverflow(wInt256 x, wInt256 y, wInt256 & res)
{
res = x + y;
return (y > 0 && x > std::numeric_limits<wInt256>::max() - y) ||
(y < 0 && x < std::numeric_limits<wInt256>::min() - y);
}
template <>
inline bool addOverflow(wUInt256 x, wUInt256 y, wUInt256 & res)
{
res = x + y;
return x > std::numeric_limits<wUInt256>::max() - y;
}
template <typename T>
inline bool subOverflow(T x, T y, T & res)
{
@ -62,12 +79,27 @@ namespace common
template <>
inline bool subOverflow(__int128 x, __int128 y, __int128 & res)
{
static constexpr __int128 min_int128 = __int128(0x8000000000000000ll) << 64;
static constexpr __int128 max_int128 = (__int128(0x7fffffffffffffffll) << 64) + 0xffffffffffffffffll;
static constexpr __int128 min_int128 = minInt128();
static constexpr __int128 max_int128 = maxInt128();
res = x - y;
return (y < 0 && x > max_int128 + y) || (y > 0 && x < min_int128 + y);
}
template <>
inline bool subOverflow(wInt256 x, wInt256 y, wInt256 & res)
{
res = x - y;
return (y < 0 && x > std::numeric_limits<wInt256>::max() + y) ||
(y > 0 && x < std::numeric_limits<wInt256>::min() + y);
}
template <>
inline bool subOverflow(wUInt256 x, wUInt256 y, wUInt256 & res)
{
res = x - y;
return x < y;
}
template <typename T>
inline bool mulOverflow(T x, T y, T & res)
{
@ -103,4 +135,25 @@ namespace common
unsigned __int128 b = (y > 0) ? y : -y;
return (a * b) / b != a;
}
template <>
inline bool mulOverflow(wInt256 x, wInt256 y, wInt256 & res)
{
res = x * y;
if (!x || !y)
return false;
wInt256 a = (x > 0) ? x : -x;
wInt256 b = (y > 0) ? y : -y;
return (a * b) / b != a;
}
template <>
inline bool mulOverflow(wUInt256 x, wUInt256 y, wUInt256 & res)
{
res = x * y;
if (!x || !y)
return false;
return (x * y) / y != x;
}
}

5
base/common/coverage.cpp
View File

@ -3,12 +3,11 @@
#if WITH_COVERAGE
# include <mutex>
# include <unistd.h>
# if defined(__clang__)
extern "C" void __llvm_profile_dump();
extern "C" void __llvm_profile_dump(); // NOLINT
# elif defined(__GNUC__) || defined(__GNUG__)
extern "C" void __gcov_exit();
# endif
@ -23,7 +22,7 @@ void dumpCoverageReportIfPossible()
std::lock_guard lock(mutex);
# if defined(__clang__)
__llvm_profile_dump();
__llvm_profile_dump(); // NOLINT
# elif defined(__GNUC__) || defined(__GNUG__)
__gcov_exit();
# endif

14
base/common/defines.h
View File

@ -76,12 +76,8 @@
# define NO_SANITIZE_THREAD
#endif
#if defined __GNUC__ && !defined __clang__
# define OPTIMIZE(x) __attribute__((__optimize__(x)))
#else
# define OPTIMIZE(x)
#endif
/// A macro for suppressing warnings about unused variables or function results.
/// Useful for structured bindings which have no standard way to declare this.
#define UNUSED(...) (void)(__VA_ARGS__)
/// A template function for suppressing warnings about unused variables or function results.
template <typename... Args>
constexpr void UNUSED(Args &&... args [[maybe_unused]])
{
}

111
base/common/extended_types.h Normal file
View File

@ -0,0 +1,111 @@
#pragma once
#include <type_traits>
#include <common/types.h>
#include <common/wide_integer.h>
using Int128 = __int128;
using wInt256 = wide::integer<256, signed>;
using wUInt256 = wide::integer<256, unsigned>;
static_assert(sizeof(wInt256) == 32);
static_assert(sizeof(wUInt256) == 32);
static constexpr __int128 minInt128() { return static_cast<unsigned __int128>(1) << 127; }
static constexpr __int128 maxInt128() { return (static_cast<unsigned __int128>(1) << 127) - 1; }
/// The standard library type traits, such as std::is_arithmetic, with one exception
/// (std::common_type), are "set in stone". Attempting to specialize them causes undefined behavior.
/// So instead of using the std type_traits, we use our own version which allows extension.
template <typename T>
struct is_signed
{
static constexpr bool value = std::is_signed_v<T>;
};
template <> struct is_signed<Int128> { static constexpr bool value = true; };
template <> struct is_signed<wInt256> { static constexpr bool value = true; };
template <typename T>
inline constexpr bool is_signed_v = is_signed<T>::value;
template <typename T>
struct is_unsigned
{
static constexpr bool value = std::is_unsigned_v<T>;
};
template <> struct is_unsigned<wUInt256> { static constexpr bool value = true; };
template <typename T>
inline constexpr bool is_unsigned_v = is_unsigned<T>::value;
/// TODO: is_integral includes char, char8_t and wchar_t.
template <typename T>
struct is_integer
{
static constexpr bool value = std::is_integral_v<T>;
};
template <> struct is_integer<Int128> { static constexpr bool value = true; };
template <> struct is_integer<wInt256> { static constexpr bool value = true; };
template <> struct is_integer<wUInt256> { static constexpr bool value = true; };
template <typename T>
inline constexpr bool is_integer_v = is_integer<T>::value;
template <typename T>
struct is_arithmetic
{
static constexpr bool value = std::is_arithmetic_v<T>;
};
template <> struct is_arithmetic<__int128> { static constexpr bool value = true; };
template <typename T>
inline constexpr bool is_arithmetic_v = is_arithmetic<T>::value;
template <typename T>
struct make_unsigned
{
typedef std::make_unsigned_t<T> type;
};
template <> struct make_unsigned<Int128> { using type = unsigned __int128; };
template <> struct make_unsigned<wInt256> { using type = wUInt256; };
template <> struct make_unsigned<wUInt256> { using type = wUInt256; };
template <typename T> using make_unsigned_t = typename make_unsigned<T>::type;
template <typename T>
struct make_signed
{
typedef std::make_signed_t<T> type;
};
template <> struct make_signed<wInt256> { using type = wInt256; };
template <> struct make_signed<wUInt256> { using type = wInt256; };
template <typename T> using make_signed_t = typename make_signed<T>::type;
template <typename T>
struct is_big_int
{
static constexpr bool value = false;
};
template <> struct is_big_int<wInt256> { static constexpr bool value = true; };
template <> struct is_big_int<wUInt256> { static constexpr bool value = true; };
template <typename T>
inline constexpr bool is_big_int_v = is_big_int<T>::value;
template <typename To, typename From>
inline To bigint_cast(const From & x [[maybe_unused]])
{
return static_cast<To>(x);
}

93
base/common/getMemoryAmount.cpp
View File

@ -1,100 +1,29 @@
#include <stdexcept>
#include "common/getMemoryAmount.h"
#include "common/getPageSize.h"
// http://nadeausoftware.com/articles/2012/09/c_c_tip_how_get_physical_memory_size_system
/*
* Author: David Robert Nadeau
* Site: http://NadeauSoftware.com/
* License: Creative Commons Attribution 3.0 Unported License
* http://creativecommons.org/licenses/by/3.0/deed.en_US
*/
#if defined(WIN32) || defined(_WIN32)
#include <Windows.h>
#else
#include <unistd.h>
#include <sys/types.h>
#include <sys/param.h>
#if defined(BSD)
#include <sys/sysctl.h>
#endif
#endif
/**
* Returns the size of physical memory (RAM) in bytes.
* Returns 0 on unsupported platform
*/
/** Returns the size of physical memory (RAM) in bytes.
* Returns 0 on unsupported platform
*/
uint64_t getMemoryAmountOrZero()
{
#if defined(_WIN32) && (defined(__CYGWIN__) || defined(__CYGWIN32__))
/* Cygwin under Windows. ------------------------------------ */
/* New 64-bit MEMORYSTATUSEX isn't available. Use old 32.bit */
MEMORYSTATUS status;
status.dwLength = sizeof(status);
GlobalMemoryStatus(&status);
return status.dwTotalPhys;
int64_t num_pages = sysconf(_SC_PHYS_PAGES);
if (num_pages <= 0)
return 0;
#elif defined(WIN32) || defined(_WIN32)
/* Windows. ------------------------------------------------- */
/* Use new 64-bit MEMORYSTATUSEX, not old 32-bit MEMORYSTATUS */
MEMORYSTATUSEX status;
status.dwLength = sizeof(status);
GlobalMemoryStatusEx(&status);
return status.ullTotalPhys;
int64_t page_size = getPageSize();
if (page_size <= 0)
return 0;
#else
/* UNIX variants. ------------------------------------------- */
/* Prefer sysctl() over sysconf() except sysctl() HW_REALMEM and HW_PHYSMEM */
#if defined(CTL_HW) && (defined(HW_MEMSIZE) || defined(HW_PHYSMEM64))
int mib[2];
mib[0] = CTL_HW;
#if defined(HW_MEMSIZE)
mib[1] = HW_MEMSIZE; /* OSX. --------------------- */
#elif defined(HW_PHYSMEM64)
mib[1] = HW_PHYSMEM64; /* NetBSD, OpenBSD. --------- */
#endif
uint64_t size = 0; /* 64-bit */
size_t len = sizeof(size);
if (sysctl(mib, 2, &size, &len, nullptr, 0) == 0)
return size;
return 0; /* Failed? */
#elif defined(_SC_AIX_REALMEM)
/* AIX. ----------------------------------------------------- */
return sysconf(_SC_AIX_REALMEM) * 1024;
#elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
/* FreeBSD, Linux, OpenBSD, and Solaris. -------------------- */
return uint64_t(sysconf(_SC_PHYS_PAGES))
*uint64_t(sysconf(_SC_PAGESIZE));
#elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGE_SIZE)
/* Legacy. -------------------------------------------------- */
return uint64_t(sysconf(_SC_PHYS_PAGES))
* uint64_t(sysconf(_SC_PAGE_SIZE));
#elif defined(CTL_HW) && (defined(HW_PHYSMEM) || defined(HW_REALMEM))
/* DragonFly BSD, FreeBSD, NetBSD, OpenBSD, and OSX. -------- */
int mib[2];
mib[0] = CTL_HW;
#if defined(HW_REALMEM)
mib[1] = HW_REALMEM; /* FreeBSD. ----------------- */
#elif defined(HW_PYSMEM)
mib[1] = HW_PHYSMEM; /* Others. ------------------ */
#endif
unsigned int size = 0; /* 32-bit */
size_t len = sizeof(size);
if (sysctl(mib, 2, &size, &len, nullptr, 0) == 0)
return size;
return 0; /* Failed? */
#endif /* sysctl and sysconf variants */
#endif
return num_pages * page_size;
}

8
base/common/getPageSize.cpp Normal file
View File

@ -0,0 +1,8 @@
#include "common/getPageSize.h"
#include <unistd.h>
Int64 getPageSize()
{
return sysconf(_SC_PAGESIZE);
}

6
base/common/getPageSize.h Normal file
View File

@ -0,0 +1,6 @@
#pragma once
#include "common/types.h"
/// Get memory page size
Int64 getPageSize();

2
base/common/getResource.cpp
View File

@ -2,6 +2,7 @@
#include "unaligned.h"
#include <dlfcn.h>
#include <string>
#include <boost/algorithm/string/replace.hpp>
std::string_view getResource(std::string_view name)
@ -10,6 +11,7 @@ std::string_view getResource(std::string_view name)
std::replace(name_replaced.begin(), name_replaced.end(), '/', '_');
std::replace(name_replaced.begin(), name_replaced.end(), '-', '_');
std::replace(name_replaced.begin(), name_replaced.end(), '.', '_');
boost::replace_all(name_replaced, "+", "_PLUS_");
/// These are the names that are generated by "ld -r -b binary"
std::string symbol_name_data = "_binary_" + name_replaced + "_start";

2
base/common/itoa.h
View File

@ -372,7 +372,7 @@ static inline char * writeLeadingMinus(char * pos)
static inline char * writeSIntText(int128_t x, char * pos)
{
static const int128_t min_int128 = int128_t(0x8000000000000000ll) << 64;
static constexpr int128_t min_int128 = uint128_t(1) << 127;
if (unlikely(x == min_int128))
{

1
base/common/logger_useful.h
View File

@ -3,7 +3,6 @@
/// Macros for convenient usage of Poco logger.
#include <fmt/format.h>
#include <fmt/ostream.h>
#include <Poco/Logger.h>
#include <Poco/Message.h>
#include <Common/CurrentThread.h>

4
base/common/phdr_cache.cpp
View File

@ -14,7 +14,7 @@
# pragma clang diagnostic ignored "-Wunused-macros"
#endif
#define __msan_unpoison(X, Y)
#define __msan_unpoison(X, Y) // NOLINT
#if defined(__has_feature)
# if __has_feature(memory_sanitizer)
# undef __msan_unpoison
@ -84,7 +84,7 @@ extern "C"
#ifdef ADDRESS_SANITIZER
void __lsan_ignore_object(const void *);
#else
void __lsan_ignore_object(const void *) {}
void __lsan_ignore_object(const void *) {} // NOLINT
#endif
}

5
base/common/setTerminalEcho.cpp
View File

@ -1,6 +1,7 @@
// https://stackoverflow.com/questions/1413445/reading-a-password-from-stdcin
#include <common/setTerminalEcho.h>
#include <common/errnoToString.h>
#include <stdexcept>
#include <cstring>
#include <string>
@ -31,7 +32,7 @@ void setTerminalEcho(bool enable)
#else
struct termios tty;
if (tcgetattr(STDIN_FILENO, &tty))
throw std::runtime_error(std::string("setTerminalEcho failed get: ") + strerror(errno));
throw std::runtime_error(std::string("setTerminalEcho failed get: ") + errnoToString(errno));
if (!enable)
tty.c_lflag &= ~ECHO;
else
@ -39,6 +40,6 @@ void setTerminalEcho(bool enable)
auto ret = tcsetattr(STDIN_FILENO, TCSANOW, &tty);
if (ret)
throw std::runtime_error(std::string("setTerminalEcho failed set: ") + strerror(errno));
throw std::runtime_error(std::string("setTerminalEcho failed set: ") + errnoToString(errno));
#endif
}

37
base/common/sort.h Normal file
View File

@ -0,0 +1,37 @@
#pragma once
#if !defined(ARCADIA_BUILD)
# include <miniselect/floyd_rivest_select.h> // Y_IGNORE
#else
# include <algorithm>
#endif
template <class RandomIt>
void nth_element(RandomIt first, RandomIt nth, RandomIt last)
{
#if !defined(ARCADIA_BUILD)
::miniselect::floyd_rivest_select(first, nth, last);
#else
::std::nth_element(first, nth, last);
#endif
}
template <class RandomIt>
void partial_sort(RandomIt first, RandomIt middle, RandomIt last)
{
#if !defined(ARCADIA_BUILD)
::miniselect::floyd_rivest_partial_sort(first, middle, last);
#else
::std::partial_sort(first, middle, last);
#endif
}
template <class RandomIt, class Compare>
void partial_sort(RandomIt first, RandomIt middle, RandomIt last, Compare compare)
{
#if !defined(ARCADIA_BUILD)
::miniselect::floyd_rivest_partial_sort(first, middle, last, compare);
#else
::std::partial_sort(first, middle, last, compare);
#endif
}

2
base/common/terminalColors.h
View File

@ -1,3 +1,5 @@
#pragma once
#include <string>
#include <common/types.h>

4
base/common/tests/CMakeLists.txt
View File

@ -1,19 +1,15 @@
include (${ClickHouse_SOURCE_DIR}/cmake/add_check.cmake)
add_executable (date_lut_init date_lut_init.cpp)
add_executable (date_lut2 date_lut2.cpp)
add_executable (date_lut3 date_lut3.cpp)
add_executable (date_lut4 date_lut4.cpp)
add_executable (date_lut_default_timezone date_lut_default_timezone.cpp)
add_executable (local_date_time_comparison local_date_time_comparison.cpp)
add_executable (realloc-perf allocator.cpp)
set(PLATFORM_LIBS ${CMAKE_DL_LIBS})
target_link_libraries (date_lut_init PRIVATE common ${PLATFORM_LIBS})
target_link_libraries (date_lut2 PRIVATE common ${PLATFORM_LIBS})
target_link_libraries (date_lut3 PRIVATE common ${PLATFORM_LIBS})
target_link_libraries (date_lut4 PRIVATE common ${PLATFORM_LIBS})
target_link_libraries (date_lut_default_timezone PRIVATE common ${PLATFORM_LIBS})
target_link_libraries (local_date_time_comparison PRIVATE common)
target_link_libraries (realloc-perf PRIVATE common)

20
base/common/tests/date_lut4.cpp
View File

@ -1,20 +0,0 @@
#include <iostream>
#include <common/DateLUT.h>
int main(int, char **)
{
/** В DateLUT был глюк - для времён из дня 1970-01-01, возвращался номер часа больше 23. */
static const time_t time = 66130;
const auto & date_lut = DateLUT::instance();
std::cerr << date_lut.toHour(time) << std::endl;
std::cerr << date_lut.toDayNum(time) << std::endl;
const auto * values = reinterpret_cast<const DateLUTImpl::Values *>(&date_lut);
std::cerr << values[0].date << ", " << time_t(values[1].date - values[0].date) << std::endl;
return 0;
}

8
base/common/tests/date_lut_init.cpp
View File

@ -1,8 +0,0 @@
#include <common/DateLUT.h>
/// Позволяет проверить время инициализации DateLUT.
int main(int, char **)
{
DateLUT::instance();
return 0;
}

13
base/common/throwError.h Normal file
View File

@ -0,0 +1,13 @@
#pragma once
#include <stdexcept>
/// Throw DB::Exception-like exception before its definition.
/// DB::Exception derived from Poco::Exception derived from std::exception.
/// DB::Exception generally cought as Poco::Exception. std::exception generally has other catch blocks and could lead to other outcomes.
/// DB::Exception is not defined yet. It'd better to throw Poco::Exception but we do not want to include any big header here, even <string>.
/// So we throw some std::exception instead in the hope its catch block is the same as DB::Exception one.
template <typename T>
inline void throwError(const T & err)
{
throw std::runtime_error(err);
}

51
base/common/types.h
View File

@ -1,20 +1,18 @@
#pragma once
#include <algorithm>
#include <cstdint>
#include <cstdlib>
#include <string>
#include <type_traits>
using Int8 = int8_t;
using Int16 = int16_t;
using Int32 = int32_t;
using Int64 = int64_t;
#if __cplusplus <= 201703L
#ifndef __cpp_char8_t
using char8_t = unsigned char;
#endif
/// This is needed for more strict aliasing. https://godbolt.org/z/xpJBSb https://stackoverflow.com/a/57453713
using UInt8 = char8_t;
using UInt16 = uint16_t;
using UInt32 = uint32_t;
@ -22,41 +20,22 @@ using UInt64 = uint64_t;
using String = std::string;
/// The standard library type traits, such as std::is_arithmetic, with one exception
/// (std::common_type), are "set in stone". Attempting to specialize them causes undefined behavior.
/// So instead of using the std type_traits, we use our own version which allows extension.
template <typename T>
struct is_signed
namespace DB
{
static constexpr bool value = std::is_signed_v<T>;
};
template <typename T>
inline constexpr bool is_signed_v = is_signed<T>::value;
using UInt8 = ::UInt8;
using UInt16 = ::UInt16;
using UInt32 = ::UInt32;
using UInt64 = ::UInt64;
template <typename T>
struct is_unsigned
{
static constexpr bool value = std::is_unsigned_v<T>;
};
using Int8 = ::Int8;
using Int16 = ::Int16;
using Int32 = ::Int32;
using Int64 = ::Int64;
template <typename T>
inline constexpr bool is_unsigned_v = is_unsigned<T>::value;
using Float32 = float;
using Float64 = double;
template <typename T>
struct is_integral
{
static constexpr bool value = std::is_integral_v<T>;
};
using String = std::string;
template <typename T>
inline constexpr bool is_integral_v = is_integral<T>::value;
template <typename T>
struct is_arithmetic
{
static constexpr bool value = std::is_arithmetic_v<T>;
};
template <typename T>
inline constexpr bool is_arithmetic_v = is_arithmetic<T>::value;
}

258
base/common/wide_integer.h Normal file
View File

@ -0,0 +1,258 @@
#pragma once
///////////////////////////////////////////////////////////////
// Distributed under the Boost Software License, Version 1.0.
// (See at http://www.boost.org/LICENSE_1_0.txt)
///////////////////////////////////////////////////////////////
/* Divide and multiply
*
*
* Copyright (c) 2008
* Evan Teran
*
* Permission to use, copy, modify, and distribute this software and its
* documentation for any purpose and without fee is hereby granted, provided
* that the above copyright notice appears in all copies and that both the
* copyright notice and this permission notice appear in supporting
* documentation, and that the same name not be used in advertising or
* publicity pertaining to distribution of the software without specific,
* written prior permission. We make no representations about the
* suitability this software for any purpose. It is provided "as is"
* without express or implied warranty.
*/
#include <cstdint>
#include <limits>
#include <type_traits>
#include <initializer_list>
namespace wide
{
template <size_t Bits, typename Signed>
class integer;
}
namespace std
{
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
struct common_type<wide::integer<Bits, Signed>, wide::integer<Bits2, Signed2>>;
template <size_t Bits, typename Signed, typename Arithmetic>
struct common_type<wide::integer<Bits, Signed>, Arithmetic>;
template <typename Arithmetic, size_t Bits, typename Signed>
struct common_type<Arithmetic, wide::integer<Bits, Signed>>;
}
namespace wide
{
template <size_t Bits, typename Signed>
class integer
{
public:
using base_type = uint64_t;
using signed_base_type = int64_t;
// ctors
constexpr integer() noexcept;
template <typename T>
constexpr integer(T rhs) noexcept;
template <typename T>
constexpr integer(std::initializer_list<T> il) noexcept;
// assignment
template <size_t Bits2, typename Signed2>
constexpr integer<Bits, Signed> & operator=(const integer<Bits2, Signed2> & rhs) noexcept;
template <typename Arithmetic>
constexpr integer<Bits, Signed> & operator=(Arithmetic rhs) noexcept;
template <typename Arithmetic>
constexpr integer<Bits, Signed> & operator*=(const Arithmetic & rhs);
template <typename Arithmetic>
constexpr integer<Bits, Signed> & operator/=(const Arithmetic & rhs);
template <typename Arithmetic>
constexpr integer<Bits, Signed> & operator+=(const Arithmetic & rhs) noexcept(std::is_same_v<Signed, unsigned>);
template <typename Arithmetic>
constexpr integer<Bits, Signed> & operator-=(const Arithmetic & rhs) noexcept(std::is_same_v<Signed, unsigned>);
template <typename Integral>
constexpr integer<Bits, Signed> & operator%=(const Integral & rhs);
template <typename Integral>
constexpr integer<Bits, Signed> & operator&=(const Integral & rhs) noexcept;
template <typename Integral>
constexpr integer<Bits, Signed> & operator|=(const Integral & rhs) noexcept;
template <typename Integral>
constexpr integer<Bits, Signed> & operator^=(const Integral & rhs) noexcept;
constexpr integer<Bits, Signed> & operator<<=(int n) noexcept;
constexpr integer<Bits, Signed> & operator>>=(int n) noexcept;
constexpr integer<Bits, Signed> & operator++() noexcept(std::is_same_v<Signed, unsigned>);
constexpr integer<Bits, Signed> operator++(int) noexcept(std::is_same_v<Signed, unsigned>);
constexpr integer<Bits, Signed> & operator--() noexcept(std::is_same_v<Signed, unsigned>);
constexpr integer<Bits, Signed> operator--(int) noexcept(std::is_same_v<Signed, unsigned>);
// observers
constexpr explicit operator bool() const noexcept;
template <class T>
using __integral_not_wide_integer_class = typename std::enable_if<std::is_arithmetic<T>::value, T>::type;
template <class T, class = __integral_not_wide_integer_class<T>>
constexpr operator T() const noexcept;
constexpr operator long double() const noexcept;
constexpr operator double() const noexcept;
constexpr operator float() const noexcept;
struct _impl;
private:
template <size_t Bits2, typename Signed2>
friend class integer;
friend class std::numeric_limits<integer<Bits, signed>>;
friend class std::numeric_limits<integer<Bits, unsigned>>;
base_type items[_impl::item_count];
};
template <typename T>
static constexpr bool ArithmeticConcept() noexcept;
template <class T1, class T2>
using __only_arithmetic = typename std::enable_if<ArithmeticConcept<T1>() && ArithmeticConcept<T2>()>::type;
template <typename T>
static constexpr bool IntegralConcept() noexcept;
template <class T, class T2>
using __only_integer = typename std::enable_if<IntegralConcept<T>() && IntegralConcept<T2>()>::type;
// Unary operators
template <size_t Bits, typename Signed>
constexpr integer<Bits, Signed> operator~(const integer<Bits, Signed> & lhs) noexcept;
template <size_t Bits, typename Signed>
constexpr integer<Bits, Signed> operator-(const integer<Bits, Signed> & lhs) noexcept(std::is_same_v<Signed, unsigned>);
template <size_t Bits, typename Signed>
constexpr integer<Bits, Signed> operator+(const integer<Bits, Signed> & lhs) noexcept(std::is_same_v<Signed, unsigned>);
// Binary operators
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator*(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
std::common_type_t<Arithmetic, Arithmetic2> constexpr operator*(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator/(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
std::common_type_t<Arithmetic, Arithmetic2> constexpr operator/(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator+(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
std::common_type_t<Arithmetic, Arithmetic2> constexpr operator+(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator-(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
std::common_type_t<Arithmetic, Arithmetic2> constexpr operator-(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator%(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
std::common_type_t<Integral, Integral2> constexpr operator%(const Integral & rhs, const Integral2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator&(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
std::common_type_t<Integral, Integral2> constexpr operator&(const Integral & rhs, const Integral2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator|(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
std::common_type_t<Integral, Integral2> constexpr operator|(const Integral & rhs, const Integral2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
std::common_type_t<integer<Bits, Signed>, integer<Bits2, Signed2>> constexpr
operator^(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Integral, typename Integral2, class = __only_integer<Integral, Integral2>>
std::common_type_t<Integral, Integral2> constexpr operator^(const Integral & rhs, const Integral2 & lhs);
// TODO: Integral
template <size_t Bits, typename Signed>
constexpr integer<Bits, Signed> operator<<(const integer<Bits, Signed> & lhs, int n) noexcept;
template <size_t Bits, typename Signed>
constexpr integer<Bits, Signed> operator>>(const integer<Bits, Signed> & lhs, int n) noexcept;
template <size_t Bits, typename Signed, typename Int, typename = std::enable_if_t<!std::is_same_v<Int, int>>>
constexpr integer<Bits, Signed> operator<<(const integer<Bits, Signed> & lhs, Int n) noexcept
{
return lhs << int(n);
}
template <size_t Bits, typename Signed, typename Int, typename = std::enable_if_t<!std::is_same_v<Int, int>>>
constexpr integer<Bits, Signed> operator>>(const integer<Bits, Signed> & lhs, Int n) noexcept
{
return lhs >> int(n);
}
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
constexpr bool operator<(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
constexpr bool operator<(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
constexpr bool operator>(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
constexpr bool operator>(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
constexpr bool operator<=(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
constexpr bool operator<=(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
constexpr bool operator>=(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
constexpr bool operator>=(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
constexpr bool operator==(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
constexpr bool operator==(const Arithmetic & rhs, const Arithmetic2 & lhs);
template <size_t Bits, typename Signed, size_t Bits2, typename Signed2>
constexpr bool operator!=(const integer<Bits, Signed> & lhs, const integer<Bits2, Signed2> & rhs);
template <typename Arithmetic, typename Arithmetic2, class = __only_arithmetic<Arithmetic, Arithmetic2>>
constexpr bool operator!=(const Arithmetic & rhs, const Arithmetic2 & lhs);
}
namespace std
{
template <size_t Bits, typename Signed>
struct hash<wide::integer<Bits, Signed>>;
}
#include "wide_integer_impl.h"

1386
base/common/wide_integer_impl.h Normal file
View File

File diff suppressed because it is too large Load Diff

35
base/common/wide_integer_to_string.h Normal file
View File

@ -0,0 +1,35 @@
#pragma once
#include <string>
#include "wide_integer.h"
namespace wide
{
template <size_t Bits, typename Signed>
inline std::string to_string(const integer<Bits, Signed> & n)
{
std::string res;
if (integer<Bits, Signed>::_impl::operator_eq(n, 0U))
return "0";
integer<Bits, unsigned> t;
bool is_neg = integer<Bits, Signed>::_impl::is_negative(n);
if (is_neg)
t = integer<Bits, Signed>::_impl::operator_unary_minus(n);
else
t = n;
while (!integer<Bits, unsigned>::_impl::operator_eq(t, 0U))
{
res.insert(res.begin(), '0' + char(integer<Bits, unsigned>::_impl::operator_percent(t, 10U)));
t = integer<Bits, unsigned>::_impl::operator_slash(t, 10U);
}
if (is_neg)
res.insert(res.begin(), '-');
return res;
}
}

20
base/common/ya.make
View File

@ -1,9 +1,10 @@
# This file is generated automatically, do not edit. See 'ya.make.in' and use 'utils/generate-ya-make' to regenerate it.
OWNER(g:clickhouse)
LIBRARY()
ADDINCL(
GLOBAL clickhouse/base
GLOBAL contrib/libs/cctz/include
)
CFLAGS (GLOBAL -DARCADIA_BUILD)
@ -22,29 +23,33 @@ ELSEIF (OS_LINUX)
ENDIF ()
PEERDIR(
contrib/libs/cctz/src
contrib/libs/cctz
contrib/libs/cxxsupp/libcxx-filesystem
contrib/libs/poco/Net
contrib/libs/poco/NetSSL_OpenSSL
contrib/libs/poco/Util
contrib/libs/poco/NetSSL_OpenSSL
contrib/libs/fmt
contrib/restricted/boost
contrib/restricted/cityhash-1.0.2
)
CFLAGS(-g0)
SRCS(
argsToConfig.cpp
coverage.cpp
DateLUT.cpp
DateLUTImpl.cpp
JSON.cpp
LineReader.cpp
StringRef.cpp
argsToConfig.cpp
coverage.cpp
demangle.cpp
errnoToString.cpp
getFQDNOrHostName.cpp
getMemoryAmount.cpp
getPageSize.cpp
getResource.cpp
getThreadId.cpp
JSON.cpp
LineReader.cpp
mremap.cpp
phdr_cache.cpp
preciseExp10.cpp
@ -52,6 +57,7 @@ SRCS(
shift10.cpp
sleep.cpp
terminalColors.cpp
)
END()

9
base/common/ya.make.in
View File

@ -1,8 +1,9 @@
OWNER(g:clickhouse)
LIBRARY()
ADDINCL(
GLOBAL clickhouse/base
GLOBAL contrib/libs/cctz/include
)
CFLAGS (GLOBAL -DARCADIA_BUILD)
@ -10,6 +11,7 @@ CFLAGS (GLOBAL -DARCADIA_BUILD)
CFLAGS (GLOBAL -DUSE_CPUID=1)
CFLAGS (GLOBAL -DUSE_JEMALLOC=0)
CFLAGS (GLOBAL -DUSE_RAPIDJSON=1)
CFLAGS (GLOBAL -DUSE_SSL=1)
IF (OS_DARWIN)
CFLAGS (GLOBAL -DOS_DARWIN)
@ -20,15 +22,18 @@ ELSEIF (OS_LINUX)
ENDIF ()
PEERDIR(
contrib/libs/cctz/src
contrib/libs/cctz
contrib/libs/cxxsupp/libcxx-filesystem
contrib/libs/poco/Net
contrib/libs/poco/Util
contrib/libs/poco/NetSSL_OpenSSL
contrib/libs/fmt
contrib/restricted/boost
contrib/restricted/cityhash-1.0.2
)
CFLAGS(-g0)
SRCS(
<? find . -name '*.cpp' | grep -v -F tests/ | grep -v -F Replxx | grep -v -F Readline | sed 's/^\.\// /' | sort ?>
)

6
base/daemon/BaseDaemon.cpp
View File

@ -761,14 +761,14 @@ void BaseDaemon::initializeTerminationAndSignalProcessing()
static KillingErrorHandler killing_error_handler;
Poco::ErrorHandler::set(&killing_error_handler);
signal_pipe.setNonBlocking();
signal_pipe.setNonBlockingWrite();
signal_pipe.tryIncreaseSize(1 << 20);
signal_listener = std::make_unique<SignalListener>(*this);
signal_listener_thread.start(*signal_listener);
#if defined(__ELF__) && !defined(__FreeBSD__)
String build_id_hex = DB::SymbolIndex::instance().getBuildIDHex();
String build_id_hex = DB::SymbolIndex::instance()->getBuildIDHex();
if (build_id_hex.empty())
build_id_info = "no build id";
else
@ -781,7 +781,7 @@ void BaseDaemon::initializeTerminationAndSignalProcessing()
void BaseDaemon::logRevision() const
{
Poco::Logger::root().information("Starting " + std::string{VERSION_FULL}
+ " with revision " + std::to_string(ClickHouseRevision::get())
+ " with revision " + std::to_string(ClickHouseRevision::getVersionRevision())
+ ", " + build_id_info
+ ", PID " + std::to_string(getpid()));
}

19
base/daemon/SentryWriter.cpp
View File

@ -6,10 +6,12 @@
#include <common/defines.h>
#include <common/getFQDNOrHostName.h>
#include <common/getMemoryAmount.h>
#include <common/logger_useful.h>
#include <Common/SymbolIndex.h>
#include <Common/StackTrace.h>
#include <Common/getNumberOfPhysicalCPUCores.h>
#if !defined(ARCADIA_BUILD)
# include "Common/config_version.h"
@ -28,14 +30,13 @@ namespace
bool initialized = false;
bool anonymize = false;
std::string server_data_path;
void setExtras()
{
if (!anonymize)
{
sentry_set_extra("server_name", sentry_value_new_string(getFQDNOrHostName().c_str()));
}
sentry_set_tag("version", VERSION_STRING);
sentry_set_extra("version_githash", sentry_value_new_string(VERSION_GITHASH));
sentry_set_extra("version_describe", sentry_value_new_string(VERSION_DESCRIBE));
@ -44,6 +45,15 @@ void setExtras()
sentry_set_extra("version_major", sentry_value_new_int32(VERSION_MAJOR));
sentry_set_extra("version_minor", sentry_value_new_int32(VERSION_MINOR));
sentry_set_extra("version_patch", sentry_value_new_int32(VERSION_PATCH));
sentry_set_extra("version_official", sentry_value_new_string(VERSION_OFFICIAL));
/// Sentry does not support 64-bit integers.
sentry_set_extra("total_ram", sentry_value_new_string(formatReadableSizeWithBinarySuffix(getMemoryAmountOrZero()).c_str()));
sentry_set_extra("physical_cpu_cores", sentry_value_new_int32(getNumberOfPhysicalCPUCores()));
if (!server_data_path.empty())
sentry_set_extra("disk_free_space", sentry_value_new_string(formatReadableSizeWithBinarySuffix(
Poco::File(server_data_path).freeSpace()).c_str()));
}
void sentry_logger(sentry_level_e level, const char * message, va_list args, void *)
@ -98,6 +108,7 @@ void SentryWriter::initialize(Poco::Util::LayeredConfiguration & config)
}
if (enabled)
{
server_data_path = config.getString("path", "");
const std::filesystem::path & default_tmp_path = std::filesystem::path(config.getString("tmp_path", Poco::Path::temp())) / "sentry";
const std::string & endpoint
= config.getString("send_crash_reports.endpoint");
@ -168,7 +179,7 @@ void SentryWriter::onFault(int sig, const std::string & error_message, const Sta
sentry_set_extra("signal_number", sentry_value_new_int32(sig));
#if defined(__ELF__) && !defined(__FreeBSD__)
const String & build_id_hex = DB::SymbolIndex::instance().getBuildIDHex();
const String & build_id_hex = DB::SymbolIndex::instance()->getBuildIDHex();
sentry_set_tag("build_id", build_id_hex.c_str());
#endif

4
base/daemon/ya.make
View File

@ -1,3 +1,5 @@
OWNER(g:clickhouse)
LIBRARY()
NO_COMPILER_WARNINGS()
@ -6,6 +8,8 @@ PEERDIR(
clickhouse/src/Common
)
CFLAGS(-g0)
SRCS(
BaseDaemon.cpp
GraphiteWriter.cpp

4
base/glibc-compatibility/CMakeLists.txt
View File

@ -27,6 +27,10 @@ if (GLIBC_COMPATIBILITY)
list(APPEND glibc_compatibility_sources musl/getentropy.c)
endif()
add_library (clickhouse_memcpy OBJECT
${ClickHouse_SOURCE_DIR}/contrib/FastMemcpy/memcpy_wrapper.c
)
# Need to omit frame pointers to match the performance of glibc
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fomit-frame-pointer")

93
base/glibc-compatibility/musl/__polevll.c Normal file
View File

@ -0,0 +1,93 @@
/* origin: OpenBSD /usr/src/lib/libm/src/polevll.c */
/*
* Copyright (c) 2008 Stephen L. Moshier <steve@moshier.net>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
/*
* Evaluate polynomial
*
*
* SYNOPSIS:
*
* int N;
* long double x, y, coef[N+1], polevl[];
*
* y = polevll( x, coef, N );
*
*
* DESCRIPTION:
*
* Evaluates polynomial of degree N:
*
* 2 N
* y = C + C x + C x +...+ C x
* 0 1 2 N
*
* Coefficients are stored in reverse order:
*
* coef[0] = C , ..., coef[N] = C .
* N 0
*
* The function p1evll() assumes that coef[N] = 1.0 and is
* omitted from the array. Its calling arguments are
* otherwise the same as polevll().
*
*
* SPEED:
*
* In the interest of speed, there are no checks for out
* of bounds arithmetic. This routine is used by most of
* the functions in the library. Depending on available
* equipment features, the user may wish to rewrite the
* program in microcode or assembly language.
*
*/
#include "libm.h"
#if LDBL_MANT_DIG == 53 && LDBL_MAX_EXP == 1024
#else
/*
* Polynomial evaluator:
* P[0] x^n + P[1] x^(n-1) + ... + P[n]
*/
long double __polevll(long double x, const long double *P, int n)
{
long double y;
y = *P++;
do {
y = y * x + *P++;
} while (--n);
return y;
}
/*
* Polynomial evaluator:
* x^n + P[0] x^(n-1) + P[1] x^(n-2) + ... + P[n]
*/
long double __p1evll(long double x, const long double *P, int n)
{
long double y;
n -= 1;
y = x + *P++;
do {
y = y * x + *P++;
} while (--n);
return y;
}
#endif

19
base/glibc-compatibility/musl/accept4.c Normal file
View File

@ -0,0 +1,19 @@
#define _GNU_SOURCE
#include <sys/socket.h>
#include <errno.h>
#include <fcntl.h>
#include "syscall.h"
int accept4(int fd, struct sockaddr *restrict addr, socklen_t *restrict len, int flg)
{
if (!flg) return accept(fd, addr, len);
int ret = socketcall_cp(accept4, fd, addr, len, flg, 0, 0);
if (ret>=0 || (errno != ENOSYS && errno != EINVAL)) return ret;
ret = accept(fd, addr, len);
if (ret<0) return ret;
if (flg & SOCK_CLOEXEC)
__syscall(SYS_fcntl, ret, F_SETFD, FD_CLOEXEC);
if (flg & SOCK_NONBLOCK)
__syscall(SYS_fcntl, ret, F_SETFL, O_NONBLOCK);
return ret;
}

37
base/glibc-compatibility/musl/epoll.c Normal file
View File

@ -0,0 +1,37 @@
#include <sys/epoll.h>
#include <signal.h>
#include <errno.h>
#include "syscall.h"
int epoll_create(int size)
{
return epoll_create1(0);
}
int epoll_create1(int flags)
{
int r = __syscall(SYS_epoll_create1, flags);
#ifdef SYS_epoll_create
if (r==-ENOSYS && !flags) r = __syscall(SYS_epoll_create, 1);
#endif
return __syscall_ret(r);
}
int epoll_ctl(int fd, int op, int fd2, struct epoll_event *ev)
{
return syscall(SYS_epoll_ctl, fd, op, fd2, ev);
}
int epoll_pwait(int fd, struct epoll_event *ev, int cnt, int to, const sigset_t *sigs)
{
int r = __syscall(SYS_epoll_pwait, fd, ev, cnt, to, sigs, _NSIG/8);
#ifdef SYS_epoll_wait
if (r==-ENOSYS && !sigs) r = __syscall(SYS_epoll_wait, fd, ev, cnt, to);
#endif
return __syscall_ret(r);
}
int epoll_wait(int fd, struct epoll_event *ev, int cnt, int to)
{
return epoll_pwait(fd, ev, cnt, to, 0);
}

23
base/glibc-compatibility/musl/eventfd.c Normal file
View File

@ -0,0 +1,23 @@
#include <sys/eventfd.h>
#include <unistd.h>
#include <errno.h>
#include "syscall.h"
int eventfd(unsigned int count, int flags)
{
int r = __syscall(SYS_eventfd2, count, flags);
#ifdef SYS_eventfd
if (r==-ENOSYS && !flags) r = __syscall(SYS_eventfd, count);
#endif
return __syscall_ret(r);
}
int eventfd_read(int fd, eventfd_t *value)
{
return (sizeof(*value) == read(fd, value, sizeof(*value))) ? 0 : -1;
}
int eventfd_write(int fd, eventfd_t value)
{
return (sizeof(value) == write(fd, &value, sizeof(value))) ? 0 : -1;
}

45
base/glibc-compatibility/musl/getauxval.c Normal file
View File

@ -0,0 +1,45 @@
#include <sys/auxv.h>
#include <unistd.h> // __environ
#include <errno.h>
// We don't have libc struct available here. Compute aux vector manually.
static unsigned long * __auxv = NULL;
static unsigned long __auxv_secure = 0;
static size_t __find_auxv(unsigned long type)
{
size_t i;
for (i = 0; __auxv[i]; i += 2)
{
if (__auxv[i] == type)
return i + 1;
}
return (size_t) -1;
}
__attribute__((constructor)) static void __auxv_init()
{
size_t i;
for (i = 0; __environ[i]; i++);
__auxv = (unsigned long *) (__environ + i + 1);
size_t secure_idx = __find_auxv(AT_SECURE);
if (secure_idx != ((size_t) -1))
__auxv_secure = __auxv[secure_idx];
}
unsigned long getauxval(unsigned long type)
{
if (type == AT_SECURE)
return __auxv_secure;
if (__auxv)
{
size_t index = __find_auxv(type);
if (index != ((size_t) -1))
return __auxv[index];
}
errno = ENOENT;
return 0;
}

339
base/glibc-compatibility/musl/lgammal.c Normal file
View File

@ -0,0 +1,339 @@
/* origin: OpenBSD /usr/src/lib/libm/src/ld80/e_lgammal.c */
/*
* ====================================================
* Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved.
*
* Developed at SunPro, a Sun Microsystems, Inc. business.
* Permission to use, copy, modify, and distribute this
* software is freely granted, provided that this notice
* is preserved.
* ====================================================
*/
/*
* Copyright (c) 2008 Stephen L. Moshier <steve@moshier.net>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
/* lgammal(x)
* Reentrant version of the logarithm of the Gamma function
* with user provide pointer for the sign of Gamma(x).
*
* Method:
* 1. Argument Reduction for 0 < x <= 8
* Since gamma(1+s)=s*gamma(s), for x in [0,8], we may
* reduce x to a number in [1.5,2.5] by
* lgamma(1+s) = log(s) + lgamma(s)
* for example,
* lgamma(7.3) = log(6.3) + lgamma(6.3)
* = log(6.3*5.3) + lgamma(5.3)
* = log(6.3*5.3*4.3*3.3*2.3) + lgamma(2.3)
* 2. Polynomial approximation of lgamma around its
* minimun ymin=1.461632144968362245 to maintain monotonicity.
* On [ymin-0.23, ymin+0.27] (i.e., [1.23164,1.73163]), use
* Let z = x-ymin;
* lgamma(x) = -1.214862905358496078218 + z^2*poly(z)
* 2. Rational approximation in the primary interval [2,3]
* We use the following approximation:
* s = x-2.0;
* lgamma(x) = 0.5*s + s*P(s)/Q(s)
* Our algorithms are based on the following observation
*
* zeta(2)-1 2 zeta(3)-1 3
* lgamma(2+s) = s*(1-Euler) + --------- * s - --------- * s + ...
* 2 3
*
* where Euler = 0.5771... is the Euler constant, which is very
* close to 0.5.
*
* 3. For x>=8, we have
* lgamma(x)~(x-0.5)log(x)-x+0.5*log(2pi)+1/(12x)-1/(360x**3)+....
* (better formula:
* lgamma(x)~(x-0.5)*(log(x)-1)-.5*(log(2pi)-1) + ...)
* Let z = 1/x, then we approximation
* f(z) = lgamma(x) - (x-0.5)(log(x)-1)
* by
* 3 5 11
* w = w0 + w1*z + w2*z + w3*z + ... + w6*z
*
* 4. For negative x, since (G is gamma function)
* -x*G(-x)*G(x) = pi/sin(pi*x),
* we have
* G(x) = pi/(sin(pi*x)*(-x)*G(-x))
* since G(-x) is positive, sign(G(x)) = sign(sin(pi*x)) for x<0
* Hence, for x<0, signgam = sign(sin(pi*x)) and
* lgamma(x) = log(|Gamma(x)|)
* = log(pi/(|x*sin(pi*x)|)) - lgamma(-x);
* Note: one should avoid compute pi*(-x) directly in the
* computation of sin(pi*(-x)).
*
* 5. Special Cases
* lgamma(2+s) ~ s*(1-Euler) for tiny s
* lgamma(1)=lgamma(2)=0
* lgamma(x) ~ -log(x) for tiny x
* lgamma(0) = lgamma(inf) = inf
* lgamma(-integer) = +-inf
*
*/
#include <stdint.h>
#include <math.h>
#include "libm.h"
#if LDBL_MANT_DIG == 53 && LDBL_MAX_EXP == 1024
double lgamma_r(double x, int *sg);
long double lgammal_r(long double x, int *sg)
{
return lgamma_r(x, sg);
}
#elif LDBL_MANT_DIG == 64 && LDBL_MAX_EXP == 16384
static const long double pi = 3.14159265358979323846264L,
/* lgam(1+x) = 0.5 x + x a(x)/b(x)
-0.268402099609375 <= x <= 0
peak relative error 6.6e-22 */
a0 = -6.343246574721079391729402781192128239938E2L,
a1 = 1.856560238672465796768677717168371401378E3L,
a2 = 2.404733102163746263689288466865843408429E3L,
a3 = 8.804188795790383497379532868917517596322E2L,
a4 = 1.135361354097447729740103745999661157426E2L,
a5 = 3.766956539107615557608581581190400021285E0L,
b0 = 8.214973713960928795704317259806842490498E3L,
b1 = 1.026343508841367384879065363925870888012E4L,
b2 = 4.553337477045763320522762343132210919277E3L,
b3 = 8.506975785032585797446253359230031874803E2L,
b4 = 6.042447899703295436820744186992189445813E1L,
/* b5 = 1.000000000000000000000000000000000000000E0 */
tc = 1.4616321449683623412626595423257213284682E0L,
tf = -1.2148629053584961146050602565082954242826E-1, /* double precision */
/* tt = (tail of tf), i.e. tf + tt has extended precision. */
tt = 3.3649914684731379602768989080467587736363E-18L,
/* lgam ( 1.4616321449683623412626595423257213284682E0 ) =
-1.2148629053584960809551455717769158215135617312999903886372437313313530E-1 */
/* lgam (x + tc) = tf + tt + x g(x)/h(x)
-0.230003726999612341262659542325721328468 <= x
<= 0.2699962730003876587373404576742786715318
peak relative error 2.1e-21 */
g0 = 3.645529916721223331888305293534095553827E-18L,
g1 = 5.126654642791082497002594216163574795690E3L,
g2 = 8.828603575854624811911631336122070070327E3L,
g3 = 5.464186426932117031234820886525701595203E3L,
g4 = 1.455427403530884193180776558102868592293E3L,
g5 = 1.541735456969245924860307497029155838446E2L,
g6 = 4.335498275274822298341872707453445815118E0L,
h0 = 1.059584930106085509696730443974495979641E4L,
h1 = 2.147921653490043010629481226937850618860E4L,
h2 = 1.643014770044524804175197151958100656728E4L,
h3 = 5.869021995186925517228323497501767586078E3L,
h4 = 9.764244777714344488787381271643502742293E2L,
h5 = 6.442485441570592541741092969581997002349E1L,
/* h6 = 1.000000000000000000000000000000000000000E0 */
/* lgam (x+1) = -0.5 x + x u(x)/v(x)
-0.100006103515625 <= x <= 0.231639862060546875
peak relative error 1.3e-21 */
u0 = -8.886217500092090678492242071879342025627E1L,
u1 = 6.840109978129177639438792958320783599310E2L,
u2 = 2.042626104514127267855588786511809932433E3L,
u3 = 1.911723903442667422201651063009856064275E3L,
u4 = 7.447065275665887457628865263491667767695E2L,
u5 = 1.132256494121790736268471016493103952637E2L,
u6 = 4.484398885516614191003094714505960972894E0L,
v0 = 1.150830924194461522996462401210374632929E3L,
v1 = 3.399692260848747447377972081399737098610E3L,
v2 = 3.786631705644460255229513563657226008015E3L,
v3 = 1.966450123004478374557778781564114347876E3L,
v4 = 4.741359068914069299837355438370682773122E2L,
v5 = 4.508989649747184050907206782117647852364E1L,
/* v6 = 1.000000000000000000000000000000000000000E0 */
/* lgam (x+2) = .5 x + x s(x)/r(x)
0 <= x <= 1
peak relative error 7.2e-22 */
s0 = 1.454726263410661942989109455292824853344E6L,
s1 = -3.901428390086348447890408306153378922752E6L,
s2 = -6.573568698209374121847873064292963089438E6L,
s3 = -3.319055881485044417245964508099095984643E6L,
s4 = -7.094891568758439227560184618114707107977E5L,
s5 = -6.263426646464505837422314539808112478303E4L,
s6 = -1.684926520999477529949915657519454051529E3L,
r0 = -1.883978160734303518163008696712983134698E7L,
r1 = -2.815206082812062064902202753264922306830E7L,
r2 = -1.600245495251915899081846093343626358398E7L,
r3 = -4.310526301881305003489257052083370058799E6L,
r4 = -5.563807682263923279438235987186184968542E5L,
r5 = -3.027734654434169996032905158145259713083E4L,
r6 = -4.501995652861105629217250715790764371267E2L,
/* r6 = 1.000000000000000000000000000000000000000E0 */
/* lgam(x) = ( x - 0.5 ) * log(x) - x + LS2PI + 1/x w(1/x^2)
x >= 8
Peak relative error 1.51e-21
w0 = LS2PI - 0.5 */
w0 = 4.189385332046727417803e-1L,
w1 = 8.333333333333331447505E-2L,
w2 = -2.777777777750349603440E-3L,
w3 = 7.936507795855070755671E-4L,
w4 = -5.952345851765688514613E-4L,
w5 = 8.412723297322498080632E-4L,
w6 = -1.880801938119376907179E-3L,
w7 = 4.885026142432270781165E-3L;
long double lgammal_r(long double x, int *sg) {
long double t, y, z, nadj, p, p1, p2, q, r, w;
union ldshape u = {x};
uint32_t ix = (u.i.se & 0x7fffU)<<16 | u.i.m>>48;
int sign = u.i.se >> 15;
int i;
*sg = 1;
/* purge off +-inf, NaN, +-0, tiny and negative arguments */
if (ix >= 0x7fff0000)
return x * x;
if (ix < 0x3fc08000) { /* |x|<2**-63, return -log(|x|) */
if (sign) {
*sg = -1;
x = -x;
}
return -logl(x);
}
if (sign) {
x = -x;
t = sin(pi * x);
if (t == 0.0)
return 1.0 / (x-x); /* -integer */
if (t > 0.0)
*sg = -1;
else
t = -t;
nadj = logl(pi / (t * x));
}
/* purge off 1 and 2 (so the sign is ok with downward rounding) */
if ((ix == 0x3fff8000 || ix == 0x40008000) && u.i.m == 0) {
r = 0;
} else if (ix < 0x40008000) { /* x < 2.0 */
if (ix <= 0x3ffee666) { /* 8.99993896484375e-1 */
/* lgamma(x) = lgamma(x+1) - log(x) */
r = -logl(x);
if (ix >= 0x3ffebb4a) { /* 7.31597900390625e-1 */
y = x - 1.0;
i = 0;
} else if (ix >= 0x3ffced33) { /* 2.31639862060546875e-1 */
y = x - (tc - 1.0);
i = 1;
} else { /* x < 0.23 */
y = x;
i = 2;
}
} else {
r = 0.0;
if (ix >= 0x3fffdda6) { /* 1.73162841796875 */
/* [1.7316,2] */
y = x - 2.0;
i = 0;
} else if (ix >= 0x3fff9da6) { /* 1.23162841796875 */
/* [1.23,1.73] */
y = x - tc;
i = 1;
} else {
/* [0.9, 1.23] */
y = x - 1.0;
i = 2;
}
}
switch (i) {
case 0:
p1 = a0 + y * (a1 + y * (a2 + y * (a3 + y * (a4 + y * a5))));
p2 = b0 + y * (b1 + y * (b2 + y * (b3 + y * (b4 + y))));
r += 0.5 * y + y * p1/p2;
break;
case 1:
p1 = g0 + y * (g1 + y * (g2 + y * (g3 + y * (g4 + y * (g5 + y * g6)))));
p2 = h0 + y * (h1 + y * (h2 + y * (h3 + y * (h4 + y * (h5 + y)))));
p = tt + y * p1/p2;
r += (tf + p);
break;
case 2:
p1 = y * (u0 + y * (u1 + y * (u2 + y * (u3 + y * (u4 + y * (u5 + y * u6))))));
p2 = v0 + y * (v1 + y * (v2 + y * (v3 + y * (v4 + y * (v5 + y)))));
r += (-0.5 * y + p1 / p2);
}
} else if (ix < 0x40028000) { /* 8.0 */
/* x < 8.0 */
i = (int)x;
y = x - (double)i;
p = y * (s0 + y * (s1 + y * (s2 + y * (s3 + y * (s4 + y * (s5 + y * s6))))));
q = r0 + y * (r1 + y * (r2 + y * (r3 + y * (r4 + y * (r5 + y * (r6 + y))))));
r = 0.5 * y + p / q;
z = 1.0;
/* lgamma(1+s) = log(s) + lgamma(s) */
switch (i) {
case 7:
z *= (y + 6.0); /* FALLTHRU */
case 6:
z *= (y + 5.0); /* FALLTHRU */
case 5:
z *= (y + 4.0); /* FALLTHRU */
case 4:
z *= (y + 3.0); /* FALLTHRU */
case 3:
z *= (y + 2.0); /* FALLTHRU */
r += logl(z);
break;
}
} else if (ix < 0x40418000) { /* 2^66 */
/* 8.0 <= x < 2**66 */
t = logl(x);
z = 1.0 / x;
y = z * z;
w = w0 + z * (w1 + y * (w2 + y * (w3 + y * (w4 + y * (w5 + y * (w6 + y * w7))))));
r = (x - 0.5) * (t - 1.0) + w;
} else /* 2**66 <= x <= inf */
r = x * (logl(x) - 1.0);
if (sign)
r = nadj - r;
return r;
}
#elif LDBL_MANT_DIG == 113 && LDBL_MAX_EXP == 16384
// TODO: broken implementation to make things compile
double lgamma_r(double x, int *sg);
long double lgammal_r(long double x, int *sg)
{
return lgamma_r(x, sg);
}
#endif
int signgam_lgammal;
long double lgammal(long double x)
{
return lgammal_r(x, &signgam_lgammal);
}

185
base/glibc-compatibility/musl/powf.c Normal file
View File

@ -0,0 +1,185 @@
/*
* Copyright (c) 2017-2018, Arm Limited.
* SPDX-License-Identifier: MIT
*/
#include <math.h>
#include <stdint.h>
#include "libm.h"
#include "exp2f_data.h"
#include "powf_data.h"
/*
POWF_LOG2_POLY_ORDER = 5
EXP2F_TABLE_BITS = 5
ULP error: 0.82 (~ 0.5 + relerr*2^24)
relerr: 1.27 * 2^-26 (Relative error ~= 128*Ln2*relerr_log2 + relerr_exp2)
relerr_log2: 1.83 * 2^-33 (Relative error of logx.)
relerr_exp2: 1.69 * 2^-34 (Relative error of exp2(ylogx).)
*/
#define N (1 << POWF_LOG2_TABLE_BITS)
#define T __powf_log2_data.tab
#define A __powf_log2_data.poly
#define OFF 0x3f330000
/* Subnormal input is normalized so ix has negative biased exponent.
Output is multiplied by N (POWF_SCALE) if TOINT_INTRINICS is set. */
static inline double_t log2_inline(uint32_t ix)
{
double_t z, r, r2, r4, p, q, y, y0, invc, logc;
uint32_t iz, top, tmp;
int k, i;
/* x = 2^k z; where z is in range [OFF,2*OFF] and exact.
The range is split into N subintervals.
The ith subinterval contains z and c is near its center. */
tmp = ix - OFF;
i = (tmp >> (23 - POWF_LOG2_TABLE_BITS)) % N;
top = tmp & 0xff800000;
iz = ix - top;
k = (int32_t)top >> (23 - POWF_SCALE_BITS); /* arithmetic shift */
invc = T[i].invc;
logc = T[i].logc;
z = (double_t)asfloat(iz);
/* log2(x) = log1p(z/c-1)/ln2 + log2(c) + k */
r = z * invc - 1;
y0 = logc + (double_t)k;
/* Pipelined polynomial evaluation to approximate log1p(r)/ln2. */
r2 = r * r;
y = A[0] * r + A[1];
p = A[2] * r + A[3];
r4 = r2 * r2;
q = A[4] * r + y0;
q = p * r2 + q;
y = y * r4 + q;
return y;
}
#undef N
#undef T
#define N (1 << EXP2F_TABLE_BITS)
#define T __exp2f_data.tab
#define SIGN_BIAS (1 << (EXP2F_TABLE_BITS + 11))
/* The output of log2 and thus the input of exp2 is either scaled by N
(in case of fast toint intrinsics) or not. The unscaled xd must be
in [-1021,1023], sign_bias sets the sign of the result. */
static inline float exp2_inline(double_t xd, uint32_t sign_bias)
{
uint64_t ki, ski, t;
double_t kd, z, r, r2, y, s;
#if TOINT_INTRINSICS
#define C __exp2f_data.poly_scaled
/* N*x = k + r with r in [-1/2, 1/2] */
kd = roundtoint(xd); /* k */
ki = converttoint(xd);
#else
#define C __exp2f_data.poly
#define SHIFT __exp2f_data.shift_scaled
/* x = k/N + r with r in [-1/(2N), 1/(2N)] */
kd = eval_as_double(xd + SHIFT);
ki = asuint64(kd);
kd -= SHIFT; /* k/N */
#endif
r = xd - kd;
/* exp2(x) = 2^(k/N) * 2^r ~= s * (C0*r^3 + C1*r^2 + C2*r + 1) */
t = T[ki % N];
ski = ki + sign_bias;
t += ski << (52 - EXP2F_TABLE_BITS);
s = asdouble(t);
z = C[0] * r + C[1];
r2 = r * r;
y = C[2] * r + 1;
y = z * r2 + y;
y = y * s;
return eval_as_float(y);
}
/* Returns 0 if not int, 1 if odd int, 2 if even int. The argument is
the bit representation of a non-zero finite floating-point value. */
static inline int checkint(uint32_t iy)
{
int e = iy >> 23 & 0xff;
if (e < 0x7f)
return 0;
if (e > 0x7f + 23)
return 2;
if (iy & ((1 << (0x7f + 23 - e)) - 1))
return 0;
if (iy & (1 << (0x7f + 23 - e)))
return 1;
return 2;
}
static inline int zeroinfnan(uint32_t ix)
{
return 2 * ix - 1 >= 2u * 0x7f800000 - 1;
}
float powf(float x, float y)
{
uint32_t sign_bias = 0;
uint32_t ix, iy;
ix = asuint(x);
iy = asuint(y);
if (predict_false(ix - 0x00800000 >= 0x7f800000 - 0x00800000 ||
zeroinfnan(iy))) {
/* Either (x < 0x1p-126 or inf or nan) or (y is 0 or inf or nan). */
if (predict_false(zeroinfnan(iy))) {
if (2 * iy == 0)
return issignalingf_inline(x) ? x + y : 1.0f;
if (ix == 0x3f800000)
return issignalingf_inline(y) ? x + y : 1.0f;
if (2 * ix > 2u * 0x7f800000 ||
2 * iy > 2u * 0x7f800000)
return x + y;
if (2 * ix == 2 * 0x3f800000)
return 1.0f;
if ((2 * ix < 2 * 0x3f800000) == !(iy & 0x80000000))
return 0.0f; /* |x|<1 && y==inf or |x|>1 && y==-inf. */
return y * y;
}
if (predict_false(zeroinfnan(ix))) {
float_t x2 = x * x;
if (ix & 0x80000000 && checkint(iy) == 1)
x2 = -x2;
/* Without the barrier some versions of clang hoist the 1/x2 and
thus division by zero exception can be signaled spuriously. */
return iy & 0x80000000 ? fp_barrierf(1 / x2) : x2;
}
/* x and y are non-zero finite. */
if (ix & 0x80000000) {
/* Finite x < 0. */
int yint = checkint(iy);
if (yint == 0)
return __math_invalidf(x);
if (yint == 1)
sign_bias = SIGN_BIAS;
ix &= 0x7fffffff;
}
if (ix < 0x00800000) {
/* Normalize subnormal x so exponent becomes negative. */
ix = asuint(x * 0x1p23f);
ix &= 0x7fffffff;
ix -= 23 << 23;
}
}
double_t logx = log2_inline(ix);
double_t ylogx = y * logx; /* cannot overflow, y is single prec. */
if (predict_false((asuint64(ylogx) >> 47 & 0xffff) >=
asuint64(126.0 * POWF_SCALE) >> 47)) {
/* |y*log(x)| >= 126. */
if (ylogx > 0x1.fffffffd1d571p+6 * POWF_SCALE)
return __math_oflowf(sign_bias);
if (ylogx <= -150.0 * POWF_SCALE)
return __math_uflowf(sign_bias);
}
return exp2_inline(ylogx, sign_bias);
}

34
base/glibc-compatibility/musl/powf_data.c Normal file
View File

@ -0,0 +1,34 @@
/*
* Data definition for powf.
*
* Copyright (c) 2017-2018, Arm Limited.
* SPDX-License-Identifier: MIT
*/
#include "powf_data.h"
const struct powf_log2_data __powf_log2_data = {
.tab = {
{ 0x1.661ec79f8f3bep+0, -0x1.efec65b963019p-2 * POWF_SCALE },
{ 0x1.571ed4aaf883dp+0, -0x1.b0b6832d4fca4p-2 * POWF_SCALE },
{ 0x1.49539f0f010bp+0, -0x1.7418b0a1fb77bp-2 * POWF_SCALE },
{ 0x1.3c995b0b80385p+0, -0x1.39de91a6dcf7bp-2 * POWF_SCALE },
{ 0x1.30d190c8864a5p+0, -0x1.01d9bf3f2b631p-2 * POWF_SCALE },
{ 0x1.25e227b0b8eap+0, -0x1.97c1d1b3b7afp-3 * POWF_SCALE },
{ 0x1.1bb4a4a1a343fp+0, -0x1.2f9e393af3c9fp-3 * POWF_SCALE },
{ 0x1.12358f08ae5bap+0, -0x1.960cbbf788d5cp-4 * POWF_SCALE },
{ 0x1.0953f419900a7p+0, -0x1.a6f9db6475fcep-5 * POWF_SCALE },
{ 0x1p+0, 0x0p+0 * POWF_SCALE },
{ 0x1.e608cfd9a47acp-1, 0x1.338ca9f24f53dp-4 * POWF_SCALE },
{ 0x1.ca4b31f026aap-1, 0x1.476a9543891bap-3 * POWF_SCALE },
{ 0x1.b2036576afce6p-1, 0x1.e840b4ac4e4d2p-3 * POWF_SCALE },
{ 0x1.9c2d163a1aa2dp-1, 0x1.40645f0c6651cp-2 * POWF_SCALE },
{ 0x1.886e6037841edp-1, 0x1.88e9c2c1b9ff8p-2 * POWF_SCALE },
{ 0x1.767dcf5534862p-1, 0x1.ce0a44eb17bccp-2 * POWF_SCALE },
},
.poly = {
0x1.27616c9496e0bp-2 * POWF_SCALE, -0x1.71969a075c67ap-2 * POWF_SCALE,
0x1.ec70a6ca7baddp-2 * POWF_SCALE, -0x1.7154748bef6c8p-1 * POWF_SCALE,
0x1.71547652ab82bp0 * POWF_SCALE,
}
};

26
base/glibc-compatibility/musl/powf_data.h Normal file
View File

@ -0,0 +1,26 @@
/*
* Copyright (c) 2017-2018, Arm Limited.
* SPDX-License-Identifier: MIT
*/
#ifndef _POWF_DATA_H
#define _POWF_DATA_H
#include "libm.h"
#include "exp2f_data.h"
#define POWF_LOG2_TABLE_BITS 4
#define POWF_LOG2_POLY_ORDER 5
#if TOINT_INTRINSICS
#define POWF_SCALE_BITS EXP2F_TABLE_BITS
#else
#define POWF_SCALE_BITS 0
#endif
#define POWF_SCALE ((double)(1 << POWF_SCALE_BITS))
extern hidden const struct powf_log2_data {
struct {
double invc, logc;
} tab[1 << POWF_LOG2_TABLE_BITS];
double poly[POWF_LOG2_POLY_ORDER];
} __powf_log2_data;
#endif

525
base/glibc-compatibility/musl/powl.c Normal file
View File

@ -0,0 +1,525 @@
/* origin: OpenBSD /usr/src/lib/libm/src/ld80/e_powl.c */
/*
* Copyright (c) 2008 Stephen L. Moshier <steve@moshier.net>
*
* Permission to use, copy, modify, and distribute this software for any
* purpose with or without fee is hereby granted, provided that the above
* copyright notice and this permission notice appear in all copies.
*
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
* ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
* ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
* OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/
/* powl.c
*
* Power function, long double precision
*
*
* SYNOPSIS:
*
* long double x, y, z, powl();
*
* z = powl( x, y );
*
*
* DESCRIPTION:
*
* Computes x raised to the yth power. Analytically,
*
* x**y = exp( y log(x) ).
*
* Following Cody and Waite, this program uses a lookup table
* of 2**-i/32 and pseudo extended precision arithmetic to
* obtain several extra bits of accuracy in both the logarithm
* and the exponential.
*
*
* ACCURACY:
*
* The relative error of pow(x,y) can be estimated
* by y dl ln(2), where dl is the absolute error of
* the internally computed base 2 logarithm. At the ends
* of the approximation interval the logarithm equal 1/32
* and its relative error is about 1 lsb = 1.1e-19. Hence
* the predicted relative error in the result is 2.3e-21 y .
*
* Relative error:
* arithmetic domain # trials peak rms
*
* IEEE +-1000 40000 2.8e-18 3.7e-19
* .001 < x < 1000, with log(x) uniformly distributed.
* -1000 < y < 1000, y uniformly distributed.
*
* IEEE 0,8700 60000 6.5e-18 1.0e-18
* 0.99 < x < 1.01, 0 < y < 8700, uniformly distributed.
*
*
* ERROR MESSAGES:
*
* message condition value returned
* pow overflow x**y > MAXNUM INFINITY
* pow underflow x**y < 1/MAXNUM 0.0
* pow domain x<0 and y noninteger 0.0
*
*/
#include "libm.h"
#if LDBL_MANT_DIG == 53 && LDBL_MAX_EXP == 1024
long double powl(long double x, long double y)
{
return pow(x, y);
}
#elif LDBL_MANT_DIG == 64 && LDBL_MAX_EXP == 16384
/* Table size */
#define NXT 32
/* log(1+x) = x - .5x^2 + x^3 * P(z)/Q(z)
* on the domain 2^(-1/32) - 1 <= x <= 2^(1/32) - 1
*/
static const long double P[] = {
8.3319510773868690346226E-4L,
4.9000050881978028599627E-1L,
1.7500123722550302671919E0L,
1.4000100839971580279335E0L,
};
static const long double Q[] = {
/* 1.0000000000000000000000E0L,*/
5.2500282295834889175431E0L,
8.4000598057587009834666E0L,
4.2000302519914740834728E0L,
};
/* A[i] = 2^(-i/32), rounded to IEEE long double precision.
* If i is even, A[i] + B[i/2] gives additional accuracy.
*/
static const long double A[33] = {
1.0000000000000000000000E0L,
9.7857206208770013448287E-1L,
9.5760328069857364691013E-1L,
9.3708381705514995065011E-1L,
9.1700404320467123175367E-1L,
8.9735453750155359320742E-1L,
8.7812608018664974155474E-1L,
8.5930964906123895780165E-1L,
8.4089641525371454301892E-1L,
8.2287773907698242225554E-1L,
8.0524516597462715409607E-1L,
7.8799042255394324325455E-1L,
7.7110541270397041179298E-1L,
7.5458221379671136985669E-1L,
7.3841307296974965571198E-1L,
7.2259040348852331001267E-1L,
7.0710678118654752438189E-1L,
6.9195494098191597746178E-1L,
6.7712777346844636413344E-1L,
6.6261832157987064729696E-1L,
6.4841977732550483296079E-1L,
6.3452547859586661129850E-1L,
6.2092890603674202431705E-1L,
6.0762367999023443907803E-1L,
5.9460355750136053334378E-1L,
5.8186242938878875689693E-1L,
5.6939431737834582684856E-1L,
5.5719337129794626814472E-1L,
5.4525386633262882960438E-1L,
5.3357020033841180906486E-1L,
5.2213689121370692017331E-1L,
5.1094857432705833910408E-1L,
5.0000000000000000000000E-1L,
};
static const long double B[17] = {
0.0000000000000000000000E0L,
2.6176170809902549338711E-20L,
-1.0126791927256478897086E-20L,
1.3438228172316276937655E-21L,
1.2207982955417546912101E-20L,
-6.3084814358060867200133E-21L,
1.3164426894366316434230E-20L,
-1.8527916071632873716786E-20L,
1.8950325588932570796551E-20L,
1.5564775779538780478155E-20L,
6.0859793637556860974380E-21L,
-2.0208749253662532228949E-20L,
1.4966292219224761844552E-20L,
3.3540909728056476875639E-21L,
-8.6987564101742849540743E-22L,
-1.2327176863327626135542E-20L,
0.0000000000000000000000E0L,
};
/* 2^x = 1 + x P(x),
* on the interval -1/32 <= x <= 0
*/
static const long double R[] = {
1.5089970579127659901157E-5L,
1.5402715328927013076125E-4L,
1.3333556028915671091390E-3L,
9.6181291046036762031786E-3L,
5.5504108664798463044015E-2L,
2.4022650695910062854352E-1L,
6.9314718055994530931447E-1L,
};
#define MEXP (NXT*16384.0L)
/* The following if denormal numbers are supported, else -MEXP: */
#define MNEXP (-NXT*(16384.0L+64.0L))
/* log2(e) - 1 */
#define LOG2EA 0.44269504088896340735992L
#define F W
#define Fa Wa
#define Fb Wb
#define G W
#define Ga Wa
#define Gb u
#define H W
#define Ha Wb
#define Hb Wb
static const long double MAXLOGL = 1.1356523406294143949492E4L;
static const long double MINLOGL = -1.13994985314888605586758E4L;
static const long double LOGE2L = 6.9314718055994530941723E-1L;
static const long double huge = 0x1p10000L;
/* XXX Prevent gcc from erroneously constant folding this. */
static const volatile long double twom10000 = 0x1p-10000L;
static long double reducl(long double);
static long double powil(long double, int);
long double __polevll(long double x, const long double *P, int n);
long double __p1evll(long double x, const long double *P, int n);
long double powl(long double x, long double y)
{
/* double F, Fa, Fb, G, Ga, Gb, H, Ha, Hb */
int i, nflg, iyflg, yoddint;
long e;
volatile long double z=0;
long double w=0, W=0, Wa=0, Wb=0, ya=0, yb=0, u=0;
/* make sure no invalid exception is raised by nan comparision */
if (isnan(x)) {
if (!isnan(y) && y == 0.0)
return 1.0;
return x;
}
if (isnan(y)) {
if (x == 1.0)
return 1.0;
return y;
}
if (x == 1.0)
return 1.0; /* 1**y = 1, even if y is nan */
if (x == -1.0 && !isfinite(y))
return 1.0; /* -1**inf = 1 */
if (y == 0.0)
return 1.0; /* x**0 = 1, even if x is nan */
if (y == 1.0)
return x;
if (y >= LDBL_MAX) {
if (x > 1.0 || x < -1.0)
return INFINITY;
if (x != 0.0)
return 0.0;
}
if (y <= -LDBL_MAX) {
if (x > 1.0 || x < -1.0)
return 0.0;
if (x != 0.0 || y == -INFINITY)
return INFINITY;
}
if (x >= LDBL_MAX) {
if (y > 0.0)
return INFINITY;
return 0.0;
}
w = floorl(y);
/* Set iyflg to 1 if y is an integer. */
iyflg = 0;
if (w == y)
iyflg = 1;
/* Test for odd integer y. */
yoddint = 0;
if (iyflg) {
ya = fabsl(y);
ya = floorl(0.5 * ya);
yb = 0.5 * fabsl(w);
if( ya != yb )
yoddint = 1;
}
if (x <= -LDBL_MAX) {
if (y > 0.0) {
if (yoddint)
return -INFINITY;
return INFINITY;
}
if (y < 0.0) {
if (yoddint)
return -0.0;
return 0.0;
}
}
nflg = 0; /* (x<0)**(odd int) */
if (x <= 0.0) {
if (x == 0.0) {
if (y < 0.0) {
if (signbit(x) && yoddint)
/* (-0.0)**(-odd int) = -inf, divbyzero */
return -1.0/0.0;
/* (+-0.0)**(negative) = inf, divbyzero */
return 1.0/0.0;
}
if (signbit(x) && yoddint)
return -0.0;
return 0.0;
}
if (iyflg == 0)
return (x - x) / (x - x); /* (x<0)**(non-int) is NaN */
/* (x<0)**(integer) */
if (yoddint)
nflg = 1; /* negate result */
x = -x;
}
/* (+integer)**(integer) */
if (iyflg && floorl(x) == x && fabsl(y) < 32768.0) {
w = powil(x, (int)y);
return nflg ? -w : w;
}
/* separate significand from exponent */
x = frexpl(x, &i);
e = i;
/* find significand in antilog table A[] */
i = 1;
if (x <= A[17])
i = 17;
if (x <= A[i+8])
i += 8;
if (x <= A[i+4])
i += 4;
if (x <= A[i+2])
i += 2;
if (x >= A[1])
i = -1;
i += 1;
/* Find (x - A[i])/A[i]
* in order to compute log(x/A[i]):
*
* log(x) = log( a x/a ) = log(a) + log(x/a)
*
* log(x/a) = log(1+v), v = x/a - 1 = (x-a)/a
*/
x -= A[i];
x -= B[i/2];
x /= A[i];
/* rational approximation for log(1+v):
*
* log(1+v) = v - v**2/2 + v**3 P(v) / Q(v)
*/
z = x*x;
w = x * (z * __polevll(x, P, 3) / __p1evll(x, Q, 3));
w = w - 0.5*z;
/* Convert to base 2 logarithm:
* multiply by log2(e) = 1 + LOG2EA
*/
z = LOG2EA * w;
z += w;
z += LOG2EA * x;
z += x;
/* Compute exponent term of the base 2 logarithm. */
w = -i;
w /= NXT;
w += e;
/* Now base 2 log of x is w + z. */
/* Multiply base 2 log by y, in extended precision. */
/* separate y into large part ya
* and small part yb less than 1/NXT
*/
ya = reducl(y);
yb = y - ya;
/* (w+z)(ya+yb)
* = w*ya + w*yb + z*y
*/
F = z * y + w * yb;
Fa = reducl(F);
Fb = F - Fa;
G = Fa + w * ya;
Ga = reducl(G);
Gb = G - Ga;
H = Fb + Gb;
Ha = reducl(H);
w = (Ga + Ha) * NXT;
/* Test the power of 2 for overflow */
if (w > MEXP)
return huge * huge; /* overflow */
if (w < MNEXP)
return twom10000 * twom10000; /* underflow */
e = w;
Hb = H - Ha;
if (Hb > 0.0) {
e += 1;
Hb -= 1.0/NXT; /*0.0625L;*/
}
/* Now the product y * log2(x) = Hb + e/NXT.
*
* Compute base 2 exponential of Hb,
* where -0.0625 <= Hb <= 0.
*/
z = Hb * __polevll(Hb, R, 6); /* z = 2**Hb - 1 */
/* Express e/NXT as an integer plus a negative number of (1/NXT)ths.
* Find lookup table entry for the fractional power of 2.
*/
if (e < 0)
i = 0;
else
i = 1;
i = e/NXT + i;
e = NXT*i - e;
w = A[e];
z = w * z; /* 2**-e * ( 1 + (2**Hb-1) ) */
z = z + w;
z = scalbnl(z, i); /* multiply by integer power of 2 */
if (nflg)
z = -z;
return z;
}
/* Find a multiple of 1/NXT that is within 1/NXT of x. */
static long double reducl(long double x)
{
long double t;
t = x * NXT;
t = floorl(t);
t = t / NXT;
return t;
}
/*
* Positive real raised to integer power, long double precision
*
*
* SYNOPSIS:
*
* long double x, y, powil();
* int n;
*
* y = powil( x, n );
*
*
* DESCRIPTION:
*
* Returns argument x>0 raised to the nth power.
* The routine efficiently decomposes n as a sum of powers of
* two. The desired power is a product of two-to-the-kth
* powers of x. Thus to compute the 32767 power of x requires
* 28 multiplications instead of 32767 multiplications.
*
*
* ACCURACY:
*
* Relative error:
* arithmetic x domain n domain # trials peak rms
* IEEE .001,1000 -1022,1023 50000 4.3e-17 7.8e-18
* IEEE 1,2 -1022,1023 20000 3.9e-17 7.6e-18
* IEEE .99,1.01 0,8700 10000 3.6e-16 7.2e-17
*
* Returns MAXNUM on overflow, zero on underflow.
*/
static long double powil(long double x, int nn)
{
long double ww, y;
long double s;
int n, e, sign, lx;
if (nn == 0)
return 1.0;
if (nn < 0) {
sign = -1;
n = -nn;
} else {
sign = 1;
n = nn;
}
/* Overflow detection */
/* Calculate approximate logarithm of answer */
s = x;
s = frexpl( s, &lx);
e = (lx - 1)*n;
if ((e == 0) || (e > 64) || (e < -64)) {
s = (s - 7.0710678118654752e-1L) / (s + 7.0710678118654752e-1L);
s = (2.9142135623730950L * s - 0.5 + lx) * nn * LOGE2L;
} else {
s = LOGE2L * e;
}
if (s > MAXLOGL)
return huge * huge; /* overflow */
if (s < MINLOGL)
return twom10000 * twom10000; /* underflow */
/* Handle tiny denormal answer, but with less accuracy
* since roundoff error in 1.0/x will be amplified.
* The precise demarcation should be the gradual underflow threshold.
*/
if (s < -MAXLOGL+2.0) {
x = 1.0/x;
sign = -sign;
}
/* First bit of the power */
if (n & 1)
y = x;
else
y = 1.0;
ww = x;
n >>= 1;
while (n) {
ww = ww * ww; /* arg to the 2-to-the-kth power */
if (n & 1) /* if that bit is set, then include in product */
y *= ww;
n >>= 1;
}
if (sign < 0)
y = 1.0/y;
return y;
}
#elif LDBL_MANT_DIG == 113 && LDBL_MAX_EXP == 16384
// TODO: broken implementation to make things compile
long double powl(long double x, long double y)
{
return pow(x, y);
}
#endif

8
base/glibc-compatibility/musl/secure_getenv.c Normal file
View File

@ -0,0 +1,8 @@
#define _GNU_SOURCE
#include <stdlib.h>
#include <sys/auxv.h>
char * secure_getenv(const char * name)
{
return getauxval(AT_SECURE) ? NULL : getenv(name);
}

21
base/glibc-compatibility/musl/sync_file_range.c Normal file
View File

@ -0,0 +1,21 @@
#define _GNU_SOURCE
#include <fcntl.h>
#include <errno.h>
#include "syscall.h"
// works same in x86_64 && aarch64
#define __SYSCALL_LL_E(x) (x)
#define __SYSCALL_LL_O(x) (x)
int sync_file_range(int fd, off_t pos, off_t len, unsigned flags)
{
#if defined(SYS_sync_file_range2)
return syscall(SYS_sync_file_range2, fd, flags,
__SYSCALL_LL_E(pos), __SYSCALL_LL_E(len));
#elif defined(SYS_sync_file_range)
return __syscall(SYS_sync_file_range, fd,
__SYSCALL_LL_O(pos), __SYSCALL_LL_E(len), flags);
#else
return __syscall_ret(-ENOSYS);
#endif
}

8
base/glibc-compatibility/musl/syscall.h
View File

@ -13,3 +13,11 @@ long __syscall(syscall_arg_t, ...);
__attribute__((visibility("hidden")))
void *__vdsosym(const char *, const char *);
#define syscall(...) __syscall_ret(__syscall(__VA_ARGS__))
#define socketcall(...) __syscall_ret(__socketcall(__VA_ARGS__))
#define __socketcall(nm,a,b,c,d,e,f) __syscall(SYS_##nm, a, b, c, d, e, f)
#define socketcall_cp socketcall

16
base/glibc-compatibility/musl/vdso.c
View File

@ -40,24 +40,10 @@ static int checkver(Verdef *def, int vsym, const char *vername, char *strings)
#define OK_TYPES (1<<STT_NOTYPE | 1<<STT_OBJECT | 1<<STT_FUNC | 1<<STT_COMMON)
#define OK_BINDS (1<<STB_GLOBAL | 1<<STB_WEAK | 1<<STB_GNU_UNIQUE)
extern char** environ;
static Ehdr *eh = NULL;
void *__vdsosym(const char *vername, const char *name);
// We don't have libc struct available here. Compute aux vector manually.
__attribute__((constructor)) static void auxv_init()
{
size_t i, *auxv;
for (i=0; environ[i]; i++);
auxv = (void *)(environ+i+1);
for (i=0; auxv[i] != AT_SYSINFO_EHDR; i+=2)
if (!auxv[i]) return;
if (!auxv[i+1]) return;
eh = (void *)auxv[i+1];
}
void *__vdsosym(const char *vername, const char *name)
{
size_t i;
Ehdr * eh = (void *) getauxval(AT_SYSINFO_EHDR);
if (!eh) return 0;
Phdr *ph = (void *)((char *)eh + eh->e_phoff);
size_t *dynv=0, base=-1;

1
base/loggers/ExtendedLogChannel.cpp
View File

@ -23,6 +23,7 @@ ExtendedLogMessage ExtendedLogMessage::getFrom(const Poco::Message & base)
msg_ext.time_seconds = static_cast<UInt32>(tv.tv_sec);
msg_ext.time_microseconds = static_cast<UInt32>(tv.tv_usec);
msg_ext.time_in_microseconds = static_cast<UInt64>((tv.tv_sec) * 1000000U + (tv.tv_usec));
if (current_thread)
{

1
base/loggers/ExtendedLogChannel.h
View File

@ -23,6 +23,7 @@ public:
uint32_t time_seconds = 0;
uint32_t time_microseconds = 0;
uint64_t time_in_microseconds = 0;
uint64_t thread_id = 0;
std::string query_id;

2
base/loggers/Loggers.h
View File

@ -1,3 +1,5 @@
#pragma once
#include <optional>
#include <string>
#include <Poco/AutoPtr.h>

1
base/loggers/OwnSplitChannel.cpp
View File

@ -76,6 +76,7 @@ void OwnSplitChannel::logSplit(const Poco::Message & msg)
TextLogElement elem;
elem.event_time = msg_ext.time_seconds;
elem.event_time_microseconds = msg_ext.time_in_microseconds;
elem.microseconds = msg_ext.time_microseconds;
elem.thread_name = getThreadName();

4
base/loggers/ya.make
View File

@ -1,9 +1,13 @@
OWNER(g:clickhouse)
LIBRARY()
PEERDIR(
clickhouse/src/Common
)
CFLAGS(-g0)
SRCS(
ExtendedLogChannel.cpp
Loggers.cpp

8
base/mysqlxx/CMakeLists.txt
View File

@ -50,6 +50,14 @@ if (NOT USE_INTERNAL_MYSQL_LIBRARY AND OPENSSL_INCLUDE_DIR)
target_include_directories (mysqlxx SYSTEM PRIVATE ${OPENSSL_INCLUDE_DIR})
endif ()
if (NOT USE_INTERNAL_MYSQL_LIBRARY AND USE_STATIC_LIBRARIES)
message(WARNING "Statically linking with system mysql/mariadb only works "
"if mysql client libraries are built with same openssl version as "
"we are going to use now. It wouldn't work if GnuTLS is used. "
"Try -D\"USE_INTERNAL_MYSQL_LIBRARY\"=ON or -D\"ENABLE_MYSQL\"=OFF or "
"-D\"USE_STATIC_LIBRARIES\"=OFF")
endif ()
if (ENABLE_TESTS)
add_subdirectory (tests)
endif ()

10
base/mysqlxx/Connection.cpp
View File

@ -104,6 +104,11 @@ void Connection::connect(const char* db,
if (mysql_options(driver.get(), MYSQL_OPT_LOCAL_INFILE, &enable_local_infile_arg))
throw ConnectionFailed(errorMessage(driver.get()), mysql_errno(driver.get()));
/// Enables auto-reconnect.
bool reconnect = true;
if (mysql_options(driver.get(), MYSQL_OPT_RECONNECT, reinterpret_cast<const char *>(&reconnect)))
throw ConnectionFailed(errorMessage(driver.get()), mysql_errno(driver.get()));
/// Specifies particular ssl key and certificate if it needs
if (mysql_ssl_set(driver.get(), ifNotEmpty(ssl_key), ifNotEmpty(ssl_cert), ifNotEmpty(ssl_ca), nullptr, nullptr))
throw ConnectionFailed(errorMessage(driver.get()), mysql_errno(driver.get()));
@ -115,11 +120,6 @@ void Connection::connect(const char* db,
if (mysql_set_character_set(driver.get(), "UTF8"))
throw ConnectionFailed(errorMessage(driver.get()), mysql_errno(driver.get()));
/// Enables auto-reconnect.
my_bool reconnect = true;
if (mysql_options(driver.get(), MYSQL_OPT_RECONNECT, reinterpret_cast<const char *>(&reconnect)))
throw ConnectionFailed(errorMessage(driver.get()), mysql_errno(driver.get()));
is_connected = true;
}

73
base/mysqlxx/Pool.cpp
View File

@ -21,21 +21,20 @@ void Pool::Entry::incrementRefCount()
{
if (!data)
return;
++data->ref_count;
if (data->ref_count == 1)
/// First reference, initialize thread
if (data->ref_count.fetch_add(1) == 0)
mysql_thread_init();
}
void Pool::Entry::decrementRefCount()
{
if (!data)
return;
if (data->ref_count > 0)
{
--data->ref_count;
if (data->ref_count == 0)
mysql_thread_end();
}
/// We were the last user of this thread, deinitialize it
if (data->ref_count.fetch_sub(1) == 1)
mysql_thread_end();
}
@ -152,28 +151,39 @@ Pool::Entry Pool::tryGet()
initialize();
/// Searching for connection which was established but wasn't used.
for (auto & connection : connections)
/// Try to pick an idle connection from already allocated
for (auto connection_it = connections.cbegin(); connection_it != connections.cend();)
{
if (connection->ref_count == 0)
Connection * connection_ptr = *connection_it;
/// Fixme: There is a race condition here b/c we do not synchronize with Pool::Entry's copy-assignment operator
if (connection_ptr->ref_count == 0)
{
Entry res(connection, this);
return res.tryForceConnected() ? res : Entry();
Entry res(connection_ptr, this);
if (res.tryForceConnected()) /// Tries to reestablish connection as well
return res;
auto & logger = Poco::Util::Application::instance().logger();
logger.information("Idle connection to mysql server cannot be recovered, dropping it.");
/// This one is disconnected, cannot be reestablished and so needs to be disposed of.
connection_it = connections.erase(connection_it);
::delete connection_ptr; /// TODO: Manual memory management is awkward (matches allocConnection() method)
}
else
++connection_it;
}
/// Throws if pool is overflowed.
if (connections.size() >= max_connections)
throw Poco::Exception("mysqlxx::Pool is full");
/// Allocates new connection.
Connection * conn = allocConnection(true);
if (conn)
return Entry(conn, this);
Connection * connection_ptr = allocConnection(true);
if (connection_ptr)
return {connection_ptr, this};
return Entry();
return {};
}
void Pool::removeConnection(Connection* connection)
{
std::lock_guard<std::mutex> lock(mutex);
@ -201,11 +211,9 @@ void Pool::Entry::forceConnected() const
throw Poco::RuntimeException("Tried to access NULL database connection.");
Poco::Util::Application & app = Poco::Util::Application::instance();
if (data->conn.ping())
return;
bool first = true;
do
while (!tryForceConnected())
{
if (first)
first = false;
@ -227,7 +235,26 @@ void Pool::Entry::forceConnected() const
pool->rw_timeout,
pool->enable_local_infile);
}
while (!data->conn.ping());
}
bool Pool::Entry::tryForceConnected() const
{
auto * const mysql_driver = data->conn.getDriver();
const auto prev_connection_id = mysql_thread_id(mysql_driver);
if (data->conn.ping()) /// Attempts to reestablish lost connection
{
const auto current_connection_id = mysql_thread_id(mysql_driver);
if (prev_connection_id != current_connection_id)
{
auto & logger = Poco::Util::Application::instance().logger();
logger.information("Connection to mysql server has been reestablished. Connection id changed: %lu -> %lu",
prev_connection_id, current_connection_id);
}
return true;
}
return false;
}

10
base/mysqlxx/Pool.h
View File

@ -3,6 +3,7 @@
#include <list>
#include <memory>
#include <mutex>
#include <atomic>
#include <Poco/Exception.h>
#include <mysqlxx/Connection.h>
@ -35,7 +36,9 @@ protected:
struct Connection
{
mysqlxx::Connection conn;
int ref_count = 0;
/// Ref count modified in constructor/descructor of Entry
/// but also read in pool code.
std::atomic<int> ref_count = 0;
};
public:
@ -124,10 +127,7 @@ public:
void forceConnected() const;
/// Connects to database. If connection is failed then returns false.
bool tryForceConnected() const
{
return data->conn.ping();
}
bool tryForceConnected() const;
void incrementRefCount();
void decrementRefCount();

8
base/mysqlxx/ResultBase.cpp
View File

@ -22,4 +22,12 @@ ResultBase::~ResultBase()
mysql_free_result(res);
}
std::string ResultBase::getFieldName(size_t n) const
{
if (num_fields <= n)
throw Exception(std::string("Unknown column position ") + std::to_string(n));
return fields[n].name;
}
}

9
base/mysqlxx/ResultBase.h
View File

@ -1,9 +1,7 @@
#pragma once
#include <boost/noncopyable.hpp>
#include <mysqlxx/Types.h>
namespace mysqlxx
{
@ -22,12 +20,19 @@ class ResultBase
public:
ResultBase(MYSQL_RES * res_, Connection * conn_, const Query * query_);
ResultBase(const ResultBase &) = delete;
ResultBase & operator=(const ResultBase &) = delete;
ResultBase(ResultBase &&) = default;
ResultBase & operator=(ResultBase &&) = default;
Connection * getConnection() { return conn; }
MYSQL_FIELDS getFields() { return fields; }
unsigned getNumFields() { return num_fields; }
MYSQL_RES * getRes() { return res; }
const Query * getQuery() const { return query; }
std::string getFieldName(size_t n) const;
virtual ~ResultBase();
protected:

18
base/mysqlxx/Value.h
View File

@ -254,7 +254,23 @@ template <> inline std::string Value::get<std::string >() cons
template <> inline LocalDate Value::get<LocalDate >() const { return getDate(); }
template <> inline LocalDateTime Value::get<LocalDateTime >() const { return getDateTime(); }
template <typename T> inline T Value::get() const { return T(*this); }
namespace details
{
// To avoid stack overflow when converting to type with no appropriate c-tor,
// resulting in endless recursive calls from `Value::get<T>()` to `Value::operator T()` to `Value::get<T>()` to ...
template <typename T, typename std::enable_if_t<std::is_constructible_v<T, Value>>>
inline T contructFromValue(const Value & val)
{
return T(val);
}
}
template <typename T>
inline T Value::get() const
{
return details::contructFromValue<T>(*this);
}
inline std::ostream & operator<< (std::ostream & ostr, const Value & x)

3
base/mysqlxx/tests/CMakeLists.txt
View File

@ -1,2 +1,5 @@
add_executable (mysqlxx_test mysqlxx_test.cpp)
target_link_libraries (mysqlxx_test PRIVATE mysqlxx)
add_executable (mysqlxx_pool_test mysqlxx_pool_test.cpp)
target_link_libraries (mysqlxx_pool_test PRIVATE mysqlxx)

98
base/mysqlxx/tests/mysqlxx_pool_test.cpp Normal file
View File

@ -0,0 +1,98 @@
#include <mysqlxx/mysqlxx.h>
#include <chrono>
#include <iostream>
#include <sstream>
#include <thread>
namespace
{
mysqlxx::Pool::Entry getWithFailover(mysqlxx::Pool & connections_pool)
{
using namespace std::chrono;
constexpr size_t max_tries = 3;
mysqlxx::Pool::Entry worker_connection;
for (size_t try_no = 1; try_no <= max_tries; ++try_no)
{
try
{
worker_connection = connections_pool.tryGet();
if (!worker_connection.isNull())
{
return worker_connection;
}
}
catch (const Poco::Exception & e)
{
if (e.displayText().find("mysqlxx::Pool is full") != std::string::npos)
{
std::cerr << e.displayText() << std::endl;
}
std::cerr << "Connection to " << connections_pool.getDescription() << " failed: " << e.displayText() << std::endl;
}
std::clog << "Connection to all replicas failed " << try_no << " times" << std::endl;
std::this_thread::sleep_for(1s);
}
std::stringstream message;
message << "Connections to all replicas failed: " << connections_pool.getDescription();
throw Poco::Exception(message.str());
}
}
int main(int, char **)
{
using namespace std::chrono;
const char * remote_mysql = "localhost";
const std::string test_query = "SHOW DATABASES";
mysqlxx::Pool mysql_conn_pool("", remote_mysql, "default", "10203040", 3306);
size_t iteration = 0;
while (++iteration)
{
std::clog << "Iteration: " << iteration << std::endl;
try
{
std::clog << "Acquiring DB connection ...";
mysqlxx::Pool::Entry worker = getWithFailover(mysql_conn_pool);
std::clog << "ok" << std::endl;
std::clog << "Preparing query (5s sleep) ...";
std::this_thread::sleep_for(5s);
mysqlxx::Query query = worker->query();
query << test_query;
std::clog << "ok" << std::endl;
std::clog << "Querying result (5s sleep) ...";
std::this_thread::sleep_for(5s);
mysqlxx::UseQueryResult result = query.use();
std::clog << "ok" << std::endl;
std::clog << "Fetching result data (5s sleep) ...";
std::this_thread::sleep_for(5s);
size_t rows_count = 0;
while (result.fetch())
++rows_count;
std::clog << "ok" << std::endl;
std::clog << "Read " << rows_count << " rows." << std::endl;
}
catch (const Poco::Exception & e)
{
std::cerr << "Iteration FAILED:\n" << e.displayText() << std::endl;
}
std::clog << "====================" << std::endl;
std::this_thread::sleep_for(3s);
}
}

9
base/pcg-random/pcg_random.hpp
View File

@ -113,6 +113,12 @@
#include "pcg_extras.hpp"
namespace DB
{
struct PcgSerializer;
struct PcgDeserializer;
}
namespace pcg_detail {
using namespace pcg_extras;
@ -557,6 +563,9 @@ public:
engine<xtype1, itype1,
output_mixin1, output_previous1,
stream_mixin1, multiplier_mixin1>& rng);
friend ::DB::PcgSerializer;
friend ::DB::PcgDeserializer;
};
template <typename CharT, typename Traits,

2
base/pcg-random/ya.make
View File

@ -1,3 +1,5 @@
OWNER(g:clickhouse)
LIBRARY()
ADDINCL (GLOBAL clickhouse/base/pcg-random)

4
base/readpassphrase/ya.make
View File

@ -1,5 +1,9 @@
OWNER(g:clickhouse)
LIBRARY()
CFLAGS(-g0)
SRCS(
readpassphrase.c
)

4
base/widechar_width/ya.make
View File

@ -1,7 +1,11 @@
OWNER(g:clickhouse)
LIBRARY()
ADDINCL(GLOBAL clickhouse/base/widechar_width)
CFLAGS(-g0)
SRCS(
widechar_width.cpp
)

2
base/ya.make
View File

@ -1,3 +1,5 @@
OWNER(g:clickhouse)
RECURSE(
common
daemon

6
benchmark/greenplum/result_parser.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python
#!/usr/bin/env python3
from __future__ import print_function
import sys
import json
@ -99,7 +99,7 @@ def gen_html_json(options, arguments):
tuples = read_stats_file(options, arguments[1])
print('{')
print('"system: GreenPlum(x2),')
print('"version": "%s",' % '4.3.9.1')
print(('"version": "%s",' % '4.3.9.1'))
print('"data_size": 10000000,')
print('"time": "",')
print('"comments": "",')

127
benchmark/hardware.sh Executable file
View File

@ -0,0 +1,127 @@
#!/bin/bash -e
if [[ -n $1 ]]; then
SCALE=$1
else
SCALE=100
fi
TABLE="hits_${SCALE}m_obfuscated"
DATASET="${TABLE}_v1.tar.xz"
QUERIES_FILE="queries.sql"
TRIES=3
AMD64_BIN_URL="https://clickhouse-builds.s3.yandex.net/0/e29c4c3cc47ab2a6c4516486c1b77d57e7d42643/clickhouse_build_check/gcc-10_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse"
AARCH64_BIN_URL="https://clickhouse-builds.s3.yandex.net/0/e29c4c3cc47ab2a6c4516486c1b77d57e7d42643/clickhouse_special_build_check/clang-10-aarch64_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse"
# Note: on older Ubuntu versions, 'axel' does not support IPv6. If you are using IPv6-only servers on very old Ubuntu, just don't install 'axel'.
FASTER_DOWNLOAD=wget
if command -v axel >/dev/null; then
FASTER_DOWNLOAD=axel
else
echo "It's recommended to install 'axel' for faster downloads."
fi
if command -v pixz >/dev/null; then
TAR_PARAMS='-Ipixz'
else
echo "It's recommended to install 'pixz' for faster decompression of the dataset."
fi
mkdir -p clickhouse-benchmark-$SCALE
pushd clickhouse-benchmark-$SCALE
if [[ ! -f clickhouse ]]; then
CPU=$(uname -m)
if [[ ($CPU == x86_64) || ($CPU == amd64) ]]; then
$FASTER_DOWNLOAD "$AMD64_BIN_URL"
elif [[ $CPU == aarch64 ]]; then
$FASTER_DOWNLOAD "$AARCH64_BIN_URL"
else
echo "Unsupported CPU type: $CPU"
exit 1
fi
fi
chmod a+x clickhouse
if [[ ! -f $QUERIES_FILE ]]; then
wget "https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/clickhouse/$QUERIES_FILE"
fi
if [[ ! -d data ]]; then
if [[ ! -f $DATASET ]]; then
$FASTER_DOWNLOAD "https://clickhouse-datasets.s3.yandex.net/hits/partitions/$DATASET"
fi
tar $TAR_PARAMS --strip-components=1 --directory=. -x -v -f $DATASET
fi
uptime
echo "Starting clickhouse-server"
./clickhouse server > server.log 2>&1 &
PID=$!
function finish {
kill $PID
wait
}
trap finish EXIT
echo "Waiting for clickhouse-server to start"
for i in {1..30}; do
sleep 1
./clickhouse client --query "SELECT 'The dataset size is: ', count() FROM $TABLE" 2>/dev/null && break || echo '.'
if [[ $i == 30 ]]; then exit 1; fi
done
echo
echo "Will perform benchmark. Results:"
echo
cat "$QUERIES_FILE" | sed "s/{table}/${TABLE}/g" | while read query; do
sync
echo 3 | sudo tee /proc/sys/vm/drop_caches >/dev/null
echo -n "["
for i in $(seq 1 $TRIES); do
RES=$(./clickhouse client --max_memory_usage 100000000000 --time --format=Null --query="$query" 2>&1 ||:)
[[ "$?" == "0" ]] && echo -n "${RES}" || echo -n "null"
[[ "$i" != $TRIES ]] && echo -n ", "
done
echo "],"
done
echo
echo "Benchmark complete. System info:"
echo
echo '----Version, build id-----------'
./clickhouse local --query "SELECT format('Version: {}, build id: {}', version(), buildId())"
./clickhouse local --query "SELECT format('The number of threads is: {}', value) FROM system.settings WHERE name = 'max_threads'" --output-format TSVRaw
./clickhouse local --query "SELECT format('Current time: {}', toString(now(), 'UTC'))"
echo '----CPU-------------------------'
cat /proc/cpuinfo | grep -i -F 'model name' | uniq
lscpu
echo '----Block Devices---------------'
lsblk
echo '----Disk Free and Total--------'
df -h .
echo '----Memory Free and Total-------'
free -h
echo '----Physical Memory Amount------'
cat /proc/meminfo | grep MemTotal
echo '----RAID Info-------------------'
cat /proc/mdstat
#echo '----PCI-------------------------'
#lspci
#echo '----All Hardware Info-----------'
#lshw
echo '--------------------------------'
echo

4
benchmark/monetdb/usability.md
View File

@ -2,7 +2,7 @@ Go to https://www.monetdb.org/
The graphical design of the website is a bit old-fashioned but I do not afraid.
Dowload now.
Download now.
Latest binary releases.
Ubuntu & Debian.
@ -1103,7 +1103,7 @@ Ok, it's doing something at least for twenty minues...
clk: 28:02 min
```
Finally it has loaded data successfuly in 28 minutes. It's not fast - just below 60 000 rows per second.
Finally it has loaded data successfully in 28 minutes. It's not fast - just below 60 000 rows per second.
But the second query from the test does not work:

524
cmake/Modules/FindArrow.cmake
View File

@ -17,143 +17,417 @@
# specific language governing permissions and limitations
# under the License.
# - Find ARROW (arrow/api.h, libarrow.a, libarrow.so)
# - Find Arrow (arrow/api.h, libarrow.a, libarrow.so)
# This module defines
# ARROW_FOUND, whether Arrow has been found
# ARROW_FULL_SO_VERSION, full shared object version of found Arrow "100.0.0"
# ARROW_IMPORT_LIB, path to libarrow's import library (Windows only)
# ARROW_INCLUDE_DIR, directory containing headers
# ARROW_LIBS, directory containing arrow libraries
# ARROW_STATIC_LIB, path to libarrow.a
# ARROW_LIBS, deprecated. Use ARROW_LIB_DIR instead
# ARROW_LIB_DIR, directory containing Arrow libraries
# ARROW_SHARED_IMP_LIB, deprecated. Use ARROW_IMPORT_LIB instead
# ARROW_SHARED_LIB, path to libarrow's shared library
# ARROW_SHARED_IMP_LIB, path to libarrow's import library (MSVC only)
# ARROW_FOUND, whether arrow has been found
# ARROW_SO_VERSION, shared object version of found Arrow such as "100"
# ARROW_STATIC_LIB, path to libarrow.a
# ARROW_VERSION, version of found Arrow
# ARROW_VERSION_MAJOR, major version of found Arrow
# ARROW_VERSION_MINOR, minor version of found Arrow
# ARROW_VERSION_PATCH, patch version of found Arrow
if(DEFINED ARROW_FOUND)
return()
endif()
include(FindPkgConfig)
include(GNUInstallDirs)
include(FindPackageHandleStandardArgs)
if ("$ENV{ARROW_HOME}" STREQUAL "")
pkg_check_modules(ARROW arrow)
if (ARROW_FOUND)
pkg_get_variable(ARROW_SO_VERSION arrow so_version)
set(ARROW_ABI_VERSION ${ARROW_SO_VERSION})
message(STATUS "Arrow SO and ABI version: ${ARROW_SO_VERSION}")
pkg_get_variable(ARROW_FULL_SO_VERSION arrow full_so_version)
message(STATUS "Arrow full SO version: ${ARROW_FULL_SO_VERSION}")
if ("${ARROW_INCLUDE_DIRS}" STREQUAL "")
set(ARROW_INCLUDE_DIRS "/usr/${CMAKE_INSTALL_INCLUDEDIR}")
endif()
if ("${ARROW_LIBRARY_DIRS}" STREQUAL "")
set(ARROW_LIBRARY_DIRS "/usr/${CMAKE_INSTALL_LIBDIR}")
if (EXISTS "/etc/debian_version" AND CMAKE_LIBRARY_ARCHITECTURE)
set(ARROW_LIBRARY_DIRS
"${ARROW_LIBRARY_DIRS}/${CMAKE_LIBRARY_ARCHITECTURE}")
endif()
endif()
set(ARROW_INCLUDE_DIR ${ARROW_INCLUDE_DIRS})
set(ARROW_LIBS ${ARROW_LIBRARY_DIRS})
set(ARROW_SEARCH_LIB_PATH ${ARROW_LIBRARY_DIRS})
endif()
else()
set(ARROW_HOME "$ENV{ARROW_HOME}")
set(ARROW_SEARCH_HEADER_PATHS
${ARROW_HOME}/include
)
set(ARROW_SEARCH_LIB_PATH
${ARROW_HOME}/lib
)
find_path(ARROW_INCLUDE_DIR arrow/array.h PATHS
${ARROW_SEARCH_HEADER_PATHS}
# make sure we don't accidentally pick up a different version
NO_DEFAULT_PATH
)
set(ARROW_SEARCH_LIB_PATH_SUFFIXES)
if(CMAKE_LIBRARY_ARCHITECTURE)
list(APPEND ARROW_SEARCH_LIB_PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}")
endif()
list(APPEND ARROW_SEARCH_LIB_PATH_SUFFIXES
"lib64"
"lib32"
"lib"
"bin")
set(ARROW_CONFIG_SUFFIXES
"_RELEASE"
"_RELWITHDEBINFO"
"_MINSIZEREL"
"_DEBUG"
"")
if(CMAKE_BUILD_TYPE)
string(TOUPPER ${CMAKE_BUILD_TYPE} ARROW_CONFIG_SUFFIX_PREFERRED)
set(ARROW_CONFIG_SUFFIX_PREFERRED "_${ARROW_CONFIG_SUFFIX_PREFERRED}")
list(INSERT ARROW_CONFIG_SUFFIXES 0 "${ARROW_CONFIG_SUFFIX_PREFERRED}")
endif()
find_library(ARROW_LIB_PATH NAMES arrow
PATHS
${ARROW_SEARCH_LIB_PATH}
NO_DEFAULT_PATH)
get_filename_component(ARROW_LIBS ${ARROW_LIB_PATH} DIRECTORY)
find_library(ARROW_PYTHON_LIB_PATH NAMES arrow_python
PATHS
${ARROW_SEARCH_LIB_PATH}
NO_DEFAULT_PATH)
get_filename_component(ARROW_PYTHON_LIBS ${ARROW_PYTHON_LIB_PATH} DIRECTORY)
if (MSVC)
SET(CMAKE_FIND_LIBRARY_SUFFIXES ".lib" ".dll")
if (MSVC AND NOT DEFINED ARROW_MSVC_STATIC_LIB_SUFFIX)
if(NOT DEFINED ARROW_MSVC_STATIC_LIB_SUFFIX)
if(MSVC)
set(ARROW_MSVC_STATIC_LIB_SUFFIX "_static")
endif()
find_library(ARROW_SHARED_LIBRARIES NAMES arrow
PATHS ${ARROW_HOME} NO_DEFAULT_PATH
PATH_SUFFIXES "bin" )
find_library(ARROW_PYTHON_SHARED_LIBRARIES NAMES arrow_python
PATHS ${ARROW_HOME} NO_DEFAULT_PATH
PATH_SUFFIXES "bin" )
get_filename_component(ARROW_SHARED_LIBS ${ARROW_SHARED_LIBRARIES} PATH )
get_filename_component(ARROW_PYTHON_SHARED_LIBS ${ARROW_PYTHON_SHARED_LIBRARIES} PATH )
endif ()
if (ARROW_INCLUDE_DIR AND ARROW_LIBS)
set(ARROW_FOUND TRUE)
set(ARROW_LIB_NAME arrow)
set(ARROW_PYTHON_LIB_NAME arrow_python)
if (MSVC)
set(ARROW_STATIC_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
set(ARROW_PYTHON_STATIC_LIB ${ARROW_PYTHON_LIBS}/${ARROW_PYTHON_LIB_NAME}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
set(ARROW_SHARED_LIB ${ARROW_SHARED_LIBS}/${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
set(ARROW_PYTHON_SHARED_LIB ${ARROW_PYTHON_SHARED_LIBS}/${ARROW_PYTHON_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
set(ARROW_SHARED_IMP_LIB ${ARROW_LIBS}/${ARROW_LIB_NAME}.lib)
set(ARROW_PYTHON_SHARED_IMP_LIB ${ARROW_PYTHON_LIBS}/${ARROW_PYTHON_LIB_NAME}.lib)
else()
set(ARROW_STATIC_LIB ${ARROW_LIBS}/lib${ARROW_LIB_NAME}.a)
set(ARROW_PYTHON_STATIC_LIB ${ARROW_LIBS}/lib${ARROW_PYTHON_LIB_NAME}.a)
set(ARROW_SHARED_LIB ${ARROW_LIBS}/lib${ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
set(ARROW_PYTHON_SHARED_LIB ${ARROW_LIBS}/lib${ARROW_PYTHON_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
set(ARROW_MSVC_STATIC_LIB_SUFFIX "")
endif()
endif()
if (ARROW_FOUND)
if (NOT Arrow_FIND_QUIETLY)
message(STATUS "Found the Arrow core library: ${ARROW_LIB_PATH}")
message(STATUS "Found the Arrow Python library: ${ARROW_PYTHON_LIB_PATH}")
endif ()
else ()
if (NOT Arrow_FIND_QUIETLY)
set(ARROW_ERR_MSG "Could not find the Arrow library. Looked for headers")
set(ARROW_ERR_MSG "${ARROW_ERR_MSG} in ${ARROW_SEARCH_HEADER_PATHS}, and for libs")
set(ARROW_ERR_MSG "${ARROW_ERR_MSG} in ${ARROW_SEARCH_LIB_PATH}")
if (Arrow_FIND_REQUIRED)
message(FATAL_ERROR "${ARROW_ERR_MSG}")
else (Arrow_FIND_REQUIRED)
message(STATUS "${ARROW_ERR_MSG}")
endif (Arrow_FIND_REQUIRED)
endif ()
set(ARROW_FOUND FALSE)
endif ()
# Internal function.
#
# Set shared library name for ${base_name} to ${output_variable}.
#
# Example:
# arrow_build_shared_library_name(ARROW_SHARED_LIBRARY_NAME arrow)
# # -> ARROW_SHARED_LIBRARY_NAME=libarrow.so on Linux
# # -> ARROW_SHARED_LIBRARY_NAME=libarrow.dylib on macOS
# # -> ARROW_SHARED_LIBRARY_NAME=arrow.dll with MSVC on Windows
# # -> ARROW_SHARED_LIBRARY_NAME=libarrow.dll with MinGW on Windows
function(arrow_build_shared_library_name output_variable base_name)
set(${output_variable}
"${CMAKE_SHARED_LIBRARY_PREFIX}${base_name}${CMAKE_SHARED_LIBRARY_SUFFIX}"
PARENT_SCOPE)
endfunction()
if (MSVC)
mark_as_advanced(
ARROW_INCLUDE_DIR
ARROW_STATIC_LIB
ARROW_SHARED_LIB
ARROW_SHARED_IMP_LIB
ARROW_PYTHON_STATIC_LIB
ARROW_PYTHON_SHARED_LIB
ARROW_PYTHON_SHARED_IMP_LIB
)
else()
mark_as_advanced(
ARROW_INCLUDE_DIR
ARROW_STATIC_LIB
ARROW_SHARED_LIB
ARROW_PYTHON_STATIC_LIB
ARROW_PYTHON_SHARED_LIB
)
# Internal function.
#
# Set import library name for ${base_name} to ${output_variable}.
# This is useful only for MSVC build. Import library is used only
# with MSVC build.
#
# Example:
# arrow_build_import_library_name(ARROW_IMPORT_LIBRARY_NAME arrow)
# # -> ARROW_IMPORT_LIBRARY_NAME=arrow on Linux (meaningless)
# # -> ARROW_IMPORT_LIBRARY_NAME=arrow on macOS (meaningless)
# # -> ARROW_IMPORT_LIBRARY_NAME=arrow.lib with MSVC on Windows
# # -> ARROW_IMPORT_LIBRARY_NAME=libarrow.dll.a with MinGW on Windows
function(arrow_build_import_library_name output_variable base_name)
set(${output_variable}
"${CMAKE_IMPORT_LIBRARY_PREFIX}${base_name}${CMAKE_IMPORT_LIBRARY_SUFFIX}"
PARENT_SCOPE)
endfunction()
# Internal function.
#
# Set static library name for ${base_name} to ${output_variable}.
#
# Example:
# arrow_build_static_library_name(ARROW_STATIC_LIBRARY_NAME arrow)
# # -> ARROW_STATIC_LIBRARY_NAME=libarrow.a on Linux
# # -> ARROW_STATIC_LIBRARY_NAME=libarrow.a on macOS
# # -> ARROW_STATIC_LIBRARY_NAME=arrow.lib with MSVC on Windows
# # -> ARROW_STATIC_LIBRARY_NAME=libarrow.dll.a with MinGW on Windows
function(arrow_build_static_library_name output_variable base_name)
set(
${output_variable}
"${CMAKE_STATIC_LIBRARY_PREFIX}${base_name}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}"
PARENT_SCOPE)
endfunction()
# Internal function.
#
# Set macro value for ${macro_name} in ${header_content} to ${output_variable}.
#
# Example:
# arrow_extract_macro_value(version_major
# "ARROW_VERSION_MAJOR"
# "#define ARROW_VERSION_MAJOR 1.0.0")
# # -> version_major=1.0.0
function(arrow_extract_macro_value output_variable macro_name header_content)
string(REGEX MATCH "#define +${macro_name} +[^\r\n]+" macro_definition
"${header_content}")
string(REGEX
REPLACE "^#define +${macro_name} +(.+)$" "\\1" macro_value "${macro_definition}")
set(${output_variable} "${macro_value}" PARENT_SCOPE)
endfunction()
# Internal macro only for arrow_find_package.
#
# Find package in HOME.
macro(arrow_find_package_home)
find_path(${prefix}_include_dir "${header_path}"
PATHS "${home}"
PATH_SUFFIXES "include"
NO_DEFAULT_PATH)
set(include_dir "${${prefix}_include_dir}")
set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE)
if(MSVC)
set(CMAKE_SHARED_LIBRARY_SUFFIXES_ORIGINAL ${CMAKE_FIND_LIBRARY_SUFFIXES})
# .dll isn't found by find_library with MSVC because .dll isn't included in
# CMAKE_FIND_LIBRARY_SUFFIXES.
list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES "${CMAKE_SHARED_LIBRARY_SUFFIX}")
endif()
find_library(${prefix}_shared_lib
NAMES "${shared_lib_name}"
PATHS "${home}"
PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES}
NO_DEFAULT_PATH)
if(MSVC)
set(CMAKE_SHARED_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_ORIGINAL})
endif()
set(shared_lib "${${prefix}_shared_lib}")
set(${prefix}_SHARED_LIB "${shared_lib}" PARENT_SCOPE)
if(shared_lib)
add_library(${target_shared} SHARED IMPORTED)
set_target_properties(${target_shared} PROPERTIES IMPORTED_LOCATION "${shared_lib}")
if(include_dir)
set_target_properties(${target_shared}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}")
endif()
find_library(${prefix}_import_lib
NAMES "${import_lib_name}"
PATHS "${home}"
PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES}
NO_DEFAULT_PATH)
set(import_lib "${${prefix}_import_lib}")
set(${prefix}_IMPORT_LIB "${import_lib}" PARENT_SCOPE)
if(import_lib)
set_target_properties(${target_shared} PROPERTIES IMPORTED_IMPLIB "${import_lib}")
endif()
endif()
find_library(${prefix}_static_lib
NAMES "${static_lib_name}"
PATHS "${home}"
PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES}
NO_DEFAULT_PATH)
set(static_lib "${${prefix}_static_lib}")
set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE)
if(static_lib)
add_library(${target_static} STATIC IMPORTED)
set_target_properties(${target_static} PROPERTIES IMPORTED_LOCATION "${static_lib}")
if(include_dir)
set_target_properties(${target_static}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}")
endif()
endif()
endmacro()
# Internal macro only for arrow_find_package.
#
# Find package by CMake package configuration.
macro(arrow_find_package_cmake_package_configuration)
find_package(${cmake_package_name} CONFIG)
if(${cmake_package_name}_FOUND)
set(${prefix}_USE_CMAKE_PACKAGE_CONFIG TRUE PARENT_SCOPE)
if(TARGET ${target_shared})
foreach(suffix ${ARROW_CONFIG_SUFFIXES})
get_target_property(shared_lib ${target_shared} IMPORTED_LOCATION${suffix})
if(shared_lib)
# Remove shared library version:
# libarrow.so.100.0.0 -> libarrow.so
# Because ARROW_HOME and pkg-config approaches don't add
# shared library version.
string(REGEX
REPLACE "(${CMAKE_SHARED_LIBRARY_SUFFIX})[.0-9]+$" "\\1" shared_lib
"${shared_lib}")
set(${prefix}_SHARED_LIB "${shared_lib}" PARENT_SCOPE)
break()
endif()
endforeach()
endif()
if(TARGET ${target_static})
foreach(suffix ${ARROW_CONFIG_SUFFIXES})
get_target_property(static_lib ${target_static} IMPORTED_LOCATION${suffix})
if(static_lib)
set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE)
break()
endif()
endforeach()
endif()
endif()
endmacro()
# Internal macro only for arrow_find_package.
#
# Find package by pkg-config.
macro(arrow_find_package_pkg_config)
pkg_check_modules(${prefix}_PC ${pkg_config_name})
if(${prefix}_PC_FOUND)
set(${prefix}_USE_PKG_CONFIG TRUE PARENT_SCOPE)
set(include_dir "${${prefix}_PC_INCLUDEDIR}")
set(lib_dir "${${prefix}_PC_LIBDIR}")
set(shared_lib_paths "${${prefix}_PC_LINK_LIBRARIES}")
# Use the first shared library path as the IMPORTED_LOCATION
# for ${target_shared}. This assumes that the first shared library
# path is the shared library path for this module.
list(GET shared_lib_paths 0 first_shared_lib_path)
# Use the rest shared library paths as the INTERFACE_LINK_LIBRARIES
# for ${target_shared}. This assumes that the rest shared library
# paths are dependency library paths for this module.
list(LENGTH shared_lib_paths n_shared_lib_paths)
if(n_shared_lib_paths LESS_EQUAL 1)
set(rest_shared_lib_paths)
else()
list(SUBLIST
shared_lib_paths
1
-1
rest_shared_lib_paths)
endif()
set(${prefix}_VERSION "${${prefix}_PC_VERSION}" PARENT_SCOPE)
set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE)
set(${prefix}_SHARED_LIB "${first_shared_lib_path}" PARENT_SCOPE)
add_library(${target_shared} SHARED IMPORTED)
set_target_properties(${target_shared}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
"${include_dir}"
INTERFACE_LINK_LIBRARIES
"${rest_shared_lib_paths}"
IMPORTED_LOCATION
"${first_shared_lib_path}")
get_target_property(shared_lib ${target_shared} IMPORTED_LOCATION)
find_library(${prefix}_static_lib
NAMES "${static_lib_name}"
PATHS "${lib_dir}"
NO_DEFAULT_PATH)
set(static_lib "${${prefix}_static_lib}")
set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE)
if(static_lib)
add_library(${target_static} STATIC IMPORTED)
set_target_properties(${target_static}
PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}"
IMPORTED_LOCATION "${static_lib}")
endif()
endif()
endmacro()
function(arrow_find_package
prefix
home
base_name
header_path
cmake_package_name
pkg_config_name)
arrow_build_shared_library_name(shared_lib_name ${base_name})
arrow_build_import_library_name(import_lib_name ${base_name})
arrow_build_static_library_name(static_lib_name ${base_name})
set(target_shared ${base_name}_shared)
set(target_static ${base_name}_static)
if(home)
arrow_find_package_home()
set(${prefix}_FIND_APPROACH "HOME: ${home}" PARENT_SCOPE)
else()
arrow_find_package_cmake_package_configuration()
if(${cmake_package_name}_FOUND)
set(${prefix}_FIND_APPROACH
"CMake package configuration: ${cmake_package_name}"
PARENT_SCOPE)
else()
arrow_find_package_pkg_config()
set(${prefix}_FIND_APPROACH "pkg-config: ${pkg_config_name}" PARENT_SCOPE)
endif()
endif()
if(NOT include_dir)
if(TARGET ${target_shared})
get_target_property(include_dir ${target_shared} INTERFACE_INCLUDE_DIRECTORIES)
elseif(TARGET ${target_static})
get_target_property(include_dir ${target_static} INTERFACE_INCLUDE_DIRECTORIES)
endif()
endif()
if(include_dir)
set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE)
endif()
if(shared_lib)
get_filename_component(lib_dir "${shared_lib}" DIRECTORY)
elseif(static_lib)
get_filename_component(lib_dir "${static_lib}" DIRECTORY)
else()
set(lib_dir NOTFOUND)
endif()
set(${prefix}_LIB_DIR "${lib_dir}" PARENT_SCOPE)
# For backward compatibility
set(${prefix}_LIBS "${lib_dir}" PARENT_SCOPE)
endfunction()
if(NOT "$ENV{ARROW_HOME}" STREQUAL "")
file(TO_CMAKE_PATH "$ENV{ARROW_HOME}" ARROW_HOME)
endif()
arrow_find_package(ARROW
"${ARROW_HOME}"
arrow
arrow/api.h
Arrow
arrow)
if(ARROW_HOME)
if(ARROW_INCLUDE_DIR)
file(READ "${ARROW_INCLUDE_DIR}/arrow/util/config.h" ARROW_CONFIG_H_CONTENT)
arrow_extract_macro_value(ARROW_VERSION_MAJOR "ARROW_VERSION_MAJOR"
"${ARROW_CONFIG_H_CONTENT}")
arrow_extract_macro_value(ARROW_VERSION_MINOR "ARROW_VERSION_MINOR"
"${ARROW_CONFIG_H_CONTENT}")
arrow_extract_macro_value(ARROW_VERSION_PATCH "ARROW_VERSION_PATCH"
"${ARROW_CONFIG_H_CONTENT}")
if("${ARROW_VERSION_MAJOR}" STREQUAL ""
OR "${ARROW_VERSION_MINOR}" STREQUAL ""
OR "${ARROW_VERSION_PATCH}" STREQUAL "")
set(ARROW_VERSION "0.0.0")
else()
set(ARROW_VERSION
"${ARROW_VERSION_MAJOR}.${ARROW_VERSION_MINOR}.${ARROW_VERSION_PATCH}")
endif()
arrow_extract_macro_value(ARROW_SO_VERSION_QUOTED "ARROW_SO_VERSION"
"${ARROW_CONFIG_H_CONTENT}")
string(REGEX REPLACE "^\"(.+)\"$" "\\1" ARROW_SO_VERSION "${ARROW_SO_VERSION_QUOTED}")
arrow_extract_macro_value(ARROW_FULL_SO_VERSION_QUOTED "ARROW_FULL_SO_VERSION"
"${ARROW_CONFIG_H_CONTENT}")
string(REGEX
REPLACE "^\"(.+)\"$" "\\1" ARROW_FULL_SO_VERSION
"${ARROW_FULL_SO_VERSION_QUOTED}")
endif()
else()
if(ARROW_USE_CMAKE_PACKAGE_CONFIG)
find_package(Arrow CONFIG)
elseif(ARROW_USE_PKG_CONFIG)
pkg_get_variable(ARROW_SO_VERSION arrow so_version)
pkg_get_variable(ARROW_FULL_SO_VERSION arrow full_so_version)
endif()
endif()
set(ARROW_ABI_VERSION ${ARROW_SO_VERSION})
mark_as_advanced(ARROW_ABI_VERSION
ARROW_CONFIG_SUFFIXES
ARROW_FULL_SO_VERSION
ARROW_IMPORT_LIB
ARROW_INCLUDE_DIR
ARROW_LIBS
ARROW_LIB_DIR
ARROW_SEARCH_LIB_PATH_SUFFIXES
ARROW_SHARED_IMP_LIB
ARROW_SHARED_LIB
ARROW_SO_VERSION
ARROW_STATIC_LIB
ARROW_VERSION
ARROW_VERSION_MAJOR
ARROW_VERSION_MINOR
ARROW_VERSION_PATCH)
find_package_handle_standard_args(Arrow REQUIRED_VARS
# The first required variable is shown
# in the found message. So this list is
# not sorted alphabetically.
ARROW_INCLUDE_DIR
ARROW_LIB_DIR
ARROW_FULL_SO_VERSION
ARROW_SO_VERSION
VERSION_VAR
ARROW_VERSION)
set(ARROW_FOUND ${Arrow_FOUND})
if(Arrow_FOUND AND NOT Arrow_FIND_QUIETLY)
message(STATUS "Arrow version: ${ARROW_VERSION} (${ARROW_FIND_APPROACH})")
message(STATUS "Arrow SO and ABI version: ${ARROW_SO_VERSION}")
message(STATUS "Arrow full SO version: ${ARROW_FULL_SO_VERSION}")
message(STATUS "Found the Arrow core shared library: ${ARROW_SHARED_LIB}")
message(STATUS "Found the Arrow core import library: ${ARROW_IMPORT_LIB}")
message(STATUS "Found the Arrow core static library: ${ARROW_STATIC_LIB}")
endif()

207
cmake/Modules/FindParquet.cmake
View File

@ -17,131 +17,116 @@
# specific language governing permissions and limitations
# under the License.
# - Find PARQUET (parquet/parquet.h, libparquet.a, libparquet.so)
# - Find Parquet (parquet/api/reader.h, libparquet.a, libparquet.so)
#
# This module requires Arrow from which it uses
# arrow_find_package()
#
# This module defines
# PARQUET_FOUND, whether Parquet has been found
# PARQUET_IMPORT_LIB, path to libparquet's import library (Windows only)
# PARQUET_INCLUDE_DIR, directory containing headers
# PARQUET_LIBS, directory containing parquet libraries
# PARQUET_STATIC_LIB, path to libparquet.a
# PARQUET_LIBS, deprecated. Use PARQUET_LIB_DIR instead
# PARQUET_LIB_DIR, directory containing Parquet libraries
# PARQUET_SHARED_IMP_LIB, deprecated. Use PARQUET_IMPORT_LIB instead
# PARQUET_SHARED_LIB, path to libparquet's shared library
# PARQUET_SHARED_IMP_LIB, path to libparquet's import library (MSVC only)
# PARQUET_FOUND, whether parquet has been found
# PARQUET_SO_VERSION, shared object version of found Parquet such as "100"
# PARQUET_STATIC_LIB, path to libparquet.a
include(FindPkgConfig)
if(DEFINED PARQUET_FOUND)
return()
endif()
set(find_package_arguments)
if(${CMAKE_FIND_PACKAGE_NAME}_FIND_VERSION)
list(APPEND find_package_arguments "${${CMAKE_FIND_PACKAGE_NAME}_FIND_VERSION}")
endif()
if(${CMAKE_FIND_PACKAGE_NAME}_FIND_REQUIRED)
list(APPEND find_package_arguments REQUIRED)
endif()
if(${CMAKE_FIND_PACKAGE_NAME}_FIND_QUIETLY)
list(APPEND find_package_arguments QUIET)
endif()
find_package(Arrow ${find_package_arguments})
if(NOT "$ENV{PARQUET_HOME}" STREQUAL "")
set(PARQUET_HOME "$ENV{PARQUET_HOME}")
file(TO_CMAKE_PATH "$ENV{PARQUET_HOME}" PARQUET_HOME)
endif()
if (MSVC)
SET(CMAKE_FIND_LIBRARY_SUFFIXES ".lib" ".dll")
if((NOT PARQUET_HOME) AND ARROW_HOME)
set(PARQUET_HOME ${ARROW_HOME})
endif()
if (MSVC AND NOT DEFINED PARQUET_MSVC_STATIC_LIB_SUFFIX)
set(PARQUET_MSVC_STATIC_LIB_SUFFIX "_static")
endif()
find_library(PARQUET_SHARED_LIBRARIES NAMES parquet
PATHS ${PARQUET_HOME} NO_DEFAULT_PATH
PATH_SUFFIXES "bin" )
get_filename_component(PARQUET_SHARED_LIBS ${PARQUET_SHARED_LIBRARIES} PATH )
endif ()
if(PARQUET_HOME)
set(PARQUET_SEARCH_HEADER_PATHS
${PARQUET_HOME}/include
)
set(PARQUET_SEARCH_LIB_PATH
${PARQUET_HOME}/lib
)
find_path(PARQUET_INCLUDE_DIR parquet/api/reader.h PATHS
${PARQUET_SEARCH_HEADER_PATHS}
# make sure we don't accidentally pick up a different version
NO_DEFAULT_PATH
)
find_library(PARQUET_LIBRARIES NAMES parquet
PATHS ${PARQUET_HOME} NO_DEFAULT_PATH
PATH_SUFFIXES "lib")
get_filename_component(PARQUET_LIBS ${PARQUET_LIBRARIES} PATH )
# Try to autodiscover the Parquet ABI version
get_filename_component(PARQUET_LIB_REALPATH ${PARQUET_LIBRARIES} REALPATH)
get_filename_component(PARQUET_EXT_REALPATH ${PARQUET_LIB_REALPATH} EXT)
string(REGEX MATCH ".([0-9]+.[0-9]+.[0-9]+)" HAS_ABI_VERSION ${PARQUET_EXT_REALPATH})
if (HAS_ABI_VERSION)
if (APPLE)
string(REGEX REPLACE ".([0-9]+.[0-9]+.[0-9]+).dylib" "\\1" PARQUET_ABI_VERSION ${PARQUET_EXT_REALPATH})
if(ARROW_FOUND)
arrow_find_package(PARQUET
"${PARQUET_HOME}"
parquet
parquet/api/reader.h
Parquet
parquet)
if(PARQUET_HOME)
if(PARQUET_INCLUDE_DIR)
file(READ "${PARQUET_INCLUDE_DIR}/parquet/parquet_version.h"
PARQUET_VERSION_H_CONTENT)
arrow_extract_macro_value(PARQUET_VERSION_MAJOR "PARQUET_VERSION_MAJOR"
"${PARQUET_VERSION_H_CONTENT}")
arrow_extract_macro_value(PARQUET_VERSION_MINOR "PARQUET_VERSION_MINOR"
"${PARQUET_VERSION_H_CONTENT}")
arrow_extract_macro_value(PARQUET_VERSION_PATCH "PARQUET_VERSION_PATCH"
"${PARQUET_VERSION_H_CONTENT}")
if("${PARQUET_VERSION_MAJOR}" STREQUAL ""
OR "${PARQUET_VERSION_MINOR}" STREQUAL ""
OR "${PARQUET_VERSION_PATCH}" STREQUAL "")
set(PARQUET_VERSION "0.0.0")
else()
string(REGEX REPLACE ".so.([0-9]+.[0-9]+.[0-9]+)" "\\1" PARQUET_ABI_VERSION ${PARQUET_EXT_REALPATH})
set(PARQUET_VERSION
"${PARQUET_VERSION_MAJOR}.${PARQUET_VERSION_MINOR}.${PARQUET_VERSION_PATCH}")
endif()
string(REGEX REPLACE "([0-9]+).[0-9]+.[0-9]+" "\\1" PARQUET_SO_VERSION ${PARQUET_ABI_VERSION})
else()
set(PARQUET_ABI_VERSION "1.0.0")
set(PARQUET_SO_VERSION "1")
arrow_extract_macro_value(PARQUET_SO_VERSION_QUOTED "PARQUET_SO_VERSION"
"${PARQUET_VERSION_H_CONTENT}")
string(REGEX
REPLACE "^\"(.+)\"$" "\\1" PARQUET_SO_VERSION "${PARQUET_SO_VERSION_QUOTED}")
arrow_extract_macro_value(PARQUET_FULL_SO_VERSION_QUOTED "PARQUET_FULL_SO_VERSION"
"${PARQUET_VERSION_H_CONTENT}")
string(REGEX
REPLACE "^\"(.+)\"$" "\\1" PARQUET_FULL_SO_VERSION
"${PARQUET_FULL_SO_VERSION_QUOTED}")
endif()
else()
pkg_check_modules(PARQUET parquet)
if (PARQUET_FOUND)
pkg_get_variable(PARQUET_ABI_VERSION parquet abi_version)
message(STATUS "Parquet C++ ABI version: ${PARQUET_ABI_VERSION}")
pkg_get_variable(PARQUET_SO_VERSION parquet so_version)
message(STATUS "Parquet C++ SO version: ${PARQUET_SO_VERSION}")
set(PARQUET_INCLUDE_DIR ${PARQUET_INCLUDE_DIRS})
set(PARQUET_LIBS ${PARQUET_LIBRARY_DIRS})
set(PARQUET_SEARCH_LIB_PATH ${PARQUET_LIBRARY_DIRS})
message(STATUS "Searching for parquet libs in: ${PARQUET_SEARCH_LIB_PATH}")
find_library(PARQUET_LIBRARIES NAMES parquet
PATHS ${PARQUET_SEARCH_LIB_PATH} NO_DEFAULT_PATH)
else()
find_path(PARQUET_INCLUDE_DIR NAMES parquet/api/reader.h )
find_library(PARQUET_LIBRARIES NAMES parquet)
get_filename_component(PARQUET_LIBS ${PARQUET_LIBRARIES} PATH )
else()
if(PARQUET_USE_CMAKE_PACKAGE_CONFIG)
find_package(Parquet CONFIG)
elseif(PARQUET_USE_PKG_CONFIG)
pkg_get_variable(PARQUET_SO_VERSION parquet so_version)
pkg_get_variable(PARQUET_FULL_SO_VERSION parquet full_so_version)
endif()
endif()
set(PARQUET_ABI_VERSION "${PARQUET_SO_VERSION}")
endif()
if (PARQUET_INCLUDE_DIR AND PARQUET_LIBRARIES)
set(PARQUET_FOUND TRUE)
set(PARQUET_LIB_NAME parquet)
if (MSVC)
set(PARQUET_STATIC_LIB "${PARQUET_LIBS}/${PARQUET_LIB_NAME}${PARQUET_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}")
set(PARQUET_SHARED_LIB "${PARQUET_SHARED_LIBS}/${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}")
set(PARQUET_SHARED_IMP_LIB "${PARQUET_LIBS}/${PARQUET_LIB_NAME}.lib")
else()
set(PARQUET_STATIC_LIB ${PARQUET_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PARQUET_LIB_NAME}.a)
set(PARQUET_SHARED_LIB ${PARQUET_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${PARQUET_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
endif()
else ()
set(PARQUET_FOUND FALSE)
endif ()
mark_as_advanced(PARQUET_ABI_VERSION
PARQUET_IMPORT_LIB
PARQUET_INCLUDE_DIR
PARQUET_LIBS
PARQUET_LIB_DIR
PARQUET_SHARED_IMP_LIB
PARQUET_SHARED_LIB
PARQUET_SO_VERSION
PARQUET_STATIC_LIB
PARQUET_VERSION)
if (PARQUET_FOUND)
if (NOT Parquet_FIND_QUIETLY)
message(STATUS "Found the Parquet library: ${PARQUET_LIBRARIES}")
endif ()
else ()
if (NOT Parquet_FIND_QUIETLY)
if (NOT PARQUET_FOUND)
set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} Could not find the parquet library.")
endif()
find_package_handle_standard_args(Parquet
REQUIRED_VARS
PARQUET_INCLUDE_DIR
PARQUET_LIB_DIR
PARQUET_SO_VERSION
VERSION_VAR
PARQUET_VERSION)
set(PARQUET_FOUND ${Parquet_FOUND})
set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} Looked in ")
if ( _parquet_roots )
set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} in ${_parquet_roots}.")
else ()
set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} system search paths.")
endif ()
if (Parquet_FIND_REQUIRED)
message(FATAL_ERROR "${PARQUET_ERR_MSG}")
else (Parquet_FIND_REQUIRED)
message(STATUS "${PARQUET_ERR_MSG}")
endif (Parquet_FIND_REQUIRED)
endif ()
endif ()
mark_as_advanced(
PARQUET_FOUND
PARQUET_INCLUDE_DIR
PARQUET_LIBS
PARQUET_LIBRARIES
PARQUET_STATIC_LIB
PARQUET_SHARED_LIB
)
if(Parquet_FOUND AND NOT Parquet_FIND_QUIETLY)
message(STATUS "Parquet version: ${PARQUET_VERSION} (${PARQUET_FIND_APPROACH})")
message(STATUS "Found the Parquet shared library: ${PARQUET_SHARED_LIB}")
message(STATUS "Found the Parquet import library: ${PARQUET_IMPORT_LIB}")
message(STATUS "Found the Parquet static library: ${PARQUET_STATIC_LIB}")
endif()

44
cmake/Modules/Findbtrie.cmake
View File

@ -1,44 +0,0 @@
# - Try to find btrie headers and libraries.
#
# Usage of this module as follows:
#
# find_package(btrie)
#
# Variables used by this module, they can change the default behaviour and need
# to be set before calling find_package:
#
# BTRIE_ROOT_DIR Set this variable to the root installation of
# btrie if the module has problems finding
# the proper installation path.
#
# Variables defined by this module:
#
# BTRIE_FOUND System has btrie libs/headers
# BTRIE_LIBRARIES The btrie library/libraries
# BTRIE_INCLUDE_DIR The location of btrie headers
find_path(BTRIE_ROOT_DIR
NAMES include/btrie.h
)
find_library(BTRIE_LIBRARIES
NAMES btrie
PATHS ${BTRIE_ROOT_DIR}/lib ${BTRIE_LIBRARIES_PATHS}
)
find_path(BTRIE_INCLUDE_DIR
NAMES btrie.h
PATHS ${BTRIE_ROOT_DIR}/include ${BTRIE_INCLUDE_PATHS}
)
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(btrie DEFAULT_MSG
BTRIE_LIBRARIES
BTRIE_INCLUDE_DIR
)
mark_as_advanced(
BTRIE_ROOT_DIR
BTRIE_LIBRARIES
BTRIE_INCLUDE_DIR
)

337
cmake/Modules/FindgRPC.cmake Normal file
View File

@ -0,0 +1,337 @@
#[[
Defines the following variables:
``gRPC_FOUND``
Whether the gRPC framework is found
``gRPC_INCLUDE_DIRS``
The include directories of the gRPC framework, including the include directories of the C++ wrapper.
``gRPC_LIBRARIES``
The libraries of the gRPC framework.
``gRPC_CPP_PLUGIN``
The plugin for generating gRPC client and server C++ stubs from `.proto` files
``gRPC_PYTHON_PLUGIN``
The plugin for generating gRPC client and server Python stubs from `.proto` files
The following :prop_tgt:`IMPORTED` targets are also defined:
``grpc++``
``grpc++_unsecure``
``grpc_cpp_plugin``
``grpc_python_plugin``
Set the following variables to adjust the behaviour of this script:
``gRPC_USE_UNSECURE_LIBRARIES``
if set gRPC_LIBRARIES will be filled with the unsecure version of the libraries (i.e. without SSL)
instead of the secure ones.
``gRPC_DEBUG`
if set the debug message will be printed.
Add custom commands to process ``.proto`` files to C++::
protobuf_generate_grpc_cpp(<SRCS> <HDRS>
[DESCRIPTORS <DESC>] [EXPORT_MACRO <MACRO>] [<ARGN>...])
``SRCS``
Variable to define with autogenerated source files
``HDRS``
Variable to define with autogenerated header files
``DESCRIPTORS``
Variable to define with autogenerated descriptor files, if requested.
``EXPORT_MACRO``
is a macro which should expand to ``__declspec(dllexport)`` or
``__declspec(dllimport)`` depending on what is being compiled.
``ARGN``
``.proto`` files
#]]
# Function to generate C++ files from .proto files.
# This function is a modified version of the function PROTOBUF_GENERATE_CPP() copied from https://github.com/Kitware/CMake/blob/master/Modules/FindProtobuf.cmake.
function(PROTOBUF_GENERATE_GRPC_CPP SRCS HDRS)
cmake_parse_arguments(protobuf_generate_grpc_cpp "" "EXPORT_MACRO;DESCRIPTORS" "" ${ARGN})
set(_proto_files "${protobuf_generate_grpc_cpp_UNPARSED_ARGUMENTS}")
if(NOT _proto_files)
message(SEND_ERROR "Error: PROTOBUF_GENERATE_GRPC_CPP() called without any proto files")
return()
endif()
if(PROTOBUF_GENERATE_GRPC_CPP_APPEND_PATH)
set(_append_arg APPEND_PATH)
endif()
if(protobuf_generate_grpc_cpp_DESCRIPTORS)
set(_descriptors DESCRIPTORS)
endif()
if(DEFINED PROTOBUF_IMPORT_DIRS AND NOT DEFINED Protobuf_IMPORT_DIRS)
set(Protobuf_IMPORT_DIRS "${PROTOBUF_IMPORT_DIRS}")
endif()
if(DEFINED Protobuf_IMPORT_DIRS)
set(_import_arg IMPORT_DIRS ${Protobuf_IMPORT_DIRS})
endif()
set(_outvar)
protobuf_generate_grpc(${_append_arg} ${_descriptors} LANGUAGE cpp EXPORT_MACRO ${protobuf_generate_cpp_EXPORT_MACRO} OUT_VAR _outvar ${_import_arg} PROTOS ${_proto_files})
set(${SRCS})
set(${HDRS})
if(protobuf_generate_grpc_cpp_DESCRIPTORS)
set(${protobuf_generate_grpc_cpp_DESCRIPTORS})
endif()
foreach(_file ${_outvar})
if(_file MATCHES "cc$")
list(APPEND ${SRCS} ${_file})
elseif(_file MATCHES "desc$")
list(APPEND ${protobuf_generate_grpc_cpp_DESCRIPTORS} ${_file})
else()
list(APPEND ${HDRS} ${_file})
endif()
endforeach()
set(${SRCS} ${${SRCS}} PARENT_SCOPE)
set(${HDRS} ${${HDRS}} PARENT_SCOPE)
if(protobuf_generate_grpc_cpp_DESCRIPTORS)
set(${protobuf_generate_grpc_cpp_DESCRIPTORS} "${${protobuf_generate_grpc_cpp_DESCRIPTORS}}" PARENT_SCOPE)
endif()
endfunction()
# Helper function.
# This function is a modified version of the function protobuf_generate() copied from https://github.com/Kitware/CMake/blob/master/Modules/FindProtobuf.cmake.
function(protobuf_generate_grpc)
set(_options APPEND_PATH DESCRIPTORS)
set(_singleargs LANGUAGE OUT_VAR EXPORT_MACRO PROTOC_OUT_DIR)
if(COMMAND target_sources)
list(APPEND _singleargs TARGET)
endif()
set(_multiargs PROTOS IMPORT_DIRS GENERATE_EXTENSIONS)
cmake_parse_arguments(protobuf_generate_grpc "${_options}" "${_singleargs}" "${_multiargs}" "${ARGN}")
if(NOT protobuf_generate_grpc_PROTOS AND NOT protobuf_generate_grpc_TARGET)
message(SEND_ERROR "Error: protobuf_generate_grpc called without any targets or source files")
return()
endif()
if(NOT protobuf_generate_grpc_OUT_VAR AND NOT protobuf_generate_grpc_TARGET)
message(SEND_ERROR "Error: protobuf_generate_grpc called without a target or output variable")
return()
endif()
if(NOT protobuf_generate_grpc_LANGUAGE)
set(protobuf_generate_grpc_LANGUAGE cpp)
endif()
string(TOLOWER ${protobuf_generate_grpc_LANGUAGE} protobuf_generate_grpc_LANGUAGE)
if(NOT protobuf_generate_grpc_PROTOC_OUT_DIR)
set(protobuf_generate_grpc_PROTOC_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR})
endif()
if(protobuf_generate_grpc_EXPORT_MACRO AND protobuf_generate_grpc_LANGUAGE STREQUAL cpp)
set(_dll_export_decl "dllexport_decl=${protobuf_generate_grpc_EXPORT_MACRO}:")
endif()
if(NOT protobuf_generate_grpc_GENERATE_EXTENSIONS)
if(protobuf_generate_grpc_LANGUAGE STREQUAL cpp)
set(protobuf_generate_grpc_GENERATE_EXTENSIONS .pb.h .pb.cc .grpc.pb.h .grpc.pb.cc)
elseif(protobuf_generate_grpc_LANGUAGE STREQUAL python)
set(protobuf_generate_grpc_GENERATE_EXTENSIONS _pb2.py)
else()
message(SEND_ERROR "Error: protobuf_generate_grpc given unknown Language ${LANGUAGE}, please provide a value for GENERATE_EXTENSIONS")
return()
endif()
endif()
if(NOT protobuf_generate_grpc_PLUGIN)
if(protobuf_generate_grpc_LANGUAGE STREQUAL cpp)
set(protobuf_generate_grpc_PLUGIN "grpc_cpp_plugin")
elseif(protobuf_generate_grpc_LANGUAGE STREQUAL python)
set(protobuf_generate_grpc_PLUGIN "grpc_python_plugin")
else()
message(SEND_ERROR "Error: protobuf_generate_grpc given unknown Language ${LANGUAGE}, please provide a value for PLUGIN")
return()
endif()
endif()
if(protobuf_generate_grpc_TARGET)
get_target_property(_source_list ${protobuf_generate_grpc_TARGET} SOURCES)
foreach(_file ${_source_list})
if(_file MATCHES "proto$")
list(APPEND protobuf_generate_grpc_PROTOS ${_file})
endif()
endforeach()
endif()
if(NOT protobuf_generate_grpc_PROTOS)
message(SEND_ERROR "Error: protobuf_generate_grpc could not find any .proto files")
return()
endif()
if(protobuf_generate_grpc_APPEND_PATH)
# Create an include path for each file specified
foreach(_file ${protobuf_generate_grpc_PROTOS})
get_filename_component(_abs_file ${_file} ABSOLUTE)
get_filename_component(_abs_path ${_abs_file} PATH)
list(FIND _protobuf_include_path ${_abs_path} _contains_already)
if(${_contains_already} EQUAL -1)
list(APPEND _protobuf_include_path -I ${_abs_path})
endif()
endforeach()
else()
set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR})
endif()
foreach(DIR ${protobuf_generate_grpc_IMPORT_DIRS})
get_filename_component(ABS_PATH ${DIR} ABSOLUTE)
list(FIND _protobuf_include_path ${ABS_PATH} _contains_already)
if(${_contains_already} EQUAL -1)
list(APPEND _protobuf_include_path -I ${ABS_PATH})
endif()
endforeach()
set(_generated_srcs_all)
foreach(_proto ${protobuf_generate_grpc_PROTOS})
get_filename_component(_abs_file ${_proto} ABSOLUTE)
get_filename_component(_abs_dir ${_abs_file} DIRECTORY)
get_filename_component(_basename ${_proto} NAME_WE)
file(RELATIVE_PATH _rel_dir ${CMAKE_CURRENT_SOURCE_DIR} ${_abs_dir})
set(_possible_rel_dir)
if(NOT protobuf_generate_grpc_APPEND_PATH)
set(_possible_rel_dir ${_rel_dir}/)
endif()
set(_generated_srcs)
foreach(_ext ${protobuf_generate_grpc_GENERATE_EXTENSIONS})
list(APPEND _generated_srcs "${protobuf_generate_grpc_PROTOC_OUT_DIR}/${_possible_rel_dir}${_basename}${_ext}")
endforeach()
if(protobuf_generate_grpc_DESCRIPTORS AND protobuf_generate_grpc_LANGUAGE STREQUAL cpp)
set(_descriptor_file "${CMAKE_CURRENT_BINARY_DIR}/${_basename}.desc")
set(_dll_desc_out "--descriptor_set_out=${_descriptor_file}")
list(APPEND _generated_srcs ${_descriptor_file})
endif()
list(APPEND _generated_srcs_all ${_generated_srcs})
add_custom_command(
OUTPUT ${_generated_srcs}
COMMAND protobuf::protoc
ARGS --${protobuf_generate_grpc_LANGUAGE}_out ${_dll_export_decl}${protobuf_generate_grpc_PROTOC_OUT_DIR}
--grpc_out ${_dll_export_decl}${protobuf_generate_grpc_PROTOC_OUT_DIR}
--plugin=protoc-gen-grpc=$<TARGET_FILE:${protobuf_generate_grpc_PLUGIN}>
${_dll_desc_out} ${_protobuf_include_path} ${_abs_file}
DEPENDS ${_abs_file} protobuf::protoc ${protobuf_generate_grpc_PLUGIN}
COMMENT "Running ${protobuf_generate_grpc_LANGUAGE} protocol buffer compiler on ${_proto}"
VERBATIM)
endforeach()
set_source_files_properties(${_generated_srcs_all} PROPERTIES GENERATED TRUE)
if(protobuf_generate_grpc_OUT_VAR)
set(${protobuf_generate_grpc_OUT_VAR} ${_generated_srcs_all} PARENT_SCOPE)
endif()
if(protobuf_generate_grpc_TARGET)
target_sources(${protobuf_generate_grpc_TARGET} PRIVATE ${_generated_srcs_all})
endif()
endfunction()
# Find the libraries.
if(gRPC_USE_STATIC_LIBS)
# Support preference of static libs by adjusting CMAKE_FIND_LIBRARY_SUFFIXES
set(_gRPC_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
if(WIN32)
set(CMAKE_FIND_LIBRARY_SUFFIXES .lib .a ${CMAKE_FIND_LIBRARY_SUFFIXES})
else()
set(CMAKE_FIND_LIBRARY_SUFFIXES .a)
endif()
endif()
find_library(gRPC_LIBRARY NAMES grpc)
find_library(gRPC_CPP_LIBRARY NAMES grpc++)
find_library(gRPC_UNSECURE_LIBRARY NAMES grpc_unsecure)
find_library(gRPC_CPP_UNSECURE_LIBRARY NAMES grpc++_unsecure)
find_library(gRPC_CARES_LIBRARY NAMES cares)
set(gRPC_LIBRARIES)
if(gRPC_USE_UNSECURE_LIBRARIES)
if(gRPC_UNSECURE_LIBRARY)
set(gRPC_LIBRARIES ${gRPC_LIBRARIES} ${gRPC_UNSECURE_LIBRARY})
endif()
if(gRPC_CPP_UNSECURE_LIBRARY)
set(gRPC_LIBRARIES ${gRPC_LIBRARIES} ${gRPC_CPP_UNSECURE_LIBRARY})
endif()
else()
if(gRPC_LIBRARY)
set(gRPC_LIBRARIES ${gRPC_LIBRARIES} ${gRPC_LIBRARY})
endif()
if(gRPC_CPP_UNSECURE_LIBRARY)
set(gRPC_LIBRARIES ${gRPC_LIBRARIES} ${gRPC_CPP_LIBRARY})
endif()
endif()
set(gRPC_LIBRARIES ${gRPC_LIBRARIES} ${gRPC_CARES_LIBRARY})
# Restore the original find library ordering.
if(gRPC_USE_STATIC_LIBS)
set(CMAKE_FIND_LIBRARY_SUFFIXES ${_gRPC_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES})
endif()
# Find the include directories.
find_path(gRPC_INCLUDE_DIR grpc/grpc.h)
find_path(gRPC_CPP_INCLUDE_DIR grpc++/grpc++.h)
if(gRPC_INCLUDE_DIR AND gRPC_CPP_INCLUDE_DIR AND NOT(gRPC_INCLUDE_DIR STREQUAL gRPC_CPP_INCLUDE_DIR))
set(gRPC_INCLUDE_DIRS ${gRPC_INCLUDE_DIR} ${gRPC_CPP_INCLUDE_DIR})
elseif(gRPC_INCLUDE_DIR)
set(gRPC_INCLUDE_DIRS ${gRPC_INCLUDE_DIR})
else()
set(gRPC_INCLUDE_DIRS ${gRPC_CPP_INCLUDE_DIR})
endif()
# Get full path to plugin.
find_program(gRPC_CPP_PLUGIN
NAMES grpc_cpp_plugin
DOC "The plugin for generating gRPC client and server C++ stubs from `.proto` files")
find_program(gRPC_PYTHON_PLUGIN
NAMES grpc_python_plugin
DOC "The plugin for generating gRPC client and server Python stubs from `.proto` files")
# Add imported targets.
if(gRPC_CPP_LIBRARY AND NOT TARGET grpc++)
add_library(grpc++ UNKNOWN IMPORTED)
set_target_properties(grpc++ PROPERTIES
IMPORTED_LOCATION "${gRPC_CPP_LIBRARY}")
set_target_properties(grpc++ PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES ${gRPC_INCLUDE_DIRS})
endif()
if(gRPC_CPP_UNSECURE_LIBRARY AND NOT TARGET grpc++_unsecure)
add_library(grpc++_unsecure UNKNOWN IMPORTED)
set_target_properties(grpc++_unsecure PROPERTIES
IMPORTED_LOCATION "${gRPC_CPP_UNSECURE_LIBRARY}")
set_target_properties(grpc++_unsecure PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES ${gRPC_INCLUDE_DIRS})
endif()
if(gRPC_CPP_PLUGIN AND NOT TARGET grpc_cpp_plugin)
add_executable(grpc_cpp_plugin IMPORTED)
set_target_properties(grpc_cpp_plugin PROPERTIES
IMPORTED_LOCATION "${gRPC_CPP_PLUGIN}")
endif()
if(gRPC_PYTHON_PLUGIN AND NOT TARGET grpc_python_plugin)
add_executable(grpc_python_plugin IMPORTED)
set_target_properties(grpc_python_plugin PROPERTIES
IMPORTED_LOCATION "${gRPC_PYTHON_PLUGIN}")
endif()
#include(FindPackageHandleStandardArgs.cmake)
FIND_PACKAGE_HANDLE_STANDARD_ARGS(gRPC
REQUIRED_VARS gRPC_LIBRARY gRPC_CPP_LIBRARY gRPC_UNSECURE_LIBRARY gRPC_CPP_UNSECURE_LIBRARY gRPC_CARES_LIBRARY
gRPC_INCLUDE_DIR gRPC_CPP_INCLUDE_DIR gRPC_CPP_PLUGIN gRPC_PYTHON_PLUGIN)
if(gRPC_FOUND)
if(gRPC_DEBUG)
message(STATUS "gRPC: INCLUDE_DIRS=${gRPC_INCLUDE_DIRS}")
message(STATUS "gRPC: LIBRARIES=${gRPC_LIBRARIES}")
message(STATUS "gRPC: CPP_PLUGIN=${gRPC_CPP_PLUGIN}")
message(STATUS "gRPC: PYTHON_PLUGIN=${gRPC_PYTHON_PLUGIN}")
endif()
endif()

22
cmake/analysis.cmake
View File

@ -1,18 +1,28 @@
# This file configures static analysis tools that can be integrated to the build process
# https://clang.llvm.org/extra/clang-tidy/
option (ENABLE_CLANG_TIDY "Use clang-tidy static analyzer" OFF)
option (ENABLE_CLANG_TIDY "Use 'clang-tidy' static analyzer if present" OFF)
if (ENABLE_CLANG_TIDY)
if (${CMAKE_VERSION} VERSION_LESS "3.6.0")
message(FATAL_ERROR "clang-tidy requires CMake version at least 3.6.")
endif()
find_program (CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-10" "clang-tidy-9" "clang-tidy-8")
find_program (CLANG_TIDY_PATH NAMES "clang-tidy" "clang-tidy-11" "clang-tidy-10" "clang-tidy-9" "clang-tidy-8")
if (CLANG_TIDY_PATH)
message(STATUS "Using clang-tidy: ${CLANG_TIDY_PATH}. The checks will be run during build process. See the .clang-tidy file at the root directory to configure the checks.")
set (USE_CLANG_TIDY 1)
message(STATUS
"Using clang-tidy: ${CLANG_TIDY_PATH}.
The checks will be run during build process.
See the .clang-tidy file at the root directory to configure the checks.")
set (USE_CLANG_TIDY ON)
# The variable CMAKE_CXX_CLANG_TIDY will be set inside src and base directories with non third-party code.
# set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}")
elseif (FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION)
message(FATAL_ERROR "clang-tidy is not found")
else ()
message(STATUS "clang-tidy is not found. This is normal - the tool is used only for static code analysis and not essential for build.")
message(STATUS
"clang-tidy is not found.
This is normal - the tool is only used for code static analysis and isn't essential for the build.")
endif ()
endif ()

4
cmake/arch.cmake
View File

@ -16,8 +16,4 @@ endif ()
if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc64le.*|PPC64LE.*)")
set (ARCH_PPC64LE 1)
# FIXME: move this check into tools.cmake
if (COMPILER_CLANG OR (COMPILER_GCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8))
message(FATAL_ERROR "Only gcc-8 or higher is supported for powerpc architecture")
endif ()
endif ()

10
cmake/autogenerated_versions.txt
View File

@ -1,9 +1,9 @@
# This strings autochanged from release_lib.sh:
SET(VERSION_REVISION 54438)
SET(VERSION_REVISION 54444)
SET(VERSION_MAJOR 20)
SET(VERSION_MINOR 8)
SET(VERSION_MINOR 13)
SET(VERSION_PATCH 1)
SET(VERSION_GITHASH 5d60ab33a511efd149c7c3de77c0dd4b81e65b13)
SET(VERSION_DESCRIBE v20.8.1.1-prestable)
SET(VERSION_STRING 20.8.1.1)
SET(VERSION_GITHASH e581f9ccfc5c64867b0f488cce72412fd2966471)
SET(VERSION_DESCRIBE v20.13.1.1-prestable)
SET(VERSION_STRING 20.13.1.1)
# end of autochange

4
cmake/contrib_finder.cmake
View File

@ -8,6 +8,9 @@ macro(find_contrib_lib LIB_NAME)
if (NOT USE_INTERNAL_${LIB_NAME_UC}_LIBRARY)
find_package ("${LIB_NAME}")
if (NOT ${LIB_NAME_UC}_FOUND)
message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use system ${LIB_NAME}")
endif()
endif ()
if (NOT ${LIB_NAME_UC}_FOUND)
@ -17,5 +20,4 @@ macro(find_contrib_lib LIB_NAME)
endif ()
message (STATUS "Using ${LIB_NAME}: ${${LIB_NAME_UC}_INCLUDE_DIR} : ${${LIB_NAME_UC}_LIBRARIES}")
endmacro()

12
cmake/darwin/default_libs.cmake
View File

@ -12,17 +12,7 @@ set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS})
set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS})
# Minimal supported SDK version
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mmacosx-version-min=10.15")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mmacosx-version-min=10.15")
set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} -mmacosx-version-min=10.15")
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -mmacosx-version-min=10.15")
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -mmacosx-version-min=10.15")
# Global libraries
add_library(global-libs INTERFACE)
set(CMAKE_OSX_DEPLOYMENT_TARGET 10.15)
# Unfortunately '-pthread' doesn't work with '-nodefaultlibs'.
# Just make sure we have pthreads at all.

25
cmake/find/amqpcpp.cmake
View File

@ -1,21 +1,22 @@
option(ENABLE_AMQPCPP "Enalbe AMQP-CPP" ${ENABLE_LIBRARIES})
if (NOT ENABLE_AMQPCPP)
return()
endif()
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/CMakeLists.txt")
message (WARNING "submodule contrib/AMQP-CPP is missing. to fix try run: \n git submodule update --init --recursive")
set (ENABLE_AMQPCPP 0)
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal AMQP-CPP library")
set (USE_AMQPCPP 0)
return()
endif ()
if (ENABLE_AMQPCPP)
set (USE_AMQPCPP 1)
set (AMQPCPP_LIBRARY AMQP-CPP)
set (USE_AMQPCPP 1)
set (AMQPCPP_LIBRARY AMQP-CPP)
set (AMQPCPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/include")
list (APPEND AMQPCPP_INCLUDE_DIR
"${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/include"
"${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP")
endif()
set (AMQPCPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/include")
list (APPEND AMQPCPP_INCLUDE_DIR
"${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP/include"
"${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP")
message (STATUS "Using AMQP-CPP=${USE_AMQPCPP}: ${AMQPCPP_INCLUDE_DIR} : ${AMQPCPP_LIBRARY}")

22
cmake/find/avro.cmake
View File

@ -1,28 +1,36 @@
# Needed when using Apache Avro serialization format
option (ENABLE_AVRO "Enable Avro" ${ENABLE_LIBRARIES})
if (ENABLE_AVRO)
if (NOT ENABLE_AVRO)
if (USE_INTERNAL_AVRO_LIBRARY)
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal avro library with ENABLE_AVRO=OFF")
endif()
return()
endif()
option (USE_INTERNAL_AVRO_LIBRARY "Set to FALSE to use system avro library instead of bundled" ${NOT_UNBUNDLED})
option (USE_INTERNAL_AVRO_LIBRARY
"Set to FALSE to use system avro library instead of bundled" ON) # TODO: provide unbundled support
if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/avro/lang/c++/CMakeLists.txt")
if(USE_INTERNAL_AVRO_LIBRARY)
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/avro/lang/c++/CMakeLists.txt")
if (USE_INTERNAL_AVRO_LIBRARY)
message(WARNING "submodule contrib/avro is missing. to fix try run: \n git submodule update --init --recursive")
message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot find internal avro")
set(USE_INTERNAL_AVRO_LIBRARY 0)
endif()
set(MISSING_INTERNAL_AVRO_LIBRARY 1)
set(USE_INTERNAL_AVRO_LIBRARY 0)
endif()
if (NOT USE_INTERNAL_AVRO_LIBRARY)
message (${RECONFIGURE_MESSAGE_LEVEL} "Using system avro library is not supported yet")
elseif(NOT MISSING_INTERNAL_AVRO_LIBRARY)
include(cmake/find/snappy.cmake)
set(AVROCPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/avro/lang/c++/include")
set(AVROCPP_LIBRARY avrocpp)
set(USE_INTERNAL_AVRO_LIBRARY 1)
endif ()
if (AVROCPP_LIBRARY AND AVROCPP_INCLUDE_DIR)
set(USE_AVRO 1)
endif()
endif()
message (STATUS "Using avro=${USE_AVRO}: ${AVROCPP_INCLUDE_DIR} : ${AVROCPP_LIBRARY}")

26
cmake/find/base64.cmake
View File

@ -1,17 +1,21 @@
option (ENABLE_BASE64 "Enable base64" ${ENABLE_LIBRARIES})
if (NOT ENABLE_BASE64)
return()
endif()
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64/LICENSE")
set (MISSING_INTERNAL_BASE64_LIBRARY 1)
message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive")
endif ()
if (NOT MISSING_INTERNAL_BASE64_LIBRARY)
option (ENABLE_BASE64 "Enable base64" ${ENABLE_LIBRARIES})
endif ()
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64")
message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive")
else()
set (BASE64_LIBRARY base64)
set (USE_BASE64 1)
endif()
if (ENABLE_BASE64)
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/base64")
message (WARNING "submodule contrib/base64 is missing. to fix try run: \n git submodule update --init --recursive")
else()
set (BASE64_LIBRARY base64)
set (USE_BASE64 1)
endif()
endif ()
if (NOT USE_BASE64)
message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot enable base64")
endif()

21
cmake/find/brotli.cmake
View File

@ -1,12 +1,23 @@
option (ENABLE_BROTLI "Enable brotli" ${ENABLE_LIBRARIES})
if (ENABLE_BROTLI)
if (NOT ENABLE_BROTLI)
if (USE_INTERNAL_BROTLI_LIBRARY)
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal brotly library with ENABLE_BROTLI=OFF")
endif()
return()
endif()
option (USE_INTERNAL_BROTLI_LIBRARY "Set to FALSE to use system libbrotli library instead of bundled" ${NOT_UNBUNDLED})
if (UNBUNDLED)
# Many system ship only dynamic brotly libraries, so we back off to bundled by default
option (USE_INTERNAL_BROTLI_LIBRARY "Set to FALSE to use system libbrotli library instead of bundled" ${USE_STATIC_LIBRARIES})
else()
option (USE_INTERNAL_BROTLI_LIBRARY "Set to FALSE to use system libbrotli library instead of bundled" ON)
endif()
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/brotli/c/include/brotli/decode.h")
if (USE_INTERNAL_BROTLI_LIBRARY)
message (WARNING "submodule contrib/brotli is missing. to fix try run: \n git submodule update --init --recursive")
message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot find internal brotli")
set (USE_INTERNAL_BROTLI_LIBRARY 0)
endif ()
set (MISSING_INTERNAL_BROTLI_LIBRARY 1)
@ -19,18 +30,18 @@ if(NOT USE_INTERNAL_BROTLI_LIBRARY)
find_path(BROTLI_INCLUDE_DIR NAMES brotli/decode.h brotli/encode.h brotli/port.h brotli/types.h PATHS ${BROTLI_INCLUDE_PATHS})
if(BROTLI_LIBRARY_DEC AND BROTLI_LIBRARY_ENC AND BROTLI_LIBRARY_COMMON)
set(BROTLI_LIBRARY ${BROTLI_LIBRARY_DEC} ${BROTLI_LIBRARY_ENC} ${BROTLI_LIBRARY_COMMON})
else()
message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use system brotli")
endif()
endif()
if (BROTLI_LIBRARY AND BROTLI_INCLUDE_DIR)
set (USE_BROTLI 1)
elseif (NOT MISSING_INTERNAL_BROTLI_LIBRARY)
set (BROTLI_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/brotli/c/include)
set (BROTLI_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/brotli/c/include")
set (USE_INTERNAL_BROTLI_LIBRARY 1)
set (BROTLI_LIBRARY brotli)
set (USE_BROTLI 1)
endif ()
endif()
message (STATUS "Using brotli=${USE_BROTLI}: ${BROTLI_INCLUDE_DIR} : ${BROTLI_LIBRARY}")

26
cmake/find/capnp.cmake
View File

@ -1,15 +1,21 @@
option (ENABLE_CAPNP "Enable Cap'n Proto" ${ENABLE_LIBRARIES})
if (ENABLE_CAPNP)
if (NOT ENABLE_CAPNP)
if (USE_INTERNAL_CAPNP_LIBRARY)
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal capnproto library with ENABLE_CAPNP=OFF")
endif()
return()
endif()
option (USE_INTERNAL_CAPNP_LIBRARY "Set to FALSE to use system capnproto library instead of bundled" ${NOT_UNBUNDLED})
if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/capnproto/CMakeLists.txt")
if(USE_INTERNAL_CAPNP_LIBRARY)
message(WARNING "submodule contrib/capnproto is missing. to fix try run: \n git submodule update --init --recursive")
message(${RECONFIGURE_MESSAGE_LEVEL} "cannot find internal capnproto")
set(USE_INTERNAL_CAPNP_LIBRARY 0)
endif()
set(MISSING_INTERNAL_CAPNP_LIBRARY 1)
set(USE_INTERNAL_CAPNP_LIBRARY 0)
endif()
# FIXME: refactor to use `add_library(… IMPORTED)` if possible.
@ -18,17 +24,21 @@ if (NOT USE_INTERNAL_CAPNP_LIBRARY)
find_library (CAPNP capnp)
find_library (CAPNPC capnpc)
set (CAPNP_LIBRARIES ${CAPNPC} ${CAPNP} ${KJ})
if(KJ AND CAPNP AND CAPNPC)
set (CAPNP_LIBRARIES ${CAPNPC} ${CAPNP} ${KJ})
else()
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system capnproto")
endif()
endif()
if (CAPNP_LIBRARIES)
set (USE_CAPNP 1)
elseif(NOT MISSING_INTERNAL_CAPNP_LIBRARY)
add_subdirectory(contrib/capnproto-cmake)
set (CAPNP_LIBRARIES capnpc)
endif ()
if (CAPNP_LIBRARIES)
set (USE_CAPNP 1)
endif ()
set (USE_INTERNAL_CAPNP_LIBRARY 1)
endif ()
message (STATUS "Using capnp=${USE_CAPNP}: ${CAPNP_LIBRARIES}")

46
cmake/find/cassandra.cmake
View File

@ -1,29 +1,33 @@
option(ENABLE_CASSANDRA "Enable Cassandra" ${ENABLE_LIBRARIES})
if (ENABLE_CASSANDRA)
if (APPLE)
SET(CMAKE_MACOSX_RPATH ON)
endif()
if (NOT ENABLE_CASSANDRA)
return()
endif()
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libuv")
message (ERROR "submodule contrib/libuv is missing. to fix try run: \n git submodule update --init --recursive")
elseif (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cassandra")
message (ERROR "submodule contrib/cassandra is missing. to fix try run: \n git submodule update --init --recursive")
if (APPLE)
set(CMAKE_MACOSX_RPATH ON)
endif()
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libuv")
message (ERROR "submodule contrib/libuv is missing. to fix try run: \n git submodule update --init --recursive")
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libuv needed for Cassandra")
elseif (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cassandra")
message (ERROR "submodule contrib/cassandra is missing. to fix try run: \n git submodule update --init --recursive")
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal Cassandra")
else()
set (LIBUV_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/libuv")
set (CASSANDRA_INCLUDE_DIR
"${ClickHouse_SOURCE_DIR}/contrib/cassandra/include/")
if (MAKE_STATIC_LIBRARIES)
set (LIBUV_LIBRARY uv_a)
set (CASSANDRA_LIBRARY cassandra_static)
else()
set (LIBUV_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/libuv")
set (CASSANDRA_INCLUDE_DIR
"${ClickHouse_SOURCE_DIR}/contrib/cassandra/include/")
if (USE_STATIC_LIBRARIES)
set (LIBUV_LIBRARY uv_a)
set (CASSANDRA_LIBRARY cassandra_static)
else()
set (LIBUV_LIBRARY uv)
set (CASSANDRA_LIBRARY cassandra)
endif()
set (USE_CASSANDRA 1)
set (CASS_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/cassandra")
set (LIBUV_LIBRARY uv)
set (CASSANDRA_LIBRARY cassandra)
endif()
set (USE_CASSANDRA 1)
set (CASS_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/cassandra")
endif()
message (STATUS "Using cassandra=${USE_CASSANDRA}: ${CASSANDRA_INCLUDE_DIR} : ${CASSANDRA_LIBRARY}")

35
cmake/find/ccache.cmake
View File

@ -1,14 +1,41 @@
find_program (CCACHE_FOUND ccache)
if (CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" OR CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache")
set(COMPILER_MATCHES_CCACHE 1)
else()
set(COMPILER_MATCHES_CCACHE 0)
endif()
if (CCACHE_FOUND AND NOT CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" AND NOT CMAKE_CXX_COMPILER MATCHES "ccache")
if ((ENABLE_CCACHE OR NOT DEFINED ENABLE_CCACHE) AND NOT COMPILER_MATCHES_CCACHE)
find_program (CCACHE_FOUND ccache)
if (CCACHE_FOUND)
set(ENABLE_CCACHE_BY_DEFAULT 1)
else()
set(ENABLE_CCACHE_BY_DEFAULT 0)
endif()
endif()
if (NOT CCACHE_FOUND AND NOT DEFINED ENABLE_CCACHE AND NOT COMPILER_MATCHES_CCACHE)
message(WARNING "CCache is not found. We recommend setting it up if you build ClickHouse from source often. "
"Setting it up will significantly reduce compilation time for 2nd and consequent builds")
endif()
# https://ccache.dev/
option(ENABLE_CCACHE "Speedup re-compilations using ccache (external tool)" ${ENABLE_CCACHE_BY_DEFAULT})
if (NOT ENABLE_CCACHE)
return()
endif()
if (CCACHE_FOUND AND NOT COMPILER_MATCHES_CCACHE)
execute_process(COMMAND ${CCACHE_FOUND} "-V" OUTPUT_VARIABLE CCACHE_VERSION)
string(REGEX REPLACE "ccache version ([0-9\\.]+).*" "\\1" CCACHE_VERSION ${CCACHE_VERSION})
if (CCACHE_VERSION VERSION_GREATER "3.2.0" OR NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
#message(STATUS "Using ${CCACHE_FOUND} ${CCACHE_VERSION}")
message(STATUS "Using ${CCACHE_FOUND} ${CCACHE_VERSION}")
set_property (GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND})
else ()
message(STATUS "Not using ${CCACHE_FOUND} ${CCACHE_VERSION} bug: https://bugzilla.samba.org/show_bug.cgi?id=8118")
message(${RECONFIGURE_MESSAGE_LEVEL} "Not using ${CCACHE_FOUND} ${CCACHE_VERSION} bug: https://bugzilla.samba.org/show_bug.cgi?id=8118")
endif ()
elseif (NOT CCACHE_FOUND AND NOT COMPILER_MATCHES_CCACHE)
message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use ccache")
endif ()

37
cmake/find/curl.cmake Normal file
View File

@ -0,0 +1,37 @@
option (ENABLE_CURL "Enable curl" ${ENABLE_LIBRARIES})
if (NOT ENABLE_CURL)
if (USE_INTERNAL_CURL)
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal curl with ENABLE_CURL=OFF")
endif()
return()
endif()
option (USE_INTERNAL_CURL "Use internal curl library" ${NOT_UNBUNDLED})
if (NOT USE_INTERNAL_CURL)
find_package (CURL)
if (NOT CURL_FOUND)
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system curl")
endif()
endif()
if (NOT CURL_FOUND)
set (USE_INTERNAL_CURL 1)
set (CURL_LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/curl")
# find_package(CURL) compatibility for the following packages that uses
# find_package(CURL)/include(FindCURL):
# - mariadb-connector-c
# - aws-s3-cmake
# - sentry-native
set (CURL_FOUND ON CACHE BOOL "")
set (CURL_ROOT_DIR ${CURL_LIBRARY_DIR} CACHE PATH "")
set (CURL_INCLUDE_DIR ${CURL_LIBRARY_DIR}/include CACHE PATH "")
set (CURL_INCLUDE_DIRS ${CURL_LIBRARY_DIR}/include CACHE PATH "")
set (CURL_LIBRARY curl CACHE STRING "")
set (CURL_LIBRARIES ${CURL_LIBRARY} CACHE STRING "")
set (CURL_VERSION_STRING 7.67.0 CACHE STRING "")
endif ()
message (STATUS "Using curl: ${CURL_INCLUDE_DIRS} : ${CURL_LIBRARIES}")

76
cmake/find/cxx.cmake
View File

@ -1,41 +1,71 @@
set(USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT ${NOT_UNBUNDLED})
option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++" ${NOT_UNBUNDLED})
if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxx/CMakeLists.txt")
message(WARNING "submodule contrib/libcxx is missing. to fix try run: \n git submodule update --init --recursive")
set(USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT 0)
if (NOT USE_LIBCXX)
if (USE_INTERNAL_LIBCXX_LIBRARY)
message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal libcxx with USE_LIBCXX=OFF")
endif()
target_link_libraries(global-libs INTERFACE -l:libstdc++.a -l:libstdc++fs.a) # Always link these libraries as static
target_link_libraries(global-libs INTERFACE ${EXCEPTION_HANDLING_LIBRARY})
return()
endif()
option (USE_LIBCXX "Use libc++ and libc++abi instead of libstdc++" ${NOT_UNBUNDLED})
option (USE_INTERNAL_LIBCXX_LIBRARY "Set to FALSE to use system libcxx and libcxxabi libraries instead of bundled" ${USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT})
set(USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT ${NOT_UNBUNDLED})
if (USE_LIBCXX)
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=0") # More checks in debug build.
option (USE_INTERNAL_LIBCXX_LIBRARY "Disable to use system libcxx and libcxxabi libraries instead of bundled"
${USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT})
if (NOT USE_INTERNAL_LIBCXX_LIBRARY)
find_library (LIBCXX_LIBRARY c++)
find_library (LIBCXXFS_LIBRARY c++fs)
find_library (LIBCXXABI_LIBRARY c++abi)
if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libcxx/CMakeLists.txt")
if (USE_INTERNAL_LIBCXX_LIBRARY)
message(WARNING "submodule contrib/libcxx is missing. to fix try run: \n git submodule update --init --recursive")
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal libcxx")
set(USE_INTERNAL_LIBCXX_LIBRARY 0)
endif()
set(USE_INTERNAL_LIBCXX_LIBRARY_DEFAULT 0)
set(MISSING_INTERNAL_LIBCXX_LIBRARY 1)
endif()
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=0") # More checks in debug build.
target_link_libraries(global-libs INTERFACE ${EXCEPTION_HANDLING_LIBRARY})
if (NOT USE_INTERNAL_LIBCXX_LIBRARY)
find_library (LIBCXX_LIBRARY c++)
find_library (LIBCXXFS_LIBRARY c++fs)
find_library (LIBCXXABI_LIBRARY c++abi)
if(LIBCXX_LIBRARY AND LIBCXXABI_LIBRARY) # c++fs is now a part of the libc++
set (HAVE_LIBCXX 1)
else ()
set (LIBCXX_LIBRARY cxx)
set (LIBCXXABI_LIBRARY cxxabi)
add_subdirectory(contrib/libcxxabi-cmake)
add_subdirectory(contrib/libcxx-cmake)
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system libcxx")
endif()
# Exception handling library is embedded into libcxxabi.
endif ()
if(NOT LIBCXXFS_LIBRARY)
set(LIBCXXFS_LIBRARY ${LIBCXX_LIBRARY})
endif()
target_link_libraries(global-libs INTERFACE ${LIBCXX_LIBRARY} ${LIBCXXABI_LIBRARY} ${LIBCXXFS_LIBRARY})
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
target_link_libraries(global-libs INTERFACE ${EXCEPTION_HANDLING_LIBRARY})
endif ()
if (NOT HAVE_LIBCXX AND NOT MISSING_INTERNAL_LIBCXX_LIBRARY)
set (LIBCXX_LIBRARY cxx)
set (LIBCXXABI_LIBRARY cxxabi)
add_subdirectory(contrib/libcxxabi-cmake)
add_subdirectory(contrib/libcxx-cmake)
# Exception handling library is embedded into libcxxabi.
set (HAVE_LIBCXX 1)
set(USE_INTERNAL_LIBCXX_LIBRARY 1)
endif ()
if (HAVE_LIBCXX)
target_link_libraries(global-libs INTERFACE ${LIBCXX_LIBRARY} ${LIBCXXABI_LIBRARY} ${LIBCXXFS_LIBRARY})
message (STATUS "Using libcxx: ${LIBCXX_LIBRARY}")
message (STATUS "Using libcxxfs: ${LIBCXXFS_LIBRARY}")
message (STATUS "Using libcxxabi: ${LIBCXXABI_LIBRARY}")
else ()
else()
target_link_libraries(global-libs INTERFACE -l:libstdc++.a -l:libstdc++fs.a) # Always link these libraries as static
target_link_libraries(global-libs INTERFACE ${EXCEPTION_HANDLING_LIBRARY})
endif ()
endif()

Some files were not shown because too many files have changed in this diff Show More