mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 23:21:59 +00:00
Merge
This commit is contained in:
Clément Rodriguez
commit
cdd99d7f23
@ -52,12 +52,12 @@ IncludeCategories:
|
||||
ReflowComments: false
|
||||
AlignEscapedNewlinesLeft: false
|
||||
AlignEscapedNewlines: DontAlign
|
||||
AlignTrailingComments: true
|
||||
|
||||
# Not changed:
|
||||
AccessModifierOffset: -4
|
||||
AlignConsecutiveAssignments: false
|
||||
AlignOperands: false
|
||||
AlignTrailingComments: false
|
||||
AllowAllParametersOfDeclarationOnNextLine: true
|
||||
AllowShortBlocksOnASingleLine: false
|
||||
AllowShortCaseLabelsOnASingleLine: false
|
||||
|
||||
2
.github/CODEOWNERS
vendored
2
.github/CODEOWNERS
vendored
@ -1,4 +1,2 @@
|
||||
dbms/* @ClickHouse/core-assigner
|
||||
utils/* @ClickHouse/core-assigner
|
||||
docs/* @ClickHouse/docs
|
||||
docs/zh/* @ClickHouse/docs-zh
|
||||
|
||||
12
.github/PULL_REQUEST_TEMPLATE.md
vendored
12
.github/PULL_REQUEST_TEMPLATE.md
vendored
@ -7,16 +7,20 @@ Changelog category (leave one):
|
||||
- Performance Improvement
|
||||
- Backward Incompatible Change
|
||||
- Build/Testing/Packaging Improvement
|
||||
- Documentation
|
||||
- Documentation (changelog entry is not required)
|
||||
- Other
|
||||
- Non-significant (changelog entry is not needed)
|
||||
- Non-significant (changelog entry is not required)
|
||||
|
||||
|
||||
Changelog entry (up to few sentences, not needed for non-significant PRs):
|
||||
Changelog entry (a user-readable short description of the changes that goes to CHANGELOG.md):
|
||||
|
||||
...
|
||||
|
||||
|
||||
Detailed description (optional):
|
||||
Detailed description / Documentation draft:
|
||||
|
||||
...
|
||||
|
||||
By adding documentation, you'll allow users to try your new feature immediately, not when someone else will have time to document it later. Documentation is necessary for all features that affect user experience in any way. You can add brief documentation draft above, or add documentation right into your patch as Markdown files in [docs](https://github.com/ClickHouse/ClickHouse/tree/master/docs) folder.
|
||||
|
||||
If you are doing this for the first time, it's recommended to read the lightweight [Contributing to ClickHouse Documentation](https://github.com/ClickHouse/ClickHouse/tree/master/docs/README.md) guide first.
|
||||
|
||||
10
.gitignore
vendored
10
.gitignore
vendored
@ -13,12 +13,15 @@
|
||||
/build_*
|
||||
/build-*
|
||||
/docs/build
|
||||
/docs/publish
|
||||
/docs/edit
|
||||
/docs/website
|
||||
/docs/tools/venv/
|
||||
/docs/en/development/build/
|
||||
/docs/ru/development/build/
|
||||
/docs/en/single.md
|
||||
/docs/ru/single.md
|
||||
/docs/zh/single.md
|
||||
/docs/ja/single.md
|
||||
/docs/fa/single.md
|
||||
|
||||
# callgrind files
|
||||
callgrind.out.*
|
||||
@ -244,6 +247,9 @@ website/package-lock.json
|
||||
# ccls cache
|
||||
/.ccls-cache
|
||||
|
||||
# clangd cache
|
||||
/.clangd
|
||||
|
||||
/compile_commands.json
|
||||
|
||||
# Toolchains
|
||||
|
||||
54
.gitmodules
vendored
54
.gitmodules
vendored
@ -1,6 +1,7 @@
|
||||
[submodule "contrib/poco"]
|
||||
path = contrib/poco
|
||||
url = https://github.com/ClickHouse-Extras/poco
|
||||
branch = clickhouse
|
||||
[submodule "contrib/zstd"]
|
||||
path = contrib/zstd
|
||||
url = https://github.com/facebook/zstd.git
|
||||
@ -28,9 +29,6 @@
|
||||
[submodule "contrib/re2"]
|
||||
path = contrib/re2
|
||||
url = https://github.com/google/re2.git
|
||||
[submodule "contrib/ssl"]
|
||||
path = contrib/ssl
|
||||
url = https://github.com/ClickHouse-Extras/ssl.git
|
||||
[submodule "contrib/llvm"]
|
||||
path = contrib/llvm
|
||||
url = https://github.com/ClickHouse-Extras/llvm
|
||||
@ -51,7 +49,7 @@
|
||||
url = https://github.com/ClickHouse-Extras/boost.git
|
||||
[submodule "contrib/base64"]
|
||||
path = contrib/base64
|
||||
url = https://github.com/aklomp/base64.git
|
||||
url = https://github.com/powturbo/Turbo-Base64.git
|
||||
[submodule "contrib/arrow"]
|
||||
path = contrib/arrow
|
||||
url = https://github.com/apache/arrow
|
||||
@ -69,10 +67,10 @@
|
||||
url = https://github.com/ClickHouse-Extras/libgsasl.git
|
||||
[submodule "contrib/libcxx"]
|
||||
path = contrib/libcxx
|
||||
url = https://github.com/llvm-mirror/libcxx.git
|
||||
url = https://github.com/ClickHouse-Extras/libcxx.git
|
||||
[submodule "contrib/libcxxabi"]
|
||||
path = contrib/libcxxabi
|
||||
url = https://github.com/llvm-mirror/libcxxabi.git
|
||||
url = https://github.com/ClickHouse-Extras/libcxxabi.git
|
||||
[submodule "contrib/snappy"]
|
||||
path = contrib/snappy
|
||||
url = https://github.com/google/snappy
|
||||
@ -106,3 +104,47 @@
|
||||
[submodule "contrib/sparsehash-c11"]
|
||||
path = contrib/sparsehash-c11
|
||||
url = https://github.com/sparsehash/sparsehash-c11.git
|
||||
[submodule "contrib/grpc"]
|
||||
path = contrib/grpc
|
||||
url = https://github.com/ClickHouse-Extras/grpc.git
|
||||
branch = v1.25.0
|
||||
[submodule "contrib/aws"]
|
||||
path = contrib/aws
|
||||
url = https://github.com/aws/aws-sdk-cpp.git
|
||||
[submodule "aws-c-event-stream"]
|
||||
path = contrib/aws-c-event-stream
|
||||
url = https://github.com/awslabs/aws-c-event-stream.git
|
||||
[submodule "aws-c-common"]
|
||||
path = contrib/aws-c-common
|
||||
url = https://github.com/awslabs/aws-c-common.git
|
||||
[submodule "aws-checksums"]
|
||||
path = contrib/aws-checksums
|
||||
url = https://github.com/awslabs/aws-checksums.git
|
||||
[submodule "contrib/curl"]
|
||||
path = contrib/curl
|
||||
url = https://github.com/curl/curl.git
|
||||
[submodule "contrib/openssl"]
|
||||
path = contrib/openssl
|
||||
url = https://github.com/ClickHouse-Extras/openssl.git
|
||||
[submodule "contrib/icudata"]
|
||||
path = contrib/icudata
|
||||
url = https://github.com/ClickHouse-Extras/icudata.git
|
||||
[submodule "contrib/icu"]
|
||||
path = contrib/icu
|
||||
url = https://github.com/unicode-org/icu.git
|
||||
[submodule "contrib/flatbuffers"]
|
||||
path = contrib/flatbuffers
|
||||
url = https://github.com/google/flatbuffers.git
|
||||
[submodule "contrib/libc-headers"]
|
||||
path = contrib/libc-headers
|
||||
url = https://github.com/ClickHouse-Extras/libc-headers.git
|
||||
[submodule "contrib/replxx"]
|
||||
path = contrib/replxx
|
||||
url = https://github.com/AmokHuginnsson/replxx.git
|
||||
[submodule "contrib/ryu"]
|
||||
path = contrib/ryu
|
||||
url = https://github.com/ClickHouse-Extras/ryu.git
|
||||
[submodule "contrib/avro"]
|
||||
path = contrib/avro
|
||||
url = https://github.com/ClickHouse-Extras/avro.git
|
||||
ignore = untracked
|
||||
|
||||
45
AUTHORS
45
AUTHORS
@ -1,43 +1,2 @@
|
||||
The following authors have created the source code of "ClickHouse"
|
||||
published and distributed by YANDEX LLC as the owner:
|
||||
|
||||
Alexander Makarov <asealback@yandex-team.ru>
|
||||
Alexander Prudaev <aprudaev@yandex-team.ru>
|
||||
Alexey Arno <af-arno@yandex-team.ru>
|
||||
Alexey Milovidov <milovidov@yandex-team.ru>
|
||||
Alexey Tronov <vkusny@yandex-team.ru>
|
||||
Alexey Vasiliev <loudhorr@yandex-team.ru>
|
||||
Alexey Zatelepin <ztlpn@yandex-team.ru>
|
||||
Amy Krishnevsky <krishnevsky@yandex-team.ru>
|
||||
Andrey M <hertz@yandex-team.ru>
|
||||
Andrey Mironov <hertz@yandex-team.ru>
|
||||
Andrey Urusov <drobus@yandex-team.ru>
|
||||
Anton Tikhonov <rokerjoker@yandex-team.ru>
|
||||
Dmitry Bilunov <kmeaw@yandex-team.ru>
|
||||
Dmitry Galuza <galuza@yandex-team.ru>
|
||||
Eugene Konkov <konkov@yandex-team.ru>
|
||||
Evgeniy Gatov <egatov@yandex-team.ru>
|
||||
Ilya Khomutov <robert@yandex-team.ru>
|
||||
Ilya Korolev <breeze@yandex-team.ru>
|
||||
Ivan Blinkov <blinkov@yandex-team.ru>
|
||||
Maxim Nikulin <mnikulin@yandex-team.ru>
|
||||
Michael Kolupaev <mkolupaev@yandex-team.ru>
|
||||
Michael Razuvaev <razuvaev@yandex-team.ru>
|
||||
Nikolai Kochetov <nik-kochetov@yandex-team.ru>
|
||||
Nikolay Vasiliev <lonlylocly@yandex-team.ru>
|
||||
Nikolay Volosatov <bamx23@yandex-team.ru>
|
||||
Pavel Artemkin <stanly@yandex-team.ru>
|
||||
Pavel Kartaviy <kartavyy@yandex-team.ru>
|
||||
Roman Nozdrin <drrtuy@yandex-team.ru>
|
||||
Roman Peshkurov <peshkurov@yandex-team.ru>
|
||||
Sergey Fedorov <fets@yandex-team.ru>
|
||||
Sergey Lazarev <hamilkar@yandex-team.ru>
|
||||
Sergey Magidovich <mgsergio@yandex-team.ru>
|
||||
Sergey Serebryanik <serebrserg@yandex-team.ru>
|
||||
Sergey Veletskiy <velom@yandex-team.ru>
|
||||
Vasily Okunev <okunev@yandex-team.ru>
|
||||
Vitaliy Lyudvichenko <vludv@yandex-team.ru>
|
||||
Vladimir Chebotarev <chebotarev@yandex-team.ru>
|
||||
Vsevolod Orlov <vorloff@yandex-team.ru>
|
||||
Vyacheslav Alipov <alipov@yandex-team.ru>
|
||||
Yuriy Galitskiy <orantius@yandex-team.ru>
|
||||
To see the list of authors who created the source code of ClickHouse, published and distributed by YANDEX LLC as the owner,
|
||||
run "SELECT * FROM system.contributors;" query on any ClickHouse server.
|
||||
|
||||
1243
CHANGELOG.md
1243
CHANGELOG.md
File diff suppressed because it is too large
Load Diff
@ -95,6 +95,8 @@ if (CMAKE_GENERATOR STREQUAL "Ninja")
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fdiagnostics-color=always")
|
||||
endif ()
|
||||
|
||||
include (cmake/add_warning.cmake)
|
||||
|
||||
if (NOT MSVC)
|
||||
set (COMMON_WARNING_FLAGS "${COMMON_WARNING_FLAGS} -Wall") # -Werror is also added inside directories with our own code.
|
||||
endif ()
|
||||
@ -176,7 +178,9 @@ if (ARCH_NATIVE)
|
||||
set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=native")
|
||||
endif ()
|
||||
|
||||
set (CMAKE_CXX_STANDARD 17)
|
||||
# cmake < 3.12 doesn't supoprt 20. We'll set CMAKE_CXX_FLAGS for now
|
||||
# set (CMAKE_CXX_STANDARD 20)
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++2a")
|
||||
set (CMAKE_CXX_EXTENSIONS 0) # https://cmake.org/cmake/help/latest/prop_tgt/CXX_EXTENSIONS.html#prop_tgt:CXX_EXTENSIONS
|
||||
set (CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
|
||||
@ -198,14 +202,21 @@ if(WITH_COVERAGE AND COMPILER_GCC)
|
||||
endif()
|
||||
|
||||
set (CMAKE_BUILD_COLOR_MAKEFILE ON)
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS}")
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMPILER_FLAGS} ${PLATFORM_EXTRA_CXX_FLAG} ${COMMON_WARNING_FLAGS} ${CXX_WARNING_FLAGS}")
|
||||
set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_CXX_FLAGS_ADD}")
|
||||
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_CXX_FLAGS_ADD}")
|
||||
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} -fno-omit-frame-pointer ${COMMON_WARNING_FLAGS} ${CMAKE_C_FLAGS_ADD}")
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMPILER_FLAGS} ${COMMON_WARNING_FLAGS} ${CMAKE_C_FLAGS_ADD}")
|
||||
set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -O3 ${CMAKE_C_FLAGS_ADD}")
|
||||
set (CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O0 -g3 -ggdb3 -fno-inline ${CMAKE_C_FLAGS_ADD}")
|
||||
|
||||
if (COMPILER_CLANG)
|
||||
# Exception unwinding doesn't work in clang release build without this option
|
||||
# TODO investigate that
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fno-omit-frame-pointer")
|
||||
endif ()
|
||||
|
||||
option (ENABLE_LIBRARIES "Enable all libraries (Global default switch)" ON)
|
||||
|
||||
option (UNBUNDLED "Try find all libraries in system. We recommend to avoid this mode for production builds, because we cannot guarantee exact versions and variants of libraries your system has installed. This mode exists for enthusiastic developers who search for trouble. Also it is useful for maintainers of OS packages." OFF)
|
||||
@ -215,9 +226,13 @@ else ()
|
||||
set(NOT_UNBUNDLED 1)
|
||||
endif ()
|
||||
|
||||
# Using system libs can cause lot of warnings in includes.
|
||||
if (UNBUNDLED OR NOT (OS_LINUX OR APPLE) OR ARCH_32)
|
||||
# Using system libs can cause a lot of warnings in includes (on macro expansion).
|
||||
if (UNBUNDLED OR NOT (OS_LINUX OR OS_DARWIN) OR ARCH_32)
|
||||
option (NO_WERROR "Disable -Werror compiler option" ON)
|
||||
|
||||
if (NOT NO_WERROR)
|
||||
add_warning(error)
|
||||
endif ()
|
||||
endif ()
|
||||
|
||||
# Make this extra-checks for correct library dependencies.
|
||||
@ -241,8 +256,16 @@ endif ()
|
||||
string (TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UC)
|
||||
set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX")
|
||||
|
||||
if (NOT MAKE_STATIC_LIBRARIES)
|
||||
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
|
||||
if (MAKE_STATIC_LIBRARIES)
|
||||
set (CMAKE_POSITION_INDEPENDENT_CODE OFF)
|
||||
if (OS_LINUX)
|
||||
# Slightly more efficient code can be generated
|
||||
set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-pie")
|
||||
set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -fno-pie")
|
||||
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-no-pie")
|
||||
endif ()
|
||||
else ()
|
||||
set (CMAKE_POSITION_INDEPENDENT_CODE ON)
|
||||
endif ()
|
||||
|
||||
# Using "include-what-you-use" tool.
|
||||
@ -307,9 +330,7 @@ include (cmake/find/poco.cmake)
|
||||
include (cmake/find/lz4.cmake)
|
||||
include (cmake/find/xxhash.cmake)
|
||||
include (cmake/find/sparsehash.cmake)
|
||||
include (cmake/find/rt.cmake)
|
||||
include (cmake/find/execinfo.cmake)
|
||||
include (cmake/find/readline_edit.cmake)
|
||||
include (cmake/find/re2.cmake)
|
||||
include (cmake/find/libgsasl.cmake)
|
||||
include (cmake/find/rdkafka.cmake)
|
||||
@ -323,8 +344,10 @@ endif()
|
||||
include (cmake/find/libxml2.cmake)
|
||||
include (cmake/find/brotli.cmake)
|
||||
include (cmake/find/protobuf.cmake)
|
||||
include (cmake/find/grpc.cmake)
|
||||
include (cmake/find/pdqsort.cmake)
|
||||
include (cmake/find/hdfs3.cmake) # uses protobuf
|
||||
include (cmake/find/s3.cmake)
|
||||
include (cmake/find/consistent-hashing.cmake)
|
||||
include (cmake/find/base64.cmake)
|
||||
include (cmake/find/parquet.cmake)
|
||||
@ -333,6 +356,7 @@ include (cmake/find/simdjson.cmake)
|
||||
include (cmake/find/rapidjson.cmake)
|
||||
include (cmake/find/fastops.cmake)
|
||||
include (cmake/find/orc.cmake)
|
||||
include (cmake/find/avro.cmake)
|
||||
|
||||
find_contrib_lib(cityhash)
|
||||
find_contrib_lib(farmhash)
|
||||
@ -344,28 +368,15 @@ if (ENABLE_TESTS)
|
||||
endif ()
|
||||
|
||||
# Need to process before "contrib" dir:
|
||||
include (libs/libcommon/cmake/find_gperftools.cmake)
|
||||
include (libs/libcommon/cmake/find_jemalloc.cmake)
|
||||
include (libs/libcommon/cmake/find_cctz.cmake)
|
||||
include (libs/libmysqlxx/cmake/find_mysqlclient.cmake)
|
||||
include (cmake/find/jemalloc.cmake)
|
||||
include (cmake/find/cctz.cmake)
|
||||
include (cmake/find/mysqlclient.cmake)
|
||||
|
||||
# When testing for memory leaks with Valgrind, don't link tcmalloc or jemalloc.
|
||||
|
||||
if (USE_JEMALLOC)
|
||||
message (STATUS "Link jemalloc: ${JEMALLOC_LIBRARIES}")
|
||||
set (MALLOC_LIBRARIES ${JEMALLOC_LIBRARIES})
|
||||
elseif (USE_TCMALLOC)
|
||||
if (DEBUG_TCMALLOC AND NOT GPERFTOOLS_TCMALLOC_MINIMAL_DEBUG)
|
||||
message (FATAL_ERROR "Requested DEBUG_TCMALLOC but debug library is not found. You should install Google Perftools. Example: sudo apt-get install libgoogle-perftools-dev")
|
||||
endif ()
|
||||
|
||||
if (DEBUG_TCMALLOC AND GPERFTOOLS_TCMALLOC_MINIMAL_DEBUG)
|
||||
message (STATUS "Link libtcmalloc_minimal_debug for testing: ${GPERFTOOLS_TCMALLOC_MINIMAL_DEBUG}")
|
||||
set (MALLOC_LIBRARIES ${GPERFTOOLS_TCMALLOC_MINIMAL_DEBUG})
|
||||
else ()
|
||||
message (STATUS "Link libtcmalloc_minimal: ${GPERFTOOLS_TCMALLOC_MINIMAL}")
|
||||
set (MALLOC_LIBRARIES ${GPERFTOOLS_TCMALLOC_MINIMAL})
|
||||
endif ()
|
||||
elseif (SANITIZE)
|
||||
message (STATUS "Will use ${SANITIZE} sanitizer.")
|
||||
elseif (OS_LINUX)
|
||||
@ -382,20 +393,16 @@ add_subdirectory (contrib EXCLUDE_FROM_ALL)
|
||||
|
||||
macro (add_executable target)
|
||||
# invoke built-in add_executable
|
||||
_add_executable (${ARGV})
|
||||
# explicitly acquire and interpose malloc symbols by clickhouse_malloc
|
||||
_add_executable (${ARGV} $<TARGET_OBJECTS:clickhouse_malloc>)
|
||||
get_target_property (type ${target} TYPE)
|
||||
if (${type} STREQUAL EXECUTABLE)
|
||||
file (RELATIVE_PATH dir ${CMAKE_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR})
|
||||
if (${dir} MATCHES "^dbms")
|
||||
# Only interpose operator::new/delete for dbms executables (MemoryTracker stuff)
|
||||
target_link_libraries (${target} PRIVATE clickhouse_new_delete ${MALLOC_LIBRARIES})
|
||||
else ()
|
||||
target_link_libraries (${target} PRIVATE ${MALLOC_LIBRARIES})
|
||||
endif ()
|
||||
# operator::new/delete for executables (MemoryTracker stuff)
|
||||
target_link_libraries (${target} PRIVATE clickhouse_new_delete ${MALLOC_LIBRARIES})
|
||||
endif()
|
||||
endmacro()
|
||||
|
||||
add_subdirectory (libs)
|
||||
add_subdirectory (base)
|
||||
add_subdirectory (utils)
|
||||
add_subdirectory (dbms)
|
||||
|
||||
|
||||
@ -1,24 +1,30 @@
|
||||
# Contributing to ClickHouse
|
||||
|
||||
## Technical info
|
||||
Developer guide for writing code for ClickHouse is published on official website alongside the usage and operations documentation:
|
||||
https://clickhouse.yandex/docs/en/development/architecture/
|
||||
ClickHouse is an open project, and you can contribute to it in many ways. You can help with ideas, code, or documentation. We appreciate any efforts that help us to make the project better.
|
||||
|
||||
## Legal info
|
||||
Thank you.
|
||||
|
||||
In order for us (YANDEX LLC) to accept patches and other contributions from you, you will have to adopt our Yandex Contributor License Agreement (the "**CLA**"). The current version of the CLA you may find here:
|
||||
## Technical Info
|
||||
|
||||
We have a [developer's guide](https://clickhouse.yandex/docs/en/development/developer_instruction/) for writing code for ClickHouse. Besides this guide, you can find [Overview of ClickHouse Architecture](https://clickhouse.yandex/docs/en/development/architecture/) and instructions on how to build ClickHouse in different environments.
|
||||
|
||||
If you want to contribute to documentation, read the [Contributing to ClickHouse Documentation](docs/README.md) guide.
|
||||
|
||||
## Legal Info
|
||||
|
||||
In order for us (YANDEX LLC) to accept patches and other contributions from you, you may adopt our Yandex Contributor License Agreement (the "**CLA**"). The current version of the CLA you may find here:
|
||||
1) https://yandex.ru/legal/cla/?lang=en (in English) and
|
||||
2) https://yandex.ru/legal/cla/?lang=ru (in Russian).
|
||||
|
||||
By adopting the CLA, you state the following:
|
||||
|
||||
* You obviously wish and are willingly licensing your contributions to us for our open source projects under the terms of the CLA,
|
||||
* You has read the terms and conditions of the CLA and agree with them in full,
|
||||
* You have read the terms and conditions of the CLA and agree with them in full,
|
||||
* You are legally able to provide and license your contributions as stated,
|
||||
* We may use your contributions for our open source projects and for any other our project too,
|
||||
* We rely on your assurances concerning the rights of third parties in relation to your contributes.
|
||||
* We rely on your assurances concerning the rights of third parties in relation to your contributions.
|
||||
|
||||
If you agree with these principles, please read and adopt our CLA. By providing us your contributions, you hereby declare that you has already read and adopt our CLA, and we may freely merge your contributions with our corresponding open source project and use it in further in accordance with terms and conditions of the CLA.
|
||||
If you agree with these principles, please read and adopt our CLA. By providing us your contributions, you hereby declare that you have already read and adopt our CLA, and we may freely merge your contributions with our corresponding open source project and use it in further in accordance with terms and conditions of the CLA.
|
||||
|
||||
If you have already adopted terms and conditions of the CLA, you are able to provide your contributes. When you submit your pull request, please add the following information into it:
|
||||
|
||||
@ -31,4 +37,7 @@ Replace the bracketed text as follows:
|
||||
|
||||
It is enough to provide us such notification once.
|
||||
|
||||
If you don't agree with the CLA, you still can open a pull request to provide your contributions.
|
||||
As an alternative, you can provide DCO instead of CLA. You can find the text of DCO here: https://developercertificate.org/
|
||||
It is enough to read and copy it verbatim to your pull request.
|
||||
|
||||
If you don't agree with the CLA and don't want to provide DCO, you still can open a pull request to provide your contributions.
|
||||
|
||||
4
LICENSE
4
LICENSE
@ -1,4 +1,4 @@
|
||||
Copyright 2016-2019 Yandex LLC
|
||||
Copyright 2016-2020 Yandex LLC
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
@ -188,7 +188,7 @@ Copyright 2016-2019 Yandex LLC
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright 2016-2019 Yandex LLC
|
||||
Copyright 2016-2020 Yandex LLC
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
||||
18
README.md
18
README.md
@ -1,21 +1,17 @@
|
||||
[](https://clickhouse.yandex)
|
||||
[](https://clickhouse.tech)
|
||||
|
||||
ClickHouse is an open-source column-oriented database management system that allows generating analytical data reports in real time.
|
||||
|
||||
## Useful Links
|
||||
|
||||
* [Official website](https://clickhouse.yandex/) has quick high-level overview of ClickHouse on main page.
|
||||
* [Tutorial](https://clickhouse.yandex/tutorial.html) shows how to set up and query small ClickHouse cluster.
|
||||
* [Documentation](https://clickhouse.yandex/docs/en/) provides more in-depth information.
|
||||
* [Official website](https://clickhouse.tech/) has quick high-level overview of ClickHouse on main page.
|
||||
* [Tutorial](https://clickhouse.tech/docs/en/getting_started/tutorial/) shows how to set up and query small ClickHouse cluster.
|
||||
* [Documentation](https://clickhouse.tech/docs/en/) provides more in-depth information.
|
||||
* [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format.
|
||||
* [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announces and reports about events.
|
||||
* [Contacts](https://clickhouse.yandex/#contacts) can help to get your questions answered if there are any.
|
||||
* [Contacts](https://clickhouse.tech/#contacts) can help to get your questions answered if there are any.
|
||||
* You can also [fill this form](https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/) to meet Yandex ClickHouse team in person.
|
||||
|
||||
## Upcoming Events
|
||||
* [ClickHouse Meetup in Tokyo](https://clickhouse.connpass.com/event/147001/) on November 14.
|
||||
* [ClickHouse Meetup in Istanbul](https://www.eventbrite.com/e/clickhouse-meetup-istanbul-create-blazing-fast-experiences-w-clickhouse-tickets-73101120419) on November 19.
|
||||
* [ClickHouse Meetup in Ankara](https://www.eventbrite.com/e/clickhouse-meetup-ankara-create-blazing-fast-experiences-w-clickhouse-tickets-73100530655) on November 21.
|
||||
* [ClickHouse Meetup in Singapore](https://www.meetup.com/Singapore-Clickhouse-Meetup-Group/events/265085331/) on November 23.
|
||||
* [ClickHouse Meetup in San Francisco](https://www.eventbrite.com/e/clickhouse-december-meetup-registration-78642047481) on December 3.
|
||||
## Upcoming Events
|
||||
|
||||
* [ClickHouse Meetup in Athens](https://www.meetup.com/Athens-Big-Data/events/268379195/) on March 5.
|
||||
|
||||
17
base/CMakeLists.txt
Normal file
17
base/CMakeLists.txt
Normal file
@ -0,0 +1,17 @@
|
||||
add_subdirectory (common)
|
||||
add_subdirectory (loggers)
|
||||
add_subdirectory (daemon)
|
||||
|
||||
if (USE_INTERNAL_MEMCPY)
|
||||
add_subdirectory (memcpy)
|
||||
endif()
|
||||
|
||||
if (USE_MYSQL)
|
||||
add_subdirectory (mysqlxx)
|
||||
endif ()
|
||||
|
||||
if (USE_INTERNAL_CONSISTENT_HASHING_LIBRARY)
|
||||
add_subdirectory (consistent-hashing)
|
||||
endif ()
|
||||
add_subdirectory (consistent-hashing-sumbur)
|
||||
add_subdirectory (widechar_width)
|
||||
64
base/common/CMakeLists.txt
Normal file
64
base/common/CMakeLists.txt
Normal file
@ -0,0 +1,64 @@
|
||||
configure_file (config_common.h.in config_common.h)
|
||||
|
||||
set (SRCS
|
||||
argsToConfig.cpp
|
||||
coverage.cpp
|
||||
DateLUT.cpp
|
||||
DateLUTImpl.cpp
|
||||
demangle.cpp
|
||||
getMemoryAmount.cpp
|
||||
getThreadId.cpp
|
||||
JSON.cpp
|
||||
LineReader.cpp
|
||||
mremap.cpp
|
||||
phdr_cache.cpp
|
||||
preciseExp10.c
|
||||
setTerminalEcho.cpp
|
||||
shift10.cpp
|
||||
sleep.cpp
|
||||
)
|
||||
|
||||
if (ENABLE_REPLXX)
|
||||
set (SRCS ${SRCS}
|
||||
ReplxxLineReader.cpp
|
||||
ReplxxLineReader.h
|
||||
)
|
||||
endif ()
|
||||
|
||||
add_library (common ${SRCS})
|
||||
|
||||
target_include_directories(common PUBLIC .. ${CMAKE_CURRENT_BINARY_DIR}/..)
|
||||
|
||||
if (USE_INTERNAL_MEMCPY)
|
||||
target_link_libraries (common PRIVATE memcpy)
|
||||
endif ()
|
||||
|
||||
if(CCTZ_INCLUDE_DIR)
|
||||
target_include_directories(common BEFORE PRIVATE ${CCTZ_INCLUDE_DIR})
|
||||
endif()
|
||||
|
||||
if (NOT USE_INTERNAL_BOOST_LIBRARY)
|
||||
target_include_directories (common SYSTEM BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
|
||||
endif ()
|
||||
|
||||
if(NOT USE_INTERNAL_POCO_LIBRARY)
|
||||
target_include_directories (common SYSTEM BEFORE PUBLIC ${Poco_Foundation_INCLUDE_DIR})
|
||||
endif()
|
||||
|
||||
if(CCTZ_LIBRARY)
|
||||
target_link_libraries(common PRIVATE ${CCTZ_LIBRARY})
|
||||
endif()
|
||||
|
||||
target_link_libraries(common PUBLIC replxx)
|
||||
|
||||
target_link_libraries (common
|
||||
PUBLIC
|
||||
${Poco_Util_LIBRARY}
|
||||
${Poco_Foundation_LIBRARY}
|
||||
${CITYHASH_LIBRARIES}
|
||||
${Boost_SYSTEM_LIBRARY}
|
||||
)
|
||||
|
||||
if (ENABLE_TESTS)
|
||||
add_subdirectory (tests)
|
||||
endif ()
|
||||
@ -391,9 +391,9 @@ public:
|
||||
|
||||
/*
|
||||
The bits in week_mode has the following meaning:
|
||||
WeekModeFlag::MONDAY_FIRST (0) If not set Sunday is first day of week
|
||||
If set Monday is first day of week
|
||||
WeekModeFlag::YEAR (1) If not set Week is in range 0-53
|
||||
WeekModeFlag::MONDAY_FIRST (0) If not set Sunday is first day of week
|
||||
If set Monday is first day of week
|
||||
WeekModeFlag::YEAR (1) If not set Week is in range 0-53
|
||||
|
||||
Week 0 is returned for the the last week of the previous year (for
|
||||
a date at start of january) In this case one can get 53 for the
|
||||
@ -401,19 +401,19 @@ public:
|
||||
relevant for the given year. Note that this flag is only
|
||||
releveant if WeekModeFlag::JANUARY is not set.
|
||||
|
||||
If set Week is in range 1-53.
|
||||
If set Week is in range 1-53.
|
||||
|
||||
In this case one may get week 53 for a date in January (when
|
||||
the week is that last week of previous year) and week 1 for a
|
||||
date in December.
|
||||
|
||||
WeekModeFlag::FIRST_WEEKDAY (2) If not set Weeks are numbered according
|
||||
WeekModeFlag::FIRST_WEEKDAY (2) If not set Weeks are numbered according
|
||||
to ISO 8601:1988
|
||||
If set The week that contains the first
|
||||
If set The week that contains the first
|
||||
'first-day-of-week' is week 1.
|
||||
|
||||
WeekModeFlag::NEWYEAR_DAY (3) If not set no meaning
|
||||
If set The week that contains the January 1 is week 1.
|
||||
WeekModeFlag::NEWYEAR_DAY (3) If not set no meaning
|
||||
If set The week that contains the January 1 is week 1.
|
||||
Week is in range 1-53.
|
||||
And ignore WeekModeFlag::YEAR, WeekModeFlag::FIRST_WEEKDAY
|
||||
|
||||
@ -666,7 +666,7 @@ public:
|
||||
inline DayNum makeDayNum(UInt16 year, UInt8 month, UInt8 day_of_month) const
|
||||
{
|
||||
if (unlikely(year < DATE_LUT_MIN_YEAR || year > DATE_LUT_MAX_YEAR || month < 1 || month > 12 || day_of_month < 1 || day_of_month > 31))
|
||||
return DayNum(0);
|
||||
return DayNum(0); // TODO (nemkov, DateTime64 phase 2): implement creating real date for year outside of LUT range.
|
||||
|
||||
return DayNum(years_months_lut[(year - DATE_LUT_MIN_YEAR) * 12 + month - 1] + day_of_month - 1);
|
||||
}
|
||||
@ -12,9 +12,9 @@
|
||||
class KillingErrorHandler : public Poco::ErrorHandler
|
||||
{
|
||||
public:
|
||||
void exception(const Poco::Exception &) { std::terminate(); }
|
||||
void exception(const std::exception &) { std::terminate(); }
|
||||
void exception() { std::terminate(); }
|
||||
void exception(const Poco::Exception &) override { std::terminate(); }
|
||||
void exception(const std::exception &) override { std::terminate(); }
|
||||
void exception() override { std::terminate(); }
|
||||
};
|
||||
|
||||
|
||||
@ -23,9 +23,9 @@ public:
|
||||
class ServerErrorHandler : public Poco::ErrorHandler
|
||||
{
|
||||
public:
|
||||
void exception(const Poco::Exception &) { logException(); }
|
||||
void exception(const std::exception &) { logException(); }
|
||||
void exception() { logException(); }
|
||||
void exception(const Poco::Exception &) override { logException(); }
|
||||
void exception(const std::exception &) override { logException(); }
|
||||
void exception() override { logException(); }
|
||||
|
||||
private:
|
||||
Logger * log = &Logger::get("ServerErrorHandler");
|
||||
@ -399,7 +399,7 @@ JSON::Pos JSON::skipElement() const
|
||||
|
||||
ElementType type = getType();
|
||||
|
||||
switch(type)
|
||||
switch (type)
|
||||
{
|
||||
case TYPE_NULL:
|
||||
return skipNull();
|
||||
@ -587,7 +587,7 @@ std::string JSON::getString() const
|
||||
++s;
|
||||
checkPos(s);
|
||||
|
||||
switch(*s)
|
||||
switch (*s)
|
||||
{
|
||||
case '"':
|
||||
buf += '"';
|
||||
@ -665,7 +665,7 @@ StringRef JSON::getRawString() const
|
||||
if (*s != '"')
|
||||
throw JSONException(std::string("JSON: expected \", got ") + *s);
|
||||
while (++s != ptr_end && *s != '"');
|
||||
if (s != ptr_end )
|
||||
if (s != ptr_end)
|
||||
return StringRef(ptr_begin + 1, s - ptr_begin - 1);
|
||||
throw JSONException("JSON: incorrect syntax (expected end of string, found end of JSON).");
|
||||
}
|
||||
156
base/common/LineReader.cpp
Normal file
156
base/common/LineReader.cpp
Normal file
@ -0,0 +1,156 @@
|
||||
#include <common/LineReader.h>
|
||||
|
||||
#include <iostream>
|
||||
#include <string_view>
|
||||
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#ifdef OS_LINUX
|
||||
/// We can detect if code is linked with one or another readline variants or open the library dynamically.
|
||||
# include <dlfcn.h>
|
||||
extern "C"
|
||||
{
|
||||
char * readline(const char *) __attribute__((__weak__));
|
||||
char * (*readline_ptr)(const char *) = readline;
|
||||
}
|
||||
#endif
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
/// Trim ending whitespace inplace
|
||||
void trim(String & s)
|
||||
{
|
||||
s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); }).base(), s.end());
|
||||
}
|
||||
|
||||
/// Check if multi-line query is inserted from the paste buffer.
|
||||
/// Allows delaying the start of query execution until the entirety of query is inserted.
|
||||
bool hasInputData()
|
||||
{
|
||||
timeval timeout = {0, 0};
|
||||
fd_set fds;
|
||||
FD_ZERO(&fds);
|
||||
FD_SET(STDIN_FILENO, &fds);
|
||||
return select(1, &fds, nullptr, nullptr, &timeout) == 1;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
LineReader::Suggest::WordsRange LineReader::Suggest::getCompletions(const String & prefix, size_t prefix_length) const
|
||||
{
|
||||
if (!ready)
|
||||
return std::make_pair(words.end(), words.end());
|
||||
|
||||
std::string_view last_word;
|
||||
|
||||
auto last_word_pos = prefix.find_last_of(word_break_characters);
|
||||
if (std::string::npos == last_word_pos)
|
||||
last_word = prefix;
|
||||
else
|
||||
last_word = std::string_view(prefix).substr(last_word_pos + 1, std::string::npos);
|
||||
|
||||
/// last_word can be empty.
|
||||
|
||||
return std::equal_range(
|
||||
words.begin(), words.end(), last_word, [prefix_length](std::string_view s, std::string_view prefix_searched)
|
||||
{
|
||||
return strncmp(s.data(), prefix_searched.data(), prefix_length) < 0;
|
||||
});
|
||||
}
|
||||
|
||||
LineReader::LineReader(const String & history_file_path_, char extender_, char delimiter_)
|
||||
: history_file_path(history_file_path_), extender(extender_), delimiter(delimiter_)
|
||||
{
|
||||
/// FIXME: check extender != delimiter
|
||||
}
|
||||
|
||||
String LineReader::readLine(const String & first_prompt, const String & second_prompt)
|
||||
{
|
||||
String line;
|
||||
bool is_multiline = false;
|
||||
|
||||
while (auto status = readOneLine(is_multiline ? second_prompt : first_prompt))
|
||||
{
|
||||
if (status == RESET_LINE)
|
||||
{
|
||||
line.clear();
|
||||
is_multiline = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (input.empty())
|
||||
{
|
||||
if (!line.empty() && !delimiter && !hasInputData())
|
||||
break;
|
||||
else
|
||||
continue;
|
||||
}
|
||||
|
||||
is_multiline = (input.back() == extender) || (delimiter && input.back() != delimiter) || hasInputData();
|
||||
|
||||
if (input.back() == extender)
|
||||
{
|
||||
input = input.substr(0, input.size() - 1);
|
||||
trim(input);
|
||||
if (input.empty())
|
||||
continue;
|
||||
}
|
||||
|
||||
line += (line.empty() ? "" : " ") + input;
|
||||
|
||||
if (!is_multiline)
|
||||
break;
|
||||
}
|
||||
|
||||
if (!line.empty() && line != prev_line)
|
||||
{
|
||||
addToHistory(line);
|
||||
prev_line = line;
|
||||
}
|
||||
|
||||
return line;
|
||||
}
|
||||
|
||||
LineReader::InputStatus LineReader::readOneLine(const String & prompt)
|
||||
{
|
||||
input.clear();
|
||||
|
||||
#ifdef OS_LINUX
|
||||
if (!readline_ptr)
|
||||
{
|
||||
for (auto name : {"libreadline.so", "libreadline.so.0", "libeditline.so", "libeditline.so.0"})
|
||||
{
|
||||
void * dl_handle = dlopen(name, RTLD_LAZY);
|
||||
if (dl_handle)
|
||||
{
|
||||
readline_ptr = reinterpret_cast<char * (*)(const char *)>(dlsym(dl_handle, "readline"));
|
||||
if (readline_ptr)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Minimal support for readline
|
||||
if (readline_ptr)
|
||||
{
|
||||
char * line_read = (*readline_ptr)(prompt.c_str());
|
||||
if (!line_read)
|
||||
return ABORT;
|
||||
input = line_read;
|
||||
}
|
||||
else
|
||||
#endif
|
||||
{
|
||||
std::cout << prompt;
|
||||
std::getline(std::cin, input);
|
||||
if (!std::cin.good())
|
||||
return ABORT;
|
||||
}
|
||||
|
||||
trim(input);
|
||||
return INPUT_LINE;
|
||||
}
|
||||
55
base/common/LineReader.h
Normal file
55
base/common/LineReader.h
Normal file
@ -0,0 +1,55 @@
|
||||
#pragma once
|
||||
|
||||
#include <common/Types.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <vector>
|
||||
|
||||
class LineReader
|
||||
{
|
||||
public:
|
||||
class Suggest
|
||||
{
|
||||
protected:
|
||||
using Words = std::vector<std::string>;
|
||||
using WordsRange = std::pair<Words::const_iterator, Words::const_iterator>;
|
||||
|
||||
Words words;
|
||||
std::atomic<bool> ready{false};
|
||||
|
||||
public:
|
||||
/// Get iterators for the matched range of words if any.
|
||||
WordsRange getCompletions(const String & prefix, size_t prefix_length) const;
|
||||
};
|
||||
|
||||
LineReader(const String & history_file_path, char extender, char delimiter = 0); /// if delimiter != 0, then it's multiline mode
|
||||
virtual ~LineReader() {}
|
||||
|
||||
/// Reads the whole line until delimiter (in multiline mode) or until the last line without extender.
|
||||
/// If resulting line is empty, it means the user interrupted the input.
|
||||
/// Non-empty line is appended to history - without duplication.
|
||||
/// Typical delimiter is ';' (semicolon) and typical extender is '\' (backslash).
|
||||
String readLine(const String & first_prompt, const String & second_prompt);
|
||||
|
||||
protected:
|
||||
enum InputStatus
|
||||
{
|
||||
ABORT = 0,
|
||||
RESET_LINE,
|
||||
INPUT_LINE,
|
||||
};
|
||||
|
||||
const String history_file_path;
|
||||
static constexpr char word_break_characters[] = " \t\n\r\"\\'`@$><=;|&{(.";
|
||||
|
||||
String input;
|
||||
|
||||
private:
|
||||
const char extender;
|
||||
const char delimiter;
|
||||
|
||||
String prev_line;
|
||||
|
||||
virtual InputStatus readOneLine(const String & prompt);
|
||||
virtual void addToHistory(const String &) {}
|
||||
};
|
||||
57
base/common/ReplxxLineReader.cpp
Normal file
57
base/common/ReplxxLineReader.cpp
Normal file
@ -0,0 +1,57 @@
|
||||
#include <common/ReplxxLineReader.h>
|
||||
|
||||
#include <errno.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
/// Trim ending whitespace inplace
|
||||
void trim(String & s)
|
||||
{
|
||||
s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); }).base(), s.end());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
ReplxxLineReader::ReplxxLineReader(const Suggest & suggest, const String & history_file_path_, char extender_, char delimiter_)
|
||||
: LineReader(history_file_path_, extender_, delimiter_)
|
||||
{
|
||||
if (!history_file_path.empty())
|
||||
rx.history_load(history_file_path);
|
||||
|
||||
auto callback = [&suggest] (const String & context, size_t context_size)
|
||||
{
|
||||
auto range = suggest.getCompletions(context, context_size);
|
||||
return replxx::Replxx::completions_t(range.first, range.second);
|
||||
};
|
||||
|
||||
rx.set_completion_callback(callback);
|
||||
rx.set_complete_on_empty(false);
|
||||
rx.set_word_break_characters(word_break_characters);
|
||||
}
|
||||
|
||||
ReplxxLineReader::~ReplxxLineReader()
|
||||
{
|
||||
if (!history_file_path.empty())
|
||||
rx.history_save(history_file_path);
|
||||
}
|
||||
|
||||
LineReader::InputStatus ReplxxLineReader::readOneLine(const String & prompt)
|
||||
{
|
||||
input.clear();
|
||||
|
||||
const char* cinput = rx.input(prompt);
|
||||
if (cinput == nullptr)
|
||||
return (errno != EAGAIN) ? ABORT : RESET_LINE;
|
||||
input = cinput;
|
||||
|
||||
trim(input);
|
||||
return INPUT_LINE;
|
||||
}
|
||||
|
||||
void ReplxxLineReader::addToHistory(const String & line)
|
||||
{
|
||||
rx.history_add(line);
|
||||
}
|
||||
18
base/common/ReplxxLineReader.h
Normal file
18
base/common/ReplxxLineReader.h
Normal file
@ -0,0 +1,18 @@
|
||||
#pragma once
|
||||
|
||||
#include "LineReader.h"
|
||||
|
||||
#include <replxx.hxx>
|
||||
|
||||
class ReplxxLineReader : public LineReader
|
||||
{
|
||||
public:
|
||||
ReplxxLineReader(const Suggest & suggest, const String & history_file_path, char extender, char delimiter = 0);
|
||||
~ReplxxLineReader() override;
|
||||
|
||||
private:
|
||||
InputStatus readOneLine(const String & prompt) override;
|
||||
void addToHistory(const String & line) override;
|
||||
|
||||
replxx::Replxx rx;
|
||||
};
|
||||
@ -29,11 +29,14 @@ struct StringRef
|
||||
StringRef(const char * data_, size_t size_) : data(data_), size(size_) {}
|
||||
StringRef(const unsigned char * data_, size_t size_) : data(reinterpret_cast<const char *>(data_)), size(size_) {}
|
||||
StringRef(const std::string & s) : data(s.data()), size(s.size()) {}
|
||||
StringRef(const std::string_view & s) : data(s.data()), size(s.size()) {}
|
||||
explicit StringRef(const char * data_) : data(data_), size(strlen(data_)) {}
|
||||
StringRef() = default;
|
||||
|
||||
std::string toString() const { return std::string(data, size); }
|
||||
|
||||
explicit operator std::string() const { return toString(); }
|
||||
explicit operator std::string_view() const { return {data, size}; }
|
||||
};
|
||||
|
||||
using StringRefs = std::vector<StringRef>;
|
||||
@ -1,8 +1,10 @@
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdint>
|
||||
#include <cstdlib>
|
||||
#include <string>
|
||||
#include <type_traits>
|
||||
#include <algorithm>
|
||||
|
||||
using Int8 = int8_t;
|
||||
using Int16 = int16_t;
|
||||
@ -14,6 +16,8 @@ using UInt16 = uint16_t;
|
||||
using UInt32 = uint32_t;
|
||||
using UInt64 = uint64_t;
|
||||
|
||||
using String = std::string;
|
||||
|
||||
/// The standard library type traits, such as std::is_arithmetic, with one exception
|
||||
/// (std::common_type), are "set in stone". Attempting to specialize them causes undefined behavior.
|
||||
/// So instead of using the std type_traits, we use our own version which allows extension.
|
||||
@ -52,4 +56,3 @@ struct is_arithmetic
|
||||
|
||||
template <typename T>
|
||||
inline constexpr bool is_arithmetic_v = is_arithmetic<T>::value;
|
||||
|
||||
@ -2,10 +2,6 @@
|
||||
|
||||
// .h autogenerated by cmake !
|
||||
|
||||
#cmakedefine01 USE_TCMALLOC
|
||||
#cmakedefine01 USE_JEMALLOC
|
||||
#cmakedefine01 USE_READLINE
|
||||
#cmakedefine01 USE_LIBEDIT
|
||||
#cmakedefine01 HAVE_READLINE_HISTORY
|
||||
#cmakedefine01 UNBUNDLED
|
||||
#cmakedefine01 WITH_COVERAGE
|
||||
31
base/common/coverage.cpp
Normal file
31
base/common/coverage.cpp
Normal file
@ -0,0 +1,31 @@
|
||||
#include <common/coverage.h>
|
||||
#include <common/config_common.h>
|
||||
|
||||
#if WITH_COVERAGE
|
||||
|
||||
#include <unistd.h>
|
||||
#include <mutex>
|
||||
|
||||
#if defined(__clang__)
|
||||
extern "C" void __llvm_profile_dump();
|
||||
#elif defined(__GNUC__) || defined(__GNUG__)
|
||||
extern "C" void __gcov_exit();
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
void dumpCoverageReportIfPossible()
|
||||
{
|
||||
#if WITH_COVERAGE
|
||||
static std::mutex mutex;
|
||||
std::lock_guard lock(mutex);
|
||||
|
||||
#if defined(__clang__)
|
||||
__llvm_profile_dump();
|
||||
#elif defined(__GNUC__) || defined(__GNUG__)
|
||||
__gcov_exit();
|
||||
#endif
|
||||
|
||||
#endif
|
||||
}
|
||||
9
base/common/coverage.h
Normal file
9
base/common/coverage.h
Normal file
@ -0,0 +1,9 @@
|
||||
#pragma once
|
||||
|
||||
/// Flush coverage report to file, depending on coverage system
|
||||
/// proposed by compiler (llvm for clang and gcov for gcc).
|
||||
///
|
||||
/// Noop if build without coverage (WITH_COVERAGE=0).
|
||||
/// Thread safe (use exclusive lock).
|
||||
/// Idempotent, may be called multiple times.
|
||||
void dumpCoverageReportIfPossible();
|
||||
@ -10,6 +10,11 @@
|
||||
|
||||
#if _MSC_VER || MEMORY_SANITIZER
|
||||
|
||||
DemangleResult tryDemangle(const char * name)
|
||||
{
|
||||
return DemangleResult{};
|
||||
}
|
||||
|
||||
std::string demangle(const char * name, int & status)
|
||||
{
|
||||
status = 0;
|
||||
@ -21,28 +26,27 @@ std::string demangle(const char * name, int & status)
|
||||
#include <stdlib.h>
|
||||
#include <cxxabi.h>
|
||||
|
||||
std::string demangle(const char * name, int & status)
|
||||
static DemangleResult tryDemangle(const char * name, int & status)
|
||||
{
|
||||
std::string res;
|
||||
|
||||
char * demangled_str = abi::__cxa_demangle(name, 0, 0, &status);
|
||||
if (demangled_str)
|
||||
{
|
||||
try
|
||||
{
|
||||
res = demangled_str;
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
free(demangled_str);
|
||||
throw;
|
||||
}
|
||||
free(demangled_str);
|
||||
}
|
||||
else
|
||||
res = name;
|
||||
|
||||
return res;
|
||||
return DemangleResult(abi::__cxa_demangle(name, nullptr, nullptr, &status));
|
||||
}
|
||||
|
||||
DemangleResult tryDemangle(const char * name)
|
||||
{
|
||||
int status = 0;
|
||||
return tryDemangle(name, status);
|
||||
}
|
||||
|
||||
std::string demangle(const char * name, int & status)
|
||||
{
|
||||
auto result = tryDemangle(name, status);
|
||||
if (result)
|
||||
{
|
||||
return std::string(result.get());
|
||||
}
|
||||
|
||||
return name;
|
||||
}
|
||||
|
||||
|
||||
#endif
|
||||
@ -1,5 +1,6 @@
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
|
||||
@ -14,3 +15,18 @@ inline std::string demangle(const char * name)
|
||||
int status = 0;
|
||||
return demangle(name, status);
|
||||
}
|
||||
|
||||
// abi::__cxa_demangle returns a C string of known size that should be deleted
|
||||
// with free().
|
||||
struct FreeingDeleter
|
||||
{
|
||||
template <typename PointerType>
|
||||
void operator() (PointerType ptr)
|
||||
{
|
||||
std::free(ptr);
|
||||
}
|
||||
};
|
||||
|
||||
typedef std::unique_ptr<char, FreeingDeleter> DemangleResult;
|
||||
|
||||
DemangleResult tryDemangle(const char * name);
|
||||
@ -33,7 +33,7 @@ uint64_t getMemoryAmountOrZero()
|
||||
/* New 64-bit MEMORYSTATUSEX isn't available. Use old 32.bit */
|
||||
MEMORYSTATUS status;
|
||||
status.dwLength = sizeof(status);
|
||||
GlobalMemoryStatus( &status );
|
||||
GlobalMemoryStatus(&status);
|
||||
return status.dwTotalPhys;
|
||||
|
||||
#elif defined(WIN32) || defined(_WIN32)
|
||||
@ -41,7 +41,7 @@ uint64_t getMemoryAmountOrZero()
|
||||
/* Use new 64-bit MEMORYSTATUSEX, not old 32-bit MEMORYSTATUS */
|
||||
MEMORYSTATUSEX status;
|
||||
status.dwLength = sizeof(status);
|
||||
GlobalMemoryStatusEx( &status );
|
||||
GlobalMemoryStatusEx(&status);
|
||||
return status.ullTotalPhys;
|
||||
|
||||
#else
|
||||
@ -58,24 +58,24 @@ uint64_t getMemoryAmountOrZero()
|
||||
#endif
|
||||
uint64_t size = 0; /* 64-bit */
|
||||
size_t len = sizeof(size);
|
||||
if ( sysctl( mib, 2, &size, &len, NULL, 0 ) == 0 ) {
|
||||
if (sysctl(mib, 2, &size, &len, NULL, 0) == 0)
|
||||
return size;
|
||||
}
|
||||
|
||||
return 0; /* Failed? */
|
||||
|
||||
#elif defined(_SC_AIX_REALMEM)
|
||||
/* AIX. ----------------------------------------------------- */
|
||||
return sysconf( _SC_AIX_REALMEM ) * 1024;
|
||||
return sysconf(_SC_AIX_REALMEM) * 1024;
|
||||
|
||||
#elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGESIZE)
|
||||
/* FreeBSD, Linux, OpenBSD, and Solaris. -------------------- */
|
||||
return (uint64_t)sysconf( _SC_PHYS_PAGES )
|
||||
* (uint64_t)sysconf( _SC_PAGESIZE );
|
||||
return (uint64_t)sysconf(_SC_PHYS_PAGES)
|
||||
* (uint64_t)sysconf(_SC_PAGESIZE);
|
||||
|
||||
#elif defined(_SC_PHYS_PAGES) && defined(_SC_PAGE_SIZE)
|
||||
/* Legacy. -------------------------------------------------- */
|
||||
return (uint64_t)sysconf( _SC_PHYS_PAGES )
|
||||
* (uint64_t)sysconf( _SC_PAGE_SIZE );
|
||||
return (uint64_t)sysconf(_SC_PHYS_PAGES)
|
||||
* (uint64_t)sysconf(_SC_PAGE_SIZE);
|
||||
|
||||
#elif defined(CTL_HW) && (defined(HW_PHYSMEM) || defined(HW_REALMEM))
|
||||
/* DragonFly BSD, FreeBSD, NetBSD, OpenBSD, and OSX. -------- */
|
||||
@ -87,10 +87,10 @@ uint64_t getMemoryAmountOrZero()
|
||||
mib[1] = HW_PHYSMEM; /* Others. ------------------ */
|
||||
#endif
|
||||
unsigned int size = 0; /* 32-bit */
|
||||
size_t len = sizeof( size );
|
||||
if ( sysctl( mib, 2, &size, &len, NULL, 0 ) == 0 ) {
|
||||
size_t len = sizeof(size);
|
||||
if (sysctl(mib, 2, &size, &len, NULL, 0) == 0)
|
||||
return size;
|
||||
}
|
||||
|
||||
return 0; /* Failed? */
|
||||
#endif /* sysctl and sysconf variants */
|
||||
|
||||
26
base/common/getThreadId.cpp
Normal file
26
base/common/getThreadId.cpp
Normal file
@ -0,0 +1,26 @@
|
||||
#include <common/getThreadId.h>
|
||||
|
||||
#if OS_LINUX
|
||||
#include <unistd.h>
|
||||
#include <syscall.h>
|
||||
#else
|
||||
#include <pthread.h>
|
||||
#include <stdexcept>
|
||||
#endif
|
||||
|
||||
|
||||
static thread_local uint64_t current_tid = 0;
|
||||
uint64_t getThreadId()
|
||||
{
|
||||
if (!current_tid)
|
||||
{
|
||||
#if OS_LINUX
|
||||
current_tid = syscall(SYS_gettid); /// This call is always successful. - man gettid
|
||||
#else
|
||||
if (0 != pthread_threadid_np(nullptr, ¤t_tid))
|
||||
throw std::logic_error("pthread_threadid_np returned error");
|
||||
#endif
|
||||
}
|
||||
|
||||
return current_tid;
|
||||
}
|
||||
5
base/common/getThreadId.h
Normal file
5
base/common/getThreadId.h
Normal file
@ -0,0 +1,5 @@
|
||||
#pragma once
|
||||
#include <cstdint>
|
||||
|
||||
/// Obtain thread id from OS. The value is cached in thread local variable.
|
||||
uint64_t getThreadId();
|
||||
@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include "demangle.h"
|
||||
#include "getThreadNumber.h"
|
||||
#include "getThreadId.h"
|
||||
#include <type_traits>
|
||||
#include <tuple>
|
||||
#include <iomanip>
|
||||
@ -140,8 +140,8 @@ Out & dump(Out & out, const char * name, T && x)
|
||||
#pragma clang diagnostic ignored "-Wgnu-zero-variadic-macro-arguments"
|
||||
#endif
|
||||
|
||||
#define DUMPVAR(VAR) dump(std::cerr, #VAR, (VAR)); std::cerr << "; ";
|
||||
#define DUMPHEAD std::cerr << __FILE__ << ':' << __LINE__ << " [ " << getThreadNumber() << " ] ";
|
||||
#define DUMPVAR(VAR) ::dump(std::cerr, #VAR, (VAR)); std::cerr << "; ";
|
||||
#define DUMPHEAD std::cerr << __FILE__ << ':' << __LINE__ << " [ " << getThreadId() << " ] ";
|
||||
#define DUMPTAIL std::cerr << '\n';
|
||||
|
||||
#define DUMP1(V1) do { DUMPHEAD DUMPVAR(V1) DUMPTAIL } while(0)
|
||||
@ -6,7 +6,6 @@
|
||||
#include <Poco/Logger.h>
|
||||
#include <Poco/Message.h>
|
||||
#include <Poco/Version.h>
|
||||
#include <Core/SettingsCommon.h>
|
||||
#include <Common/CurrentThread.h>
|
||||
|
||||
#ifndef QUERY_PREVIEW_LENGTH
|
||||
@ -1,7 +1,8 @@
|
||||
#include <common/shift10.h>
|
||||
|
||||
#include <common/likely.h>
|
||||
|
||||
#include <limits>
|
||||
#include <port/ssize_t.h>
|
||||
|
||||
|
||||
template <typename T>
|
||||
7
base/common/time.h
Normal file
7
base/common/time.h
Normal file
@ -0,0 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <time.h>
|
||||
|
||||
#if defined (OS_DARWIN)
|
||||
# define CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC
|
||||
#endif
|
||||
@ -26,7 +26,7 @@ inline uint32_t HI_32(uint64_t x) { return static_cast<uint32_t>(x >> 32); }
|
||||
return std::numeric_limits<unsigned long long>::digits - __builtin_clzll(value);
|
||||
}
|
||||
#else
|
||||
/// Stupid realization for non GCC-like compilers. Can use BSR from x86 instructions set.
|
||||
/// Stupid implementation for non GCC-like compilers. Can use BSR from x86 instructions set.
|
||||
template <typename T>
|
||||
inline unsigned GetValueBitCountImpl(T value) noexcept {
|
||||
unsigned result = 1; // result == 0 - impossible value, since value cannot be zero
|
||||
@ -1,7 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <port/unistd.h>
|
||||
#include <unistd.h>
|
||||
#include <iostream>
|
||||
#include <memory>
|
||||
#include <functional>
|
||||
@ -19,10 +19,12 @@
|
||||
#include <Poco/Version.h>
|
||||
#include <common/Types.h>
|
||||
#include <common/logger_useful.h>
|
||||
#include <common/getThreadId.h>
|
||||
#include <daemon/GraphiteWriter.h>
|
||||
#include <Common/Config/ConfigProcessor.h>
|
||||
#include <loggers/Loggers.h>
|
||||
|
||||
|
||||
namespace Poco { class TaskManager; }
|
||||
|
||||
|
||||
@ -166,22 +168,11 @@ protected:
|
||||
{
|
||||
std::string file;
|
||||
|
||||
/// Создать объект, не создавая PID файл
|
||||
PID() {}
|
||||
|
||||
/// Создать объект, создать PID файл
|
||||
PID(const std::string & file_) { seed(file_); }
|
||||
|
||||
/// Создать PID файл
|
||||
void seed(const std::string & file_);
|
||||
|
||||
/// Удалить PID файл
|
||||
void clear();
|
||||
|
||||
~PID() { clear(); }
|
||||
PID(const std::string & file_);
|
||||
~PID();
|
||||
};
|
||||
|
||||
PID pid;
|
||||
std::optional<PID> pid;
|
||||
|
||||
std::atomic_bool is_cancelled{false};
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
#include <daemon/BaseDaemon.h>
|
||||
#include <Common/Config/ConfigProcessor.h>
|
||||
|
||||
#include <sys/stat.h>
|
||||
#include <sys/types.h>
|
||||
#include <sys/time.h>
|
||||
@ -12,19 +12,16 @@
|
||||
#include <unistd.h>
|
||||
|
||||
#include <typeinfo>
|
||||
#include <common/logger_useful.h>
|
||||
#include <common/ErrorHandlers.h>
|
||||
#include <common/Pipe.h>
|
||||
#include <Common/StackTrace.h>
|
||||
#include <sys/time.h>
|
||||
#include <sys/resource.h>
|
||||
#include <iostream>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
#include <memory>
|
||||
#include <ext/scope_guard.h>
|
||||
|
||||
#include <Poco/Observer.h>
|
||||
#include <Poco/AutoPtr.h>
|
||||
#include <common/getThreadNumber.h>
|
||||
#include <Poco/PatternFormatter.h>
|
||||
#include <Poco/TaskManager.h>
|
||||
#include <Poco/File.h>
|
||||
@ -36,16 +33,25 @@
|
||||
#include <Poco/Condition.h>
|
||||
#include <Poco/SyslogChannel.h>
|
||||
#include <Poco/DirectoryIterator.h>
|
||||
#include <Common/Exception.h>
|
||||
|
||||
#include <common/logger_useful.h>
|
||||
#include <common/ErrorHandlers.h>
|
||||
#include <common/argsToConfig.h>
|
||||
#include <common/getThreadId.h>
|
||||
#include <common/coverage.h>
|
||||
|
||||
#include <IO/WriteBufferFromFile.h>
|
||||
#include <IO/WriteBufferFromFileDescriptorDiscardOnFailure.h>
|
||||
#include <IO/ReadBufferFromFileDescriptor.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Common/Exception.h>
|
||||
#include <Common/PipeFDs.h>
|
||||
#include <Common/StackTrace.h>
|
||||
#include <Common/getMultipleKeysFromConfig.h>
|
||||
#include <Common/ClickHouseRevision.h>
|
||||
#include <Common/Config/ConfigProcessor.h>
|
||||
#include <Common/config_version.h>
|
||||
#include <common/argsToConfig.h>
|
||||
|
||||
#ifdef __APPLE__
|
||||
// ucontext is not available without _XOPEN_SOURCE
|
||||
@ -54,7 +60,7 @@
|
||||
#include <ucontext.h>
|
||||
|
||||
|
||||
Pipe signal_pipe;
|
||||
DB::PipeFDs signal_pipe;
|
||||
|
||||
|
||||
/** Reset signal handler to the default and send signal to itself.
|
||||
@ -67,8 +73,16 @@ static void call_default_signal_handler(int sig)
|
||||
}
|
||||
|
||||
|
||||
using ThreadNumber = decltype(getThreadNumber());
|
||||
static const size_t buf_size = sizeof(int) + sizeof(siginfo_t) + sizeof(ucontext_t) + sizeof(StackTrace) + sizeof(ThreadNumber);
|
||||
static constexpr size_t max_query_id_size = 127;
|
||||
|
||||
static const size_t buf_size =
|
||||
sizeof(int)
|
||||
+ sizeof(siginfo_t)
|
||||
+ sizeof(ucontext_t)
|
||||
+ sizeof(StackTrace)
|
||||
+ sizeof(UInt32)
|
||||
+ max_query_id_size + 1; /// query_id + varint encoded length
|
||||
|
||||
|
||||
using signal_function = void(int, siginfo_t*, void*);
|
||||
|
||||
@ -92,9 +106,9 @@ static void terminateRequestedSignalHandler(int sig, siginfo_t * info, void * co
|
||||
}
|
||||
|
||||
|
||||
/** Handler for "fault" signals. Send data about fault to separate thread to write into log.
|
||||
/** Handler for "fault" or diagnostic signals. Send data about fault to separate thread to write into log.
|
||||
*/
|
||||
static void faultSignalHandler(int sig, siginfo_t * info, void * context)
|
||||
static void signalHandler(int sig, siginfo_t * info, void * context)
|
||||
{
|
||||
char buf[buf_size];
|
||||
DB::WriteBufferFromFileDescriptorDiscardOnFailure out(signal_pipe.fds_rw[1], buf_size, buf);
|
||||
@ -102,15 +116,19 @@ static void faultSignalHandler(int sig, siginfo_t * info, void * context)
|
||||
const ucontext_t signal_context = *reinterpret_cast<ucontext_t *>(context);
|
||||
const StackTrace stack_trace(signal_context);
|
||||
|
||||
StringRef query_id = CurrentThread::getQueryId(); /// This is signal safe.
|
||||
query_id.size = std::min(query_id.size, max_query_id_size);
|
||||
|
||||
DB::writeBinary(sig, out);
|
||||
DB::writePODBinary(*info, out);
|
||||
DB::writePODBinary(signal_context, out);
|
||||
DB::writePODBinary(stack_trace, out);
|
||||
DB::writeBinary(getThreadNumber(), out);
|
||||
DB::writeBinary(UInt32(getThreadId()), out);
|
||||
DB::writeStringBinary(query_id, out);
|
||||
|
||||
out.next();
|
||||
|
||||
if (sig != SIGPROF) /// This signal is used for debugging.
|
||||
if (sig != SIGTSTP) /// This signal is used for debugging.
|
||||
{
|
||||
/// The time that is usually enough for separate thread to print info into log.
|
||||
::sleep(10);
|
||||
@ -162,7 +180,7 @@ public:
|
||||
}
|
||||
else if (sig == Signals::StdTerminate)
|
||||
{
|
||||
ThreadNumber thread_num;
|
||||
UInt32 thread_num;
|
||||
std::string message;
|
||||
|
||||
DB::readBinary(thread_num, in);
|
||||
@ -181,16 +199,18 @@ public:
|
||||
siginfo_t info;
|
||||
ucontext_t context;
|
||||
StackTrace stack_trace(NoCapture{});
|
||||
ThreadNumber thread_num;
|
||||
UInt32 thread_num;
|
||||
std::string query_id;
|
||||
|
||||
DB::readPODBinary(info, in);
|
||||
DB::readPODBinary(context, in);
|
||||
DB::readPODBinary(stack_trace, in);
|
||||
DB::readBinary(thread_num, in);
|
||||
DB::readBinary(query_id, in);
|
||||
|
||||
/// This allows to receive more signals if failure happens inside onFault function.
|
||||
/// Example: segfault while symbolizing stack trace.
|
||||
std::thread([=] { onFault(sig, info, context, stack_trace, thread_num); }).detach();
|
||||
std::thread([=, this] { onFault(sig, info, context, stack_trace, thread_num, query_id); }).detach();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -200,16 +220,33 @@ private:
|
||||
BaseDaemon & daemon;
|
||||
|
||||
private:
|
||||
void onTerminate(const std::string & message, ThreadNumber thread_num) const
|
||||
void onTerminate(const std::string & message, UInt32 thread_num) const
|
||||
{
|
||||
LOG_FATAL(log, "(version " << VERSION_STRING << VERSION_OFFICIAL << ") (from thread " << thread_num << ") " << message);
|
||||
}
|
||||
|
||||
void onFault(int sig, const siginfo_t & info, const ucontext_t & context, const StackTrace & stack_trace, ThreadNumber thread_num) const
|
||||
void onFault(
|
||||
int sig,
|
||||
const siginfo_t & info,
|
||||
const ucontext_t & context,
|
||||
const StackTrace & stack_trace,
|
||||
UInt32 thread_num,
|
||||
const std::string & query_id) const
|
||||
{
|
||||
LOG_FATAL(log, "########################################");
|
||||
LOG_FATAL(log, "(version " << VERSION_STRING << VERSION_OFFICIAL << ") (from thread " << thread_num << ") "
|
||||
<< "Received signal " << strsignal(sig) << " (" << sig << ")" << ".");
|
||||
|
||||
{
|
||||
std::stringstream message;
|
||||
message << "(version " << VERSION_STRING << VERSION_OFFICIAL << ")";
|
||||
message << " (from thread " << thread_num << ")";
|
||||
if (query_id.empty())
|
||||
message << " (no query)";
|
||||
else
|
||||
message << " (query_id: " << query_id << ")";
|
||||
message << " Received signal " << strsignal(sig) << " (" << sig << ")" << ".";
|
||||
|
||||
LOG_FATAL(log, message.rdbuf());
|
||||
}
|
||||
|
||||
LOG_FATAL(log, signalToErrorMessage(sig, info, context));
|
||||
|
||||
@ -264,7 +301,7 @@ static void terminate_handler()
|
||||
DB::WriteBufferFromFileDescriptor out(signal_pipe.fds_rw[1], buf_size, buf);
|
||||
|
||||
DB::writeBinary(static_cast<int>(SignalListener::StdTerminate), out);
|
||||
DB::writeBinary(getThreadNumber(), out);
|
||||
DB::writeBinary(UInt32(getThreadId()), out);
|
||||
DB::writeBinary(log_message, out);
|
||||
out.next();
|
||||
|
||||
@ -345,7 +382,7 @@ enum class InstructionFail
|
||||
|
||||
static std::string instructionFailToString(InstructionFail fail)
|
||||
{
|
||||
switch(fail)
|
||||
switch (fail)
|
||||
{
|
||||
case InstructionFail::NONE:
|
||||
return "NONE";
|
||||
@ -461,7 +498,8 @@ void BaseDaemon::terminate()
|
||||
|
||||
void BaseDaemon::kill()
|
||||
{
|
||||
pid.clear();
|
||||
dumpCoverageReportIfPossible();
|
||||
pid.reset();
|
||||
if (::raise(SIGKILL) != 0)
|
||||
throw Poco::SystemException("cannot kill process");
|
||||
}
|
||||
@ -631,7 +669,7 @@ void BaseDaemon::initialize(Application & self)
|
||||
|
||||
/// Create pid file.
|
||||
if (config().has("pid"))
|
||||
pid.seed(config().getString("pid"));
|
||||
pid.emplace(config().getString("pid"));
|
||||
|
||||
/// Change path for logging.
|
||||
if (!log_path.empty())
|
||||
@ -649,7 +687,7 @@ void BaseDaemon::initialize(Application & self)
|
||||
}
|
||||
|
||||
// sensitive data masking rules are not used here
|
||||
buildLoggers(config(), logger());
|
||||
buildLoggers(config(), logger(), self.commandName());
|
||||
|
||||
if (is_daemon)
|
||||
{
|
||||
@ -719,9 +757,9 @@ void BaseDaemon::initializeTerminationAndSignalProcessing()
|
||||
}
|
||||
};
|
||||
|
||||
/// SIGPROF is added for debugging purposes. To output a stack trace of any running thread at anytime.
|
||||
/// SIGTSTP is added for debugging purposes. To output a stack trace of any running thread at anytime.
|
||||
|
||||
add_signal_handler({SIGABRT, SIGSEGV, SIGILL, SIGBUS, SIGSYS, SIGFPE, SIGPIPE, SIGPROF}, faultSignalHandler);
|
||||
add_signal_handler({SIGABRT, SIGSEGV, SIGILL, SIGBUS, SIGSYS, SIGFPE, SIGPIPE, SIGTSTP}, signalHandler);
|
||||
add_signal_handler({SIGHUP, SIGUSR1}, closeLogsSignalHandler);
|
||||
add_signal_handler({SIGINT, SIGQUIT, SIGTERM}, terminateRequestedSignalHandler);
|
||||
|
||||
@ -729,9 +767,11 @@ void BaseDaemon::initializeTerminationAndSignalProcessing()
|
||||
static KillingErrorHandler killing_error_handler;
|
||||
Poco::ErrorHandler::set(&killing_error_handler);
|
||||
|
||||
signal_pipe.setNonBlocking();
|
||||
signal_pipe.tryIncreaseSize(1 << 20);
|
||||
|
||||
signal_listener.reset(new SignalListener(*this));
|
||||
signal_listener_thread.start(*signal_listener);
|
||||
|
||||
}
|
||||
|
||||
void BaseDaemon::logRevision() const
|
||||
@ -796,7 +836,7 @@ bool isPidRunning(pid_t pid)
|
||||
return 0;
|
||||
}
|
||||
|
||||
void BaseDaemon::PID::seed(const std::string & file_)
|
||||
BaseDaemon::PID::PID(const std::string & file_)
|
||||
{
|
||||
file = Poco::Path(file_).absolute().toString();
|
||||
Poco::File poco_file(file);
|
||||
@ -823,34 +863,28 @@ void BaseDaemon::PID::seed(const std::string & file_)
|
||||
|
||||
if (-1 == fd)
|
||||
{
|
||||
file.clear();
|
||||
if (EEXIST == errno)
|
||||
throw Poco::Exception("Pid file exists, should not start daemon.");
|
||||
throw Poco::CreateFileException("Cannot create pid file.");
|
||||
}
|
||||
|
||||
SCOPE_EXIT({ close(fd); });
|
||||
|
||||
std::stringstream s;
|
||||
s << getpid();
|
||||
if (static_cast<ssize_t>(s.str().size()) != write(fd, s.str().c_str(), s.str().size()))
|
||||
throw Poco::Exception("Cannot write to pid file.");
|
||||
}
|
||||
|
||||
BaseDaemon::PID::~PID()
|
||||
{
|
||||
try
|
||||
{
|
||||
std::stringstream s;
|
||||
s << getpid();
|
||||
if (static_cast<ssize_t>(s.str().size()) != write(fd, s.str().c_str(), s.str().size()))
|
||||
throw Poco::Exception("Cannot write to pid file.");
|
||||
Poco::File(file).remove();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
close(fd);
|
||||
throw;
|
||||
}
|
||||
|
||||
close(fd);
|
||||
}
|
||||
|
||||
void BaseDaemon::PID::clear()
|
||||
{
|
||||
if (!file.empty())
|
||||
{
|
||||
Poco::File(file).remove();
|
||||
file.clear();
|
||||
DB::tryLogCurrentException(__PRETTY_FUNCTION__);
|
||||
}
|
||||
}
|
||||
|
||||
@ -891,4 +925,3 @@ void BaseDaemon::waitForTerminationRequest()
|
||||
std::unique_lock<std::mutex> lock(signal_handler_mutex);
|
||||
signal_event.wait(lock, [this](){ return terminate_signals_counter > 0; });
|
||||
}
|
||||
|
||||
37
base/ext/chrono_io.h
Normal file
37
base/ext/chrono_io.h
Normal file
@ -0,0 +1,37 @@
|
||||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <string>
|
||||
#include <common/DateLUT.h>
|
||||
|
||||
|
||||
namespace ext
|
||||
{
|
||||
template <typename Clock, typename Duration = typename Clock::duration>
|
||||
std::string to_string(const std::chrono::time_point<Clock, Duration> & tp)
|
||||
{
|
||||
return DateLUT::instance().timeToString(std::chrono::system_clock::to_time_t(tp));
|
||||
}
|
||||
|
||||
template <typename Rep, typename Period = std::ratio<1>>
|
||||
std::string to_string(const std::chrono::duration<Rep, Period> & dur)
|
||||
{
|
||||
auto seconds_as_int = std::chrono::duration_cast<std::chrono::seconds>(dur);
|
||||
if (seconds_as_int == dur)
|
||||
return std::to_string(seconds_as_int.count()) + "s";
|
||||
auto seconds_as_double = std::chrono::duration_cast<std::chrono::duration<double>>(dur);
|
||||
return std::to_string(seconds_as_double.count()) + "s";
|
||||
}
|
||||
|
||||
template <typename Clock, typename Duration = typename Clock::duration>
|
||||
std::ostream & operator<<(std::ostream & o, const std::chrono::time_point<Clock, Duration> & tp)
|
||||
{
|
||||
return o << to_string(tp);
|
||||
}
|
||||
|
||||
template <typename Rep, typename Period = std::ratio<1>>
|
||||
std::ostream & operator<<(std::ostream & o, const std::chrono::duration<Rep, Period> & dur)
|
||||
{
|
||||
return o << to_string(dur);
|
||||
}
|
||||
}
|
||||
25
base/ext/push_back.h
Normal file
25
base/ext/push_back.h
Normal file
@ -0,0 +1,25 @@
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
|
||||
namespace ext
|
||||
{
|
||||
|
||||
/// Moves all arguments starting from the second to the end of the vector.
|
||||
/// For example, `push_back(vec, a1, a2, a3)` is a more compact way to write
|
||||
/// `vec.push_back(a1); vec.push_back(a2); vec.push_back(a3);`
|
||||
/// This function is like boost::range::push_back() but works for noncopyable types too.
|
||||
template <typename T>
|
||||
void push_back(std::vector<T> &)
|
||||
{
|
||||
}
|
||||
|
||||
template <typename T, typename FirstArg, typename... OtherArgs>
|
||||
void push_back(std::vector<T> & vec, FirstArg && first, OtherArgs &&... other)
|
||||
{
|
||||
vec.reserve(vec.size() + sizeof...(other) + 1);
|
||||
vec.emplace_back(std::move(first));
|
||||
push_back(vec, std::move(other)...);
|
||||
}
|
||||
|
||||
}
|
||||
42
base/ext/range.h
Normal file
42
base/ext/range.h
Normal file
@ -0,0 +1,42 @@
|
||||
#pragma once
|
||||
|
||||
#include <type_traits>
|
||||
#include <boost/range/counting_range.hpp>
|
||||
#include <boost/range/adaptor/transformed.hpp>
|
||||
|
||||
|
||||
namespace ext
|
||||
{
|
||||
/// For loop adaptor which is used to iterate through a half-closed interval [begin, end).
|
||||
template <typename BeginType, typename EndType>
|
||||
inline auto range(BeginType begin, EndType end)
|
||||
{
|
||||
using CommonType = typename std::common_type<BeginType, EndType>::type;
|
||||
return boost::counting_range<CommonType>(begin, end);
|
||||
}
|
||||
|
||||
template <typename Type>
|
||||
inline auto range(Type end)
|
||||
{
|
||||
return range<Type, Type>(static_cast<Type>(0), end);
|
||||
}
|
||||
|
||||
/// The same as range(), but every value is casted statically to a specified `ValueType`.
|
||||
/// This is useful to iterate through all constants of a enum.
|
||||
template <typename ValueType, typename BeginType, typename EndType>
|
||||
inline auto range_with_static_cast(BeginType begin, EndType end)
|
||||
{
|
||||
using CommonType = typename std::common_type<BeginType, EndType>::type;
|
||||
if constexpr (std::is_same_v<ValueType, CommonType>)
|
||||
return boost::counting_range<CommonType>(begin, end);
|
||||
else
|
||||
return boost::counting_range<CommonType>(begin, end)
|
||||
| boost::adaptors::transformed([](CommonType x) -> ValueType { return static_cast<ValueType>(x); });
|
||||
}
|
||||
|
||||
template <typename ValueType, typename EndType>
|
||||
inline auto range_with_static_cast(EndType end)
|
||||
{
|
||||
return range_with_static_cast<ValueType, EndType, EndType>(static_cast<EndType>(0), end);
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user