mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-10 01:25:21 +00:00
Merge remote-tracking branch 'origin/master' into ordinary_distinct_small_refact
This commit is contained in:
Igor Nikonov
commit
9fddf6efde
14
.github/workflows/master.yml
vendored
14
.github/workflows/master.yml
vendored
@ -151,8 +151,8 @@ jobs:
|
||||
# shellcheck disable=SC2046
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
SplitBuildSmokeTest:
|
||||
needs: [BuilderDebSplitted]
|
||||
SharedBuildSmokeTest:
|
||||
needs: [BuilderDebShared]
|
||||
runs-on: [self-hosted, style-checker]
|
||||
steps:
|
||||
- name: Set envs
|
||||
@ -171,7 +171,7 @@ jobs:
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
path: ${{ env.REPORTS_PATH }}
|
||||
- name: Split build check
|
||||
- name: Shared build check
|
||||
run: |
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
mkdir -p "$TEMP_PATH"
|
||||
@ -598,7 +598,7 @@ jobs:
|
||||
##########################################################################################
|
||||
##################################### SPECIAL BUILDS #####################################
|
||||
##########################################################################################
|
||||
BuilderDebSplitted:
|
||||
BuilderDebShared:
|
||||
needs: [DockerHubPush]
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
@ -609,7 +609,7 @@ jobs:
|
||||
IMAGES_PATH=${{runner.temp}}/images_path
|
||||
REPO_COPY=${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH=${{runner.temp}}/../ccaches
|
||||
BUILD_NAME=binary_splitted
|
||||
BUILD_NAME=binary_shared
|
||||
EOF
|
||||
- name: Download changed images
|
||||
uses: actions/download-artifact@v2
|
||||
@ -1012,7 +1012,7 @@ jobs:
|
||||
# - BuilderBinGCC
|
||||
- BuilderBinPPC64
|
||||
- BuilderBinClangTidy
|
||||
- BuilderDebSplitted
|
||||
- BuilderDebShared
|
||||
runs-on: [self-hosted, style-checker]
|
||||
steps:
|
||||
- name: Set envs
|
||||
@ -3153,7 +3153,7 @@ jobs:
|
||||
- UnitTestsMsan
|
||||
- UnitTestsUBsan
|
||||
- UnitTestsReleaseClang
|
||||
- SplitBuildSmokeTest
|
||||
- SharedBuildSmokeTest
|
||||
runs-on: [self-hosted, style-checker]
|
||||
steps:
|
||||
- name: Clear repository
|
||||
|
||||
19
.github/workflows/pull_request.yml
vendored
19
.github/workflows/pull_request.yml
vendored
@ -216,8 +216,8 @@ jobs:
|
||||
# shellcheck disable=SC2046
|
||||
docker rm -f $(docker ps -a -q) ||:
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
SplitBuildSmokeTest:
|
||||
needs: [BuilderDebSplitted]
|
||||
SharedBuildSmokeTest:
|
||||
needs: [BuilderDebShared]
|
||||
runs-on: [self-hosted, style-checker]
|
||||
steps:
|
||||
- name: Set envs
|
||||
@ -236,7 +236,7 @@ jobs:
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
path: ${{ env.REPORTS_PATH }}
|
||||
- name: Split build check
|
||||
- name: Shared build check
|
||||
run: |
|
||||
sudo rm -fr "$TEMP_PATH"
|
||||
mkdir -p "$TEMP_PATH"
|
||||
@ -620,7 +620,7 @@ jobs:
|
||||
##########################################################################################
|
||||
##################################### SPECIAL BUILDS #####################################
|
||||
##########################################################################################
|
||||
BuilderDebSplitted:
|
||||
BuilderDebShared:
|
||||
needs: [DockerHubPush, FastTest, StyleCheck]
|
||||
runs-on: [self-hosted, builder]
|
||||
steps:
|
||||
@ -631,7 +631,7 @@ jobs:
|
||||
IMAGES_PATH=${{runner.temp}}/images_path
|
||||
REPO_COPY=${{runner.temp}}/build_check/ClickHouse
|
||||
CACHES_PATH=${{runner.temp}}/../ccaches
|
||||
BUILD_NAME=binary_splitted
|
||||
BUILD_NAME=binary_shared
|
||||
EOF
|
||||
- name: Download changed images
|
||||
uses: actions/download-artifact@v2
|
||||
@ -1024,7 +1024,7 @@ jobs:
|
||||
# - BuilderBinGCC
|
||||
- BuilderBinPPC64
|
||||
- BuilderBinClangTidy
|
||||
- BuilderDebSplitted
|
||||
- BuilderDebShared
|
||||
runs-on: [self-hosted, style-checker]
|
||||
if: ${{ success() || failure() }}
|
||||
steps:
|
||||
@ -3348,6 +3348,10 @@ jobs:
|
||||
###################################### JEPSEN TESTS #########################################
|
||||
#############################################################################################
|
||||
Jepsen:
|
||||
# This is special test NOT INCLUDED in FinishCheck
|
||||
# When it's skipped, all dependent tasks will be skipped too.
|
||||
# DO NOT add it there
|
||||
if: contains(github.event.pull_request.labels.*.name, 'jepsen-test')
|
||||
needs: [BuilderBinRelease]
|
||||
uses: ./.github/workflows/jepsen.yml
|
||||
|
||||
@ -3416,10 +3420,9 @@ jobs:
|
||||
- UnitTestsMsan
|
||||
- UnitTestsUBsan
|
||||
- UnitTestsReleaseClang
|
||||
- SplitBuildSmokeTest
|
||||
- SharedBuildSmokeTest
|
||||
- CompatibilityCheck
|
||||
- IntegrationTestsFlakyCheck
|
||||
- Jepsen
|
||||
runs-on: [self-hosted, style-checker]
|
||||
steps:
|
||||
- name: Clear repository
|
||||
|
||||
2
.github/workflows/tags_stable.yml
vendored
2
.github/workflows/tags_stable.yml
vendored
@ -29,7 +29,7 @@ jobs:
|
||||
fetch-depth: 0
|
||||
- name: Generate versions
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GITHUB_TOKEN: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }}
|
||||
run: |
|
||||
./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv
|
||||
GID=$(id -g "${UID}")
|
||||
|
||||
2
.gitmodules
vendored
2
.gitmodules
vendored
@ -201,7 +201,7 @@
|
||||
[submodule "contrib/boringssl"]
|
||||
path = contrib/boringssl
|
||||
url = https://github.com/ClickHouse/boringssl.git
|
||||
branch = MergeWithUpstream
|
||||
branch = unknown_branch_from_artur
|
||||
[submodule "contrib/NuRaft"]
|
||||
path = contrib/NuRaft
|
||||
url = https://github.com/ClickHouse/NuRaft.git
|
||||
|
||||
@ -77,10 +77,9 @@ option(USE_STATIC_LIBRARIES "Disable to use shared libraries" ON)
|
||||
# DEVELOPER ONLY.
|
||||
# Faster linking if turned on.
|
||||
option(SPLIT_SHARED_LIBRARIES "Keep all internal libraries as separate .so files" OFF)
|
||||
option(CLICKHOUSE_SPLIT_BINARY "Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled" OFF)
|
||||
|
||||
if (USE_STATIC_LIBRARIES AND (SPLIT_SHARED_LIBRARIES OR CLICKHOUSE_SPLIT_BINARY))
|
||||
message(FATAL_ERROR "SPLIT_SHARED_LIBRARIES=1 or CLICKHOUSE_SPLIT_BINARY=1 must not be used together with USE_STATIC_LIBRARIES=1")
|
||||
if (USE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES)
|
||||
message(FATAL_ERROR "SPLIT_SHARED_LIBRARIES=1 must not be used together with USE_STATIC_LIBRARIES=1")
|
||||
endif()
|
||||
|
||||
if (NOT USE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES)
|
||||
@ -502,7 +501,7 @@ endif ()
|
||||
message (STATUS
|
||||
"Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ;
|
||||
USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES}
|
||||
SPLIT_SHARED=${SPLIT_SHARED_LIBRARIES}")
|
||||
SPLIT_SHARED_LIBRARIES=${SPLIT_SHARED_LIBRARIES}")
|
||||
|
||||
include (GNUInstallDirs)
|
||||
|
||||
|
||||
@ -15,4 +15,4 @@ ClickHouse® is an open-source column-oriented database management system that a
|
||||
* [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any.
|
||||
|
||||
## Upcoming events
|
||||
* **v22.8 Release Webinar** Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap.
|
||||
* [**v22.8 Release Webinar**](https://clickhouse.com/company/events/v22-8-release-webinar) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap.
|
||||
|
||||
@ -10,9 +10,10 @@ The following versions of ClickHouse server are currently being supported with s
|
||||
|
||||
| Version | Supported |
|
||||
|:-|:-|
|
||||
| 22.7 | ✔️ |
|
||||
| 22.6 | ✔️ |
|
||||
| 22.5 | ✔️ |
|
||||
| 22.4 | ✔️ |
|
||||
| 22.4 | ❌ |
|
||||
| 22.3 | ✔️ |
|
||||
| 22.2 | ❌ |
|
||||
| 22.1 | ❌ |
|
||||
@ -57,5 +58,5 @@ As the security issue moves from triage, to identified fix, to release planning
|
||||
|
||||
## Public Disclosure Timing
|
||||
|
||||
A public disclosure date is negotiated by the ClickHouse maintainers and the bug submitter. We prefer to fully disclose the bug as soon as possible once a user mitigation is available. It is reasonable to delay disclosure when the bug or the fix is not yet fully understood, the solution is not well-tested, or for vendor coordination. The timeframe for disclosure is from immediate (especially if it's already publicly known) to 90 days. For a vulnerability with a straightforward mitigation, we expect report date to disclosure date to be on the order of 7 days.
|
||||
A public disclosure date is negotiated by the ClickHouse maintainers and the bug submitter. We prefer to fully disclose the bug as soon as possible once a user mitigation is available. It is reasonable to delay disclosure when the bug or the fix is not yet fully understood, the solution is not well-tested, or for vendor coordination. The timeframe for disclosure is from immediate (especially if it's already publicly known) to 90 days. For a vulnerability with a straightforward mitigation, we expect the report date to disclosure date to be on the order of 7 days.
|
||||
|
||||
|
||||
@ -2,8 +2,47 @@
|
||||
|
||||
#include <cstring>
|
||||
#include <type_traits>
|
||||
#include <bit>
|
||||
|
||||
|
||||
inline void reverseMemcpy(void * dst, const void * src, size_t size)
|
||||
{
|
||||
uint8_t * uint_dst = reinterpret_cast<uint8_t *>(dst);
|
||||
const uint8_t * uint_src = reinterpret_cast<const uint8_t *>(src);
|
||||
|
||||
uint_dst += size;
|
||||
while (size)
|
||||
{
|
||||
--uint_dst;
|
||||
*uint_dst = *uint_src;
|
||||
++uint_src;
|
||||
--size;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline T unalignedLoadLE(const void * address)
|
||||
{
|
||||
T res {};
|
||||
if constexpr (std::endian::native == std::endian::little)
|
||||
memcpy(&res, address, sizeof(res));
|
||||
else
|
||||
reverseMemcpy(&res, address, sizeof(res));
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
template <typename T>
|
||||
inline void unalignedStoreLE(void * address,
|
||||
const typename std::enable_if<true, T>::type & src)
|
||||
{
|
||||
static_assert(std::is_trivially_copyable_v<T>);
|
||||
if constexpr (std::endian::native == std::endian::little)
|
||||
memcpy(address, &src, sizeof(src));
|
||||
else
|
||||
reverseMemcpy(address, &src, sizeof(src));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
inline T unalignedLoad(const void * address)
|
||||
{
|
||||
|
||||
2
contrib/NuRaft
vendored
2
contrib/NuRaft
vendored
@ -1 +1 @@
|
||||
Subproject commit e1dc47c1cfd529801a8c94a396a3921a71ae3ccf
|
||||
Subproject commit 2ef198694e10c86175ee6ead389346d199060437
|
||||
2
contrib/azure
vendored
2
contrib/azure
vendored
@ -1 +1 @@
|
||||
Subproject commit ac4b763d4ca40122275f1497cbdc5451337461d9
|
||||
Subproject commit ef75afc075fc71fbcd8fe28dcda3794ae265fd1c
|
||||
@ -1,6 +1,6 @@
|
||||
option (ENABLE_AZURE_BLOB_STORAGE "Enable Azure blob storage" ${ENABLE_LIBRARIES})
|
||||
|
||||
if (NOT ENABLE_AZURE_BLOB_STORAGE)
|
||||
if (NOT ENABLE_AZURE_BLOB_STORAGE OR BUILD_STANDALONE_KEEPER OR OS_FREEBSD)
|
||||
message(STATUS "Not using Azure blob storage")
|
||||
return()
|
||||
endif()
|
||||
|
||||
2
contrib/boringssl
vendored
2
contrib/boringssl
vendored
@ -1 +1 @@
|
||||
Subproject commit c1e01a441d6db234f4f12e63a7657d1f9e6db9c1
|
||||
Subproject commit 8061ac62d67953e61b793042e33baf1352e67510
|
||||
@ -44,6 +44,8 @@
|
||||
#define HAVE_SETJMP_H
|
||||
#define HAVE_SYS_STAT_H
|
||||
#define HAVE_UNISTD_H
|
||||
#define HAVE_POLL_H
|
||||
#define HAVE_PTHREAD_H
|
||||
|
||||
#define ENABLE_IPV6
|
||||
#define USE_OPENSSL
|
||||
|
||||
2
contrib/krb5
vendored
2
contrib/krb5
vendored
@ -1 +1 @@
|
||||
Subproject commit 5149dea4e2be0f67707383d2682b897c14631374
|
||||
Subproject commit d879821c7a4c70b0c3ad739d9951d1a2b1903df7
|
||||
2
contrib/nats-io
vendored
2
contrib/nats-io
vendored
@ -1 +1 @@
|
||||
Subproject commit 6b2227f36757da090321e2d317569d2bd42c4cc1
|
||||
Subproject commit 1e2597c54616015077e53a26d56b6bac448eb1b6
|
||||
@ -18,6 +18,8 @@ elseif(WIN32)
|
||||
set(NATS_PLATFORM_INCLUDE "apple")
|
||||
endif()
|
||||
|
||||
add_definitions(-DNATS_HAS_TLS)
|
||||
|
||||
file(GLOB PS_SOURCES "${NATS_IO_SOURCE_DIR}/${NATS_PLATFORM_INCLUDE}/*.c")
|
||||
set(SRCS
|
||||
"${NATS_IO_SOURCE_DIR}/asynccb.c"
|
||||
|
||||
@ -29,6 +29,7 @@
|
||||
"docker/test/util": {
|
||||
"name": "clickhouse/test-util",
|
||||
"dependent": [
|
||||
"docker/packager/binary",
|
||||
"docker/test/base",
|
||||
"docker/test/fasttest"
|
||||
]
|
||||
|
||||
@ -1,62 +1,7 @@
|
||||
# rebuild in #33610
|
||||
# docker build -t clickhouse/binary-builder .
|
||||
FROM ubuntu:20.04
|
||||
|
||||
# ARG for quick switch to a given ubuntu mirror
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=14
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install \
|
||||
apt-transport-https \
|
||||
apt-utils \
|
||||
ca-certificates \
|
||||
dnsutils \
|
||||
gnupg \
|
||||
iputils-ping \
|
||||
lsb-release \
|
||||
wget \
|
||||
--yes --no-install-recommends --verbose-versions \
|
||||
&& export LLVM_PUBKEY_HASH="bda960a8da687a275a2078d43c111d66b1c6a893a3275271beedf266c1ff4a0cdecb429c7a5cccf9f486ea7aa43fd27f" \
|
||||
&& wget -nv -O /tmp/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key \
|
||||
&& echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \
|
||||
&& apt-key add /tmp/llvm-snapshot.gpg.key \
|
||||
&& export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \
|
||||
&& echo "deb [trusted=yes] https://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \
|
||||
/etc/apt/sources.list \
|
||||
&& apt-get clean
|
||||
|
||||
# initial packages
|
||||
RUN apt-get update \
|
||||
&& apt-get install \
|
||||
bash \
|
||||
build-essential \
|
||||
ccache \
|
||||
clang-${LLVM_VERSION} \
|
||||
clang-tidy-${LLVM_VERSION} \
|
||||
cmake \
|
||||
curl \
|
||||
fakeroot \
|
||||
gdb \
|
||||
git \
|
||||
gperf \
|
||||
lld-${LLVM_VERSION} \
|
||||
llvm-${LLVM_VERSION} \
|
||||
llvm-${LLVM_VERSION}-dev \
|
||||
moreutils \
|
||||
ninja-build \
|
||||
pigz \
|
||||
rename \
|
||||
software-properties-common \
|
||||
tzdata \
|
||||
nasm \
|
||||
--yes --no-install-recommends \
|
||||
&& apt-get clean
|
||||
|
||||
# This symlink required by gcc to find lld compiler
|
||||
RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
|
||||
ARG FROM_TAG=latest
|
||||
FROM clickhouse/test-util:$FROM_TAG
|
||||
|
||||
ENV CC=clang-${LLVM_VERSION}
|
||||
ENV CXX=clang++-${LLVM_VERSION}
|
||||
@ -119,18 +64,18 @@ ENV GOCACHE=/workdir/
|
||||
RUN mkdir /workdir && chmod 777 /workdir
|
||||
WORKDIR /workdir
|
||||
|
||||
# FIXME: thread sanitizer is broken in clang-14, we have to build it with clang-13
|
||||
# NOTE: thread sanitizer is broken in clang-14, we have to build it with clang-15
|
||||
# https://github.com/ClickHouse/ClickHouse/pull/39450
|
||||
# https://github.com/google/sanitizers/issues/1540
|
||||
# https://github.com/google/sanitizers/issues/1552
|
||||
|
||||
RUN export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \
|
||||
&& echo "deb [trusted=yes] https://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-13 main" >> \
|
||||
&& echo "deb [trusted=yes] https://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-15 main" >> \
|
||||
/etc/apt/sources.list.d/clang.list \
|
||||
&& apt-get update \
|
||||
&& apt-get install \
|
||||
clang-13 \
|
||||
clang-tidy-13 \
|
||||
clang-15 \
|
||||
clang-tidy-15 \
|
||||
--yes --no-install-recommends \
|
||||
&& apt-get clean
|
||||
|
||||
|
||||
@ -3,7 +3,7 @@ set -x -e
|
||||
|
||||
exec &> >(ts)
|
||||
|
||||
cache_status () {
|
||||
ccache_status () {
|
||||
ccache --show-config ||:
|
||||
ccache --show-stats ||:
|
||||
}
|
||||
@ -48,7 +48,7 @@ if [ -n "$MAKE_DEB" ]; then
|
||||
fi
|
||||
|
||||
|
||||
cache_status
|
||||
ccache_status
|
||||
# clear cache stats
|
||||
ccache --zero-stats ||:
|
||||
|
||||
@ -92,7 +92,7 @@ $SCAN_WRAPPER ninja $NINJA_FLAGS $BUILD_TARGET
|
||||
|
||||
ls -la ./programs
|
||||
|
||||
cache_status
|
||||
ccache_status
|
||||
|
||||
if [ -n "$MAKE_DEB" ]; then
|
||||
# No quotes because I want it to expand to nothing if empty.
|
||||
@ -178,7 +178,8 @@ then
|
||||
mv "coverity-scan.tgz" /output
|
||||
fi
|
||||
|
||||
cache_status
|
||||
ccache_status
|
||||
ccache --evict-older-than 1d
|
||||
|
||||
if [ "${CCACHE_DEBUG:-}" == "1" ]
|
||||
then
|
||||
|
||||
@ -62,7 +62,7 @@ def pre_build(repo_path: str, env_variables: List[str]):
|
||||
f"git -C {repo_path} fetch --no-recurse-submodules "
|
||||
"--no-tags origin master:master"
|
||||
)
|
||||
logging.info("Getting master branch for performance artifact: ''%s'", cmd)
|
||||
logging.info("Getting master branch for performance artifact: '%s'", cmd)
|
||||
subprocess.check_call(cmd, shell=True)
|
||||
|
||||
|
||||
@ -100,12 +100,12 @@ def run_docker_image_with_env(
|
||||
subprocess.check_call(cmd, shell=True)
|
||||
|
||||
|
||||
def is_release_build(build_type, package_type, sanitizer, split_binary):
|
||||
def is_release_build(build_type, package_type, sanitizer, shared_libraries):
|
||||
return (
|
||||
build_type == ""
|
||||
and package_type == "deb"
|
||||
and sanitizer == ""
|
||||
and not split_binary
|
||||
and not shared_libraries
|
||||
)
|
||||
|
||||
|
||||
@ -116,7 +116,7 @@ def parse_env_variables(
|
||||
package_type,
|
||||
cache,
|
||||
distcc_hosts,
|
||||
split_binary,
|
||||
shared_libraries,
|
||||
clang_tidy,
|
||||
version,
|
||||
author,
|
||||
@ -202,7 +202,7 @@ def parse_env_variables(
|
||||
cmake_flags.append("-DCMAKE_INSTALL_PREFIX=/usr")
|
||||
cmake_flags.append("-DCMAKE_INSTALL_SYSCONFDIR=/etc")
|
||||
cmake_flags.append("-DCMAKE_INSTALL_LOCALSTATEDIR=/var")
|
||||
if is_release_build(build_type, package_type, sanitizer, split_binary):
|
||||
if is_release_build(build_type, package_type, sanitizer, shared_libraries):
|
||||
cmake_flags.append("-DSPLIT_DEBUG_SYMBOLS=ON")
|
||||
result.append("WITH_PERFORMANCE=1")
|
||||
if is_cross_arm:
|
||||
@ -215,11 +215,11 @@ def parse_env_variables(
|
||||
cmake_flags.append(f"-DCMAKE_C_COMPILER={cc}")
|
||||
cmake_flags.append(f"-DCMAKE_CXX_COMPILER={cxx}")
|
||||
|
||||
# Create combined output archive for split build and for performance tests.
|
||||
# Create combined output archive for shared library build and for performance tests.
|
||||
if package_type == "coverity":
|
||||
result.append("COMBINED_OUTPUT=coverity")
|
||||
result.append('COVERITY_TOKEN="$COVERITY_TOKEN"')
|
||||
elif split_binary:
|
||||
elif shared_libraries:
|
||||
result.append("COMBINED_OUTPUT=shared_build")
|
||||
|
||||
if sanitizer:
|
||||
@ -234,6 +234,7 @@ def parse_env_variables(
|
||||
|
||||
if cache:
|
||||
result.append("CCACHE_DIR=/ccache")
|
||||
result.append("CCACHE_COMPRESSLEVEL=5")
|
||||
result.append("CCACHE_BASEDIR=/build")
|
||||
result.append("CCACHE_NOHASHDIR=true")
|
||||
result.append("CCACHE_COMPILERCHECK=content")
|
||||
@ -242,7 +243,6 @@ def parse_env_variables(
|
||||
# 15G is not enough for tidy build
|
||||
cache_maxsize = "25G"
|
||||
result.append(f"CCACHE_MAXSIZE={cache_maxsize}")
|
||||
# result.append("CCACHE_UMASK=777")
|
||||
|
||||
if distcc_hosts:
|
||||
hosts_with_params = [f"{host}/24,lzo" for host in distcc_hosts] + [
|
||||
@ -264,14 +264,13 @@ def parse_env_variables(
|
||||
result.append("BINARY_OUTPUT=tests")
|
||||
cmake_flags.append("-DENABLE_TESTS=1")
|
||||
|
||||
if split_binary:
|
||||
if shared_libraries:
|
||||
cmake_flags.append(
|
||||
"-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 "
|
||||
"-DCLICKHOUSE_SPLIT_BINARY=1"
|
||||
"-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1"
|
||||
)
|
||||
# We can't always build utils because it requires too much space, but
|
||||
# we have to build them at least in some way in CI. The split build is
|
||||
# probably the least heavy disk-wise.
|
||||
# we have to build them at least in some way in CI. The shared library
|
||||
# build is probably the least heavy disk-wise.
|
||||
cmake_flags.append("-DENABLE_UTILS=1")
|
||||
# utils are not included into clickhouse-bundle, so build everything
|
||||
build_target = "all"
|
||||
@ -334,7 +333,7 @@ if __name__ == "__main__":
|
||||
parser.add_argument(
|
||||
"--compiler",
|
||||
choices=(
|
||||
"clang-13", # For TSAN builds, see #39450
|
||||
"clang-15", # For TSAN builds, see #39450
|
||||
"clang-14",
|
||||
"clang-14-darwin",
|
||||
"clang-14-darwin-aarch64",
|
||||
@ -352,7 +351,7 @@ if __name__ == "__main__":
|
||||
default="",
|
||||
)
|
||||
|
||||
parser.add_argument("--split-binary", action="store_true")
|
||||
parser.add_argument("--shared-libraries", action="store_true")
|
||||
parser.add_argument("--clang-tidy", action="store_true")
|
||||
parser.add_argument("--cache", choices=("ccache", "distcc", ""), default="")
|
||||
parser.add_argument(
|
||||
@ -405,7 +404,7 @@ if __name__ == "__main__":
|
||||
args.package_type,
|
||||
args.cache,
|
||||
args.distcc_hosts,
|
||||
args.split_binary,
|
||||
args.shared_libraries,
|
||||
args.clang_tidy,
|
||||
args.version,
|
||||
args.author,
|
||||
|
||||
@ -38,6 +38,7 @@ FORMAT_SCHEMA_PATH="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_
|
||||
|
||||
# There could be many disks declared in config
|
||||
readarray -t FILESYSTEM_CACHE_PATHS < <(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key='storage_configuration.disks.*.data_cache_path' || true)
|
||||
readarray -t DISKS_PATHS < <(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key='storage_configuration.disks.*.path' || true)
|
||||
|
||||
CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}"
|
||||
CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-}"
|
||||
@ -50,7 +51,8 @@ for dir in "$DATA_DIR" \
|
||||
"$TMP_DIR" \
|
||||
"$USER_PATH" \
|
||||
"$FORMAT_SCHEMA_PATH" \
|
||||
"${FILESYSTEM_CACHE_PATHS[@]}"
|
||||
"${FILESYSTEM_CACHE_PATHS[@]}" \
|
||||
"${DISKS_PATHS[@]}"
|
||||
do
|
||||
# check if variable not empty
|
||||
[ -z "$dir" ] && continue
|
||||
|
||||
@ -3,59 +3,12 @@
|
||||
ARG FROM_TAG=latest
|
||||
FROM clickhouse/test-util:$FROM_TAG
|
||||
|
||||
# ARG for quick switch to a given ubuntu mirror
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=14
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \
|
||||
--yes --no-install-recommends --verbose-versions \
|
||||
&& export LLVM_PUBKEY_HASH="bda960a8da687a275a2078d43c111d66b1c6a893a3275271beedf266c1ff4a0cdecb429c7a5cccf9f486ea7aa43fd27f" \
|
||||
&& wget -nv -O /tmp/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key \
|
||||
&& echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \
|
||||
&& apt-key add /tmp/llvm-snapshot.gpg.key \
|
||||
&& export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \
|
||||
&& echo "deb [trusted=yes] http://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \
|
||||
/etc/apt/sources.list
|
||||
|
||||
# initial packages
|
||||
RUN apt-get update \
|
||||
&& apt-get install \
|
||||
bash \
|
||||
fakeroot \
|
||||
ccache \
|
||||
curl \
|
||||
software-properties-common \
|
||||
--yes --no-install-recommends
|
||||
|
||||
# Architecture of the image when BuildKit/buildx is used
|
||||
ARG TARGETARCH
|
||||
|
||||
# Special dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) version which is able
|
||||
# to compress files using pigz (https://zlib.net/pigz/) instead of gzip.
|
||||
# Significantly increase deb packaging speed and compatible with old systems
|
||||
RUN arch=${TARGETARCH:-amd64} \
|
||||
&& curl -Lo /usr/bin/dpkg-deb https://github.com/ClickHouse-Extras/dpkg/releases/download/1.21.1-clickhouse/dpkg-deb-${arch}
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install \
|
||||
clang-${LLVM_VERSION} \
|
||||
debhelper \
|
||||
devscripts \
|
||||
gdb \
|
||||
git \
|
||||
gperf \
|
||||
lcov \
|
||||
llvm-${LLVM_VERSION} \
|
||||
moreutils \
|
||||
netbase \
|
||||
perl \
|
||||
pigz \
|
||||
pkg-config \
|
||||
tzdata \
|
||||
pv \
|
||||
nasm \
|
||||
--yes --no-install-recommends
|
||||
|
||||
# Sanitizer options for services (clickhouse-server)
|
||||
|
||||
@ -3,83 +3,23 @@
|
||||
ARG FROM_TAG=latest
|
||||
FROM clickhouse/test-util:$FROM_TAG
|
||||
|
||||
# ARG for quick switch to a given ubuntu mirror
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=14
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \
|
||||
--yes --no-install-recommends --verbose-versions \
|
||||
&& export LLVM_PUBKEY_HASH="bda960a8da687a275a2078d43c111d66b1c6a893a3275271beedf266c1ff4a0cdecb429c7a5cccf9f486ea7aa43fd27f" \
|
||||
&& wget -nv -O /tmp/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key \
|
||||
&& echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \
|
||||
&& apt-key add /tmp/llvm-snapshot.gpg.key \
|
||||
&& export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \
|
||||
&& echo "deb [trusted=yes] https://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \
|
||||
/etc/apt/sources.list
|
||||
|
||||
# initial packages
|
||||
RUN apt-get update \
|
||||
&& apt-get install \
|
||||
bash \
|
||||
fakeroot \
|
||||
ccache \
|
||||
curl \
|
||||
software-properties-common \
|
||||
--yes --no-install-recommends
|
||||
|
||||
# Architecture of the image when BuildKit/buildx is used
|
||||
ARG TARGETARCH
|
||||
|
||||
# Special dpkg-deb (https://github.com/ClickHouse-Extras/dpkg) version which is able
|
||||
# to compress files using pigz (https://zlib.net/pigz/) instead of gzip.
|
||||
# Significantly increase deb packaging speed and compatible with old systems
|
||||
RUN arch=${TARGETARCH:-amd64} \
|
||||
&& curl -Lo /usr/bin/dpkg-deb https://github.com/ClickHouse-Extras/dpkg/releases/download/1.21.1-clickhouse/dpkg-deb-${arch}
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install \
|
||||
apt-transport-https \
|
||||
bash \
|
||||
brotli \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
ccache \
|
||||
clang-${LLVM_VERSION} \
|
||||
clang-tidy-${LLVM_VERSION} \
|
||||
cmake \
|
||||
curl \
|
||||
expect \
|
||||
fakeroot \
|
||||
gdb \
|
||||
git \
|
||||
gperf \
|
||||
lld-${LLVM_VERSION} \
|
||||
llvm-${LLVM_VERSION} \
|
||||
file \
|
||||
lsof \
|
||||
moreutils \
|
||||
ninja-build \
|
||||
psmisc \
|
||||
python3 \
|
||||
python3-lxml \
|
||||
python3-pip \
|
||||
python3-requests \
|
||||
python3-termcolor \
|
||||
rename \
|
||||
software-properties-common \
|
||||
tzdata \
|
||||
unixodbc \
|
||||
file \
|
||||
nasm \
|
||||
--yes --no-install-recommends
|
||||
|
||||
RUN pip3 install numpy scipy pandas Jinja2
|
||||
|
||||
# This symlink required by gcc to find lld compiler
|
||||
RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
|
||||
|
||||
ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.4.20200302/clickhouse-odbc-1.1.4-Linux.tar.gz"
|
||||
|
||||
RUN mkdir -p /tmp/clickhouse-odbc-tmp \
|
||||
|
||||
@ -160,9 +160,8 @@ function run_cmake
|
||||
"-DENABLE_REPLXX=1"
|
||||
)
|
||||
|
||||
# TODO remove this? we don't use ccache anyway. An option would be to download it
|
||||
# from S3 simultaneously with cloning.
|
||||
export CCACHE_DIR="$FASTTEST_WORKSPACE/ccache"
|
||||
export CCACHE_COMPRESSLEVEL=5
|
||||
export CCACHE_BASEDIR="$FASTTEST_SOURCE"
|
||||
export CCACHE_NOHASHDIR=true
|
||||
export CCACHE_COMPILERCHECK=content
|
||||
@ -191,6 +190,7 @@ function build
|
||||
gzip "$FASTTEST_OUTPUT/clickhouse-stripped"
|
||||
fi
|
||||
ccache --show-stats ||:
|
||||
ccache --evict-older-than 1d ||:
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@ -218,6 +218,12 @@ clickhouse-client --query "SELECT 'Server successfully started', 'OK'" >> /test_
|
||||
|| (echo -e 'Server failed to start (see application_errors.txt and clickhouse-server.clean.log)\tFAIL' >> /test_output/test_results.tsv \
|
||||
&& grep -a "<Error>.*Application" /var/log/clickhouse-server/clickhouse-server.log > /test_output/application_errors.txt)
|
||||
|
||||
echo "Get previous release tag"
|
||||
previous_release_tag=$(clickhouse-client --query="SELECT version()" | get_previous_release_tag)
|
||||
echo $previous_release_tag
|
||||
|
||||
stop
|
||||
|
||||
[ -f /var/log/clickhouse-server/clickhouse-server.log ] || echo -e "Server log does not exist\tFAIL"
|
||||
[ -f /var/log/clickhouse-server/stderr.log ] || echo -e "Stderr log does not exist\tFAIL"
|
||||
|
||||
@ -265,10 +271,6 @@ zgrep -Fa " received signal " /test_output/gdb.log > /dev/null \
|
||||
|
||||
echo -e "Backward compatibility check\n"
|
||||
|
||||
echo "Get previous release tag"
|
||||
previous_release_tag=$(clickhouse-client --query="SELECT version()" | get_previous_release_tag)
|
||||
echo $previous_release_tag
|
||||
|
||||
echo "Clone previous release repository"
|
||||
git clone https://github.com/ClickHouse/ClickHouse.git --no-tags --progress --branch=$previous_release_tag --no-recurse-submodules --depth=1 previous_release_repository
|
||||
|
||||
@ -278,7 +280,6 @@ mkdir previous_release_package_folder
|
||||
echo $previous_release_tag | download_release_packets && echo -e 'Download script exit code\tOK' >> /test_output/test_results.tsv \
|
||||
|| echo -e 'Download script failed\tFAIL' >> /test_output/test_results.tsv
|
||||
|
||||
stop
|
||||
mv /var/log/clickhouse-server/clickhouse-server.log /var/log/clickhouse-server/clickhouse-server.clean.log
|
||||
|
||||
# Check if we cloned previous release repository successfully
|
||||
|
||||
@ -77,7 +77,7 @@ def run_func_test(
|
||||
pipes = []
|
||||
for i in range(0, len(output_paths)):
|
||||
f = open(output_paths[i], "w")
|
||||
full_command = "{} {} {} {} {}".format(
|
||||
full_command = "{} {} {} {} {} --stress".format(
|
||||
cmd,
|
||||
get_options(i, backward_compatibility_check),
|
||||
global_time_limit_option,
|
||||
|
||||
@ -1,5 +1,82 @@
|
||||
# rebuild in #33610
|
||||
# docker build -t clickhouse/test-util .
|
||||
|
||||
FROM ubuntu:20.04
|
||||
|
||||
# ARG for quick switch to a given ubuntu mirror
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=14
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install \
|
||||
apt-transport-https \
|
||||
apt-utils \
|
||||
ca-certificates \
|
||||
dnsutils \
|
||||
gnupg \
|
||||
iputils-ping \
|
||||
lsb-release \
|
||||
wget \
|
||||
--yes --no-install-recommends --verbose-versions \
|
||||
&& export LLVM_PUBKEY_HASH="bda960a8da687a275a2078d43c111d66b1c6a893a3275271beedf266c1ff4a0cdecb429c7a5cccf9f486ea7aa43fd27f" \
|
||||
&& wget -nv -O /tmp/llvm-snapshot.gpg.key https://apt.llvm.org/llvm-snapshot.gpg.key \
|
||||
&& echo "${LLVM_PUBKEY_HASH} /tmp/llvm-snapshot.gpg.key" | sha384sum -c \
|
||||
&& apt-key add /tmp/llvm-snapshot.gpg.key \
|
||||
&& export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \
|
||||
&& echo "deb [trusted=yes] https://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \
|
||||
/etc/apt/sources.list \
|
||||
&& apt-get clean
|
||||
|
||||
# initial packages
|
||||
RUN apt-get update \
|
||||
&& apt-get install \
|
||||
bash \
|
||||
bsdmainutils \
|
||||
build-essential \
|
||||
clang-${LLVM_VERSION} \
|
||||
clang-tidy-${LLVM_VERSION} \
|
||||
cmake \
|
||||
curl \
|
||||
fakeroot \
|
||||
gdb \
|
||||
git \
|
||||
gperf \
|
||||
lld-${LLVM_VERSION} \
|
||||
llvm-${LLVM_VERSION} \
|
||||
llvm-${LLVM_VERSION}-dev \
|
||||
moreutils \
|
||||
nasm \
|
||||
ninja-build \
|
||||
pigz \
|
||||
rename \
|
||||
software-properties-common \
|
||||
tzdata \
|
||||
--yes --no-install-recommends \
|
||||
&& apt-get clean
|
||||
|
||||
# This symlink required by gcc to find lld compiler
|
||||
RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
|
||||
|
||||
ARG CCACHE_VERSION=4.6.1
|
||||
RUN mkdir /tmp/ccache \
|
||||
&& cd /tmp/ccache \
|
||||
&& curl -L \
|
||||
-O https://github.com/ccache/ccache/releases/download/v$CCACHE_VERSION/ccache-$CCACHE_VERSION.tar.xz \
|
||||
-O https://github.com/ccache/ccache/releases/download/v$CCACHE_VERSION/ccache-$CCACHE_VERSION.tar.xz.asc \
|
||||
&& gpg --recv-keys --keyserver hkps://keyserver.ubuntu.com 5A939A71A46792CF57866A51996DDA075594ADB8 \
|
||||
&& gpg --verify ccache-4.6.1.tar.xz.asc \
|
||||
&& tar xf ccache-$CCACHE_VERSION.tar.xz \
|
||||
&& cd /tmp/ccache/ccache-$CCACHE_VERSION \
|
||||
&& cmake -DCMAKE_INSTALL_PREFIX=/usr \
|
||||
-DCMAKE_BUILD_TYPE=None \
|
||||
-DZSTD_FROM_INTERNET=ON \
|
||||
-DREDIS_STORAGE_BACKEND=OFF \
|
||||
-Wno-dev \
|
||||
-B build \
|
||||
-S . \
|
||||
&& make VERBOSE=1 -C build \
|
||||
&& make install -C build \
|
||||
&& cd / \
|
||||
&& rm -rf /tmp/ccache
|
||||
|
||||
COPY process_functional_tests_result.py /
|
||||
|
||||
24
docs/changelogs/v22.7.2.15-stable.md
Normal file
24
docs/changelogs/v22.7.2.15-stable.md
Normal file
@ -0,0 +1,24 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2022
|
||||
---
|
||||
|
||||
# 2022 Changelog
|
||||
|
||||
### ClickHouse release v22.7.2.15-stable (f843089624e) FIXME as compared to v22.7.1.2484-stable (f4f05ec786a)
|
||||
|
||||
#### Bug Fix
|
||||
* Backported in [#39750](https://github.com/ClickHouse/ClickHouse/issues/39750): Fix seeking while reading from encrypted disk. This PR fixes [#38381](https://github.com/ClickHouse/ClickHouse/issues/38381). [#39687](https://github.com/ClickHouse/ClickHouse/pull/39687) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in official stable or prestable release)
|
||||
|
||||
* Backported in [#39591](https://github.com/ClickHouse/ClickHouse/issues/39591): Fix data race and possible heap-buffer-overflow in Avro format. Closes [#39094](https://github.com/ClickHouse/ClickHouse/issues/39094) Closes [#33652](https://github.com/ClickHouse/ClickHouse/issues/33652). [#39498](https://github.com/ClickHouse/ClickHouse/pull/39498) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Backported in [#39613](https://github.com/ClickHouse/ClickHouse/issues/39613): Fix bug with maxsplit argument for splitByChar, which was not working correctly. [#39552](https://github.com/ClickHouse/ClickHouse/pull/39552) ([filimonov](https://github.com/filimonov)).
|
||||
* Backported in [#39792](https://github.com/ClickHouse/ClickHouse/issues/39792): Fix wrong index analysis with tuples and operator `IN`, which could lead to wrong query result. [#39752](https://github.com/ClickHouse/ClickHouse/pull/39752) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Backported in [#39837](https://github.com/ClickHouse/ClickHouse/issues/39837): Fix `CANNOT_READ_ALL_DATA` exception with `local_filesystem_read_method=pread_threadpool`. This bug affected only Linux kernel version 5.9 and 5.10 according to [man](https://manpages.debian.org/testing/manpages-dev/preadv2.2.en.html#BUGS). [#39800](https://github.com/ClickHouse/ClickHouse/pull/39800) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Replace MemoryTrackerBlockerInThread to LockMemoryExceptionInThread [#39619](https://github.com/ClickHouse/ClickHouse/pull/39619) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Change mysql-odbc url [#39702](https://github.com/ClickHouse/ClickHouse/pull/39702) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
|
||||
@ -171,11 +171,11 @@ concurrency-related errors. If it fails:
|
||||
|
||||
## Split Build Smoke Test
|
||||
|
||||
Checks that the server build in [split build](../development/build.md#split-build)
|
||||
Checks that the server build in [split build](../development/developer-instruction.md#split-build)
|
||||
configuration can start and run simple queries. If it fails:
|
||||
|
||||
* Fix other test errors first;
|
||||
* Build the server in [split build](../development/build.md#split-build) configuration
|
||||
* Build the server in [split build](../development/developer-instruction.md#split-build) configuration
|
||||
locally and check whether it can start and run `select 1`.
|
||||
|
||||
|
||||
|
||||
@ -267,19 +267,19 @@ The system will prepare ClickHouse binary builds for your pull request individua
|
||||
|
||||
Most probably some of the builds will fail at first times. This is due to the fact that we check builds both with gcc as well as with clang, with almost all of existing warnings (always with the `-Werror` flag) enabled for clang. On that same page, you can find all of the build logs so that you do not have to build ClickHouse in all of the possible ways.
|
||||
|
||||
## Browse ClickHouse Source Code {#browse-clickhouse-source-code}
|
||||
|
||||
You can use the **Woboq** online code browser available [here](https://clickhouse.com/codebrowser/ClickHouse/src/index.html). It provides code navigation, semantic highlighting, search and indexing. The code snapshot is updated daily.
|
||||
|
||||
Also, you can browse sources on [GitHub](https://github.com/ClickHouse/ClickHouse) as usual.
|
||||
|
||||
## Faster builds for development: Split build configuration {#split-build}
|
||||
|
||||
ClickHouse is normally statically linked into a single static `clickhouse` binary with minimal dependencies. This is convenient for distribution, but it means that for every change the entire binary needs to be re-linked, which is slow and inconvenient for development. As an alternative, you can instead build dynamically linked shared libraries and separate binaries `clickhouse-server`, `clickhouse-client` etc., allowing for faster incremental builds. To use it, add the following flags to your `cmake` invocation:
|
||||
ClickHouse is normally statically linked into a single static `clickhouse` binary with minimal dependencies. This is convenient for distribution, but it means that for every change the entire binary needs to be re-linked, which is slow and inconvenient for development. As an alternative, you can instead build dynamically linked shared libraries, allowing for faster incremental builds. To use it, add the following flags to your `cmake` invocation:
|
||||
```
|
||||
-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1
|
||||
-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1
|
||||
```
|
||||
|
||||
Note that the split build has several drawbacks:
|
||||
* There is no single `clickhouse` binary, and you have to run `clickhouse-server`, `clickhouse-client`, etc.
|
||||
* Risk of segfault if you run any of the programs while rebuilding the project.
|
||||
* You cannot run the integration tests since they only work a single complete binary.
|
||||
* You can't easily copy the binaries elsewhere. Instead of moving a single binary you'll need to copy all binaries and libraries.
|
||||
|
||||
If you are not interested in functionality provided by third-party libraries, you can further speed up the build using `cmake` options
|
||||
```
|
||||
-DENABLE_LIBRARIES=0 -DENABLE_EMBEDDED_COMPILER=0
|
||||
|
||||
@ -29,7 +29,7 @@ ENGINE = PostgreSQL('host:port', 'database', 'user', 'password'[, `schema`, `use
|
||||
|
||||
## Data Types Support {#data_types-support}
|
||||
|
||||
| PostgerSQL | ClickHouse |
|
||||
| PostgreSQL | ClickHouse |
|
||||
|------------------|--------------------------------------------------------------|
|
||||
| DATE | [Date](../../sql-reference/data-types/date.md) |
|
||||
| TIMESTAMP | [DateTime](../../sql-reference/data-types/datetime.md) |
|
||||
|
||||
@ -878,8 +878,6 @@ User can assign new big parts to different disks of a [JBOD](https://en.wikipedi
|
||||
|
||||
`MergeTree` family table engines can store data to [S3](https://aws.amazon.com/s3/) using a disk with type `s3`.
|
||||
|
||||
This feature is under development and not ready for production. There are known drawbacks such as very low performance.
|
||||
|
||||
Configuration markup:
|
||||
``` xml
|
||||
<storage_configuration>
|
||||
|
||||
@ -389,12 +389,6 @@ SETTINGS mutations_sync = 1;
|
||||
|
||||
Let's run the same 3 queries.
|
||||
|
||||
[Enable](../../operations/settings/settings.md#allow-experimental-projection-optimization) projections for selects:
|
||||
|
||||
```sql
|
||||
SET allow_experimental_projection_optimization = 1;
|
||||
```
|
||||
|
||||
### Query 1. Average Price Per Year {#average-price-projections}
|
||||
|
||||
Query:
|
||||
|
||||
@ -438,6 +438,18 @@ For more information, see the section “[Configuration files](../../operations/
|
||||
<include_from>/etc/metrica.xml</include_from>
|
||||
```
|
||||
|
||||
## interserver_listen_host {#interserver-listen-host}
|
||||
|
||||
Restriction on hosts that can exchange data between ClickHouse servers.
|
||||
The default value equals to `listen_host` setting.
|
||||
|
||||
Examples:
|
||||
|
||||
``` xml
|
||||
<interserver_listen_host>::ffff:a00:1</interserver_listen_host>
|
||||
<interserver_listen_host>10.0.0.1</interserver_listen_host>
|
||||
```
|
||||
|
||||
## interserver_http_port {#interserver-http-port}
|
||||
|
||||
Port for exchanging data between ClickHouse servers.
|
||||
@ -970,7 +982,7 @@ Default value: 2.
|
||||
**Example**
|
||||
|
||||
```xml
|
||||
<background_merges_mutations_concurrency_ratio>3</background_pbackground_merges_mutations_concurrency_ratio>
|
||||
<background_merges_mutations_concurrency_ratio>3</background_merges_mutations_concurrency_ratio>
|
||||
```
|
||||
|
||||
## background_move_pool_size {#background_move_pool_size}
|
||||
|
||||
@ -12,12 +12,13 @@ Reads file as a String. The file content is not parsed, so any information is re
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
file(path)
|
||||
file(path[, default])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `path` — The relative path to the file from [user_files_path](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_files_path). Path to file support following wildcards: `*`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc', 'def'` — strings.
|
||||
- `default` — The value that will be returned in the case when a file does not exist or cannot be accessed. Data types supported: [String](../../sql-reference/data-types/string.md) and [NULL](../../sql-reference/syntax.md#null-literal).
|
||||
|
||||
**Example**
|
||||
|
||||
|
||||
@ -22,7 +22,7 @@ Consider using the [sipHash64](#hash_functions-siphash64) function instead.
|
||||
|
||||
**Arguments**
|
||||
|
||||
The function takes a variable number of input parameters. Arguments can be any of the [supported data types](../../sql-reference/data-types/index.md).
|
||||
The function takes a variable number of input parameters. Arguments can be any of the [supported data types](../../sql-reference/data-types/index.md). For some data types calculated value of hash function may be the same for the same values even if types of arguments differ (integers of different size, named and unnamed `Tuple` with the same data, `Map` and the corresponding `Array(Tuple(key, value))` type with the same data).
|
||||
|
||||
**Returned Value**
|
||||
|
||||
@ -69,7 +69,7 @@ Function [interprets](../../sql-reference/functions/type-conversion-functions.md
|
||||
|
||||
**Arguments**
|
||||
|
||||
The function takes a variable number of input parameters. Arguments can be any of the [supported data types](../../sql-reference/data-types/index.md).
|
||||
The function takes a variable number of input parameters. Arguments can be any of the [supported data types](../../sql-reference/data-types/index.md). For some data types calculated value of hash function may be the same for the same values even if types of arguments differ (integers of different size, named and unnamed `Tuple` with the same data, `Map` and the corresponding `Array(Tuple(key, value))` type with the same data).
|
||||
|
||||
**Returned Value**
|
||||
|
||||
@ -99,7 +99,7 @@ sipHash128(par1,...)
|
||||
|
||||
**Arguments**
|
||||
|
||||
The function takes a variable number of input parameters. Arguments can be any of the [supported data types](../../sql-reference/data-types/index.md).
|
||||
The function takes a variable number of input parameters. Arguments can be any of the [supported data types](../../sql-reference/data-types/index.md). For some data types calculated value of hash function may be the same for the same values even if types of arguments differ (integers of different size, named and unnamed `Tuple` with the same data, `Map` and the corresponding `Array(Tuple(key, value))` type with the same data).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -135,7 +135,7 @@ This is a fast non-cryptographic hash function. It uses the CityHash algorithm f
|
||||
|
||||
**Arguments**
|
||||
|
||||
The function takes a variable number of input parameters. Arguments can be any of the [supported data types](../../sql-reference/data-types/index.md).
|
||||
The function takes a variable number of input parameters. Arguments can be any of the [supported data types](../../sql-reference/data-types/index.md). For some data types calculated value of hash function may be the same for the same values even if types of arguments differ (integers of different size, named and unnamed `Tuple` with the same data, `Map` and the corresponding `Array(Tuple(key, value))` type with the same data).
|
||||
|
||||
**Returned Value**
|
||||
|
||||
@ -275,7 +275,7 @@ These functions use the `Fingerprint64` and `Hash64` methods respectively from a
|
||||
|
||||
**Arguments**
|
||||
|
||||
The function takes a variable number of input parameters. Arguments can be any of the [supported data types](../../sql-reference/data-types/index.md).
|
||||
The function takes a variable number of input parameters. Arguments can be any of the [supported data types](../../sql-reference/data-types/index.md). For some data types calculated value of hash function may be the same for the same values even if types of arguments differ (integers of different size, named and unnamed `Tuple` with the same data, `Map` and the corresponding `Array(Tuple(key, value))` type with the same data)..
|
||||
|
||||
**Returned Value**
|
||||
|
||||
@ -401,7 +401,7 @@ metroHash64(par1, ...)
|
||||
|
||||
**Arguments**
|
||||
|
||||
The function takes a variable number of input parameters. Arguments can be any of the [supported data types](../../sql-reference/data-types/index.md).
|
||||
The function takes a variable number of input parameters. Arguments can be any of the [supported data types](../../sql-reference/data-types/index.md). For some data types calculated value of hash function may be the same for the same values even if types of arguments differ (integers of different size, named and unnamed `Tuple` with the same data, `Map` and the corresponding `Array(Tuple(key, value))` type with the same data).
|
||||
|
||||
**Returned Value**
|
||||
|
||||
@ -436,7 +436,7 @@ murmurHash2_64(par1, ...)
|
||||
|
||||
**Arguments**
|
||||
|
||||
Both functions take a variable number of input parameters. Arguments can be any of the [supported data types](../../sql-reference/data-types/index.md).
|
||||
Both functions take a variable number of input parameters. Arguments can be any of the [supported data types](../../sql-reference/data-types/index.md). For some data types calculated value of hash function may be the same for the same values even if types of arguments differ (integers of different size, named and unnamed `Tuple` with the same data, `Map` and the corresponding `Array(Tuple(key, value))` type with the same data).
|
||||
|
||||
**Returned Value**
|
||||
|
||||
@ -504,7 +504,7 @@ murmurHash3_64(par1, ...)
|
||||
|
||||
**Arguments**
|
||||
|
||||
Both functions take a variable number of input parameters. Arguments can be any of the [supported data types](../../sql-reference/data-types/index.md).
|
||||
Both functions take a variable number of input parameters. Arguments can be any of the [supported data types](../../sql-reference/data-types/index.md). For some data types calculated value of hash function may be the same for the same values even if types of arguments differ (integers of different size, named and unnamed `Tuple` with the same data, `Map` and the corresponding `Array(Tuple(key, value))` type with the same data).
|
||||
|
||||
**Returned Value**
|
||||
|
||||
|
||||
@ -34,7 +34,7 @@ CREATE TABLE table_with_ttl
|
||||
)
|
||||
ENGINE MergeTree()
|
||||
ORDER BY tuple()
|
||||
TTL event_time + INTERVAL 3 MONTH;
|
||||
TTL event_time + INTERVAL 3 MONTH
|
||||
SETTINGS min_bytes_for_wide_part = 0;
|
||||
|
||||
INSERT INTO table_with_ttl VALUES (now(), 1, 'username1');
|
||||
|
||||
@ -170,7 +170,7 @@ sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
|
||||
|
||||
В случае использования на разработческой машине старого HDD или SSD, а также при желании использовать меньше места для артефактов сборки можно использовать следующую команду:
|
||||
```bash
|
||||
cmake -DUSE_DEBUG_HELPERS=1 -DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1 ..
|
||||
cmake -DUSE_DEBUG_HELPERS=1 -DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 ..
|
||||
```
|
||||
При этом надо учесть, что получаемые в результате сборки исполнимые файлы будут динамически слинкованы с библиотеками, и поэтому фактически станут непереносимыми на другие компьютеры (либо для этого нужно будет предпринять значительно больше усилий по сравнению со статической сборкой). Плюсом же в данном случае является значительно меньшее время сборки (это проявляется не на первой сборке, а на последующих, после внесения изменений в исходный код - тратится меньшее время на линковку по сравнению со статической сборкой) и значительно меньшее использование места на жёстком диске (экономия более, чем в 3 раза по сравнению со статической сборкой). Для целей разработки, когда планируются только отладочные запуски на том же компьютере, где осуществлялась сборка, это может быть наиболее удобным вариантом.
|
||||
|
||||
@ -285,3 +285,9 @@ Pull request можно создать, даже если работа над з
|
||||
Система подготовит сборки ClickHouse специально для вашего pull request. Для их получения, нажмите на ссылку «Details» у проверки «Clickhouse build check». Там вы сможете найти прямые ссылки на собранные .deb пакеты ClickHouse, которые, при желании, вы даже сможете установить на свои продакшен серверы (если не страшно).
|
||||
|
||||
Вероятнее всего, часть сборок не будет успешной с первого раза. Ведь мы проверяем сборку кода и gcc и clang, а при сборке с помощью clang включаются почти все существующие в природе warnings (всегда с флагом `-Werror`). На той же странице, вы сможете найти логи сборки - вам не обязательно самому собирать ClickHouse всеми возможными способами.
|
||||
|
||||
## Навигация по коду ClickHouse {#navigatsiia-po-kodu-clickhouse}
|
||||
|
||||
Для навигации по коду онлайн доступен **Woboq**, он расположен [здесь](https://clickhouse.com/codebrowser/ClickHouse/src/index.html). В нём реализовано удобное перемещение между исходными файлами, семантическая подсветка, подсказки, индексация и поиск. Слепок кода обновляется ежедневно.
|
||||
|
||||
Также вы можете просматривать исходники на [GitHub](https://github.com/ClickHouse/ClickHouse).
|
||||
|
||||
@ -29,7 +29,7 @@ ENGINE = PostgreSQL('host:port', 'database', 'user', 'password'[, `schema`, `use
|
||||
|
||||
## Поддерживаемые типы данных {#data_types-support}
|
||||
|
||||
| PostgerSQL | ClickHouse |
|
||||
| PostgreSQL | ClickHouse |
|
||||
|------------------|--------------------------------------------------------------|
|
||||
| DATE | [Date](../../sql-reference/data-types/date.md) |
|
||||
| TIMESTAMP | [DateTime](../../sql-reference/data-types/datetime.md) |
|
||||
|
||||
@ -389,12 +389,6 @@ SETTINGS mutations_sync = 1;
|
||||
|
||||
Давайте выполним те же 3 запроса.
|
||||
|
||||
[Включите](../../operations/settings/settings.md#allow-experimental-projection-optimization) поддержку проекций:
|
||||
|
||||
```sql
|
||||
SET allow_experimental_projection_optimization = 1;
|
||||
```
|
||||
|
||||
### Запрос 1. Средняя цена за год {#average-price-projections}
|
||||
|
||||
Запрос:
|
||||
@ -647,4 +641,3 @@ no projection: 100 rows in set. Elapsed: 0.069 sec. Processed 26.32 million rows
|
||||
### Online Playground {#playground}
|
||||
|
||||
Этот набор данных доступен в [Online Playground](https://gh-api.clickhouse.tech/play?user=play#U0VMRUNUIHRvd24sIGRpc3RyaWN0LCBjb3VudCgpIEFTIGMsIHJvdW5kKGF2ZyhwcmljZSkpIEFTIHByaWNlLCBiYXIocHJpY2UsIDAsIDUwMDAwMDAsIDEwMCkgRlJPTSB1a19wcmljZV9wYWlkIFdIRVJFIGRhdGUgPj0gJzIwMjAtMDEtMDEnIEdST1VQIEJZIHRvd24sIGRpc3RyaWN0IEhBVklORyBjID49IDEwMCBPUkRFUiBCWSBwcmljZSBERVNDIExJTUlUIDEwMA==).
|
||||
|
||||
|
||||
@ -407,6 +407,18 @@ ClickHouse проверяет условия для `min_part_size` и `min_part
|
||||
<include_from>/etc/metrica.xml</include_from>
|
||||
```
|
||||
|
||||
## interserver_listen_host {#interserver-listen-host}
|
||||
|
||||
Ограничение по хостам, для обмена между серверами ClickHouse.
|
||||
Значение по умолчанию совпадает со значением параметра listen_host
|
||||
|
||||
Примеры:
|
||||
|
||||
``` xml
|
||||
<interserver_listen_host>::ffff:a00:1</interserver_listen_host>
|
||||
<interserver_listen_host>10.0.0.1</interserver_listen_host>
|
||||
```
|
||||
|
||||
## interserver_http_port {#interserver-http-port}
|
||||
|
||||
Порт для обмена между серверами ClickHouse.
|
||||
|
||||
@ -12,12 +12,13 @@ sidebar_label: "Функции для работы с файлами"
|
||||
**Синтаксис**
|
||||
|
||||
``` sql
|
||||
file(path)
|
||||
file(path[, default])
|
||||
```
|
||||
|
||||
**Аргументы**
|
||||
|
||||
- `path` — относительный путь до файла от [user_files_path](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_files_path). Путь к файлу может включать следующие символы подстановки и шаблоны: `*`, `?`, `{abc,def}` и `{N..M}`, где `N`, `M` — числа, `'abc', 'def'` — строки.
|
||||
- `default` — Значение возвращаемое в случае, если указанный файл не существует. Поддерживаемые типы данных: [String](../../sql-reference/data-types/string.md) и [NULL](../../sql-reference/syntax.md#null-literal).
|
||||
|
||||
**Примеры**
|
||||
|
||||
|
||||
@ -36,15 +36,4 @@ then
|
||||
# Push to GitHub rewriting the existing contents.
|
||||
# Sometimes it does not work with error message "! [remote rejected] master -> master (cannot lock ref 'refs/heads/master': is at 42a0f6b6b6c7be56a469441b4bf29685c1cebac3 but expected 520e9b02c0d4678a2a5f41d2f561e6532fb98cc1)"
|
||||
for _ in {1..10}; do git push --force origin master && break; sleep 5; done
|
||||
|
||||
# Turn off logging.
|
||||
set +x
|
||||
|
||||
if [[ -n "${CLOUDFLARE_TOKEN}" ]]
|
||||
then
|
||||
sleep 1m
|
||||
# https://api.cloudflare.com/#zone-purge-files-by-cache-tags,-host-or-prefix
|
||||
POST_DATA='{"hosts":["clickhouse.com"]}'
|
||||
curl -X POST "https://api.cloudflare.com/client/v4/zones/4fc6fb1d46e87851605aa7fa69ca6fe0/purge_cache" -H "Authorization: Bearer ${CLOUDFLARE_TOKEN}" -H "Content-Type:application/json" --data "${POST_DATA}"
|
||||
fi
|
||||
fi
|
||||
|
||||
@ -264,3 +264,9 @@ ClickHouse成员一旦在您的拉取请求上贴上«可以测试»标签，就
|
||||
系统将分别为您的拉取请求准备ClickHouse二进制版本。若要检索这些构建信息，请在检查列表中单击« ClickHouse构建检查»旁边的«详细信息»链接。在这里，您会找到指向ClickHouse的.deb软件包的直接链接，此外，甚至可以将其部署在生产服务器上（如果您不担心）。
|
||||
|
||||
某些构建项很可能会在首次构建时失败。这是因为我们同时检查了基于gcc和clang的构建，几乎所有现有的被clang启用的警告（总是带有`-Werror`标志）。在同一页面上，您可以找到所有构建的日志，因此不必以所有可能的方式构建ClickHouse。
|
||||
|
||||
## 浏览ClickHouse源代码 {#browse-clickhouse-source-code}
|
||||
|
||||
您可以使用 **Woboq** 在线代码浏览器 [点击这里](https://clickhouse.com/codebrowser/ClickHouse/src/index.html). 它提供了代码导航和语义突出显示、搜索和索引。 代码快照每天更新。
|
||||
|
||||
此外，您还可以像往常一样浏览源代码 [GitHub](https://github.com/ClickHouse/ClickHouse)
|
||||
|
||||
@ -29,7 +29,7 @@ ENGINE = PostgreSQL('host:port', 'database', 'user', 'password'[, `use_table_cac
|
||||
|
||||
## 支持的数据类型 {#data_types-support}
|
||||
|
||||
| PostgerSQL | ClickHouse |
|
||||
| PostgreSQL | ClickHouse |
|
||||
|------------------|--------------------------------------------------------------|
|
||||
| DATE | [Date](../../sql-reference/data-types/date.md) |
|
||||
| TIMESTAMP | [DateTime](../../sql-reference/data-types/datetime.md) |
|
||||
|
||||
@ -0,0 +1,8 @@
|
||||
position: 10
|
||||
label: '优化查询性能'
|
||||
collapsible: true
|
||||
collapsed: true
|
||||
link:
|
||||
type: generated-index
|
||||
title: Improving Query Performance
|
||||
slug: /zh/guides/improving-query-performance
|
||||
167
docs/zh/guides/improving-query-performance/skipping-indexes.md
Normal file
167
docs/zh/guides/improving-query-performance/skipping-indexes.md
Normal file
@ -0,0 +1,167 @@
|
||||
---
|
||||
sidebar_label: Data Skipping Indexes
|
||||
sidebar_position: 2
|
||||
---
|
||||
|
||||
# 深入理解ClickHouse跳数索引
|
||||
|
||||
### 跳数索引
|
||||
|
||||
影响ClickHouse查询性能的因素很多。在大多数场景中，关键因素是ClickHouse在计算查询WHERE子句条件时是否可以使用主键。因此，选择适用于最常见查询模式的主键对于表的设计至关重要。
|
||||
|
||||
然而，无论如何仔细地调优主键，不可避免地会出现不能有效使用它的查询用例。用户通常依赖于ClickHouse获得时间序列类型的数据，但他们通常希望根据其他业务维度(如客户id、网站URL或产品编号)分析同一批数据。在这种情况下，查询性能可能会相当差，因为应用WHERE子句条件可能需要对每个列值进行完整扫描。虽然ClickHouse在这些情况下仍然相对较快，但计算数百万或数十亿个单独的值将导致“非索引”查询的执行速度比基于主键的查询慢得多。
|
||||
|
||||
在传统的关系数据库中，解决这个问题的一种方法是将一个或多个“二级”索引附加到表上。这是一个b-树结构，允许数据库在O(log(n))时间内找到磁盘上所有匹配的行，而不是O(n)时间(一次表扫描)，其中n是行数。但是，这种类型的二级索引不适用于ClickHouse(或其他面向列的数据库)，因为磁盘上没有单独的行可以添加到索引中。
|
||||
|
||||
相反，ClickHouse提供了一种不同类型的索引，在特定情况下可以显著提高查询速度。这些结构被标记为跳数索引，因为它们使ClickHouse能够跳过保证没有匹配值的数据块。
|
||||
### 基本操作
|
||||
|
||||
用户只能在MergeTree表引擎上使用数据跳数索引。每个跳数索引都有四个主要参数：
|
||||
|
||||
- 索引名称。索引名用于在每个分区中创建索引文件。此外，在删除或具体化索引时需要将其作为参数。
|
||||
- 索引的表达式。索引表达式用于计算存储在索引中的值集。它可以是列、简单操作符、函数的子集的组合。
|
||||
- 类型。索引的类型控制计算，该计算决定是否可以跳过读取和计算每个索引块。
|
||||
- GRANULARITY。每个索引块由颗粒（granule）组成。例如，如果主表索引粒度为8192行，GRANULARITY为4，则每个索引“块”将为32768行。
|
||||
|
||||
当用户创建数据跳数索引时，表的每个数据部分目录中将有两个额外的文件。
|
||||
|
||||
- skp_idx_{index_name}.idx：包含排序的表达式值。
|
||||
- skp_idx_{index_name}.mrk2：包含关联数据列文件中的相应偏移量。
|
||||
|
||||
如果在执行查询并读取相关列文件时，WHERE子句过滤条件的某些部分与跳数索引表达式匹配，ClickHouse将使用索引文件数据来确定每个相关的数据块是必须被处理还是可以被绕过(假设块还没有通过应用主键索引被排除)。这里用一个非常简单的示例：考虑以下加载了可预测数据的表。
|
||||
|
||||
```
|
||||
CREATE TABLE skip_table
|
||||
(
|
||||
my_key UInt64,
|
||||
my_value UInt64
|
||||
)
|
||||
ENGINE MergeTree primary key my_key
|
||||
SETTINGS index_granularity=8192;
|
||||
|
||||
INSERT INTO skip_table SELECT number, intDiv(number,4096) FROM numbers(100000000);
|
||||
```
|
||||
|
||||
当执行一个不使用主键的简单查询时，将扫描my_value列所有的一亿条记录：
|
||||
|
||||
```
|
||||
SELECT * FROM skip_table WHERE my_value IN (125, 700)
|
||||
|
||||
┌─my_key─┬─my_value─┐
|
||||
│ 512000 │ 125 │
|
||||
│ 512001 │ 125 │
|
||||
│ ... | ... |
|
||||
└────────┴──────────┘
|
||||
|
||||
8192 rows in set. Elapsed: 0.079 sec. Processed 100.00 million rows, 800.10 MB (1.26 billion rows/s., 10.10 GB/s.
|
||||
```
|
||||
|
||||
增加一个基本的跳数索引：
|
||||
|
||||
```
|
||||
ALTER TABLE skip_table ADD INDEX vix my_value TYPE set(100) GRANULARITY 2;
|
||||
```
|
||||
|
||||
通常，跳数索引只应用于新插入的数据，所以仅仅添加索引不会影响上述查询。
|
||||
|
||||
要使已经存在的数据生效，那执行：
|
||||
|
||||
```
|
||||
ALTER TABLE skip_table MATERIALIZE INDEX vix;
|
||||
```
|
||||
|
||||
重跑SQL：
|
||||
|
||||
```
|
||||
SELECT * FROM skip_table WHERE my_value IN (125, 700)
|
||||
|
||||
┌─my_key─┬─my_value─┐
|
||||
│ 512000 │ 125 │
|
||||
│ 512001 │ 125 │
|
||||
│ ... | ... |
|
||||
└────────┴──────────┘
|
||||
|
||||
8192 rows in set. Elapsed: 0.051 sec. Processed 32.77 thousand rows, 360.45 KB (643.75 thousand rows/s., 7.08 MB/s.)
|
||||
```
|
||||
|
||||
这次没有再去处理1亿行800MB的数据，ClickHouse只读取和分析32768行360KB的数据—4个granule的数据。
|
||||
|
||||
下图是更直观的展示，这就是如何读取和选择my_value为125的4096行，以及如何跳过以下行而不从磁盘读取:
|
||||
|
||||
|
||||
|
||||
通过在执行查询时启用跟踪，用户可以看到关于跳数索引使用情况的详细信息。在clickhouse-client中设置send_logs_level:
|
||||
|
||||
```
|
||||
SET send_logs_level='trace';
|
||||
```
|
||||
这将在尝试调优查询SQL和表索引时提供有用的调试信息。上面的例子中，调试日志显示跳数索引过滤了大部分granule，只读取了两个:
|
||||
|
||||
```
|
||||
<Debug> default.skip_table (933d4b2c-8cea-4bf9-8c93-c56e900eefd1) (SelectExecutor): Index `vix` has dropped 6102/6104 granules.
|
||||
```
|
||||
### 跳数索引类型
|
||||
|
||||
#### minmax
|
||||
|
||||
这种轻量级索引类型不需要参数。它存储每个块的索引表达式的最小值和最大值(如果表达式是一个元组，它分别存储元组元素的每个成员的值)。对于倾向于按值松散排序的列，这种类型非常理想。在查询处理期间，这种索引类型的开销通常是最小的。
|
||||
|
||||
这种类型的索引只适用于标量或元组表达式——索引永远不适用于返回数组或map数据类型的表达式。
|
||||
|
||||
#### set
|
||||
|
||||
这种轻量级索引类型接受单个参数max_size，即每个块的值集(0允许无限数量的离散值)。这个集合包含块中的所有值(如果值的数量超过max_size则为空)。这种索引类型适用于每组颗粒中基数较低(本质上是“聚集在一起”)但总体基数较高的列。
|
||||
|
||||
该索引的成本、性能和有效性取决于块中的基数。如果每个块包含大量惟一值，那么针对大型索引集计算查询条件将非常昂贵，或者由于索引超过max_size而为空，因此索引将不应用。
|
||||
|
||||
#### Bloom Filter Types
|
||||
|
||||
Bloom filter是一种数据结构，它允许对集合成员进行高效的是否存在测试，但代价是有轻微的误报。在跳数索引的使用场景，假阳性不是一个大问题，因为惟一的问题只是读取一些不必要的块。潜在的假阳性意味着索引表达式应该为真，否则有效的数据可能会被跳过。
|
||||
|
||||
因为Bloom filter可以更有效地处理大量离散值的测试，所以它们可以适用于大量条件表达式判断的场景。特别的是Bloom filter索引可以应用于数组，数组中的每个值都被测试，也可以应用于map，通过使用mapKeys或mapValues函数将键或值转换为数组。
|
||||
|
||||
有三种基于Bloom过滤器的数据跳数索引类型：
|
||||
|
||||
* 基本的**bloom_filter**接受一个可选参数，该参数表示在0到1之间允许的“假阳性”率(如果未指定，则使用.025)。
|
||||
|
||||
* 更专业的**tokenbf_v1**。需要三个参数，用来优化布隆过滤器：（1）过滤器的大小字节(大过滤器有更少的假阳性，有更高的存储成本)，（2）哈希函数的个数(更多的散列函数可以减少假阳性)。（3）布隆过滤器哈希函数的种子。有关这些参数如何影响布隆过滤器功能的更多细节，请参阅 [这里](https://hur.st/bloomfilter/) 。此索引仅适用于String、FixedString和Map类型的数据。输入表达式被分割为由非字母数字字符分隔的字符序列。例如，列值`This is a candidate for a "full text" search`将被分割为`This` `is` `a` `candidate` `for` `full` `text` `search`。它用于LIKE、EQUALS、in、hasToken()和类似的长字符串中单词和其他值的搜索。例如，一种可能的用途是在非结构的应用程序日志行列中搜索少量的类名或行号。
|
||||
|
||||
* 更专业的**ngrambf_v1**。该索引的功能与tokenbf_v1相同。在Bloom filter设置之前需要一个额外的参数，即要索引的ngram的大小。一个ngram是长度为n的任何字符串，比如如果n是4，`A short string`会被分割为`A sh`` sho`, `shor`, `hort`, `ort s`, `or st`, `r str`, ` stri`, `trin`, `ring`。这个索引对于文本搜索也很有用，特别是没有单词间断的语言，比如中文。
|
||||
|
||||
### 跳数索引函数
|
||||
|
||||
跳数索引核心目的是限制流行查询分析的数据量。鉴于ClickHouse数据的分析特性，这些查询的模式在大多数情况下都包含函数表达式。因此，跳数索引必须与常用函数正确交互才能提高效率。这种情况可能发生在:
|
||||
* 插入数据并将索引定义为一个函数表达式(表达式的结果存储在索引文件中)或者
|
||||
* 处理查询，并将表达式应用于存储的索引值，以确定是否排除数据块。
|
||||
|
||||
每种类型的跳数索引支持的函数列表可以查看 [这里](https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree/#functions-support) 。通常，集合索引和基于Bloom filter的索引(另一种类型的集合索引)都是无序的，因此不能用于范围。相反，最大最小值索引在范围中工作得特别好，因为确定范围是否相交非常快。部分匹配函数LIKE、startsWith、endsWith和hasToken的有效性取决于使用的索引类型、索引表达式和数据的特定形状。
|
||||
|
||||
### 跳数索引的配置
|
||||
|
||||
有两个可用的设置可应用于跳数索引。
|
||||
|
||||
* **use_skip_indexes** (0或1，默认为1)。不是所有查询都可以有效地使用跳过索引。如果一个特定的过滤条件可能包含很多颗粒，那么应用数据跳过索引将导致不必要的、有时甚至是非常大的成本。对于不太可能从任何跳过索引中获益的查询，将该值设置为0。
|
||||
* **force_data_skipping_indexes** (以逗号分隔的索引名列表)。此设置可用于防止某些类型的低效查询。在某些情况下，除非使用跳过索引，否则查询表的开销太大，如果将此设置与一个或多个索引名一起使用，则对于任何没有使用所列索引的查询将返回一个异常。这将防止编写糟糕的查询消耗服务器资源。
|
||||
|
||||
### 最佳实践
|
||||
|
||||
跳数索引并不直观，特别是对于来自RDMS领域并且习惯二级行索引或来自文档存储的反向索引的用户来说。要获得任何优化，应用ClickHouse数据跳数索引必须避免足够多的颗粒读取，以抵消计算索引的成本。关键是，如果一个值在一个索引块中只出现一次，就意味着整个块必须读入内存并计算，而索引开销是不必要的。
|
||||
|
||||
考虑以下数据分布：
|
||||
|
||||
|
||||
|
||||
|
||||
假设主键/顺序是时间戳，并且在visitor_id上有一个索引。考虑下面的查询:
|
||||
|
||||
`SELECT timestamp, url FROM table WHERE visitor_id = 1001`
|
||||
|
||||
对于这种数据分布，传统的二级索引非常有利。不是读取所有的32678行来查找具有请求的visitor_id的5行，而是二级索引只包含5行位置，并且只从磁盘读取这5行。ClickHouse数据跳过索引的情况正好相反。无论跳转索引的类型是什么，visitor_id列中的所有32678值都将被测试。
|
||||
|
||||
因此，试图通过简单地向键列添加索引来加速ClickHouse查询的冲动通常是不正确的。只有在研究了其他替代方法之后，才应该使用此高级功能，例如修改主键(查看 [如何选择主键](../improving-query-performance/sparse-primary-indexes.md))、使用投影或使用实体化视图。即使跳数索引是合适的，也经常需要对索引和表进行仔细的调优。
|
||||
|
||||
在大多数情况下，一个有用的跳数索引需要主键和目标的非主列/表达式之间具有很强的相关性。如果没有相关性(如上图所示)，那么在包含数千个值的块中，至少有一行满足过滤条件的可能性很高，并且只有几个块会被跳过。相反，如果主键的值范围(如一天中的时间)与潜在索引列中的值强相关(如电视观众年龄)，则最小值类型的索引可能是有益的。注意，在插入数据时，可以增加这种相关性，方法是在sort /ORDER by键中包含额外的列，或者以在插入时对与主键关联的值进行分组的方式对插入进行批处理。例如，即使主键是一个包含大量站点事件的时间戳，特定site_id的所有事件也都可以被分组并由写入进程插入到一起，这将导致许多只包含少量站点id的颗粒，因此当根据特定的site_id值搜索时，可以跳过许多块。
|
||||
|
||||
跳数索引的另一个候选者是高基数表达式，其中任何一个值在数据中都相对稀疏。一个可能的例子是跟踪API请求中的错误代码的可观察性平台。某些错误代码虽然在数据中很少出现，但对搜索来说可能特别重要。error_code列上的set skip索引将允许绕过绝大多数不包含错误的块，从而显著改善针对错误的查询。
|
||||
|
||||
最后，关键的最佳实践是测试、测试、再测试。同样，与用于搜索文档的b-树二级索引或倒排索引不同，跳数索引行为是不容易预测的。将它们添加到表中会在数据摄取和查询方面产生很大的成本，这些查询由于各种原因不能从索引中受益。它们应该总是在真实世界的数据类型上进行测试，测试应该包括类型、粒度大小和其他参数的变化。测试通常会暴露仅仅通过思考不能发现的陷阱。
|
||||
1172
docs/zh/guides/improving-query-performance/sparse-primary-indexes.md
Normal file
1172
docs/zh/guides/improving-query-performance/sparse-primary-indexes.md
Normal file
File diff suppressed because it is too large
Load Diff
@ -26,7 +26,7 @@ if [ "$1" = configure ] || [ -n "$not_deb_os" ]; then
|
||||
|
||||
${CLICKHOUSE_GENERIC_PROGRAM} install --user "${CLICKHOUSE_USER}" --group "${CLICKHOUSE_GROUP}" --pid-path "${CLICKHOUSE_PIDDIR}" --config-path "${CLICKHOUSE_CONFDIR}" --binary-path "${CLICKHOUSE_BINDIR}" --log-path "${CLICKHOUSE_LOGDIR}" --data-path "${CLICKHOUSE_DATADIR}"
|
||||
|
||||
if [ -x "/bin/systemctl" ] && [ -f /etc/systemd/system/clickhouse-server.service ] && [ -d /run/systemd/system ]; then
|
||||
if [ -x "/bin/systemctl" ] && [ -f /lib/systemd/system/clickhouse-server.service ] && [ -d /run/systemd/system ]; then
|
||||
# if old rc.d service present - remove it
|
||||
if [ -x "/etc/init.d/clickhouse-server" ] && [ -x "/usr/sbin/update-rc.d" ]; then
|
||||
/usr/sbin/update-rc.d clickhouse-server remove
|
||||
@ -44,4 +44,16 @@ if [ "$1" = configure ] || [ -n "$not_deb_os" ]; then
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# /etc/systemd/system/clickhouse-server.service shouldn't be distributed by the package, but it was
|
||||
# here we delete the service file if it was from our package
|
||||
if [ -f /etc/systemd/system/clickhouse-server.service ]; then
|
||||
SHA256=$(sha256sum /etc/systemd/system/clickhouse-server.service | cut -d' ' -f1)
|
||||
for ref_sum in 7769a14773e811a56f67fd70f7960147217f5e68f746010aec96722e24d289bb 22890012047ea84fbfcebd6e291fe2ef2185cbfdd94a0294e13c8bf9959f58f8 b7790ae57156663c723f92e75ac2508453bf0a7b7e8313bb8081da99e5e88cd3 d1dcc1dbe92dab3ae17baa395f36abf1876b4513df272bf021484923e0111eef ac29ddd32a02eb31670bf5f0018c5d8a3cc006ca7ea572dcf717cb42310dcad7 c62d23052532a70115414833b500b266647d3924eb006a6f3eb673ff0d55f8fa b6b200ffb517afc2b9cf9e25ad8a4afdc0dad5a045bddbfb0174f84cc5a959ed; do
|
||||
if [ "$SHA256" = "$ref_sum" ]; then
|
||||
rm /etc/systemd/system/clickhouse-server.service
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
@ -18,11 +18,7 @@ option (ENABLE_CLICKHOUSE_SERVER "Server mode (main mode)" ${ENABLE_CLICKHOUSE_A
|
||||
option (ENABLE_CLICKHOUSE_CLIENT "Client mode (interactive tui/shell that connects to the server)"
|
||||
${ENABLE_CLICKHOUSE_ALL})
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
option (ENABLE_CLICKHOUSE_SELF_EXTRACTING "Self-extracting executable" OFF)
|
||||
else ()
|
||||
option (ENABLE_CLICKHOUSE_SELF_EXTRACTING "Self-extracting executable" ON)
|
||||
endif ()
|
||||
option (ENABLE_CLICKHOUSE_SELF_EXTRACTING "Self-extracting executable" ON)
|
||||
|
||||
# https://clickhouse.com/docs/en/operations/utilities/clickhouse-local/
|
||||
option (ENABLE_CLICKHOUSE_LOCAL "Local files fast processing mode" ${ENABLE_CLICKHOUSE_ALL})
|
||||
@ -80,12 +76,7 @@ if (NOT ENABLE_NURAFT)
|
||||
set(ENABLE_CLICKHOUSE_KEEPER_CONVERTER OFF)
|
||||
endif()
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
option(ENABLE_CLICKHOUSE_INSTALL "Install ClickHouse without .deb/.rpm/.tgz packages (having the binary only)" OFF)
|
||||
else ()
|
||||
option(ENABLE_CLICKHOUSE_INSTALL "Install ClickHouse without .deb/.rpm/.tgz packages (having the binary only)"
|
||||
${ENABLE_CLICKHOUSE_ALL})
|
||||
endif ()
|
||||
option(ENABLE_CLICKHOUSE_INSTALL "Install ClickHouse without .deb/.rpm/.tgz packages (having the binary only)" ${ENABLE_CLICKHOUSE_ALL})
|
||||
|
||||
message(STATUS "ClickHouse modes:")
|
||||
|
||||
@ -211,10 +202,6 @@ macro(clickhouse_target_link_split_lib target name)
|
||||
endif()
|
||||
endmacro()
|
||||
|
||||
macro(clickhouse_program_link_split_binary name)
|
||||
clickhouse_target_link_split_lib(clickhouse-${name} ${name})
|
||||
endmacro()
|
||||
|
||||
macro(clickhouse_program_add_library name)
|
||||
string(TOUPPER ${name} name_uc)
|
||||
string(REPLACE "-" "_" name_uc ${name_uc})
|
||||
@ -239,17 +226,8 @@ macro(clickhouse_program_add_library name)
|
||||
endif()
|
||||
endmacro()
|
||||
|
||||
macro(clickhouse_program_add_executable name)
|
||||
if(CLICKHOUSE_SPLIT_BINARY)
|
||||
clickhouse_add_executable(clickhouse-${name} clickhouse-${name}.cpp)
|
||||
clickhouse_program_link_split_binary(${name})
|
||||
install(TARGETS clickhouse-${name} ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
endif()
|
||||
endmacro()
|
||||
|
||||
macro(clickhouse_program_add name)
|
||||
clickhouse_program_add_library(${name})
|
||||
clickhouse_program_add_executable(${name})
|
||||
endmacro()
|
||||
|
||||
add_subdirectory (server)
|
||||
@ -342,210 +320,173 @@ if (CLICKHOUSE_ONE_SHARED)
|
||||
install (TARGETS clickhouse-lib LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT clickhouse)
|
||||
endif()
|
||||
|
||||
if (CLICKHOUSE_SPLIT_BINARY)
|
||||
set (CLICKHOUSE_ALL_TARGETS
|
||||
clickhouse-server
|
||||
clickhouse-client
|
||||
clickhouse-local
|
||||
clickhouse-benchmark
|
||||
clickhouse-extract-from-config
|
||||
clickhouse-compressor
|
||||
clickhouse-format
|
||||
clickhouse-obfuscator
|
||||
clickhouse-git-import
|
||||
clickhouse-copier
|
||||
clickhouse-static-files-disk-uploader
|
||||
clickhouse-disks)
|
||||
clickhouse_add_executable (clickhouse main.cpp)
|
||||
|
||||
if (ENABLE_CLICKHOUSE_ODBC_BRIDGE)
|
||||
list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-odbc-bridge)
|
||||
endif ()
|
||||
if (NOT USE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES)
|
||||
# Shared split (dev) build: In CI, the server is run with custom LD_LIBRARY_PATH. This makes the harmful env check re-execute the
|
||||
# process in a clean environment but as in CI the containing directory is not included in DT_RUNPATH/DT_RPATH, the server won't come up.
|
||||
target_compile_definitions(clickhouse PRIVATE DISABLE_HARMFUL_ENV_VAR_CHECK)
|
||||
endif ()
|
||||
|
||||
if (ENABLE_CLICKHOUSE_LIBRARY_BRIDGE)
|
||||
list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-library-bridge)
|
||||
endif ()
|
||||
# A library that prevent usage of several functions from libc.
|
||||
if (ARCH_AMD64 AND OS_LINUX AND NOT OS_ANDROID)
|
||||
set (HARMFUL_LIB harmful)
|
||||
endif ()
|
||||
|
||||
if (ENABLE_CLICKHOUSE_KEEPER)
|
||||
list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-keeper)
|
||||
endif ()
|
||||
target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils ${HARMFUL_LIB})
|
||||
target_include_directories (clickhouse PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
|
||||
|
||||
if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
|
||||
list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-keeper-converter)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_SERVER)
|
||||
clickhouse_target_link_split_lib(clickhouse server)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_CLIENT)
|
||||
clickhouse_target_link_split_lib(clickhouse client)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_LOCAL)
|
||||
clickhouse_target_link_split_lib(clickhouse local)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_BENCHMARK)
|
||||
clickhouse_target_link_split_lib(clickhouse benchmark)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_COPIER)
|
||||
clickhouse_target_link_split_lib(clickhouse copier)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG)
|
||||
clickhouse_target_link_split_lib(clickhouse extract-from-config)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_COMPRESSOR)
|
||||
clickhouse_target_link_split_lib(clickhouse compressor)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_FORMAT)
|
||||
clickhouse_target_link_split_lib(clickhouse format)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_OBFUSCATOR)
|
||||
clickhouse_target_link_split_lib(clickhouse obfuscator)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_GIT_IMPORT)
|
||||
clickhouse_target_link_split_lib(clickhouse git-import)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER)
|
||||
clickhouse_target_link_split_lib(clickhouse static-files-disk-uploader)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_SU)
|
||||
clickhouse_target_link_split_lib(clickhouse su)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_KEEPER)
|
||||
clickhouse_target_link_split_lib(clickhouse keeper)
|
||||
endif()
|
||||
if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
|
||||
clickhouse_target_link_split_lib(clickhouse keeper-converter)
|
||||
endif()
|
||||
if (ENABLE_CLICKHOUSE_INSTALL)
|
||||
clickhouse_target_link_split_lib(clickhouse install)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_DISKS)
|
||||
clickhouse_target_link_split_lib(clickhouse disks)
|
||||
endif ()
|
||||
|
||||
if (ENABLE_CLICKHOUSE_SU)
|
||||
list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-su)
|
||||
endif ()
|
||||
set (CLICKHOUSE_BUNDLE)
|
||||
if (ENABLE_CLICKHOUSE_SELF_EXTRACTING)
|
||||
list(APPEND CLICKHOUSE_BUNDLE self-extracting)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_SERVER)
|
||||
add_custom_target (clickhouse-server ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-server DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-server" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-server)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_CLIENT)
|
||||
add_custom_target (clickhouse-client ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-client DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-client" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-client)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_LOCAL)
|
||||
add_custom_target (clickhouse-local ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-local DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-local" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-local)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_BENCHMARK)
|
||||
add_custom_target (clickhouse-benchmark ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-benchmark DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-benchmark" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-benchmark)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_COPIER)
|
||||
add_custom_target (clickhouse-copier ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-copier DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-copier" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-copier)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG)
|
||||
add_custom_target (clickhouse-extract-from-config ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-extract-from-config DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-extract-from-config" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-extract-from-config)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_COMPRESSOR)
|
||||
add_custom_target (clickhouse-compressor ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-compressor DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-compressor" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-compressor)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_FORMAT)
|
||||
add_custom_target (clickhouse-format ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-format DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-format" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-format)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_OBFUSCATOR)
|
||||
add_custom_target (clickhouse-obfuscator ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-obfuscator DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-obfuscator" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-obfuscator)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_GIT_IMPORT)
|
||||
add_custom_target (clickhouse-git-import ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-git-import DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-git-import" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-git-import)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER)
|
||||
add_custom_target (clickhouse-static-files-disk-uploader ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-static-files-disk-uploader DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-static-files-disk-uploader" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-static-files-disk-uploader)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_SU)
|
||||
add_custom_target (clickhouse-su ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-su DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-su" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-su)
|
||||
endif ()
|
||||
|
||||
set_target_properties(${CLICKHOUSE_ALL_TARGETS} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..)
|
||||
|
||||
add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_ALL_TARGETS})
|
||||
add_custom_target (clickhouse ALL DEPENDS clickhouse-bundle)
|
||||
|
||||
install(PROGRAMS clickhouse-split-helper DESTINATION ${CMAKE_INSTALL_BINDIR} RENAME clickhouse COMPONENT clickhouse)
|
||||
else ()
|
||||
clickhouse_add_executable (clickhouse main.cpp)
|
||||
|
||||
# A library that prevent usage of several functions from libc.
|
||||
if (ARCH_AMD64 AND OS_LINUX AND NOT OS_ANDROID)
|
||||
set (HARMFUL_LIB harmful)
|
||||
endif ()
|
||||
|
||||
target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils ${HARMFUL_LIB})
|
||||
target_include_directories (clickhouse PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
|
||||
|
||||
if (ENABLE_CLICKHOUSE_SERVER)
|
||||
clickhouse_target_link_split_lib(clickhouse server)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_CLIENT)
|
||||
clickhouse_target_link_split_lib(clickhouse client)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_LOCAL)
|
||||
clickhouse_target_link_split_lib(clickhouse local)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_BENCHMARK)
|
||||
clickhouse_target_link_split_lib(clickhouse benchmark)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_COPIER)
|
||||
clickhouse_target_link_split_lib(clickhouse copier)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG)
|
||||
clickhouse_target_link_split_lib(clickhouse extract-from-config)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_COMPRESSOR)
|
||||
clickhouse_target_link_split_lib(clickhouse compressor)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_FORMAT)
|
||||
clickhouse_target_link_split_lib(clickhouse format)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_OBFUSCATOR)
|
||||
clickhouse_target_link_split_lib(clickhouse obfuscator)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_GIT_IMPORT)
|
||||
clickhouse_target_link_split_lib(clickhouse git-import)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER)
|
||||
clickhouse_target_link_split_lib(clickhouse static-files-disk-uploader)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_SU)
|
||||
clickhouse_target_link_split_lib(clickhouse su)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_KEEPER)
|
||||
clickhouse_target_link_split_lib(clickhouse keeper)
|
||||
endif()
|
||||
if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
|
||||
clickhouse_target_link_split_lib(clickhouse keeper-converter)
|
||||
endif()
|
||||
if (ENABLE_CLICKHOUSE_INSTALL)
|
||||
clickhouse_target_link_split_lib(clickhouse install)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_DISKS)
|
||||
clickhouse_target_link_split_lib(clickhouse disks)
|
||||
endif ()
|
||||
|
||||
set (CLICKHOUSE_BUNDLE)
|
||||
if (ENABLE_CLICKHOUSE_SELF_EXTRACTING)
|
||||
list(APPEND CLICKHOUSE_BUNDLE self-extracting)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_SERVER)
|
||||
add_custom_target (clickhouse-server ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-server DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-server" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-server)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_CLIENT)
|
||||
add_custom_target (clickhouse-client ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-client DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-client" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-client)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_LOCAL)
|
||||
add_custom_target (clickhouse-local ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-local DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-local" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-local)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_BENCHMARK)
|
||||
add_custom_target (clickhouse-benchmark ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-benchmark DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-benchmark" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-benchmark)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_COPIER)
|
||||
add_custom_target (clickhouse-copier ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-copier DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-copier" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-copier)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG)
|
||||
add_custom_target (clickhouse-extract-from-config ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-extract-from-config DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-extract-from-config" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-extract-from-config)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_COMPRESSOR)
|
||||
add_custom_target (clickhouse-compressor ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-compressor DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-compressor" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-compressor)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_FORMAT)
|
||||
add_custom_target (clickhouse-format ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-format DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-format" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-format)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_OBFUSCATOR)
|
||||
add_custom_target (clickhouse-obfuscator ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-obfuscator DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-obfuscator" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-obfuscator)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_GIT_IMPORT)
|
||||
add_custom_target (clickhouse-git-import ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-git-import DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-git-import" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-git-import)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER)
|
||||
add_custom_target (clickhouse-static-files-disk-uploader ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-static-files-disk-uploader DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-static-files-disk-uploader" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-static-files-disk-uploader)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_SU)
|
||||
add_custom_target (clickhouse-su ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-su DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-su" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-su)
|
||||
endif ()
|
||||
|
||||
if (ENABLE_CLICKHOUSE_KEEPER)
|
||||
if (NOT BUILD_STANDALONE_KEEPER AND CREATE_KEEPER_SYMLINK)
|
||||
add_custom_target (clickhouse-keeper ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-keeper DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
endif()
|
||||
|
||||
# otherwise we don't build keeper
|
||||
if (BUILD_STANDALONE_KEEPER OR CREATE_KEEPER_SYMLINK)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-keeper)
|
||||
endif()
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
|
||||
add_custom_target (clickhouse-keeper-converter ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-keeper-converter DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper-converter" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-keeper-converter)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_DISKS)
|
||||
add_custom_target (clickhouse-disks ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-disks DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-disks" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-disks)
|
||||
endif ()
|
||||
|
||||
add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_BUNDLE})
|
||||
|
||||
if (USE_GDB_ADD_INDEX)
|
||||
add_custom_command(TARGET clickhouse POST_BUILD COMMAND ${GDB_ADD_INDEX_EXE} clickhouse COMMENT "Adding .gdb-index to clickhouse" VERBATIM)
|
||||
if (ENABLE_CLICKHOUSE_KEEPER)
|
||||
if (NOT BUILD_STANDALONE_KEEPER AND CREATE_KEEPER_SYMLINK)
|
||||
add_custom_target (clickhouse-keeper ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-keeper DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
endif()
|
||||
|
||||
if (USE_BINARY_HASH)
|
||||
add_custom_command(TARGET clickhouse POST_BUILD COMMAND ./clickhouse hash-binary > hash && ${OBJCOPY_PATH} --add-section .clickhouse.hash=hash clickhouse COMMENT "Adding section '.clickhouse.hash' to clickhouse binary" VERBATIM)
|
||||
# otherwise we don't build keeper
|
||||
if (BUILD_STANDALONE_KEEPER OR CREATE_KEEPER_SYMLINK)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-keeper)
|
||||
endif()
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
|
||||
add_custom_target (clickhouse-keeper-converter ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-keeper-converter DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper-converter" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-keeper-converter)
|
||||
endif ()
|
||||
if (ENABLE_CLICKHOUSE_DISKS)
|
||||
add_custom_target (clickhouse-disks ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-disks DEPENDS clickhouse)
|
||||
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-disks" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
list(APPEND CLICKHOUSE_BUNDLE clickhouse-disks)
|
||||
endif ()
|
||||
|
||||
if (SPLIT_DEBUG_SYMBOLS)
|
||||
clickhouse_split_debug_symbols(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${SPLITTED_DEBUG_SYMBOLS_DIR} BINARY_PATH clickhouse)
|
||||
else()
|
||||
clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${SPLITTED_DEBUG_SYMBOLS_DIR})
|
||||
install (TARGETS clickhouse RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
endif()
|
||||
add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_BUNDLE})
|
||||
|
||||
if (USE_GDB_ADD_INDEX)
|
||||
add_custom_command(TARGET clickhouse POST_BUILD COMMAND ${GDB_ADD_INDEX_EXE} clickhouse COMMENT "Adding .gdb-index to clickhouse" VERBATIM)
|
||||
endif()
|
||||
|
||||
if (USE_BINARY_HASH)
|
||||
add_custom_command(TARGET clickhouse POST_BUILD COMMAND ./clickhouse hash-binary > hash && ${OBJCOPY_PATH} --add-section .clickhouse.hash=hash clickhouse COMMENT "Adding section '.clickhouse.hash' to clickhouse binary" VERBATIM)
|
||||
endif()
|
||||
|
||||
if (SPLIT_DEBUG_SYMBOLS)
|
||||
clickhouse_split_debug_symbols(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${SPLITTED_DEBUG_SYMBOLS_DIR} BINARY_PATH clickhouse)
|
||||
else()
|
||||
clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${SPLITTED_DEBUG_SYMBOLS_DIR})
|
||||
install (TARGETS clickhouse RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
|
||||
endif()
|
||||
|
||||
if (ENABLE_TESTS)
|
||||
|
||||
@ -3,6 +3,7 @@ package database
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/ClickHouse/ClickHouse/programs/diagnostics/internal/platform/data"
|
||||
@ -17,7 +18,7 @@ type ClickhouseNativeClient struct {
|
||||
|
||||
func NewNativeClient(host string, port uint16, username string, password string) (*ClickhouseNativeClient, error) {
|
||||
// debug output ?debug=true
|
||||
connection, err := sql.Open("clickhouse", fmt.Sprintf("clickhouse://%s:%s@%s:%d/", username, password, host, port))
|
||||
connection, err := sql.Open("clickhouse", fmt.Sprintf("clickhouse://%s:%s@%s:%d/", url.QueryEscape(username), url.QueryEscape(password), host, port))
|
||||
if err != nil {
|
||||
return &ClickhouseNativeClient{}, err
|
||||
}
|
||||
|
||||
@ -323,12 +323,28 @@ void LocalServer::setupUsers()
|
||||
auto & access_control = global_context->getAccessControl();
|
||||
access_control.setNoPasswordAllowed(config().getBool("allow_no_password", true));
|
||||
access_control.setPlaintextPasswordAllowed(config().getBool("allow_plaintext_password", true));
|
||||
if (config().has("users_config") || config().has("config-file") || fs::exists("config.xml"))
|
||||
if (config().has("config-file") || fs::exists("config.xml"))
|
||||
{
|
||||
const auto users_config_path = config().getString("users_config", config().getString("config-file", "config.xml"));
|
||||
ConfigProcessor config_processor(users_config_path);
|
||||
const auto loaded_config = config_processor.loadConfig();
|
||||
users_config = loaded_config.configuration;
|
||||
String config_path = config().getString("config-file", "");
|
||||
bool has_user_directories = config().has("user_directories");
|
||||
const auto config_dir = fs::path{config_path}.remove_filename().string();
|
||||
String users_config_path = config().getString("users_config", "");
|
||||
|
||||
if (users_config_path.empty() && has_user_directories)
|
||||
{
|
||||
users_config_path = config().getString("user_directories.users_xml.path");
|
||||
if (fs::path(users_config_path).is_relative() && fs::exists(fs::path(config_dir) / users_config_path))
|
||||
users_config_path = fs::path(config_dir) / users_config_path;
|
||||
}
|
||||
|
||||
if (users_config_path.empty())
|
||||
users_config = getConfigurationFromXMLString(minimal_default_user_xml);
|
||||
else
|
||||
{
|
||||
ConfigProcessor config_processor(users_config_path);
|
||||
const auto loaded_config = config_processor.loadConfig();
|
||||
users_config = loaded_config.configuration;
|
||||
}
|
||||
}
|
||||
else
|
||||
users_config = getConfigurationFromXMLString(minimal_default_user_xml);
|
||||
@ -338,7 +354,6 @@ void LocalServer::setupUsers()
|
||||
throw Exception("Can't load config for users", ErrorCodes::CANNOT_LOAD_CONFIG);
|
||||
}
|
||||
|
||||
|
||||
void LocalServer::connect()
|
||||
{
|
||||
connection_parameters = ConnectionParameters(config());
|
||||
@ -352,6 +367,9 @@ try
|
||||
{
|
||||
UseSSL use_ssl;
|
||||
ThreadStatus thread_status;
|
||||
|
||||
StackTrace::setShowAddresses(config().getBool("show_addresses_in_stack_traces", true));
|
||||
|
||||
setupSignalHandler();
|
||||
|
||||
std::cout << std::fixed << std::setprecision(3);
|
||||
|
||||
@ -345,6 +345,7 @@ struct Checker
|
||||
;
|
||||
|
||||
|
||||
#ifndef DISABLE_HARMFUL_ENV_VAR_CHECK
|
||||
/// NOTE: We will migrate to full static linking or our own dynamic loader to make this code obsolete.
|
||||
void checkHarmfulEnvironmentVariables(char ** argv)
|
||||
{
|
||||
@ -396,6 +397,7 @@ void checkHarmfulEnvironmentVariables(char ** argv)
|
||||
_exit(error);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
}
|
||||
|
||||
@ -422,7 +424,9 @@ int main(int argc_, char ** argv_)
|
||||
/// will work only after additional call of this function.
|
||||
updatePHDRCache();
|
||||
|
||||
#ifndef DISABLE_HARMFUL_ENV_VAR_CHECK
|
||||
checkHarmfulEnvironmentVariables(argv_);
|
||||
#endif
|
||||
|
||||
/// Reset new handler to default (that throws std::bad_alloc)
|
||||
/// It is needed because LLVM library clobbers it.
|
||||
|
||||
@ -46,6 +46,7 @@
|
||||
#include <boost/algorithm/string.hpp>
|
||||
#include <boost/container/flat_map.hpp>
|
||||
#include <Common/TerminalSize.h>
|
||||
#include <bit>
|
||||
|
||||
|
||||
static const char * documentation = R"(
|
||||
@ -186,7 +187,7 @@ static UInt64 transform(UInt64 x, UInt64 seed)
|
||||
if (x == 2 || x == 3)
|
||||
return x ^ (seed & 1);
|
||||
|
||||
size_t num_leading_zeros = __builtin_clzll(x);
|
||||
size_t num_leading_zeros = std::countl_zero(x);
|
||||
|
||||
return feistelNetwork(x, 64 - num_leading_zeros - 1, seed);
|
||||
}
|
||||
|
||||
@ -367,7 +367,7 @@ Poco::Net::SocketAddress Server::socketBindListen(
|
||||
return address;
|
||||
}
|
||||
|
||||
std::vector<std::string> getListenHosts(const Poco::Util::AbstractConfiguration & config)
|
||||
Strings getListenHosts(const Poco::Util::AbstractConfiguration & config)
|
||||
{
|
||||
auto listen_hosts = DB::getMultipleValuesFromConfig(config, "", "listen_host");
|
||||
if (listen_hosts.empty())
|
||||
@ -378,6 +378,16 @@ std::vector<std::string> getListenHosts(const Poco::Util::AbstractConfiguration
|
||||
return listen_hosts;
|
||||
}
|
||||
|
||||
Strings getInterserverListenHosts(const Poco::Util::AbstractConfiguration & config)
|
||||
{
|
||||
auto interserver_listen_hosts = DB::getMultipleValuesFromConfig(config, "", "interserver_listen_host");
|
||||
if (!interserver_listen_hosts.empty())
|
||||
return interserver_listen_hosts;
|
||||
|
||||
/// Use more general restriction in case of emptiness
|
||||
return getListenHosts(config);
|
||||
}
|
||||
|
||||
bool getListenTry(const Poco::Util::AbstractConfiguration & config)
|
||||
{
|
||||
bool listen_try = config.getBool("listen_try", false);
|
||||
@ -627,6 +637,8 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
|
||||
MainThreadStatus::getInstance();
|
||||
|
||||
StackTrace::setShowAddresses(config().getBool("show_addresses_in_stack_traces", true));
|
||||
|
||||
registerFunctions();
|
||||
registerAggregateFunctions();
|
||||
registerTableFunctions();
|
||||
@ -1232,6 +1244,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
/* already_loaded = */ false); /// Reload it right now (initial loading)
|
||||
|
||||
const auto listen_hosts = getListenHosts(config());
|
||||
const auto interserver_listen_hosts = getInterserverListenHosts(config());
|
||||
const auto listen_try = getListenTry(config());
|
||||
|
||||
if (config().has("keeper_server"))
|
||||
@ -1627,7 +1640,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
|
||||
{
|
||||
std::lock_guard lock(servers_lock);
|
||||
createServers(config(), listen_hosts, listen_try, server_pool, async_metrics, servers);
|
||||
createServers(config(), listen_hosts, interserver_listen_hosts, listen_try, server_pool, async_metrics, servers);
|
||||
if (servers.empty())
|
||||
throw Exception(
|
||||
"No servers started (add valid listen_host and 'tcp_port' or 'http_port' to configuration file.)",
|
||||
@ -1809,7 +1822,8 @@ int Server::main(const std::vector<std::string> & /*args*/)
|
||||
|
||||
void Server::createServers(
|
||||
Poco::Util::AbstractConfiguration & config,
|
||||
const std::vector<std::string> & listen_hosts,
|
||||
const Strings & listen_hosts,
|
||||
const Strings & interserver_listen_hosts,
|
||||
bool listen_try,
|
||||
Poco::ThreadPool & server_pool,
|
||||
AsynchronousMetrics & async_metrics,
|
||||
@ -1927,51 +1941,6 @@ void Server::createServers(
|
||||
#endif
|
||||
});
|
||||
|
||||
/// Interserver IO HTTP
|
||||
port_name = "interserver_http_port";
|
||||
createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter
|
||||
{
|
||||
Poco::Net::ServerSocket socket;
|
||||
auto address = socketBindListen(config, socket, listen_host, port);
|
||||
socket.setReceiveTimeout(settings.http_receive_timeout);
|
||||
socket.setSendTimeout(settings.http_send_timeout);
|
||||
return ProtocolServerAdapter(
|
||||
listen_host,
|
||||
port_name,
|
||||
"replica communication (interserver): http://" + address.toString(),
|
||||
std::make_unique<HTTPServer>(
|
||||
context(),
|
||||
createHandlerFactory(*this, async_metrics, "InterserverIOHTTPHandler-factory"),
|
||||
server_pool,
|
||||
socket,
|
||||
http_params));
|
||||
});
|
||||
|
||||
port_name = "interserver_https_port";
|
||||
createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter
|
||||
{
|
||||
#if USE_SSL
|
||||
Poco::Net::SecureServerSocket socket;
|
||||
auto address = socketBindListen(config, socket, listen_host, port, /* secure = */ true);
|
||||
socket.setReceiveTimeout(settings.http_receive_timeout);
|
||||
socket.setSendTimeout(settings.http_send_timeout);
|
||||
return ProtocolServerAdapter(
|
||||
listen_host,
|
||||
port_name,
|
||||
"secure replica communication (interserver): https://" + address.toString(),
|
||||
std::make_unique<HTTPServer>(
|
||||
context(),
|
||||
createHandlerFactory(*this, async_metrics, "InterserverIOHTTPSHandler-factory"),
|
||||
server_pool,
|
||||
socket,
|
||||
http_params));
|
||||
#else
|
||||
UNUSED(port);
|
||||
throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.",
|
||||
ErrorCodes::SUPPORT_IS_DISABLED};
|
||||
#endif
|
||||
});
|
||||
|
||||
port_name = "mysql_port";
|
||||
createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter
|
||||
{
|
||||
@ -2030,6 +1999,55 @@ void Server::createServers(
|
||||
});
|
||||
}
|
||||
|
||||
/// Now iterate over interserver_listen_hosts
|
||||
for (const auto & interserver_listen_host : interserver_listen_hosts)
|
||||
{
|
||||
/// Interserver IO HTTP
|
||||
const char * port_name = "interserver_http_port";
|
||||
createServer(config, interserver_listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter
|
||||
{
|
||||
Poco::Net::ServerSocket socket;
|
||||
auto address = socketBindListen(config, socket, interserver_listen_host, port);
|
||||
socket.setReceiveTimeout(settings.http_receive_timeout);
|
||||
socket.setSendTimeout(settings.http_send_timeout);
|
||||
return ProtocolServerAdapter(
|
||||
interserver_listen_host,
|
||||
port_name,
|
||||
"replica communication (interserver): http://" + address.toString(),
|
||||
std::make_unique<HTTPServer>(
|
||||
context(),
|
||||
createHandlerFactory(*this, async_metrics, "InterserverIOHTTPHandler-factory"),
|
||||
server_pool,
|
||||
socket,
|
||||
http_params));
|
||||
});
|
||||
|
||||
port_name = "interserver_https_port";
|
||||
createServer(config, interserver_listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter
|
||||
{
|
||||
#if USE_SSL
|
||||
Poco::Net::SecureServerSocket socket;
|
||||
auto address = socketBindListen(config, socket, interserver_listen_host, port, /* secure = */ true);
|
||||
socket.setReceiveTimeout(settings.http_receive_timeout);
|
||||
socket.setSendTimeout(settings.http_send_timeout);
|
||||
return ProtocolServerAdapter(
|
||||
interserver_listen_host,
|
||||
port_name,
|
||||
"secure replica communication (interserver): https://" + address.toString(),
|
||||
std::make_unique<HTTPServer>(
|
||||
context(),
|
||||
createHandlerFactory(*this, async_metrics, "InterserverIOHTTPSHandler-factory"),
|
||||
server_pool,
|
||||
socket,
|
||||
http_params));
|
||||
#else
|
||||
UNUSED(port);
|
||||
throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.",
|
||||
ErrorCodes::SUPPORT_IS_DISABLED};
|
||||
#endif
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void Server::updateServers(
|
||||
@ -2041,6 +2059,7 @@ void Server::updateServers(
|
||||
Poco::Logger * log = &logger();
|
||||
|
||||
const auto listen_hosts = getListenHosts(config);
|
||||
const auto interserver_listen_hosts = getInterserverListenHosts(config);
|
||||
const auto listen_try = getListenTry(config);
|
||||
|
||||
/// Remove servers once all their connections are closed
|
||||
@ -2073,7 +2092,7 @@ void Server::updateServers(
|
||||
}
|
||||
}
|
||||
|
||||
createServers(config, listen_hosts, listen_try, server_pool, async_metrics, servers, /* start_servers= */ true);
|
||||
createServers(config, listen_hosts, interserver_listen_hosts, listen_try, server_pool, async_metrics, servers, /* start_servers= */ true);
|
||||
|
||||
std::erase_if(servers, std::bind_front(check_server, ""));
|
||||
}
|
||||
|
||||
@ -86,7 +86,8 @@ private:
|
||||
|
||||
void createServers(
|
||||
Poco::Util::AbstractConfiguration & config,
|
||||
const std::vector<std::string> & listen_hosts,
|
||||
const Strings & listen_hosts,
|
||||
const Strings & interserver_listen_hosts,
|
||||
bool listen_try,
|
||||
Poco::ThreadPool & server_pool,
|
||||
AsynchronousMetrics & async_metrics,
|
||||
|
||||
@ -188,6 +188,10 @@
|
||||
<listen_host>127.0.0.1</listen_host>
|
||||
-->
|
||||
|
||||
<!-- <interserver_listen_host>::</interserver_listen_host> -->
|
||||
<!-- Listen host for communication between replicas. Used for data exchange -->
|
||||
<!-- Default values - equal to listen_host -->
|
||||
|
||||
<!-- Don't exit if IPv6 or IPv4 networks are unavailable while trying to listen. -->
|
||||
<!-- <listen_try>0</listen_try> -->
|
||||
|
||||
@ -1386,4 +1390,13 @@
|
||||
<lru_cache_size>268435456</lru_cache_size>
|
||||
<continue_if_corrupted>true</continue_if_corrupted>
|
||||
</merge_tree_metadata_cache-->
|
||||
|
||||
<!-- This allows to disable exposing addresses in stack traces for security reasons.
|
||||
Please be aware that it does not improve security much, but makes debugging much harder.
|
||||
The addresses that are small offsets from zero will be displayed nevertheless to show nullptr dereferences.
|
||||
Regardless of this configuration, the addresses are visible in the system.stack_trace and system.trace_log tables
|
||||
if the user has access to these tables.
|
||||
I don't recommend to change this setting.
|
||||
-->
|
||||
<show_addresses_in_stack_traces>false</show_addresses_in_stack_traces>
|
||||
</clickhouse>
|
||||
|
||||
@ -49,6 +49,16 @@ public:
|
||||
return nested_func->getReturnType();
|
||||
}
|
||||
|
||||
const IAggregateFunction & getBaseAggregateFunctionWithSameStateRepresentation() const override
|
||||
{
|
||||
return nested_func->getBaseAggregateFunctionWithSameStateRepresentation();
|
||||
}
|
||||
|
||||
DataTypePtr getNormalizedStateType() const override
|
||||
{
|
||||
return nested_func->getNormalizedStateType();
|
||||
}
|
||||
|
||||
bool isVersioned() const override
|
||||
{
|
||||
return nested_func->isVersioned();
|
||||
|
||||
@ -5,9 +5,11 @@
|
||||
|
||||
#include <array>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypeAggregateFunction.h>
|
||||
#include <Columns/ColumnNullable.h>
|
||||
#include <Columns/ColumnsCommon.h>
|
||||
#include <AggregateFunctions/IAggregateFunction.h>
|
||||
#include <AggregateFunctions/AggregateFunctionFactory.h>
|
||||
#include <Common/assert_cast.h>
|
||||
|
||||
#include <Common/config.h>
|
||||
@ -102,6 +104,19 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
bool haveSameStateRepresentationImpl(const IAggregateFunction & rhs) const override
|
||||
{
|
||||
return this->getName() == rhs.getName();
|
||||
}
|
||||
|
||||
DataTypePtr getNormalizedStateType() const override
|
||||
{
|
||||
/// Return normalized state type: count()
|
||||
AggregateFunctionProperties properties;
|
||||
return std::make_shared<DataTypeAggregateFunction>(
|
||||
AggregateFunctionFactory::instance().get(getName(), {}, {}, properties), DataTypes{}, Array{});
|
||||
}
|
||||
|
||||
void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override
|
||||
{
|
||||
data(place).count += data(rhs).count;
|
||||
@ -240,6 +255,19 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
bool haveSameStateRepresentationImpl(const IAggregateFunction & rhs) const override
|
||||
{
|
||||
return this->getName() == rhs.getName();
|
||||
}
|
||||
|
||||
DataTypePtr getNormalizedStateType() const override
|
||||
{
|
||||
/// Return normalized state type: count()
|
||||
AggregateFunctionProperties properties;
|
||||
return std::make_shared<DataTypeAggregateFunction>(
|
||||
AggregateFunctionFactory::instance().get(getName(), {}, {}, properties), DataTypes{}, Array{});
|
||||
}
|
||||
|
||||
void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override
|
||||
{
|
||||
data(place).count += data(rhs).count;
|
||||
|
||||
@ -152,8 +152,8 @@ template <typename Data>
|
||||
class AggregateFunctionDistinct : public IAggregateFunctionDataHelper<Data, AggregateFunctionDistinct<Data>>
|
||||
{
|
||||
private:
|
||||
static constexpr auto prefix_size = sizeof(Data);
|
||||
AggregateFunctionPtr nested_func;
|
||||
size_t prefix_size;
|
||||
size_t arguments_num;
|
||||
|
||||
AggregateDataPtr getNestedPlace(AggregateDataPtr __restrict place) const noexcept
|
||||
@ -170,7 +170,11 @@ public:
|
||||
AggregateFunctionDistinct(AggregateFunctionPtr nested_func_, const DataTypes & arguments, const Array & params_)
|
||||
: IAggregateFunctionDataHelper<Data, AggregateFunctionDistinct>(arguments, params_)
|
||||
, nested_func(nested_func_)
|
||||
, arguments_num(arguments.size()) {}
|
||||
, arguments_num(arguments.size())
|
||||
{
|
||||
size_t nested_size = nested_func->alignOfData();
|
||||
prefix_size = (sizeof(Data) + nested_size - 1) / nested_size * nested_size;
|
||||
}
|
||||
|
||||
void add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const override
|
||||
{
|
||||
|
||||
@ -56,6 +56,16 @@ public:
|
||||
return nested_func->getReturnType();
|
||||
}
|
||||
|
||||
const IAggregateFunction & getBaseAggregateFunctionWithSameStateRepresentation() const override
|
||||
{
|
||||
return nested_func->getBaseAggregateFunctionWithSameStateRepresentation();
|
||||
}
|
||||
|
||||
DataTypePtr getNormalizedStateType() const override
|
||||
{
|
||||
return nested_func->getNormalizedStateType();
|
||||
}
|
||||
|
||||
bool isVersioned() const override
|
||||
{
|
||||
return nested_func->isVersioned();
|
||||
|
||||
@ -23,14 +23,20 @@ public:
|
||||
DataTypes transformArguments(const DataTypes & arguments) const override
|
||||
{
|
||||
if (arguments.size() != 1)
|
||||
throw Exception("Incorrect number of arguments for aggregate function with " + getName() + " suffix", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
throw Exception(
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
|
||||
"Incorrect number of arguments for aggregate function with {} suffix",
|
||||
getName());
|
||||
|
||||
const DataTypePtr & argument = arguments[0];
|
||||
|
||||
const DataTypeAggregateFunction * function = typeid_cast<const DataTypeAggregateFunction *>(argument.get());
|
||||
if (!function)
|
||||
throw Exception("Illegal type " + argument->getName() + " of argument for aggregate function with " + getName() + " suffix"
|
||||
+ " must be AggregateFunction(...)", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument for aggregate function with {} suffix. It must be AggregateFunction(...)",
|
||||
argument->getName(),
|
||||
getName());
|
||||
|
||||
return function->getArgumentsDataTypes();
|
||||
}
|
||||
@ -45,13 +51,21 @@ public:
|
||||
|
||||
const DataTypeAggregateFunction * function = typeid_cast<const DataTypeAggregateFunction *>(argument.get());
|
||||
if (!function)
|
||||
throw Exception("Illegal type " + argument->getName() + " of argument for aggregate function with " + getName() + " suffix"
|
||||
+ " must be AggregateFunction(...)", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument for aggregate function with {} suffix. It must be AggregateFunction(...)",
|
||||
argument->getName(),
|
||||
getName());
|
||||
|
||||
if (nested_function->getName() != function->getFunctionName())
|
||||
throw Exception("Illegal type " + argument->getName() + " of argument for aggregate function with " + getName() + " suffix"
|
||||
+ ", because it corresponds to different aggregate function: " + function->getFunctionName() + " instead of " + nested_function->getName(),
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
if (!nested_function->haveSameStateRepresentation(*function->getFunction()))
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument for aggregate function with {} suffix. because it corresponds to different aggregate "
|
||||
"function: {} instead of {}",
|
||||
argument->getName(),
|
||||
getName(),
|
||||
function->getFunctionName(),
|
||||
nested_function->getName());
|
||||
|
||||
return std::make_shared<AggregateFunctionMerge>(nested_function, argument, params);
|
||||
}
|
||||
|
||||
@ -50,6 +50,11 @@ public:
|
||||
return nested_func->getReturnType();
|
||||
}
|
||||
|
||||
const IAggregateFunction & getBaseAggregateFunctionWithSameStateRepresentation() const override
|
||||
{
|
||||
return nested_func->getBaseAggregateFunctionWithSameStateRepresentation();
|
||||
}
|
||||
|
||||
bool isVersioned() const override
|
||||
{
|
||||
return nested_func->isVersioned();
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <AggregateFunctions/FactoryHelpers.h>
|
||||
#include <AggregateFunctions/AggregateFunctionFactory.h>
|
||||
|
||||
/// These must be exposed in header for the purpose of dynamic compilation.
|
||||
#include <AggregateFunctions/QuantileReservoirSampler.h>
|
||||
@ -20,9 +21,11 @@
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypeAggregateFunction.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Common/assert_cast.h>
|
||||
#include <Interpreters/GatherFunctionQuantileVisitor.h>
|
||||
|
||||
#include <type_traits>
|
||||
|
||||
@ -61,10 +64,9 @@ template <
|
||||
typename FloatReturnType,
|
||||
/// If true, the function will accept multiple parameters with quantile levels
|
||||
/// and return an Array filled with many values of that quantiles.
|
||||
bool returns_many
|
||||
>
|
||||
class AggregateFunctionQuantile final : public IAggregateFunctionDataHelper<Data,
|
||||
AggregateFunctionQuantile<Value, Data, Name, has_second_arg, FloatReturnType, returns_many>>
|
||||
bool returns_many>
|
||||
class AggregateFunctionQuantile final
|
||||
: public IAggregateFunctionDataHelper<Data, AggregateFunctionQuantile<Value, Data, Name, has_second_arg, FloatReturnType, returns_many>>
|
||||
{
|
||||
private:
|
||||
using ColVecType = ColumnVectorOrDecimal<Value>;
|
||||
@ -81,11 +83,14 @@ private:
|
||||
|
||||
public:
|
||||
AggregateFunctionQuantile(const DataTypes & argument_types_, const Array & params)
|
||||
: IAggregateFunctionDataHelper<Data, AggregateFunctionQuantile<Value, Data, Name, has_second_arg, FloatReturnType, returns_many>>(argument_types_, params)
|
||||
, levels(params, returns_many), level(levels.levels[0]), argument_type(this->argument_types[0])
|
||||
: IAggregateFunctionDataHelper<Data, AggregateFunctionQuantile<Value, Data, Name, has_second_arg, FloatReturnType, returns_many>>(
|
||||
argument_types_, params)
|
||||
, levels(params, returns_many)
|
||||
, level(levels.levels[0])
|
||||
, argument_type(this->argument_types[0])
|
||||
{
|
||||
if (!returns_many && levels.size() > 1)
|
||||
throw Exception("Aggregate function " + getName() + " require one parameter or less", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Aggregate function {} require one parameter or less", getName());
|
||||
}
|
||||
|
||||
String getName() const override { return Name::name; }
|
||||
@ -105,9 +110,22 @@ public:
|
||||
return res;
|
||||
}
|
||||
|
||||
bool haveSameStateRepresentation(const IAggregateFunction & rhs) const override
|
||||
bool haveSameStateRepresentationImpl(const IAggregateFunction & rhs) const override
|
||||
{
|
||||
return getName() == rhs.getName() && this->haveEqualArgumentTypes(rhs);
|
||||
return GatherFunctionQuantileData::toFusedNameOrSelf(getName()) == GatherFunctionQuantileData::toFusedNameOrSelf(rhs.getName())
|
||||
&& this->haveEqualArgumentTypes(rhs);
|
||||
}
|
||||
|
||||
DataTypePtr getNormalizedStateType() const override
|
||||
{
|
||||
/// Return normalized state type: quantiles*(1)(...)
|
||||
Array params{1};
|
||||
AggregateFunctionProperties properties;
|
||||
return std::make_shared<DataTypeAggregateFunction>(
|
||||
AggregateFunctionFactory::instance().get(
|
||||
GatherFunctionQuantileData::toFusedNameOrSelf(getName()), this->argument_types, params, properties),
|
||||
this->argument_types,
|
||||
params);
|
||||
}
|
||||
|
||||
bool allocatesMemoryInArena() const override { return false; }
|
||||
@ -124,9 +142,7 @@ public:
|
||||
}
|
||||
|
||||
if constexpr (has_second_arg)
|
||||
this->data(place).add(
|
||||
value,
|
||||
columns[1]->getUInt(row_num));
|
||||
this->data(place).add(value, columns[1]->getUInt(row_num));
|
||||
else
|
||||
this->data(place).add(value);
|
||||
}
|
||||
@ -149,7 +165,6 @@ public:
|
||||
|
||||
void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override
|
||||
{
|
||||
/// const_cast is required because some data structures apply finalizaton (like sorting) for obtain a result.
|
||||
auto & data = this->data(place);
|
||||
|
||||
if constexpr (returns_many)
|
||||
@ -195,7 +210,11 @@ public:
|
||||
{
|
||||
assertBinary(Name::name, types);
|
||||
if (!isUnsignedInteger(types[1]))
|
||||
throw Exception("Second argument (weight) for function " + std::string(Name::name) + " must be unsigned integer, but it has type " + types[1]->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Second argument (weight) for function {} must be unsigned integer, but it has type {}",
|
||||
Name::name,
|
||||
types[1]->getName());
|
||||
}
|
||||
else
|
||||
assertUnary(Name::name, types);
|
||||
|
||||
@ -163,7 +163,7 @@ public:
|
||||
this->data(place).deserialize(buf);
|
||||
}
|
||||
|
||||
bool haveSameStateRepresentation(const IAggregateFunction & rhs) const override
|
||||
bool haveSameStateRepresentationImpl(const IAggregateFunction & rhs) const override
|
||||
{
|
||||
return this->getName() == rhs.getName() && this->haveEqualArgumentTypes(rhs);
|
||||
}
|
||||
|
||||
@ -194,7 +194,7 @@ public:
|
||||
|
||||
DataTypePtr getReturnType() const override { return data_type; }
|
||||
|
||||
bool haveSameStateRepresentation(const IAggregateFunction & rhs) const override
|
||||
bool haveSameStateRepresentationImpl(const IAggregateFunction & rhs) const override
|
||||
{
|
||||
return this->getName() == rhs.getName() && this->haveEqualArgumentTypes(rhs);
|
||||
}
|
||||
|
||||
@ -37,6 +37,11 @@ public:
|
||||
return getStateType();
|
||||
}
|
||||
|
||||
const IAggregateFunction & getBaseAggregateFunctionWithSameStateRepresentation() const override
|
||||
{
|
||||
return nested_func->getBaseAggregateFunctionWithSameStateRepresentation();
|
||||
}
|
||||
|
||||
DataTypePtr getStateType() const override
|
||||
{
|
||||
return nested_func->getStateType();
|
||||
|
||||
@ -59,6 +59,13 @@ bool IAggregateFunction::haveEqualArgumentTypes(const IAggregateFunction & rhs)
|
||||
}
|
||||
|
||||
bool IAggregateFunction::haveSameStateRepresentation(const IAggregateFunction & rhs) const
|
||||
{
|
||||
const auto & lhs_base = getBaseAggregateFunctionWithSameStateRepresentation();
|
||||
const auto & rhs_base = rhs.getBaseAggregateFunctionWithSameStateRepresentation();
|
||||
return lhs_base.haveSameStateRepresentationImpl(rhs_base);
|
||||
}
|
||||
|
||||
bool IAggregateFunction::haveSameStateRepresentationImpl(const IAggregateFunction & rhs) const
|
||||
{
|
||||
bool res = getName() == rhs.getName()
|
||||
&& parameters == rhs.parameters
|
||||
|
||||
@ -73,13 +73,19 @@ public:
|
||||
/// Get the data type of internal state. By default it is AggregateFunction(name(params), argument_types...).
|
||||
virtual DataTypePtr getStateType() const;
|
||||
|
||||
/// Same as the above but normalize state types so that variants with the same binary representation will use the same type.
|
||||
virtual DataTypePtr getNormalizedStateType() const { return getStateType(); }
|
||||
|
||||
/// Returns true if two aggregate functions have the same state representation in memory and the same serialization,
|
||||
/// so state of one aggregate function can be safely used with another.
|
||||
/// Examples:
|
||||
/// - quantile(x), quantile(a)(x), quantile(b)(x) - parameter doesn't affect state and used for finalization only
|
||||
/// - foo(x) and fooIf(x) - If combinator doesn't affect state
|
||||
/// By default returns true only if functions have exactly the same names, combinators and parameters.
|
||||
virtual bool haveSameStateRepresentation(const IAggregateFunction & rhs) const;
|
||||
bool haveSameStateRepresentation(const IAggregateFunction & rhs) const;
|
||||
virtual bool haveSameStateRepresentationImpl(const IAggregateFunction & rhs) const;
|
||||
|
||||
virtual const IAggregateFunction & getBaseAggregateFunctionWithSameStateRepresentation() const { return *this; }
|
||||
|
||||
bool haveEqualArgumentTypes(const IAggregateFunction & rhs) const;
|
||||
|
||||
|
||||
@ -144,8 +144,8 @@ endif ()
|
||||
list (APPEND clickhouse_common_io_sources ${CONFIG_BUILD})
|
||||
list (APPEND clickhouse_common_io_headers ${CONFIG_VERSION} ${CONFIG_COMMON})
|
||||
|
||||
list (APPEND dbms_sources Functions/IFunction.cpp Functions/FunctionFactory.cpp Functions/FunctionHelpers.cpp Functions/extractTimeZoneFromFunctionArguments.cpp Functions/FunctionsLogical.cpp)
|
||||
list (APPEND dbms_headers Functions/IFunction.h Functions/FunctionFactory.h Functions/FunctionHelpers.h Functions/extractTimeZoneFromFunctionArguments.h Functions/FunctionsLogical.h)
|
||||
list (APPEND dbms_sources Functions/IFunction.cpp Functions/FunctionFactory.cpp Functions/FunctionHelpers.cpp Functions/extractTimeZoneFromFunctionArguments.cpp Functions/FunctionsLogical.cpp Functions/indexHint.cpp)
|
||||
list (APPEND dbms_headers Functions/IFunction.h Functions/FunctionFactory.h Functions/FunctionHelpers.h Functions/extractTimeZoneFromFunctionArguments.h Functions/FunctionsLogical.h Functions/indexHint.h)
|
||||
|
||||
list (APPEND dbms_sources
|
||||
AggregateFunctions/IAggregateFunction.cpp
|
||||
@ -488,7 +488,7 @@ if (TARGET ch_contrib::aws_s3)
|
||||
endif()
|
||||
|
||||
if (TARGET ch_contrib::azure_sdk)
|
||||
target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::azure_sdk)
|
||||
dbms_target_link_libraries (PRIVATE ch_contrib::azure_sdk)
|
||||
endif()
|
||||
|
||||
if (TARGET ch_contrib::s2)
|
||||
@ -611,4 +611,3 @@ if (ENABLE_TESTS)
|
||||
|
||||
add_check(unit_tests_dbms)
|
||||
endif ()
|
||||
|
||||
|
||||
@ -583,9 +583,14 @@ try
|
||||
if (has_vertical_output_suffix)
|
||||
current_format = "Vertical";
|
||||
|
||||
/// It is not clear how to write progress intermixed with data with parallel formatting.
|
||||
bool logs_into_stdout = server_logs_file == "-";
|
||||
bool extras_into_stdout = need_render_progress || logs_into_stdout;
|
||||
bool select_only_into_file = select_into_file && !select_into_file_and_stdout;
|
||||
|
||||
/// It is not clear how to write progress and logs
|
||||
/// intermixed with data with parallel formatting.
|
||||
/// It may increase code complexity significantly.
|
||||
if (!need_render_progress || (select_into_file && !select_into_file_and_stdout))
|
||||
if (!extras_into_stdout || select_only_into_file)
|
||||
output_format = global_context->getOutputFormatParallelIfPossible(
|
||||
current_format, out_file_buf ? *out_file_buf : *out_buf, block);
|
||||
else
|
||||
|
||||
@ -263,11 +263,6 @@ public:
|
||||
}
|
||||
}
|
||||
|
||||
SerializationInfoPtr getSerializationInfo() const override
|
||||
{
|
||||
return data->getSerializationInfo();
|
||||
}
|
||||
|
||||
bool isNullable() const override { return isColumnNullable(*data); }
|
||||
bool onlyNull() const override { return data->isNullAt(0); }
|
||||
bool isNumeric() const override { return data->isNumeric(); }
|
||||
|
||||
@ -298,7 +298,7 @@ ColumnPtr ColumnDecimal<T>::filter(const IColumn::Filter & filt, ssize_t result_
|
||||
{
|
||||
while (mask)
|
||||
{
|
||||
size_t index = __builtin_ctzll(mask);
|
||||
size_t index = std::countr_zero(mask);
|
||||
res_data.push_back(data_pos[index]);
|
||||
#ifdef __BMI__
|
||||
mask = _blsr_u64(mask);
|
||||
|
||||
@ -240,7 +240,7 @@ ColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result
|
||||
size_t res_chars_size = res->chars.size();
|
||||
while (mask)
|
||||
{
|
||||
size_t index = __builtin_ctzll(mask);
|
||||
size_t index = std::countr_zero(mask);
|
||||
res->chars.resize(res_chars_size + n);
|
||||
memcpySmallAllowReadWriteOverflow15(&res->chars[res_chars_size], data_pos + index * n, n);
|
||||
res_chars_size += n;
|
||||
|
||||
@ -561,15 +561,4 @@ void ColumnTuple::getIndicesOfNonDefaultRows(Offsets & indices, size_t from, siz
|
||||
return getIndicesOfNonDefaultRowsImpl<ColumnTuple>(indices, from, limit);
|
||||
}
|
||||
|
||||
SerializationInfoPtr ColumnTuple::getSerializationInfo() const
|
||||
{
|
||||
MutableSerializationInfos infos;
|
||||
infos.reserve(columns.size());
|
||||
|
||||
for (const auto & column : columns)
|
||||
infos.push_back(const_pointer_cast<SerializationInfo>(column->getSerializationInfo()));
|
||||
|
||||
return std::make_shared<SerializationInfoTuple>(std::move(infos), SerializationInfo::Settings{});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -102,7 +102,6 @@ public:
|
||||
ColumnPtr compress() const override;
|
||||
double getRatioOfDefaultRows(double sample_ratio) const override;
|
||||
void getIndicesOfNonDefaultRows(Offsets & indices, size_t from, size_t limit) const override;
|
||||
SerializationInfoPtr getSerializationInfo() const override;
|
||||
|
||||
size_t tupleSize() const { return columns.size(); }
|
||||
|
||||
|
||||
@ -12,12 +12,14 @@
|
||||
#include <Common/RadixSort.h>
|
||||
#include <Common/SipHash.h>
|
||||
#include <Common/WeakHash.h>
|
||||
#include <Common/TargetSpecific.h>
|
||||
#include <Common/assert_cast.h>
|
||||
#include <base/sort.h>
|
||||
#include <base/unaligned.h>
|
||||
#include <base/bit_cast.h>
|
||||
#include <base/scope_guard.h>
|
||||
|
||||
#include <bit>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
|
||||
@ -25,6 +27,10 @@
|
||||
# include <emmintrin.h>
|
||||
#endif
|
||||
|
||||
#if USE_MULTITARGET_CODE
|
||||
# include <immintrin.h>
|
||||
#endif
|
||||
|
||||
#if USE_EMBEDDED_COMPILER
|
||||
#include <DataTypes/Native.h>
|
||||
#include <llvm/IR/IRBuilder.h>
|
||||
@ -471,6 +477,115 @@ void ColumnVector<T>::insertRangeFrom(const IColumn & src, size_t start, size_t
|
||||
memcpy(data.data() + old_size, &src_vec.data[start], length * sizeof(data[0]));
|
||||
}
|
||||
|
||||
static inline UInt64 blsr(UInt64 mask)
|
||||
{
|
||||
#ifdef __BMI__
|
||||
return _blsr_u64(mask);
|
||||
#else
|
||||
return mask & (mask-1);
|
||||
#endif
|
||||
}
|
||||
|
||||
DECLARE_DEFAULT_CODE(
|
||||
template <typename T, typename Container, size_t SIMD_BYTES>
|
||||
inline void doFilterAligned(const UInt8 *& filt_pos, const UInt8 *& filt_end_aligned, const T *& data_pos, Container & res_data)
|
||||
{
|
||||
while (filt_pos < filt_end_aligned)
|
||||
{
|
||||
UInt64 mask = bytes64MaskToBits64Mask(filt_pos);
|
||||
|
||||
if (0xffffffffffffffff == mask)
|
||||
{
|
||||
res_data.insert(data_pos, data_pos + SIMD_BYTES);
|
||||
}
|
||||
else
|
||||
{
|
||||
while (mask)
|
||||
{
|
||||
size_t index = std::countr_zero(mask);
|
||||
res_data.push_back(data_pos[index]);
|
||||
mask = blsr(mask);
|
||||
}
|
||||
}
|
||||
|
||||
filt_pos += SIMD_BYTES;
|
||||
data_pos += SIMD_BYTES;
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
DECLARE_AVX512VBMI2_SPECIFIC_CODE(
|
||||
template <size_t ELEMENT_WIDTH>
|
||||
inline void compressStoreAVX512(const void *src, void *dst, const UInt64 mask)
|
||||
{
|
||||
__m512i vsrc = _mm512_loadu_si512(src);
|
||||
if constexpr (ELEMENT_WIDTH == 1)
|
||||
_mm512_mask_compressstoreu_epi8(dst, static_cast<__mmask64>(mask), vsrc);
|
||||
else if constexpr (ELEMENT_WIDTH == 2)
|
||||
_mm512_mask_compressstoreu_epi16(dst, static_cast<__mmask32>(mask), vsrc);
|
||||
else if constexpr (ELEMENT_WIDTH == 4)
|
||||
_mm512_mask_compressstoreu_epi32(dst, static_cast<__mmask16>(mask), vsrc);
|
||||
else if constexpr (ELEMENT_WIDTH == 8)
|
||||
_mm512_mask_compressstoreu_epi64(dst, static_cast<__mmask8>(mask), vsrc);
|
||||
}
|
||||
|
||||
template <typename T, typename Container, size_t SIMD_BYTES>
|
||||
inline void doFilterAligned(const UInt8 *& filt_pos, const UInt8 *& filt_end_aligned, const T *& data_pos, Container & res_data)
|
||||
{
|
||||
static constexpr size_t VEC_LEN = 64; /// AVX512 vector length - 64 bytes
|
||||
static constexpr size_t ELEMENT_WIDTH = sizeof(T);
|
||||
static constexpr size_t ELEMENTS_PER_VEC = VEC_LEN / ELEMENT_WIDTH;
|
||||
static constexpr UInt64 KMASK = 0xffffffffffffffff >> (64 - ELEMENTS_PER_VEC);
|
||||
|
||||
size_t current_offset = res_data.size();
|
||||
size_t reserve_size = res_data.size();
|
||||
size_t alloc_size = SIMD_BYTES * 2;
|
||||
|
||||
while (filt_pos < filt_end_aligned)
|
||||
{
|
||||
/// to avoid calling resize too frequently, resize to reserve buffer.
|
||||
if (reserve_size - current_offset < SIMD_BYTES)
|
||||
{
|
||||
reserve_size += alloc_size;
|
||||
res_data.resize(reserve_size);
|
||||
alloc_size *= 2;
|
||||
}
|
||||
|
||||
UInt64 mask = bytes64MaskToBits64Mask(filt_pos);
|
||||
|
||||
if (0xffffffffffffffff == mask)
|
||||
{
|
||||
for (size_t i = 0; i < SIMD_BYTES; i += ELEMENTS_PER_VEC)
|
||||
_mm512_storeu_si512(reinterpret_cast<void *>(&res_data[current_offset + i]),
|
||||
_mm512_loadu_si512(reinterpret_cast<const void *>(data_pos + i)));
|
||||
current_offset += SIMD_BYTES;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (mask)
|
||||
{
|
||||
for (size_t i = 0; i < SIMD_BYTES; i += ELEMENTS_PER_VEC)
|
||||
{
|
||||
compressStoreAVX512<ELEMENT_WIDTH>(reinterpret_cast<const void *>(data_pos + i),
|
||||
reinterpret_cast<void *>(&res_data[current_offset]), mask & KMASK);
|
||||
current_offset += std::popcount(mask & KMASK);
|
||||
/// prepare mask for next iter, if ELEMENTS_PER_VEC = 64, no next iter
|
||||
if (ELEMENTS_PER_VEC < 64)
|
||||
{
|
||||
mask >>= ELEMENTS_PER_VEC;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
filt_pos += SIMD_BYTES;
|
||||
data_pos += SIMD_BYTES;
|
||||
}
|
||||
/// resize to the real size.
|
||||
res_data.resize(current_offset);
|
||||
}
|
||||
)
|
||||
|
||||
template <typename T>
|
||||
ColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_size_hint) const
|
||||
{
|
||||
@ -496,31 +611,13 @@ ColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_s
|
||||
static constexpr size_t SIMD_BYTES = 64;
|
||||
const UInt8 * filt_end_aligned = filt_pos + size / SIMD_BYTES * SIMD_BYTES;
|
||||
|
||||
while (filt_pos < filt_end_aligned)
|
||||
{
|
||||
UInt64 mask = bytes64MaskToBits64Mask(filt_pos);
|
||||
|
||||
if (0xffffffffffffffff == mask)
|
||||
{
|
||||
res_data.insert(data_pos, data_pos + SIMD_BYTES);
|
||||
}
|
||||
else
|
||||
{
|
||||
while (mask)
|
||||
{
|
||||
size_t index = __builtin_ctzll(mask);
|
||||
res_data.push_back(data_pos[index]);
|
||||
#ifdef __BMI__
|
||||
mask = _blsr_u64(mask);
|
||||
#else
|
||||
mask = mask & (mask-1);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
filt_pos += SIMD_BYTES;
|
||||
data_pos += SIMD_BYTES;
|
||||
}
|
||||
#if USE_MULTITARGET_CODE
|
||||
static constexpr bool VBMI2_CAPABLE = sizeof(T) == 1 || sizeof(T) == 2 || sizeof(T) == 4 || sizeof(T) == 8;
|
||||
if (VBMI2_CAPABLE && isArchSupported(TargetArch::AVX512VBMI2))
|
||||
TargetSpecific::AVX512VBMI2::doFilterAligned<T, Container, SIMD_BYTES>(filt_pos, filt_end_aligned, data_pos, res_data);
|
||||
else
|
||||
#endif
|
||||
TargetSpecific::Default::doFilterAligned<T, Container, SIMD_BYTES>(filt_pos, filt_end_aligned, data_pos, res_data);
|
||||
|
||||
while (filt_pos < filt_end)
|
||||
{
|
||||
|
||||
@ -2,13 +2,14 @@
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/HashTable/HashSet.h>
|
||||
#include <bit>
|
||||
#include "ColumnsCommon.h"
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
#if defined(__SSE2__) && defined(__POPCNT__)
|
||||
#if defined(__SSE2__)
|
||||
/// Transform 64-byte mask to 64-bit mask.
|
||||
static UInt64 toBits64(const Int8 * bytes64)
|
||||
{
|
||||
@ -41,11 +42,11 @@ size_t countBytesInFilter(const UInt8 * filt, size_t start, size_t end)
|
||||
|
||||
const Int8 * end_pos = pos + (end - start);
|
||||
|
||||
#if defined(__SSE2__) && defined(__POPCNT__)
|
||||
#if defined(__SSE2__)
|
||||
const Int8 * end_pos64 = pos + (end - start) / 64 * 64;
|
||||
|
||||
for (; pos < end_pos64; pos += 64)
|
||||
count += __builtin_popcountll(toBits64(pos));
|
||||
count += std::popcount(toBits64(pos));
|
||||
|
||||
/// TODO Add duff device for tail?
|
||||
#endif
|
||||
@ -74,11 +75,11 @@ size_t countBytesInFilterWithNull(const IColumn::Filter & filt, const UInt8 * nu
|
||||
const Int8 * pos2 = reinterpret_cast<const Int8 *>(null_map) + start;
|
||||
const Int8 * end_pos = pos + (end - start);
|
||||
|
||||
#if defined(__SSE2__) && defined(__POPCNT__)
|
||||
#if defined(__SSE2__)
|
||||
const Int8 * end_pos64 = pos + (end - start) / 64 * 64;
|
||||
|
||||
for (; pos < end_pos64; pos += 64, pos2 += 64)
|
||||
count += __builtin_popcountll(toBits64(pos) & ~toBits64(pos2));
|
||||
count += std::popcount(toBits64(pos) & ~toBits64(pos2));
|
||||
|
||||
/// TODO Add duff device for tail?
|
||||
#endif
|
||||
@ -259,7 +260,7 @@ namespace
|
||||
{
|
||||
while (mask)
|
||||
{
|
||||
size_t index = __builtin_ctzll(mask);
|
||||
size_t index = std::countr_zero(mask);
|
||||
copy_array(offsets_pos + index);
|
||||
#ifdef __BMI__
|
||||
mask = _blsr_u64(mask);
|
||||
|
||||
@ -27,17 +27,17 @@ namespace ErrorCodes
|
||||
inline UInt64 bytes64MaskToBits64Mask(const UInt8 * bytes64)
|
||||
{
|
||||
#if defined(__AVX512F__) && defined(__AVX512BW__)
|
||||
static const __m512i zero64 = _mm512_setzero_epi32();
|
||||
UInt64 res = _mm512_cmp_epi8_mask(_mm512_loadu_si512(reinterpret_cast<const __m512i *>(bytes64)), zero64, _MM_CMPINT_EQ);
|
||||
const __m512i vbytes = _mm512_loadu_si512(reinterpret_cast<const void *>(bytes64));
|
||||
UInt64 res = _mm512_testn_epi8_mask(vbytes, vbytes);
|
||||
#elif defined(__AVX__) && defined(__AVX2__)
|
||||
static const __m256i zero32 = _mm256_setzero_si256();
|
||||
const __m256i zero32 = _mm256_setzero_si256();
|
||||
UInt64 res =
|
||||
(static_cast<UInt64>(_mm256_movemask_epi8(_mm256_cmpeq_epi8(
|
||||
_mm256_loadu_si256(reinterpret_cast<const __m256i *>(bytes64)), zero32))) & 0xffffffff)
|
||||
| (static_cast<UInt64>(_mm256_movemask_epi8(_mm256_cmpeq_epi8(
|
||||
_mm256_loadu_si256(reinterpret_cast<const __m256i *>(bytes64+32)), zero32))) << 32);
|
||||
#elif defined(__SSE2__) && defined(__POPCNT__)
|
||||
static const __m128i zero16 = _mm_setzero_si128();
|
||||
#elif defined(__SSE2__)
|
||||
const __m128i zero16 = _mm_setzero_si128();
|
||||
UInt64 res =
|
||||
(static_cast<UInt64>(_mm_movemask_epi8(_mm_cmpeq_epi8(
|
||||
_mm_loadu_si128(reinterpret_cast<const __m128i *>(bytes64)), zero16))) & 0xffff)
|
||||
|
||||
@ -64,11 +64,6 @@ ColumnPtr IColumn::createWithOffsets(const Offsets & offsets, const Field & defa
|
||||
return res;
|
||||
}
|
||||
|
||||
SerializationInfoPtr IColumn::getSerializationInfo() const
|
||||
{
|
||||
return std::make_shared<SerializationInfo>(ISerialization::getKind(*this), SerializationInfo::Settings{});
|
||||
}
|
||||
|
||||
bool isColumnNullable(const IColumn & column)
|
||||
{
|
||||
return checkColumn<ColumnNullable>(column);
|
||||
|
||||
@ -35,9 +35,6 @@ class ColumnGathererStream;
|
||||
class Field;
|
||||
class WeakHash32;
|
||||
|
||||
class SerializationInfo;
|
||||
using SerializationInfoPtr = std::shared_ptr<const SerializationInfo>;
|
||||
|
||||
/*
|
||||
* Represents a set of equal ranges in previous column to perform sorting in current column.
|
||||
* Used in sorting by tuples.
|
||||
@ -445,8 +442,6 @@ public:
|
||||
/// Used to create full column from sparse.
|
||||
[[nodiscard]] virtual Ptr createWithOffsets(const Offsets & offsets, const Field & default_field, size_t total_rows, size_t shift) const;
|
||||
|
||||
[[nodiscard]] virtual SerializationInfoPtr getSerializationInfo() const;
|
||||
|
||||
/// Compress column in memory to some representation that allows to decompress it back.
|
||||
/// Return itself if compression is not applicable for this column type.
|
||||
[[nodiscard]] virtual Ptr compress() const
|
||||
|
||||
91
src/Columns/tests/gtest_column_vector.cpp
Normal file
91
src/Columns/tests/gtest_column_vector.cpp
Normal file
@ -0,0 +1,91 @@
|
||||
#include <typeinfo>
|
||||
#include <vector>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Common/randomSeed.h>
|
||||
#include <gtest/gtest.h>
|
||||
|
||||
|
||||
using namespace DB;
|
||||
|
||||
static pcg64 rng(randomSeed());
|
||||
static constexpr int error_code = 12345;
|
||||
static constexpr size_t TEST_RUNS = 500;
|
||||
static constexpr size_t MAX_ROWS = 10000;
|
||||
static const std::vector<size_t> filter_ratios = {1, 2, 5, 11, 32, 64, 100, 1000};
|
||||
static const size_t K = filter_ratios.size();
|
||||
|
||||
template <typename T>
|
||||
static MutableColumnPtr createColumn(size_t n)
|
||||
{
|
||||
auto column = ColumnVector<T>::create();
|
||||
auto & values = column->getData();
|
||||
|
||||
for (size_t i = 0; i < n; ++i)
|
||||
{
|
||||
values.push_back(i);
|
||||
}
|
||||
|
||||
return column;
|
||||
}
|
||||
|
||||
bool checkFilter(const PaddedPODArray<UInt8> &flit, const IColumn & src, const IColumn & dst)
|
||||
{
|
||||
size_t n = flit.size();
|
||||
size_t dst_size = dst.size();
|
||||
size_t j = 0; /// index of dest
|
||||
for (size_t i = 0; i < n; ++i)
|
||||
{
|
||||
if (flit[i] != 0)
|
||||
{
|
||||
if ((dst_size <= j) || (src.compareAt(i, j, dst, 0) != 0))
|
||||
return false;
|
||||
j++;
|
||||
}
|
||||
}
|
||||
return dst_size == j; /// filtered size check
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
static void testFilter()
|
||||
{
|
||||
auto test_case = [&](size_t rows, size_t filter_ratio)
|
||||
{
|
||||
auto vector_column = createColumn<T>(rows);
|
||||
PaddedPODArray<UInt8> flit(rows);
|
||||
for (size_t i = 0; i < rows; ++i)
|
||||
flit[i] = rng() % filter_ratio == 0;
|
||||
auto res_column = vector_column->filter(flit, -1);
|
||||
|
||||
if (!checkFilter(flit, *vector_column, *res_column))
|
||||
throw Exception(error_code, "VectorColumn filter failure, type: {}", typeid(T).name());
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
for (size_t i = 0; i < TEST_RUNS; ++i)
|
||||
{
|
||||
size_t rows = rng() % MAX_ROWS + 1;
|
||||
size_t filter_ratio = filter_ratios[rng() % K];
|
||||
|
||||
test_case(rows, filter_ratio);
|
||||
}
|
||||
}
|
||||
catch (const Exception & e)
|
||||
{
|
||||
FAIL() << e.displayText();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
TEST(ColumnVector, Filter)
|
||||
{
|
||||
testFilter<UInt8>();
|
||||
testFilter<Int16>();
|
||||
testFilter<UInt32>();
|
||||
testFilter<Int64>();
|
||||
testFilter<UInt128>();
|
||||
testFilter<Int256>();
|
||||
testFilter<Float32>();
|
||||
testFilter<Float64>();
|
||||
testFilter<UUID>();
|
||||
}
|
||||
@ -13,8 +13,6 @@ namespace fs = std::filesystem;
|
||||
namespace DB
|
||||
{
|
||||
|
||||
constexpr decltype(ConfigReloader::reload_interval) ConfigReloader::reload_interval;
|
||||
|
||||
ConfigReloader::ConfigReloader(
|
||||
const std::string & path_,
|
||||
const std::string & include_from_path_,
|
||||
|
||||
@ -82,6 +82,7 @@ inline bool cpuid(UInt32 op, UInt32 * res) noexcept /// NOLINT
|
||||
OP(AVX512BW) \
|
||||
OP(AVX512VL) \
|
||||
OP(AVX512VBMI) \
|
||||
OP(AVX512VBMI2) \
|
||||
OP(PREFETCHWT1) \
|
||||
OP(SHA) \
|
||||
OP(ADX) \
|
||||
@ -302,6 +303,11 @@ bool haveAVX512VBMI() noexcept
|
||||
return haveAVX512F() && ((CpuInfo(0x7, 0).registers.ecx >> 1) & 1u);
|
||||
}
|
||||
|
||||
bool haveAVX512VBMI2() noexcept
|
||||
{
|
||||
return haveAVX512F() && ((CpuInfo(0x7, 0).registers.ecx >> 6) & 1u);
|
||||
}
|
||||
|
||||
bool haveRDRAND() noexcept
|
||||
{
|
||||
return CpuInfo(0x0).registers.eax >= 0x7 && ((CpuInfo(0x1).registers.ecx >> 30) & 1u);
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
#include <Common/HashTable/HashMap.h>
|
||||
#include <Common/HashTable/HashTable.h>
|
||||
|
||||
#include <bit>
|
||||
#include <new>
|
||||
#include <variant>
|
||||
|
||||
@ -21,17 +22,17 @@ struct StringKey24
|
||||
inline StringRef ALWAYS_INLINE toStringRef(const StringKey8 & n)
|
||||
{
|
||||
assert(n != 0);
|
||||
return {reinterpret_cast<const char *>(&n), 8ul - (__builtin_clzll(n) >> 3)};
|
||||
return {reinterpret_cast<const char *>(&n), 8ul - (std::countl_zero(n) >> 3)};
|
||||
}
|
||||
inline StringRef ALWAYS_INLINE toStringRef(const StringKey16 & n)
|
||||
{
|
||||
assert(n.items[1] != 0);
|
||||
return {reinterpret_cast<const char *>(&n), 16ul - (__builtin_clzll(n.items[1]) >> 3)};
|
||||
return {reinterpret_cast<const char *>(&n), 16ul - (std::countl_zero(n.items[1]) >> 3)};
|
||||
}
|
||||
inline StringRef ALWAYS_INLINE toStringRef(const StringKey24 & n)
|
||||
{
|
||||
assert(n.c != 0);
|
||||
return {reinterpret_cast<const char *>(&n), 24ul - (__builtin_clzll(n.c) >> 3)};
|
||||
return {reinterpret_cast<const char *>(&n), 24ul - (std::countl_zero(n.c) >> 3)};
|
||||
}
|
||||
|
||||
struct StringHashTableHash
|
||||
|
||||
@ -11,6 +11,7 @@
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Core/Defines.h>
|
||||
|
||||
#include <bit>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
|
||||
@ -205,7 +206,7 @@ struct TrailingZerosCounter<UInt32>
|
||||
{
|
||||
static int apply(UInt32 val)
|
||||
{
|
||||
return __builtin_ctz(val);
|
||||
return std::countr_zero(val);
|
||||
}
|
||||
};
|
||||
|
||||
@ -214,7 +215,7 @@ struct TrailingZerosCounter<UInt64>
|
||||
{
|
||||
static int apply(UInt64 val)
|
||||
{
|
||||
return __builtin_ctzll(val);
|
||||
return std::countr_zero(val);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -5,6 +5,7 @@
|
||||
#include <Common/formatIPv6.h>
|
||||
|
||||
#include <cstring>
|
||||
#include <bit>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -89,7 +90,7 @@ bool matchIPv6Subnet(const uint8_t * addr, const uint8_t * cidr_addr, UInt8 pref
|
||||
|
||||
if (mask)
|
||||
{
|
||||
auto offset = __builtin_ctz(mask);
|
||||
auto offset = std::countr_zero(mask);
|
||||
|
||||
if (prefix / 8 != offset)
|
||||
return prefix / 8 < offset;
|
||||
|
||||
@ -57,17 +57,17 @@ inline std::string_view toDescription(OvercommitResult result)
|
||||
switch (result)
|
||||
{
|
||||
case OvercommitResult::NONE:
|
||||
return "Memory overcommit isn't used. OvercommitTracker isn't set.";
|
||||
return "Memory overcommit isn't used. OvercommitTracker isn't set";
|
||||
case OvercommitResult::DISABLED:
|
||||
return "Memory overcommit isn't used. Waiting time or orvercommit denominator are set to zero.";
|
||||
return "Memory overcommit isn't used. Waiting time or overcommit denominator are set to zero";
|
||||
case OvercommitResult::MEMORY_FREED:
|
||||
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "OvercommitResult::MEMORY_FREED shouldn't be asked for description");
|
||||
case OvercommitResult::SELECTED:
|
||||
return "Query was selected to stop by OvercommitTracker.";
|
||||
return "Query was selected to stop by OvercommitTracker";
|
||||
case OvercommitResult::TIMEOUTED:
|
||||
return "Waiting timeout for memory to be freed is reached.";
|
||||
return "Waiting timeout for memory to be freed is reached";
|
||||
case OvercommitResult::NOT_ENOUGH_FREED:
|
||||
return "Memory overcommit has freed not enough memory.";
|
||||
return "Memory overcommit has freed not enough memory";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -25,6 +25,10 @@
|
||||
M(WriteBufferFromFileDescriptorWrite, "Number of writes (write/pwrite) to a file descriptor. Does not include sockets.") \
|
||||
M(WriteBufferFromFileDescriptorWriteFailed, "Number of times the write (write/pwrite) to a file descriptor have failed.") \
|
||||
M(WriteBufferFromFileDescriptorWriteBytes, "Number of bytes written to file descriptors. If the file is compressed, this will show compressed data size.") \
|
||||
M(FileSync, "Number of times the F_FULLFSYNC/fsync/fdatasync function was called for files.") \
|
||||
M(DirectorySync, "Number of times the F_FULLFSYNC/fsync/fdatasync function was called for directories.") \
|
||||
M(FileSyncElapsedMicroseconds, "Total time spent waiting for F_FULLFSYNC/fsync/fdatasync syscall for files.") \
|
||||
M(DirectorySyncElapsedMicroseconds, "Total time spent waiting for F_FULLFSYNC/fsync/fdatasync syscall for directories.") \
|
||||
M(ReadCompressedBytes, "Number of bytes (the number of bytes before decompression) read from compressed sources (files, network).") \
|
||||
M(CompressedReadBufferBlocks, "Number of compressed blocks (the blocks of data that are compressed independent of each other) read from compressed sources (files, network).") \
|
||||
M(CompressedReadBufferBytes, "Number of uncompressed bytes (the number of bytes after decompression) read from compressed sources (files, network).") \
|
||||
|
||||
@ -111,7 +111,7 @@ public:
|
||||
|
||||
while (data + 8 <= end)
|
||||
{
|
||||
current_word = unalignedLoad<UInt64>(data);
|
||||
current_word = unalignedLoadLE<UInt64>(data);
|
||||
|
||||
v3 ^= current_word;
|
||||
SIPROUND;
|
||||
@ -157,8 +157,8 @@ public:
|
||||
void get128(char * out)
|
||||
{
|
||||
finalize();
|
||||
unalignedStore<UInt64>(out, v0 ^ v1);
|
||||
unalignedStore<UInt64>(out + 8, v2 ^ v3);
|
||||
unalignedStoreLE<UInt64>(out, v0 ^ v1);
|
||||
unalignedStoreLE<UInt64>(out + 8, v2 ^ v3);
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
|
||||
@ -78,7 +78,7 @@ private:
|
||||
constexpr uint64_t nextAlphaSize(uint64_t x)
|
||||
{
|
||||
constexpr uint64_t alpha_map_elements_per_counter = 6;
|
||||
return 1ULL << (sizeof(uint64_t) * 8 - __builtin_clzll(x * alpha_map_elements_per_counter));
|
||||
return 1ULL << (sizeof(uint64_t) * 8 - std::countl_zero(x * alpha_map_elements_per_counter));
|
||||
}
|
||||
|
||||
public:
|
||||
|
||||
@ -1,6 +1,5 @@
|
||||
#include <Common/StackTrace.h>
|
||||
|
||||
#include <Core/Defines.h>
|
||||
#include <Common/Dwarf.h>
|
||||
#include <Common/Elf.h>
|
||||
#include <Common/SymbolIndex.h>
|
||||
@ -8,6 +7,7 @@
|
||||
#include <base/CachedFn.h>
|
||||
#include <base/demangle.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <cstring>
|
||||
#include <filesystem>
|
||||
#include <sstream>
|
||||
@ -19,6 +19,32 @@
|
||||
# include <libunwind.h>
|
||||
#endif
|
||||
|
||||
|
||||
namespace
|
||||
{
|
||||
/// Currently this variable is set up once on server startup.
|
||||
/// But we use atomic just in case, so it is possible to be modified at runtime.
|
||||
std::atomic<bool> show_addresses = true;
|
||||
|
||||
bool shouldShowAddress(const void * addr)
|
||||
{
|
||||
/// If the address is less than 4096, most likely it is a nullptr dereference with offset,
|
||||
/// and showing this offset is secure nevertheless.
|
||||
/// NOTE: 4096 is the page size on x86 and it can be different on other systems,
|
||||
/// but for the purpose of this branch, it does not matter.
|
||||
if (reinterpret_cast<uintptr_t>(addr) < 4096)
|
||||
return true;
|
||||
|
||||
return show_addresses.load(std::memory_order_relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
void StackTrace::setShowAddresses(bool show)
|
||||
{
|
||||
show_addresses.store(show, std::memory_order_relaxed);
|
||||
}
|
||||
|
||||
|
||||
std::string signalToErrorMessage(int sig, const siginfo_t & info, [[maybe_unused]] const ucontext_t & context)
|
||||
{
|
||||
std::stringstream error; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
|
||||
@ -30,7 +56,7 @@ std::string signalToErrorMessage(int sig, const siginfo_t & info, [[maybe_unused
|
||||
/// Print info about address and reason.
|
||||
if (nullptr == info.si_addr)
|
||||
error << "Address: NULL pointer.";
|
||||
else
|
||||
else if (shouldShowAddress(info.si_addr))
|
||||
error << "Address: " << info.si_addr;
|
||||
|
||||
#if defined(__x86_64__) && !defined(OS_FREEBSD) && !defined(OS_DARWIN) && !defined(__arm__) && !defined(__powerpc__)
|
||||
@ -372,7 +398,9 @@ static void toStringEveryLineImpl(
|
||||
else
|
||||
out << "?";
|
||||
|
||||
out << " @ " << physical_addr;
|
||||
if (shouldShowAddress(physical_addr))
|
||||
out << " @ " << physical_addr;
|
||||
|
||||
out << " in " << (object ? object->name : "?");
|
||||
|
||||
for (size_t j = 0; j < inline_frames.size(); ++j)
|
||||
@ -393,10 +421,13 @@ static void toStringEveryLineImpl(
|
||||
for (size_t i = offset; i < size; ++i)
|
||||
{
|
||||
const void * addr = frame_pointers[i];
|
||||
out << i << ". " << addr;
|
||||
if (shouldShowAddress(addr))
|
||||
{
|
||||
out << i << ". " << addr;
|
||||
|
||||
callback(out.str());
|
||||
out.str({});
|
||||
callback(out.str());
|
||||
out.str({});
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
@ -67,6 +67,11 @@ public:
|
||||
|
||||
void toStringEveryLine(std::function<void(const std::string &)> callback) const;
|
||||
|
||||
/// Displaying the addresses can be disabled for security reasons.
|
||||
/// If you turn off addresses, it will be more secure, but we will be unable to help you with debugging.
|
||||
/// Please note: addresses are also available in the system.stack_trace and system.trace_log tables.
|
||||
static void setShowAddresses(bool show);
|
||||
|
||||
protected:
|
||||
void tryCapture();
|
||||
|
||||
|
||||
@ -20,6 +20,8 @@ UInt32 getSupportedArchs()
|
||||
result |= static_cast<UInt32>(TargetArch::AVX512BW);
|
||||
if (Cpu::CpuFlagsCache::have_AVX512VBMI)
|
||||
result |= static_cast<UInt32>(TargetArch::AVX512VBMI);
|
||||
if (Cpu::CpuFlagsCache::have_AVX512VBMI2)
|
||||
result |= static_cast<UInt32>(TargetArch::AVX512VBMI2);
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -38,8 +40,9 @@ String toString(TargetArch arch)
|
||||
case TargetArch::AVX: return "avx";
|
||||
case TargetArch::AVX2: return "avx2";
|
||||
case TargetArch::AVX512F: return "avx512f";
|
||||
case TargetArch::AVX512BW: return "avx512bw";
|
||||
case TargetArch::AVX512VBMI: return "avx512vbmi";
|
||||
case TargetArch::AVX512BW: return "avx512bw";
|
||||
case TargetArch::AVX512VBMI: return "avx512vbmi";
|
||||
case TargetArch::AVX512VBMI2: return "avx512vbmi";
|
||||
}
|
||||
|
||||
__builtin_unreachable();
|
||||
|
||||
@ -31,7 +31,7 @@
|
||||
* int funcImpl() {
|
||||
* return 2;
|
||||
* }
|
||||
* ) // DECLARE_DEFAULT_CODE
|
||||
* ) // DECLARE_AVX2_SPECIFIC_CODE
|
||||
*
|
||||
* int func() {
|
||||
* #if USE_MULTITARGET_CODE
|
||||
@ -80,8 +80,9 @@ enum class TargetArch : UInt32
|
||||
AVX = (1 << 1),
|
||||
AVX2 = (1 << 2),
|
||||
AVX512F = (1 << 3),
|
||||
AVX512BW = (1 << 4),
|
||||
AVX512VBMI = (1 << 5),
|
||||
AVX512BW = (1 << 4),
|
||||
AVX512VBMI = (1 << 5),
|
||||
AVX512VBMI2 = (1 << 6),
|
||||
};
|
||||
|
||||
/// Runtime detection.
|
||||
@ -100,6 +101,7 @@ String toString(TargetArch arch);
|
||||
|
||||
#if defined(__clang__)
|
||||
|
||||
#define AVX512VBMI2_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi,avx512vbmi2")))
|
||||
#define AVX512VBMI_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi")))
|
||||
#define AVX512BW_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw")))
|
||||
#define AVX512_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f")))
|
||||
@ -108,6 +110,8 @@ String toString(TargetArch arch);
|
||||
#define SSE42_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt")))
|
||||
#define DEFAULT_FUNCTION_SPECIFIC_ATTRIBUTE
|
||||
|
||||
# define BEGIN_AVX512VBMI2_SPECIFIC_CODE \
|
||||
_Pragma("clang attribute push(__attribute__((target(\"sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi,avx512vbmi2\"))),apply_to=function)")
|
||||
# define BEGIN_AVX512VBMI_SPECIFIC_CODE \
|
||||
_Pragma("clang attribute push(__attribute__((target(\"sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi\"))),apply_to=function)")
|
||||
# define BEGIN_AVX512BW_SPECIFIC_CODE \
|
||||
@ -129,6 +133,7 @@ String toString(TargetArch arch);
|
||||
# define DUMMY_FUNCTION_DEFINITION [[maybe_unused]] void _dummy_function_definition();
|
||||
#else
|
||||
|
||||
#define AVX512VBMI2_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi,avx512vbmi2,tune=native")))
|
||||
#define AVX512VBMI_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi,tune=native")))
|
||||
#define AVX512BW_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,tune=native")))
|
||||
#define AVX512_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,tune=native")))
|
||||
@ -137,6 +142,9 @@ String toString(TargetArch arch);
|
||||
#define SSE42_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt",tune=native)))
|
||||
#define DEFAULT_FUNCTION_SPECIFIC_ATTRIBUTE
|
||||
|
||||
# define BEGIN_AVX512VBMI2_SPECIFIC_CODE \
|
||||
_Pragma("GCC push_options") \
|
||||
_Pragma("GCC target(\"sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi,avx512vbmi2,tune=native\")")
|
||||
# define BEGIN_AVX512VBMI_SPECIFIC_CODE \
|
||||
_Pragma("GCC push_options") \
|
||||
_Pragma("GCC target(\"sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi,tune=native\")")
|
||||
@ -217,6 +225,16 @@ namespace TargetSpecific::AVX512VBMI { \
|
||||
} \
|
||||
END_TARGET_SPECIFIC_CODE
|
||||
|
||||
#define DECLARE_AVX512VBMI2_SPECIFIC_CODE(...) \
|
||||
BEGIN_AVX512VBMI2_SPECIFIC_CODE \
|
||||
namespace TargetSpecific::AVX512VBMI2 { \
|
||||
DUMMY_FUNCTION_DEFINITION \
|
||||
using namespace DB::TargetSpecific::AVX512VBMI2; \
|
||||
__VA_ARGS__ \
|
||||
} \
|
||||
END_TARGET_SPECIFIC_CODE
|
||||
|
||||
|
||||
#else
|
||||
|
||||
#define USE_MULTITARGET_CODE 0
|
||||
@ -229,6 +247,7 @@ END_TARGET_SPECIFIC_CODE
|
||||
#define DECLARE_AVX512F_SPECIFIC_CODE(...)
|
||||
#define DECLARE_AVX512BW_SPECIFIC_CODE(...)
|
||||
#define DECLARE_AVX512VBMI_SPECIFIC_CODE(...)
|
||||
#define DECLARE_AVX512VBMI2_SPECIFIC_CODE(...)
|
||||
|
||||
#endif
|
||||
|
||||
@ -245,8 +264,9 @@ DECLARE_SSE42_SPECIFIC_CODE (__VA_ARGS__) \
|
||||
DECLARE_AVX_SPECIFIC_CODE (__VA_ARGS__) \
|
||||
DECLARE_AVX2_SPECIFIC_CODE (__VA_ARGS__) \
|
||||
DECLARE_AVX512F_SPECIFIC_CODE(__VA_ARGS__) \
|
||||
DECLARE_AVX512BW_SPECIFIC_CODE(__VA_ARGS__) \
|
||||
DECLARE_AVX512VBMI_SPECIFIC_CODE(__VA_ARGS__)
|
||||
DECLARE_AVX512BW_SPECIFIC_CODE (__VA_ARGS__) \
|
||||
DECLARE_AVX512VBMI_SPECIFIC_CODE (__VA_ARGS__) \
|
||||
DECLARE_AVX512VBMI2_SPECIFIC_CODE (__VA_ARGS__)
|
||||
|
||||
DECLARE_DEFAULT_CODE(
|
||||
constexpr auto BuildArch = TargetArch::Default; /// NOLINT
|
||||
@ -276,6 +296,9 @@ DECLARE_AVX512VBMI_SPECIFIC_CODE(
|
||||
constexpr auto BuildArch = TargetArch::AVX512VBMI; /// NOLINT
|
||||
) // DECLARE_AVX512VBMI_SPECIFIC_CODE
|
||||
|
||||
DECLARE_AVX512VBMI2_SPECIFIC_CODE(
|
||||
constexpr auto BuildArch = TargetArch::AVX512VBMI2; /// NOLINT
|
||||
) // DECLARE_AVX512VBMI2_SPECIFIC_CODE
|
||||
|
||||
/** Runtime Dispatch helpers for class members.
|
||||
*
|
||||
|
||||
@ -238,27 +238,36 @@ TaskStatsInfoGetter::TaskStatsInfoGetter()
|
||||
if (netlink_socket_fd < 0)
|
||||
throwFromErrno("Can't create PF_NETLINK socket", ErrorCodes::NETLINK_ERROR);
|
||||
|
||||
/// On some containerized environments, operation on Netlink socket could hang forever.
|
||||
/// We set reasonably small timeout to overcome this issue.
|
||||
|
||||
struct timeval tv;
|
||||
tv.tv_sec = 0;
|
||||
tv.tv_usec = 50000;
|
||||
|
||||
if (0 != ::setsockopt(netlink_socket_fd, SOL_SOCKET, SO_RCVTIMEO, reinterpret_cast<const char *>(&tv), sizeof(tv)))
|
||||
throwFromErrno("Can't set timeout on PF_NETLINK socket", ErrorCodes::NETLINK_ERROR);
|
||||
|
||||
union
|
||||
try
|
||||
{
|
||||
::sockaddr_nl addr{};
|
||||
::sockaddr sockaddr;
|
||||
};
|
||||
addr.nl_family = AF_NETLINK;
|
||||
/// On some containerized environments, operation on Netlink socket could hang forever.
|
||||
/// We set reasonably small timeout to overcome this issue.
|
||||
|
||||
if (::bind(netlink_socket_fd, &sockaddr, sizeof(addr)) < 0)
|
||||
throwFromErrno("Can't bind PF_NETLINK socket", ErrorCodes::NETLINK_ERROR);
|
||||
struct timeval tv;
|
||||
tv.tv_sec = 0;
|
||||
tv.tv_usec = 50000;
|
||||
|
||||
taskstats_family_id = getFamilyId(netlink_socket_fd);
|
||||
if (0 != ::setsockopt(netlink_socket_fd, SOL_SOCKET, SO_RCVTIMEO, reinterpret_cast<const char *>(&tv), sizeof(tv)))
|
||||
throwFromErrno("Can't set timeout on PF_NETLINK socket", ErrorCodes::NETLINK_ERROR);
|
||||
|
||||
union
|
||||
{
|
||||
::sockaddr_nl addr{};
|
||||
::sockaddr sockaddr;
|
||||
};
|
||||
addr.nl_family = AF_NETLINK;
|
||||
|
||||
if (::bind(netlink_socket_fd, &sockaddr, sizeof(addr)) < 0)
|
||||
throwFromErrno("Can't bind PF_NETLINK socket", ErrorCodes::NETLINK_ERROR);
|
||||
|
||||
taskstats_family_id = getFamilyId(netlink_socket_fd);
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
if (netlink_socket_fd >= 0)
|
||||
close(netlink_socket_fd);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
#include <Common/StringUtils/StringUtils.h>
|
||||
|
||||
#include <widechar_width.h>
|
||||
#include <bit>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -124,7 +125,7 @@ size_t computeWidthImpl(const UInt8 * data, size_t size, size_t prefix, size_t l
|
||||
|
||||
if (non_regular_width_mask)
|
||||
{
|
||||
auto num_regular_chars = __builtin_ctz(non_regular_width_mask);
|
||||
auto num_regular_chars = std::countr_zero(non_regular_width_mask);
|
||||
width += num_regular_chars;
|
||||
i += num_regular_chars;
|
||||
break;
|
||||
|
||||
@ -83,7 +83,7 @@ inline size_t countCodePoints(const UInt8 * data, size_t size)
|
||||
const auto threshold = vdupq_n_s8(0xBF);
|
||||
|
||||
for (; data < src_end_sse; data += bytes_sse)
|
||||
res += __builtin_popcountll(get_nibble_mask(vcgtq_s8(vld1q_s8(reinterpret_cast<const int8_t *>(data)), threshold)));
|
||||
res += std::popcount(get_nibble_mask(vcgtq_s8(vld1q_s8(reinterpret_cast<const int8_t *>(data)), threshold)));
|
||||
res >>= 2;
|
||||
#endif
|
||||
|
||||
|
||||
@ -18,7 +18,6 @@
|
||||
|
||||
#cmakedefine01 USE_DATASKETCHES
|
||||
#cmakedefine01 USE_YAML_CPP
|
||||
#cmakedefine01 CLICKHOUSE_SPLIT_BINARY
|
||||
#cmakedefine01 USE_BZIP2
|
||||
#cmakedefine01 USE_MINIZIP
|
||||
#cmakedefine01 USE_SNAPPY
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <bit>
|
||||
|
||||
#include <fmt/format.h>
|
||||
|
||||
@ -561,7 +562,7 @@ int main(int argc, char ** argv)
|
||||
/// Fill source data
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
keys[i] = __builtin_ctz(i + 1); /// Make keys to have just slightly more realistic distribution.
|
||||
keys[i] = std::countr_zero(i + 1); /// Make keys to have just slightly more realistic distribution.
|
||||
values[i] = 1234.5; /// The distribution of values does not affect execution speed.
|
||||
}
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user