thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-26 01:22:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into replicated_database_improvements

Browse Source
This commit is contained in:
Alexander Tokmakov 2022-08-01 19:12:37 +02:00
commit 1567b65473
789 changed files with 3674 additions and 3606 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
CMakeLists.txt
View File

@ -77,10 +77,9 @@ option(USE_STATIC_LIBRARIES "Disable to use shared libraries" ON)
# DEVELOPER ONLY. # DEVELOPER ONLY.
# Faster linking if turned on. # Faster linking if turned on.
option(SPLIT_SHARED_LIBRARIES "Keep all internal libraries as separate .so files" OFF) option(SPLIT_SHARED_LIBRARIES "Keep all internal libraries as separate .so files" OFF)
option(CLICKHOUSE_SPLIT_BINARY "Make several binaries (clickhouse-server, clickhouse-client etc.) instead of one bundled" OFF)
if (USE_STATIC_LIBRARIES AND (SPLIT_SHARED_LIBRARIES OR CLICKHOUSE_SPLIT_BINARY)) if (USE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES)
message(FATAL_ERROR "SPLIT_SHARED_LIBRARIES=1 or CLICKHOUSE_SPLIT_BINARY=1 must not be used together with USE_STATIC_LIBRARIES=1") message(FATAL_ERROR "SPLIT_SHARED_LIBRARIES=1 must not be used together with USE_STATIC_LIBRARIES=1")
endif() endif()
if (NOT USE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES) if (NOT USE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES)
@ -502,7 +501,7 @@ endif ()
message (STATUS message (STATUS
"Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ; "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE} ;
USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES} USE_STATIC_LIBRARIES=${USE_STATIC_LIBRARIES}
SPLIT_SHARED=${SPLIT_SHARED_LIBRARIES}") SPLIT_SHARED_LIBRARIES=${SPLIT_SHARED_LIBRARIES}")
include (GNUInstallDirs) include (GNUInstallDirs)

8
README.md
View File

@ -12,11 +12,7 @@ ClickHouse® is an open-source column-oriented database management system that a
* [Blog](https://clickhouse.com/blog/en/) contains various ClickHouse-related articles, as well as announcements and reports about events. * [Blog](https://clickhouse.com/blog/en/) contains various ClickHouse-related articles, as well as announcements and reports about events.
* [Code Browser (Woboq)](https://clickhouse.com/codebrowser/ClickHouse/index.html) with syntax highlight and navigation. * [Code Browser (Woboq)](https://clickhouse.com/codebrowser/ClickHouse/index.html) with syntax highlight and navigation.
* [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlight, powered by github.dev. * [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlight, powered by github.dev.
* [Contacts](https://clickhouse.com/company/#contact) can help to get your questions answered if there are any. * [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any.
## Upcoming events ## Upcoming events
* [v22.7 Release Webinar](https://clickhouse.com/company/events/v22-7-release-webinar/) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap. * **v22.8 Release Webinar** Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap.
* [ClickHouse Meetup at the Cloudflare office in London](https://www.meetup.com/clickhouse-london-user-group/events/286891586/) ClickHouse meetup at the Cloudflare office space in central London
* [ClickHouse Meetup at the Metoda office in Munich](https://www.meetup.com/clickhouse-meetup-munich/events/286891667/) ClickHouse meetup at the Metoda office in Munich

2
cmake/cpu_features.cmake
View File

@ -16,7 +16,7 @@ option (ENABLE_SSE41 "Use SSE4.1 instructions on x86_64" 1)
option (ENABLE_SSE42 "Use SSE4.2 instructions on x86_64" 1) option (ENABLE_SSE42 "Use SSE4.2 instructions on x86_64" 1)
option (ENABLE_PCLMULQDQ "Use pclmulqdq instructions on x86_64" 1) option (ENABLE_PCLMULQDQ "Use pclmulqdq instructions on x86_64" 1)
option (ENABLE_POPCNT "Use popcnt instructions on x86_64" 1) option (ENABLE_POPCNT "Use popcnt instructions on x86_64" 1)
option (ENABLE_AVX "Use AVX instructions on x86_64" 0) option (ENABLE_AVX "Use AVX instructions on x86_64" 1)
option (ENABLE_AVX2 "Use AVX2 instructions on x86_64" 0) option (ENABLE_AVX2 "Use AVX2 instructions on x86_64" 0)
option (ENABLE_AVX512 "Use AVX512 instructions on x86_64" 0) option (ENABLE_AVX512 "Use AVX512 instructions on x86_64" 0)
option (ENABLE_AVX512_VBMI "Use AVX512_VBMI instruction on x86_64 (depends on ENABLE_AVX512)" 0) option (ENABLE_AVX512_VBMI "Use AVX512_VBMI instruction on x86_64 (depends on ENABLE_AVX512)" 0)

2
contrib/NuRaft vendored

@ -1 +1 @@
Subproject commit 1334b9ae72576821a698d657d08838861cf33007 Subproject commit e1dc47c1cfd529801a8c94a396a3921a71ae3ccf

2
contrib/arrow vendored

@ -1 +1 @@
Subproject commit efdcd015cfdee1b6aa349c9ca227ca12c3d697f5 Subproject commit 3e03c6de41a86df2fc54a61e9be1abaefeff6b0e

2
contrib/libgsasl vendored

@ -1 +1 @@
Subproject commit 383ee28e82f69fa16ed43b48bd9c8ee5b313ab84 Subproject commit 0324680f13f22bb43df5353a08e26453d7d640ac

8
docker/packager/binary/build.sh
View File

@ -55,7 +55,7 @@ ccache --zero-stats ||:
if [ "$BUILD_MUSL_KEEPER" == "1" ] if [ "$BUILD_MUSL_KEEPER" == "1" ]
then then
# build keeper with musl separately # build keeper with musl separately
cmake --debug-trycompile --verbose=1 -DBUILD_STANDALONE_KEEPER=1 -DENABLE_CLICKHOUSE_KEEPER=1 -DCMAKE_VERBOSE_MAKEFILE=1 -DUSE_MUSL=1 -LA -DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" .. cmake --debug-trycompile -DBUILD_STANDALONE_KEEPER=1 -DENABLE_CLICKHOUSE_KEEPER=1 -DCMAKE_VERBOSE_MAKEFILE=1 -DUSE_MUSL=1 -LA -DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
# shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty. # shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty.
ninja $NINJA_FLAGS clickhouse-keeper ninja $NINJA_FLAGS clickhouse-keeper
@ -70,10 +70,10 @@ then
rm -f CMakeCache.txt rm -f CMakeCache.txt
# Build the rest of binaries # Build the rest of binaries
cmake --debug-trycompile --verbose=1 -DBUILD_STANDALONE_KEEPER=0 -DCREATE_KEEPER_SYMLINK=0 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" .. cmake --debug-trycompile -DBUILD_STANDALONE_KEEPER=0 -DCREATE_KEEPER_SYMLINK=0 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
else else
# Build everything # Build everything
cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" .. cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
fi fi
if [ "coverity" == "$COMBINED_OUTPUT" ] if [ "coverity" == "$COMBINED_OUTPUT" ]
@ -88,7 +88,7 @@ fi
# No quotes because I want it to expand to nothing if empty. # No quotes because I want it to expand to nothing if empty.
# shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty. # shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty.
$SCAN_WRAPPER ninja $NINJA_FLAGS clickhouse-bundle $SCAN_WRAPPER ninja $NINJA_FLAGS $BUILD_TARGET
ls -la ./programs ls -la ./programs

18
docker/packager/packager
View File

@ -134,6 +134,7 @@ def parse_env_variables(
result = [] result = []
result.append("OUTPUT_DIR=/output") result.append("OUTPUT_DIR=/output")
cmake_flags = ["$CMAKE_FLAGS"] cmake_flags = ["$CMAKE_FLAGS"]
build_target = "clickhouse-bundle"
is_cross_darwin = compiler.endswith(DARWIN_SUFFIX) is_cross_darwin = compiler.endswith(DARWIN_SUFFIX)
is_cross_darwin_arm = compiler.endswith(DARWIN_ARM_SUFFIX) is_cross_darwin_arm = compiler.endswith(DARWIN_ARM_SUFFIX)
@ -236,7 +237,11 @@ def parse_env_variables(
result.append("CCACHE_BASEDIR=/build") result.append("CCACHE_BASEDIR=/build")
result.append("CCACHE_NOHASHDIR=true") result.append("CCACHE_NOHASHDIR=true")
result.append("CCACHE_COMPILERCHECK=content") result.append("CCACHE_COMPILERCHECK=content")
result.append("CCACHE_MAXSIZE=15G") cache_maxsize = "15G"
if clang_tidy:
# 15G is not enough for tidy build
cache_maxsize = "25G"
result.append(f"CCACHE_MAXSIZE={cache_maxsize}")
# result.append("CCACHE_UMASK=777") # result.append("CCACHE_UMASK=777")
if distcc_hosts: if distcc_hosts:
@ -261,22 +266,26 @@ def parse_env_variables(
if split_binary: if split_binary:
cmake_flags.append( cmake_flags.append(
"-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 " "-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1"
"-DCLICKHOUSE_SPLIT_BINARY=1"
) )
# We can't always build utils because it requires too much space, but # We can't always build utils because it requires too much space, but
# we have to build them at least in some way in CI. The split build is # we have to build them at least in some way in CI. The split build is
# probably the least heavy disk-wise. # probably the least heavy disk-wise.
cmake_flags.append("-DENABLE_UTILS=1") cmake_flags.append("-DENABLE_UTILS=1")
# utils are not included into clickhouse-bundle, so build everything
build_target = "all"
if clang_tidy: if clang_tidy:
cmake_flags.append("-DENABLE_CLANG_TIDY=1") cmake_flags.append("-DENABLE_CLANG_TIDY=1")
cmake_flags.append("-DENABLE_UTILS=1")
cmake_flags.append("-DENABLE_TESTS=1") cmake_flags.append("-DENABLE_TESTS=1")
cmake_flags.append("-DENABLE_EXAMPLES=1") cmake_flags.append("-DENABLE_EXAMPLES=1")
# Don't stop on first error to find more clang-tidy errors in one run. # Don't stop on first error to find more clang-tidy errors in one run.
result.append("NINJA_FLAGS=-k0") result.append("NINJA_FLAGS=-k0")
cmake_flags.append("-DENABLE_UTILS=1")
# utils are not included into clickhouse-bundle, so build everything
build_target = "all"
if with_coverage: if with_coverage:
cmake_flags.append("-DWITH_COVERAGE=1") cmake_flags.append("-DWITH_COVERAGE=1")
@ -290,6 +299,7 @@ def parse_env_variables(
cmake_flags.append("-DCLICKHOUSE_OFFICIAL_BUILD=1") cmake_flags.append("-DCLICKHOUSE_OFFICIAL_BUILD=1")
result.append('CMAKE_FLAGS="' + " ".join(cmake_flags) + '"') result.append('CMAKE_FLAGS="' + " ".join(cmake_flags) + '"')
result.append(f"BUILD_TARGET={build_target}")
return result return result

4
docker/server/entrypoint.sh
View File

@ -38,6 +38,7 @@ FORMAT_SCHEMA_PATH="$(clickhouse extract-from-config --config-file "$CLICKHOUSE_
# There could be many disks declared in config # There could be many disks declared in config
readarray -t FILESYSTEM_CACHE_PATHS < <(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key='storage_configuration.disks.*.data_cache_path' || true) readarray -t FILESYSTEM_CACHE_PATHS < <(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key='storage_configuration.disks.*.data_cache_path' || true)
readarray -t DISKS_PATHS < <(clickhouse extract-from-config --config-file "$CLICKHOUSE_CONFIG" --key='storage_configuration.disks.*.path' || true)
CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}" CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}"
CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-}" CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-}"
@ -50,7 +51,8 @@ for dir in "$DATA_DIR" \
"$TMP_DIR" \ "$TMP_DIR" \
"$USER_PATH" \ "$USER_PATH" \
"$FORMAT_SCHEMA_PATH" \ "$FORMAT_SCHEMA_PATH" \
"${FILESYSTEM_CACHE_PATHS[@]}" "${FILESYSTEM_CACHE_PATHS[@]}" \
"${DISKS_PATHS[@]}"
do do
# check if variable not empty # check if variable not empty
[ -z "$dir" ] && continue [ -z "$dir" ] && continue

3
docker/test/integration/base/Dockerfile
View File

@ -34,13 +34,14 @@ RUN apt-get update \
ARG TARGETARCH ARG TARGETARCH
# Install MySQL ODBC driver from RHEL rpm # Install MySQL ODBC driver from RHEL rpm
# For reference https://downloads.mysql.com/archives/c-odbc/ RHEL
RUN arch=${TARGETARCH:-amd64} \ RUN arch=${TARGETARCH:-amd64} \
&& case $arch in \ && case $arch in \
amd64) rarch=x86_64 ;; \ amd64) rarch=x86_64 ;; \
arm64) rarch=aarch64 ;; \ arm64) rarch=aarch64 ;; \
esac \ esac \
&& cd /tmp \ && cd /tmp \
&& curl -o mysql-odbc.rpm "https://cdn.mysql.com/Downloads/Connector-ODBC/8.0/mysql-connector-odbc-8.0.27-1.el8.${rarch}.rpm" \ && curl -o mysql-odbc.rpm "https://cdn.mysql.com/archives/mysql-connector-odbc-8.0/mysql-connector-odbc-8.0.27-1.el8.${rarch}.rpm" \
&& rpm2archive mysql-odbc.rpm \ && rpm2archive mysql-odbc.rpm \
&& tar xf mysql-odbc.rpm.tgz -C / ./usr/lib64/ \ && tar xf mysql-odbc.rpm.tgz -C / ./usr/lib64/ \
&& LINK_DIR=$(dpkg -L libodbc1 | grep '^/usr/lib/.*-linux-gnu/odbc$') \ && LINK_DIR=$(dpkg -L libodbc1 | grep '^/usr/lib/.*-linux-gnu/odbc$') \

4
docs/en/development/continuous-integration.md
View File

@ -171,11 +171,11 @@ concurrency-related errors. If it fails:
## Split Build Smoke Test ## Split Build Smoke Test
Checks that the server build in [split build](../development/build.md#split-build) Checks that the server build in [split build](../development/developer-instruction.md#split-build)
configuration can start and run simple queries. If it fails: configuration can start and run simple queries. If it fails:
* Fix other test errors first; * Fix other test errors first;
* Build the server in [split build](../development/build.md#split-build) configuration * Build the server in [split build](../development/developer-instruction.md#split-build) configuration
locally and check whether it can start and run `select 1`. locally and check whether it can start and run `select 1`.

16
docs/en/development/developer-instruction.md
View File

@ -267,19 +267,19 @@ The system will prepare ClickHouse binary builds for your pull request individua
Most probably some of the builds will fail at first times. This is due to the fact that we check builds both with gcc as well as with clang, with almost all of existing warnings (always with the `-Werror` flag) enabled for clang. On that same page, you can find all of the build logs so that you do not have to build ClickHouse in all of the possible ways. Most probably some of the builds will fail at first times. This is due to the fact that we check builds both with gcc as well as with clang, with almost all of existing warnings (always with the `-Werror` flag) enabled for clang. On that same page, you can find all of the build logs so that you do not have to build ClickHouse in all of the possible ways.
## Browse ClickHouse Source Code {#browse-clickhouse-source-code}
You can use the **Woboq** online code browser available [here](https://clickhouse.com/codebrowser/ClickHouse/src/index.html). It provides code navigation, semantic highlighting, search and indexing. The code snapshot is updated daily.
Also, you can browse sources on [GitHub](https://github.com/ClickHouse/ClickHouse) as usual.
## Faster builds for development: Split build configuration {#split-build} ## Faster builds for development: Split build configuration {#split-build}
ClickHouse is normally statically linked into a single static `clickhouse` binary with minimal dependencies. This is convenient for distribution, but it means that for every change the entire binary needs to be re-linked, which is slow and inconvenient for development. As an alternative, you can instead build dynamically linked shared libraries and separate binaries `clickhouse-server`, `clickhouse-client` etc., allowing for faster incremental builds. To use it, add the following flags to your `cmake` invocation: ClickHouse is normally statically linked into a single static `clickhouse` binary with minimal dependencies. This is convenient for distribution, but it means that for every change the entire binary needs to be re-linked, which is slow and inconvenient for development. As an alternative, you can instead build dynamically linked shared libraries, allowing for faster incremental builds. To use it, add the following flags to your `cmake` invocation:
``` ```
-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1 -DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1
``` ```
Note that the split build has several drawbacks:
* There is no single `clickhouse` binary, and you have to run `clickhouse-server`, `clickhouse-client`, etc.
* Risk of segfault if you run any of the programs while rebuilding the project.
* You cannot run the integration tests since they only work a single complete binary.
* You can't easily copy the binaries elsewhere. Instead of moving a single binary you'll need to copy all binaries and libraries.
If you are not interested in functionality provided by third-party libraries, you can further speed up the build using `cmake` options If you are not interested in functionality provided by third-party libraries, you can further speed up the build using `cmake` options
``` ```
-DENABLE_LIBRARIES=0 -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_LIBRARIES=0 -DENABLE_EMBEDDED_COMPILER=0

6
docs/en/getting-started/example-datasets/uk-price-paid.md
View File

@ -389,12 +389,6 @@ SETTINGS mutations_sync = 1;
Let's run the same 3 queries. Let's run the same 3 queries.
[Enable](../../operations/settings/settings.md#allow-experimental-projection-optimization) projections for selects:
```sql
SET allow_experimental_projection_optimization = 1;
```
### Query 1. Average Price Per Year {#average-price-projections} ### Query 1. Average Price Per Year {#average-price-projections}
Query: Query:

2
docs/en/interfaces/third-party/gui.md vendored
View File

@ -149,7 +149,7 @@ Features:
### ClickCat {#clickcat} ### ClickCat {#clickcat}
[ClickCat](https://github.com/open-botech/ClickCat) is a firendly user interface that lets you search, explore and visualize your ClickHouse Data. [ClickCat](https://github.com/clickcat-project/ClickCat) is a friendly user interface that lets you search, explore and visualize your ClickHouse Data.
Features: Features:

59
docs/en/operations/performance-test.md
View File

@ -5,12 +5,12 @@ sidebar_label: Testing Hardware
# How to Test Your Hardware with ClickHouse # How to Test Your Hardware with ClickHouse
You can run basic ClickHouse performance test on any server without installation of ClickHouse packages. You can run a basic ClickHouse performance test on any server without installation of ClickHouse packages.
## Automated Run ## Automated Run
You can run benchmark with a single script. You can run the benchmark with a single script.
1. Download the script. 1. Download the script.
``` ```
@ -26,58 +26,3 @@ chmod a+x ./hardware.sh
3. Copy the output and send it to feedback@clickhouse.com 3. Copy the output and send it to feedback@clickhouse.com
All the results are published here: https://clickhouse.com/benchmark/hardware/ All the results are published here: https://clickhouse.com/benchmark/hardware/
## Manual Run
Alternatively you can perform benchmark in the following steps.
1. ssh to the server and download the binary with wget:
```bash
# For amd64:
wget https://builds.clickhouse.com/master/amd64/clickhouse
# For aarch64:
wget https://builds.clickhouse.com/master/aarch64/clickhouse
# For powerpc64le:
wget https://builds.clickhouse.com/master/powerpc64le/clickhouse
# For freebsd:
wget https://builds.clickhouse.com/master/freebsd/clickhouse
# For freebsd-aarch64:
wget https://builds.clickhouse.com/master/freebsd-aarch64/clickhouse
# For freebsd-powerpc64le:
wget https://builds.clickhouse.com/master/freebsd-powerpc64le/clickhouse
# For macos:
wget https://builds.clickhouse.com/master/macos/clickhouse
# For macos-aarch64:
wget https://builds.clickhouse.com/master/macos-aarch64/clickhouse
# Then do:
chmod a+x clickhouse
```
2. Download benchmark files:
```bash
wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/hardware/benchmark-new.sh
chmod a+x benchmark-new.sh
wget https://raw.githubusercontent.com/ClickHouse/ClickHouse/master/benchmark/hardware/queries.sql
```
3. Download the [web analytics dataset](../getting-started/example-datasets/metrica.md) (“hits” table containing 100 million rows).
```bash
wget https://datasets.clickhouse.com/hits/partitions/hits_100m_obfuscated_v1.tar.xz
tar xvf hits_100m_obfuscated_v1.tar.xz -C .
mv hits_100m_obfuscated_v1/* .
```
4. Run the server:
```bash
./clickhouse server
```
5. Check the data: ssh to the server in another terminal
```bash
./clickhouse client --query "SELECT count() FROM hits_100m_obfuscated"
100000000
```
6. Run the benchmark:
```bash
./benchmark-new.sh hits_100m_obfuscated
```
7. Send the numbers and the info about your hardware configuration to feedback@clickhouse.com
All the results are published here: https://clickhouse.com/benchmark/hardware/

14
docs/en/operations/server-configuration-parameters/settings.md
View File

@ -438,6 +438,18 @@ For more information, see the section “[Configuration files](../../operations/
<include_from>/etc/metrica.xml</include_from> <include_from>/etc/metrica.xml</include_from>
``` ```
## interserver_listen_host {#interserver-listen-host}
Restriction on hosts that can exchange data between ClickHouse servers.
The default value equals to `listen_host` setting.
Examples:
``` xml
<interserver_listen_host>::ffff:a00:1</interserver_listen_host>
<interserver_listen_host>10.0.0.1</interserver_listen_host>
```
## interserver_http_port {#interserver-http-port} ## interserver_http_port {#interserver-http-port}
Port for exchanging data between ClickHouse servers. Port for exchanging data between ClickHouse servers.
@ -970,7 +982,7 @@ Default value: 2.
**Example** **Example**
```xml ```xml
<background_merges_mutations_concurrency_ratio>3</background_pbackground_merges_mutations_concurrency_ratio> <background_merges_mutations_concurrency_ratio>3</background_merges_mutations_concurrency_ratio>
``` ```
## background_move_pool_size {#background_move_pool_size} ## background_move_pool_size {#background_move_pool_size}

25
docs/en/sql-reference/functions/date-time-functions.md
View File

@ -956,9 +956,28 @@ SELECT
## timeSlots(StartTime, Duration,\[, Size\]) ## timeSlots(StartTime, Duration,\[, Size\])
For a time interval starting at ‘StartTime’ and continuing for ‘Duration’ seconds, it returns an array of moments in time, consisting of points from this interval rounded down to the ‘Size’ in seconds. ‘Size’ is an optional parameter: a constant UInt32, set to 1800 by default. For a time interval starting at ‘StartTime’ and continuing for ‘Duration’ seconds, it returns an array of moments in time, consisting of points from this interval rounded down to the ‘Size’ in seconds. ‘Size’ is an optional parameter set to 1800 (30 minutes) by default.
For example, `timeSlots(toDateTime('2012-01-01 12:20:00'), 600) = [toDateTime('2012-01-01 12:00:00'), toDateTime('2012-01-01 12:30:00')]`. This is necessary, for example, when searching for pageviews in the corresponding session.
This is necessary for searching for pageviews in the corresponding session. Accepts DateTime and DateTime64 as ’StartTime’ argument. For DateTime, ’Duration’ and ’Size’ arguments must be `UInt32`. For ’DateTime64’ they must be `Decimal64`.
Returns an array of DateTime/DateTime64 (return type matches the type of ’StartTime’). For DateTime64, the return value's scale can differ from the scale of ’StartTime’ --- the highest scale among all given arguments is taken.
Example:
```sql
SELECT timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600));
SELECT timeSlots(toDateTime('1980-12-12 21:01:02', 'UTC'), toUInt32(600), 299);
SELECT timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64(600.1, 1), toDecimal64(299, 0));
```
``` text
┌─timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600))─┐
│ ['2012-01-01 12:00:00','2012-01-01 12:30:00'] │
└─────────────────────────────────────────────────────────────┘
┌─timeSlots(toDateTime('1980-12-12 21:01:02', 'UTC'), toUInt32(600), 299)─┐
│ ['1980-12-12 20:56:13','1980-12-12 21:01:12','1980-12-12 21:06:11'] │
└─────────────────────────────────────────────────────────────────────────┘
┌─timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64(600.1, 1), toDecimal64(299, 0))─┐
│ ['1980-12-12 20:56:13.0000','1980-12-12 21:01:12.0000','1980-12-12 21:06:11.0000'] │
└───────────────────────────────────────────────────────────────────────────────────────────────────────────┘
```
## formatDateTime ## formatDateTime

3
docs/en/sql-reference/functions/files.md
View File

@ -12,12 +12,13 @@ Reads file as a String. The file content is not parsed, so any information is re
**Syntax** **Syntax**
``` sql ``` sql
file(path) file(path[, default])
``` ```
**Arguments** **Arguments**
- `path` — The relative path to the file from [user_files_path](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_files_path). Path to file support following wildcards: `*`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc', 'def'` — strings. - `path` — The relative path to the file from [user_files_path](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_files_path). Path to file support following wildcards: `*`, `?`, `{abc,def}` and `{N..M}` where `N`, `M` — numbers, `'abc', 'def'` — strings.
- `default` — The value that will be returned in the case when a file does not exist or cannot be accessed. Data types supported: [String](../../sql-reference/data-types/string.md) and [NULL](../../sql-reference/syntax.md#null-literal).
**Example** **Example**

8
docs/ru/development/developer-instruction.md
View File

@ -170,7 +170,7 @@ sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
В случае использования на разработческой машине старого HDD или SSD, а также при желании использовать меньше места для артефактов сборки можно использовать следующую команду: В случае использования на разработческой машине старого HDD или SSD, а также при желании использовать меньше места для артефактов сборки можно использовать следующую команду:
```bash ```bash
cmake -DUSE_DEBUG_HELPERS=1 -DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1 .. cmake -DUSE_DEBUG_HELPERS=1 -DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 ..
``` ```
При этом надо учесть, что получаемые в результате сборки исполнимые файлы будут динамически слинкованы с библиотеками, и поэтому фактически станут непереносимыми на другие компьютеры (либо для этого нужно будет предпринять значительно больше усилий по сравнению со статической сборкой). Плюсом же в данном случае является значительно меньшее время сборки (это проявляется не на первой сборке, а на последующих, после внесения изменений в исходный код - тратится меньшее время на линковку по сравнению со статической сборкой) и значительно меньшее использование места на жёстком диске (экономия более, чем в 3 раза по сравнению со статической сборкой). Для целей разработки, когда планируются только отладочные запуски на том же компьютере, где осуществлялась сборка, это может быть наиболее удобным вариантом. При этом надо учесть, что получаемые в результате сборки исполнимые файлы будут динамически слинкованы с библиотеками, и поэтому фактически станут непереносимыми на другие компьютеры (либо для этого нужно будет предпринять значительно больше усилий по сравнению со статической сборкой). Плюсом же в данном случае является значительно меньшее время сборки (это проявляется не на первой сборке, а на последующих, после внесения изменений в исходный код - тратится меньшее время на линковку по сравнению со статической сборкой) и значительно меньшее использование места на жёстком диске (экономия более, чем в 3 раза по сравнению со статической сборкой). Для целей разработки, когда планируются только отладочные запуски на том же компьютере, где осуществлялась сборка, это может быть наиболее удобным вариантом.
@ -285,3 +285,9 @@ Pull request можно создать, даже если работа над з
Система подготовит сборки ClickHouse специально для вашего pull request. Для их получения, нажмите на ссылку «Details» у проверки «Clickhouse build check». Там вы сможете найти прямые ссылки на собранные .deb пакеты ClickHouse, которые, при желании, вы даже сможете установить на свои продакшен серверы (если не страшно). Система подготовит сборки ClickHouse специально для вашего pull request. Для их получения, нажмите на ссылку «Details» у проверки «Clickhouse build check». Там вы сможете найти прямые ссылки на собранные .deb пакеты ClickHouse, которые, при желании, вы даже сможете установить на свои продакшен серверы (если не страшно).
Вероятнее всего, часть сборок не будет успешной с первого раза. Ведь мы проверяем сборку кода и gcc и clang, а при сборке с помощью clang включаются почти все существующие в природе warnings (всегда с флагом `-Werror`). На той же странице, вы сможете найти логи сборки - вам не обязательно самому собирать ClickHouse всеми возможными способами. Вероятнее всего, часть сборок не будет успешной с первого раза. Ведь мы проверяем сборку кода и gcc и clang, а при сборке с помощью clang включаются почти все существующие в природе warnings (всегда с флагом `-Werror`). На той же странице, вы сможете найти логи сборки - вам не обязательно самому собирать ClickHouse всеми возможными способами.
## Навигация по коду ClickHouse {#navigatsiia-po-kodu-clickhouse}
Для навигации по коду онлайн доступен **Woboq**, он расположен [здесь](https://clickhouse.com/codebrowser/ClickHouse/src/index.html). В нём реализовано удобное перемещение между исходными файлами, семантическая подсветка, подсказки, индексация и поиск. Слепок кода обновляется ежедневно.
Также вы можете просматривать исходники на [GitHub](https://github.com/ClickHouse/ClickHouse).

7
docs/ru/getting-started/example-datasets/uk-price-paid.md
View File

@ -389,12 +389,6 @@ SETTINGS mutations_sync = 1;
Давайте выполним те же 3 запроса. Давайте выполним те же 3 запроса.
[Включите](../../operations/settings/settings.md#allow-experimental-projection-optimization) поддержку проекций:
```sql
SET allow_experimental_projection_optimization = 1;
```
### Запрос 1. Средняя цена за год {#average-price-projections} ### Запрос 1. Средняя цена за год {#average-price-projections}
Запрос: Запрос:
@ -647,4 +641,3 @@ no projection: 100 rows in set. Elapsed: 0.069 sec. Processed 26.32 million rows
### Online Playground {#playground} ### Online Playground {#playground}
Этот набор данных доступен в [Online Playground](https://gh-api.clickhouse.tech/play?user=play#U0VMRUNUIHRvd24sIGRpc3RyaWN0LCBjb3VudCgpIEFTIGMsIHJvdW5kKGF2ZyhwcmljZSkpIEFTIHByaWNlLCBiYXIocHJpY2UsIDAsIDUwMDAwMDAsIDEwMCkgRlJPTSB1a19wcmljZV9wYWlkIFdIRVJFIGRhdGUgPj0gJzIwMjAtMDEtMDEnIEdST1VQIEJZIHRvd24sIGRpc3RyaWN0IEhBVklORyBjID49IDEwMCBPUkRFUiBCWSBwcmljZSBERVNDIExJTUlUIDEwMA==). Этот набор данных доступен в [Online Playground](https://gh-api.clickhouse.tech/play?user=play#U0VMRUNUIHRvd24sIGRpc3RyaWN0LCBjb3VudCgpIEFTIGMsIHJvdW5kKGF2ZyhwcmljZSkpIEFTIHByaWNlLCBiYXIocHJpY2UsIDAsIDUwMDAwMDAsIDEwMCkgRlJPTSB1a19wcmljZV9wYWlkIFdIRVJFIGRhdGUgPj0gJzIwMjAtMDEtMDEnIEdST1VQIEJZIHRvd24sIGRpc3RyaWN0IEhBVklORyBjID49IDEwMCBPUkRFUiBCWSBwcmljZSBERVNDIExJTUlUIDEwMA==).

12
docs/ru/operations/server-configuration-parameters/settings.md
View File

@ -407,6 +407,18 @@ ClickHouse проверяет условия для `min_part_size` и `min_part
<include_from>/etc/metrica.xml</include_from> <include_from>/etc/metrica.xml</include_from>
``` ```
## interserver_listen_host {#interserver-listen-host}
Ограничение по хостам, для обмена между серверами ClickHouse.
Значение по умолчанию совпадает со значением параметра listen_host
Примеры:
``` xml
<interserver_listen_host>::ffff:a00:1</interserver_listen_host>
<interserver_listen_host>10.0.0.1</interserver_listen_host>
```
## interserver_http_port {#interserver-http-port} ## interserver_http_port {#interserver-http-port}
Порт для обмена между серверами ClickHouse. Порт для обмена между серверами ClickHouse.

27
docs/ru/sql-reference/functions/date-time-functions.md
View File

@ -944,14 +944,31 @@ SELECT now('Europe/Moscow');
## timeSlot {#timeslot} ## timeSlot {#timeslot}
Округляет время до получаса. Округляет время до получаса.
Эта функция является специфичной для Яндекс.Метрики, так как пол часа - минимальное время, для которого, если соседние по времени хиты одного посетителя на одном счётчике отстоят друг от друга строго более, чем на это время, визит может быть разбит на два визита. То есть, кортежи (номер счётчика, идентификатор посетителя, тайм-слот) могут использоваться для поиска хитов, входящий в соответствующий визит. Эта функция является специфичной для Яндекс.Метрики, так как полчаса - минимальное время, для которого, если соседние по времени хиты одного посетителя на одном счётчике отстоят друг от друга строго более, чем на это время, визит может быть разбит на два визита. То есть, кортежи (номер счётчика, идентификатор посетителя, тайм-слот) могут использоваться для поиска хитов, входящий в соответствующий визит.
## timeSlots(StartTime, Duration,\[, Size\]) {#timeslotsstarttime-duration-size} ## timeSlots(StartTime, Duration,\[, Size\]) {#timeslotsstarttime-duration-size}
Для интервала, начинающегося в `StartTime` и длящегося `Duration` секунд, возвращает массив моментов времени, кратных `Size`. Параметр `Size` указывать необязательно, по умолчанию он равен 1800 секундам (30 минутам) - необязательный параметр.
Данная функция может использоваться, например, для анализа количества просмотров страницы за соответствующую сессию.
Аргумент `StartTime` может иметь тип `DateTime` или `DateTime64`. В случае, если используется `DateTime`, аргументы `Duration` и `Size` должны иметь тип `UInt32`; Для DateTime64 они должны быть типа `Decimal64`.
Возвращает массив DateTime/DateTime64 (тип будет совпадать с типом параметра ’StartTime’). Для DateTime64 масштаб(scale) возвращаемой величины может отличаться от масштаба фргумента ’StartTime’ --- результат будет иметь наибольший масштаб среди всех данных аргументов.
Для интервала времени, начинающегося в ‘StartTime’ и продолжающегося ‘Duration’ секунд, возвращает массив моментов времени, состоящий из округлений вниз до ‘Size’ точек в секундах из этого интервала. ‘Size’ - необязательный параметр, константный UInt32, по умолчанию равен 1800. Пример использования:
```sql
Например, `timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600)) = [toDateTime('2012-01-01 12:00:00'), toDateTime('2012-01-01 12:30:00')]`. SELECT timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600));
Это нужно для поиска хитов, входящих в соответствующий визит. SELECT timeSlots(toDateTime('1980-12-12 21:01:02', 'UTC'), toUInt32(600), 299);
SELECT timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64(600.1, 1), toDecimal64(299, 0));
```
``` text
┌─timeSlots(toDateTime('2012-01-01 12:20:00'), toUInt32(600))─┐
│ ['2012-01-01 12:00:00','2012-01-01 12:30:00'] │
└─────────────────────────────────────────────────────────────┘
┌─timeSlots(toDateTime('1980-12-12 21:01:02', 'UTC'), toUInt32(600), 299)─┐
│ ['1980-12-12 20:56:13','1980-12-12 21:01:12','1980-12-12 21:06:11'] │
└─────────────────────────────────────────────────────────────────────────┘
┌─timeSlots(toDateTime64('1980-12-12 21:01:02.1234', 4, 'UTC'), toDecimal64(600.1, 1), toDecimal64(299, 0))─┐
│ ['1980-12-12 20:56:13.0000','1980-12-12 21:01:12.0000','1980-12-12 21:06:11.0000'] │
└───────────────────────────────────────────────────────────────────────────────────────────────────────────┘
```
## toYYYYMM ## toYYYYMM

3
docs/ru/sql-reference/functions/files.md
View File

@ -12,12 +12,13 @@ sidebar_label: "Функции для работы с файлами"
**Синтаксис** **Синтаксис**
``` sql ``` sql
file(path) file(path[, default])
``` ```
**Аргументы** **Аргументы**
- `path` — относительный путь до файла от [user_files_path](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_files_path). Путь к файлу может включать следующие символы подстановки и шаблоны: `*`, `?`, `{abc,def}` и `{N..M}`, где `N`, `M` — числа, `'abc', 'def'` — строки. - `path` — относительный путь до файла от [user_files_path](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-user_files_path). Путь к файлу может включать следующие символы подстановки и шаблоны: `*`, `?`, `{abc,def}` и `{N..M}`, где `N`, `M` — числа, `'abc', 'def'` — строки.
- `default` — Значение возвращаемое в случае, если указанный файл не существует. Поддерживаемые типы данных: [String](../../sql-reference/data-types/string.md) и [NULL](../../sql-reference/syntax.md#null-literal).
**Примеры** **Примеры**

6
docs/zh/development/developer-instruction.md
View File

@ -264,3 +264,9 @@ ClickHouse成员一旦在您的拉取请求上贴上«可以测试»标签，就
系统将分别为您的拉取请求准备ClickHouse二进制版本。若要检索这些构建信息，请在检查列表中单击« ClickHouse构建检查»旁边的«详细信息»链接。在这里，您会找到指向ClickHouse的.deb软件包的直接链接，此外，甚至可以将其部署在生产服务器上（如果您不担心）。 系统将分别为您的拉取请求准备ClickHouse二进制版本。若要检索这些构建信息，请在检查列表中单击« ClickHouse构建检查»旁边的«详细信息»链接。在这里，您会找到指向ClickHouse的.deb软件包的直接链接，此外，甚至可以将其部署在生产服务器上（如果您不担心）。
某些构建项很可能会在首次构建时失败。这是因为我们同时检查了基于gcc和clang的构建，几乎所有现有的被clang启用的警告（总是带有`-Werror`标志）。在同一页面上，您可以找到所有构建的日志，因此不必以所有可能的方式构建ClickHouse。 某些构建项很可能会在首次构建时失败。这是因为我们同时检查了基于gcc和clang的构建，几乎所有现有的被clang启用的警告（总是带有`-Werror`标志）。在同一页面上，您可以找到所有构建的日志，因此不必以所有可能的方式构建ClickHouse。
## 浏览ClickHouse源代码 {#browse-clickhouse-source-code}
您可以使用 **Woboq** 在线代码浏览器 [点击这里](https://clickhouse.com/codebrowser/ClickHouse/src/index.html). 它提供了代码导航和语义突出显示、搜索和索引。 代码快照每天更新。
此外，您还可以像往常一样浏览源代码 [GitHub](https://github.com/ClickHouse/ClickHouse)

2
docs/zh/engines/table-engines/mergetree-family/mergetree.md
View File

@ -62,6 +62,8 @@ ORDER BY expr
- `PARTITION BY` — [分区键](custom-partitioning-key.md) ，可选项。 - `PARTITION BY` — [分区键](custom-partitioning-key.md) ，可选项。
大多数情况下，不需要分使用区键。即使需要使用，也不需要使用比月更细粒度的分区键。分区不会加快查询（这与 ORDER BY 表达式不同）。永远也别使用过细粒度的分区键。不要使用客户端指定分区标识符或分区字段名称来对数据进行分区（而是将分区字段标识或名称作为 ORDER BY 表达式的第一列来指定分区）。
要按月分区，可以使用表达式 `toYYYYMM(date_column)` ，这里的 `date_column` 是一个 [Date](../../../engines/table-engines/mergetree-family/mergetree.md) 类型的列。分区名的格式会是 `"YYYYMM"` 。 要按月分区，可以使用表达式 `toYYYYMM(date_column)` ，这里的 `date_column` 是一个 [Date](../../../engines/table-engines/mergetree-family/mergetree.md) 类型的列。分区名的格式会是 `"YYYYMM"` 。
- `PRIMARY KEY` - 如果要 [选择与排序键不同的主键](#choosing-a-primary-key-that-differs-from-the-sorting-key)，在这里指定，可选项。 - `PRIMARY KEY` - 如果要 [选择与排序键不同的主键](#choosing-a-primary-key-that-differs-from-the-sorting-key)，在这里指定，可选项。

379
programs/CMakeLists.txt
View File

@ -18,11 +18,7 @@ option (ENABLE_CLICKHOUSE_SERVER "Server mode (main mode)" ${ENABLE_CLICKHOUSE_A
option (ENABLE_CLICKHOUSE_CLIENT "Client mode (interactive tui/shell that connects to the server)" option (ENABLE_CLICKHOUSE_CLIENT "Client mode (interactive tui/shell that connects to the server)"
${ENABLE_CLICKHOUSE_ALL}) ${ENABLE_CLICKHOUSE_ALL})
if (CLICKHOUSE_SPLIT_BINARY) option (ENABLE_CLICKHOUSE_SELF_EXTRACTING "Self-extracting executable" ON)
option (ENABLE_CLICKHOUSE_SELF_EXTRACTING "Self-extracting executable" OFF)
else ()
option (ENABLE_CLICKHOUSE_SELF_EXTRACTING "Self-extracting executable" ON)
endif ()
# https://clickhouse.com/docs/en/operations/utilities/clickhouse-local/ # https://clickhouse.com/docs/en/operations/utilities/clickhouse-local/
option (ENABLE_CLICKHOUSE_LOCAL "Local files fast processing mode" ${ENABLE_CLICKHOUSE_ALL}) option (ENABLE_CLICKHOUSE_LOCAL "Local files fast processing mode" ${ENABLE_CLICKHOUSE_ALL})
@ -80,12 +76,7 @@ if (NOT ENABLE_NURAFT)
set(ENABLE_CLICKHOUSE_KEEPER_CONVERTER OFF) set(ENABLE_CLICKHOUSE_KEEPER_CONVERTER OFF)
endif() endif()
if (CLICKHOUSE_SPLIT_BINARY) option(ENABLE_CLICKHOUSE_INSTALL "Install ClickHouse without .deb/.rpm/.tgz packages (having the binary only)" ${ENABLE_CLICKHOUSE_ALL})
option(ENABLE_CLICKHOUSE_INSTALL "Install ClickHouse without .deb/.rpm/.tgz packages (having the binary only)" OFF)
else ()
option(ENABLE_CLICKHOUSE_INSTALL "Install ClickHouse without .deb/.rpm/.tgz packages (having the binary only)"
${ENABLE_CLICKHOUSE_ALL})
endif ()
message(STATUS "ClickHouse modes:") message(STATUS "ClickHouse modes:")
@ -195,6 +186,12 @@ else()
message(STATUS "ClickHouse disks mode: OFF") message(STATUS "ClickHouse disks mode: OFF")
endif() endif()
if (ENABLE_CLICKHOUSE_SU)
message(STATUS "ClickHouse su: ON")
else()
message(STATUS "ClickHouse su: OFF")
endif()
configure_file (config_tools.h.in ${ConfigIncludePath}/config_tools.h) configure_file (config_tools.h.in ${ConfigIncludePath}/config_tools.h)
macro(clickhouse_target_link_split_lib target name) macro(clickhouse_target_link_split_lib target name)
@ -205,10 +202,6 @@ macro(clickhouse_target_link_split_lib target name)
endif() endif()
endmacro() endmacro()
macro(clickhouse_program_link_split_binary name)
clickhouse_target_link_split_lib(clickhouse-${name} ${name})
endmacro()
macro(clickhouse_program_add_library name) macro(clickhouse_program_add_library name)
string(TOUPPER ${name} name_uc) string(TOUPPER ${name} name_uc)
string(REPLACE "-" "_" name_uc ${name_uc}) string(REPLACE "-" "_" name_uc ${name_uc})
@ -233,17 +226,8 @@ macro(clickhouse_program_add_library name)
endif() endif()
endmacro() endmacro()
macro(clickhouse_program_add_executable name)
if(CLICKHOUSE_SPLIT_BINARY)
clickhouse_add_executable(clickhouse-${name} clickhouse-${name}.cpp)
clickhouse_program_link_split_binary(${name})
install(TARGETS clickhouse-${name} ${CLICKHOUSE_ALL_TARGETS} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
endif()
endmacro()
macro(clickhouse_program_add name) macro(clickhouse_program_add name)
clickhouse_program_add_library(${name}) clickhouse_program_add_library(${name})
clickhouse_program_add_executable(${name})
endmacro() endmacro()
add_subdirectory (server) add_subdirectory (server)
@ -336,206 +320,173 @@ if (CLICKHOUSE_ONE_SHARED)
install (TARGETS clickhouse-lib LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT clickhouse) install (TARGETS clickhouse-lib LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT clickhouse)
endif() endif()
if (CLICKHOUSE_SPLIT_BINARY) clickhouse_add_executable (clickhouse main.cpp)
set (CLICKHOUSE_ALL_TARGETS
clickhouse-server
clickhouse-client
clickhouse-local
clickhouse-benchmark
clickhouse-extract-from-config
clickhouse-compressor
clickhouse-format
clickhouse-obfuscator
clickhouse-git-import
clickhouse-copier
clickhouse-static-files-disk-uploader
clickhouse-disks)
if (ENABLE_CLICKHOUSE_ODBC_BRIDGE) if (NOT USE_STATIC_LIBRARIES AND SPLIT_SHARED_LIBRARIES)
list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-odbc-bridge) # Shared split (dev) build: In CI, the server is run with custom LD_LIBRARY_PATH. This makes the harmful env check re-execute the
endif () # process in a clean environment but as in CI the containing directory is not included in DT_RUNPATH/DT_RPATH, the server won't come up.
target_compile_definitions(clickhouse PRIVATE DISABLE_HARMFUL_ENV_VAR_CHECK)
endif ()
if (ENABLE_CLICKHOUSE_LIBRARY_BRIDGE) # A library that prevent usage of several functions from libc.
list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-library-bridge) if (ARCH_AMD64 AND OS_LINUX AND NOT OS_ANDROID)
endif () set (HARMFUL_LIB harmful)
endif ()
if (ENABLE_CLICKHOUSE_KEEPER) target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils ${HARMFUL_LIB})
list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-keeper) target_include_directories (clickhouse PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
endif ()
if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER) if (ENABLE_CLICKHOUSE_SERVER)
list (APPEND CLICKHOUSE_ALL_TARGETS clickhouse-keeper-converter) clickhouse_target_link_split_lib(clickhouse server)
endif () endif ()
if (ENABLE_CLICKHOUSE_CLIENT)
clickhouse_target_link_split_lib(clickhouse client)
endif ()
if (ENABLE_CLICKHOUSE_LOCAL)
clickhouse_target_link_split_lib(clickhouse local)
endif ()
if (ENABLE_CLICKHOUSE_BENCHMARK)
clickhouse_target_link_split_lib(clickhouse benchmark)
endif ()
if (ENABLE_CLICKHOUSE_COPIER)
clickhouse_target_link_split_lib(clickhouse copier)
endif ()
if (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG)
clickhouse_target_link_split_lib(clickhouse extract-from-config)
endif ()
if (ENABLE_CLICKHOUSE_COMPRESSOR)
clickhouse_target_link_split_lib(clickhouse compressor)
endif ()
if (ENABLE_CLICKHOUSE_FORMAT)
clickhouse_target_link_split_lib(clickhouse format)
endif ()
if (ENABLE_CLICKHOUSE_OBFUSCATOR)
clickhouse_target_link_split_lib(clickhouse obfuscator)
endif ()
if (ENABLE_CLICKHOUSE_GIT_IMPORT)
clickhouse_target_link_split_lib(clickhouse git-import)
endif ()
if (ENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER)
clickhouse_target_link_split_lib(clickhouse static-files-disk-uploader)
endif ()
if (ENABLE_CLICKHOUSE_SU)
clickhouse_target_link_split_lib(clickhouse su)
endif ()
if (ENABLE_CLICKHOUSE_KEEPER)
clickhouse_target_link_split_lib(clickhouse keeper)
endif()
if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
clickhouse_target_link_split_lib(clickhouse keeper-converter)
endif()
if (ENABLE_CLICKHOUSE_INSTALL)
clickhouse_target_link_split_lib(clickhouse install)
endif ()
if (ENABLE_CLICKHOUSE_DISKS)
clickhouse_target_link_split_lib(clickhouse disks)
endif ()
set_target_properties(${CLICKHOUSE_ALL_TARGETS} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ..) set (CLICKHOUSE_BUNDLE)
if (ENABLE_CLICKHOUSE_SELF_EXTRACTING)
list(APPEND CLICKHOUSE_BUNDLE self-extracting)
endif ()
if (ENABLE_CLICKHOUSE_SERVER)
add_custom_target (clickhouse-server ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-server DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-server" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-server)
endif ()
if (ENABLE_CLICKHOUSE_CLIENT)
add_custom_target (clickhouse-client ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-client DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-client" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-client)
endif ()
if (ENABLE_CLICKHOUSE_LOCAL)
add_custom_target (clickhouse-local ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-local DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-local" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-local)
endif ()
if (ENABLE_CLICKHOUSE_BENCHMARK)
add_custom_target (clickhouse-benchmark ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-benchmark DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-benchmark" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-benchmark)
endif ()
if (ENABLE_CLICKHOUSE_COPIER)
add_custom_target (clickhouse-copier ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-copier DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-copier" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-copier)
endif ()
if (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG)
add_custom_target (clickhouse-extract-from-config ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-extract-from-config DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-extract-from-config" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-extract-from-config)
endif ()
if (ENABLE_CLICKHOUSE_COMPRESSOR)
add_custom_target (clickhouse-compressor ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-compressor DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-compressor" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-compressor)
endif ()
if (ENABLE_CLICKHOUSE_FORMAT)
add_custom_target (clickhouse-format ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-format DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-format" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-format)
endif ()
if (ENABLE_CLICKHOUSE_OBFUSCATOR)
add_custom_target (clickhouse-obfuscator ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-obfuscator DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-obfuscator" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-obfuscator)
endif ()
if (ENABLE_CLICKHOUSE_GIT_IMPORT)
add_custom_target (clickhouse-git-import ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-git-import DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-git-import" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-git-import)
endif ()
if (ENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER)
add_custom_target (clickhouse-static-files-disk-uploader ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-static-files-disk-uploader DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-static-files-disk-uploader" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-static-files-disk-uploader)
endif ()
if (ENABLE_CLICKHOUSE_SU)
add_custom_target (clickhouse-su ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-su DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-su" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-su)
endif ()
add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_ALL_TARGETS}) if (ENABLE_CLICKHOUSE_KEEPER)
add_custom_target (clickhouse ALL DEPENDS clickhouse-bundle) if (NOT BUILD_STANDALONE_KEEPER AND CREATE_KEEPER_SYMLINK)
add_custom_target (clickhouse-keeper ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-keeper DEPENDS clickhouse)
install(PROGRAMS clickhouse-split-helper DESTINATION ${CMAKE_INSTALL_BINDIR} RENAME clickhouse COMPONENT clickhouse) install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
else ()
clickhouse_add_executable (clickhouse main.cpp)
# A library that prevent usage of several functions from libc.
if (ARCH_AMD64 AND OS_LINUX AND NOT OS_ANDROID)
set (HARMFUL_LIB harmful)
endif ()
target_link_libraries (clickhouse PRIVATE clickhouse_common_io string_utils ${HARMFUL_LIB})
target_include_directories (clickhouse PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
if (ENABLE_CLICKHOUSE_SERVER)
clickhouse_target_link_split_lib(clickhouse server)
endif ()
if (ENABLE_CLICKHOUSE_CLIENT)
clickhouse_target_link_split_lib(clickhouse client)
endif ()
if (ENABLE_CLICKHOUSE_LOCAL)
clickhouse_target_link_split_lib(clickhouse local)
endif ()
if (ENABLE_CLICKHOUSE_BENCHMARK)
clickhouse_target_link_split_lib(clickhouse benchmark)
endif ()
if (ENABLE_CLICKHOUSE_COPIER)
clickhouse_target_link_split_lib(clickhouse copier)
endif ()
if (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG)
clickhouse_target_link_split_lib(clickhouse extract-from-config)
endif ()
if (ENABLE_CLICKHOUSE_COMPRESSOR)
clickhouse_target_link_split_lib(clickhouse compressor)
endif ()
if (ENABLE_CLICKHOUSE_FORMAT)
clickhouse_target_link_split_lib(clickhouse format)
endif ()
if (ENABLE_CLICKHOUSE_OBFUSCATOR)
clickhouse_target_link_split_lib(clickhouse obfuscator)
endif ()
if (ENABLE_CLICKHOUSE_GIT_IMPORT)
clickhouse_target_link_split_lib(clickhouse git-import)
endif ()
if (ENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER)
clickhouse_target_link_split_lib(clickhouse static-files-disk-uploader)
endif ()
if (ENABLE_CLICKHOUSE_SU)
clickhouse_target_link_split_lib(clickhouse su)
endif ()
if (ENABLE_CLICKHOUSE_KEEPER)
clickhouse_target_link_split_lib(clickhouse keeper)
endif()
if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
clickhouse_target_link_split_lib(clickhouse keeper-converter)
endif()
if (ENABLE_CLICKHOUSE_INSTALL)
clickhouse_target_link_split_lib(clickhouse install)
endif ()
if (ENABLE_CLICKHOUSE_DISKS)
clickhouse_target_link_split_lib(clickhouse disks)
endif ()
set (CLICKHOUSE_BUNDLE)
if (ENABLE_CLICKHOUSE_SELF_EXTRACTING)
list(APPEND CLICKHOUSE_BUNDLE self-extracting)
endif ()
if (ENABLE_CLICKHOUSE_SERVER)
add_custom_target (clickhouse-server ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-server DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-server" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-server)
endif ()
if (ENABLE_CLICKHOUSE_CLIENT)
add_custom_target (clickhouse-client ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-client DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-client" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-client)
endif ()
if (ENABLE_CLICKHOUSE_LOCAL)
add_custom_target (clickhouse-local ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-local DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-local" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-local)
endif ()
if (ENABLE_CLICKHOUSE_BENCHMARK)
add_custom_target (clickhouse-benchmark ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-benchmark DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-benchmark" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-benchmark)
endif ()
if (ENABLE_CLICKHOUSE_COPIER)
add_custom_target (clickhouse-copier ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-copier DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-copier" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-copier)
endif ()
if (ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG)
add_custom_target (clickhouse-extract-from-config ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-extract-from-config DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-extract-from-config" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-extract-from-config)
endif ()
if (ENABLE_CLICKHOUSE_COMPRESSOR)
add_custom_target (clickhouse-compressor ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-compressor DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-compressor" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-compressor)
endif ()
if (ENABLE_CLICKHOUSE_FORMAT)
add_custom_target (clickhouse-format ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-format DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-format" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-format)
endif ()
if (ENABLE_CLICKHOUSE_OBFUSCATOR)
add_custom_target (clickhouse-obfuscator ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-obfuscator DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-obfuscator" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-obfuscator)
endif ()
if (ENABLE_CLICKHOUSE_GIT_IMPORT)
add_custom_target (clickhouse-git-import ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-git-import DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-git-import" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-git-import)
endif ()
if (ENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER)
add_custom_target (clickhouse-static-files-disk-uploader ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-static-files-disk-uploader DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-static-files-disk-uploader" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-static-files-disk-uploader)
endif ()
if (ENABLE_CLICKHOUSE_SU)
add_custom_target (clickhouse-su ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-su DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-su" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-su)
endif ()
if (ENABLE_CLICKHOUSE_KEEPER)
if (NOT BUILD_STANDALONE_KEEPER AND CREATE_KEEPER_SYMLINK)
add_custom_target (clickhouse-keeper ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-keeper DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
endif()
# otherwise we don't build keeper
if (BUILD_STANDALONE_KEEPER OR CREATE_KEEPER_SYMLINK)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-keeper)
endif()
endif ()
if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
add_custom_target (clickhouse-keeper-converter ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-keeper-converter DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper-converter" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-keeper-converter)
endif ()
if (ENABLE_CLICKHOUSE_DISKS)
add_custom_target (clickhouse-disks ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-disks DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-disks" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-disks)
endif ()
add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_BUNDLE})
if (USE_GDB_ADD_INDEX)
add_custom_command(TARGET clickhouse POST_BUILD COMMAND ${GDB_ADD_INDEX_EXE} clickhouse COMMENT "Adding .gdb-index to clickhouse" VERBATIM)
endif() endif()
if (USE_BINARY_HASH) # otherwise we don't build keeper
add_custom_command(TARGET clickhouse POST_BUILD COMMAND ./clickhouse hash-binary > hash && ${OBJCOPY_PATH} --add-section .clickhouse.hash=hash clickhouse COMMENT "Adding section '.clickhouse.hash' to clickhouse binary" VERBATIM) if (BUILD_STANDALONE_KEEPER OR CREATE_KEEPER_SYMLINK)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-keeper)
endif() endif()
endif ()
if (ENABLE_CLICKHOUSE_KEEPER_CONVERTER)
add_custom_target (clickhouse-keeper-converter ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-keeper-converter DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-keeper-converter" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-keeper-converter)
endif ()
if (ENABLE_CLICKHOUSE_DISKS)
add_custom_target (clickhouse-disks ALL COMMAND ${CMAKE_COMMAND} -E create_symlink clickhouse clickhouse-disks DEPENDS clickhouse)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-disks" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-disks)
endif ()
if (SPLIT_DEBUG_SYMBOLS) add_custom_target (clickhouse-bundle ALL DEPENDS ${CLICKHOUSE_BUNDLE})
clickhouse_split_debug_symbols(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${SPLITTED_DEBUG_SYMBOLS_DIR} BINARY_PATH clickhouse)
else() if (USE_GDB_ADD_INDEX)
clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${SPLITTED_DEBUG_SYMBOLS_DIR}) add_custom_command(TARGET clickhouse POST_BUILD COMMAND ${GDB_ADD_INDEX_EXE} clickhouse COMMENT "Adding .gdb-index to clickhouse" VERBATIM)
install (TARGETS clickhouse RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse) endif()
endif()
if (USE_BINARY_HASH)
add_custom_command(TARGET clickhouse POST_BUILD COMMAND ./clickhouse hash-binary > hash && ${OBJCOPY_PATH} --add-section .clickhouse.hash=hash clickhouse COMMENT "Adding section '.clickhouse.hash' to clickhouse binary" VERBATIM)
endif()
if (SPLIT_DEBUG_SYMBOLS)
clickhouse_split_debug_symbols(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${SPLITTED_DEBUG_SYMBOLS_DIR} BINARY_PATH clickhouse)
else()
clickhouse_make_empty_debug_info_for_nfpm(TARGET clickhouse DESTINATION_DIR ${CMAKE_CURRENT_BINARY_DIR}/${SPLITTED_DEBUG_SYMBOLS_DIR})
install (TARGETS clickhouse RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
endif() endif()
if (ENABLE_TESTS) if (ENABLE_TESTS)

3
programs/local/LocalServer.cpp
View File

@ -352,6 +352,9 @@ try
{ {
UseSSL use_ssl; UseSSL use_ssl;
ThreadStatus thread_status; ThreadStatus thread_status;
StackTrace::setShowAddresses(config().getBool("show_addresses_in_stack_traces", true));
setupSignalHandler(); setupSignalHandler();
std::cout << std::fixed << std::setprecision(3); std::cout << std::fixed << std::setprecision(3);

4
programs/main.cpp
View File

@ -345,6 +345,7 @@ struct Checker
; ;
#ifndef DISABLE_HARMFUL_ENV_VAR_CHECK
/// NOTE: We will migrate to full static linking or our own dynamic loader to make this code obsolete. /// NOTE: We will migrate to full static linking or our own dynamic loader to make this code obsolete.
void checkHarmfulEnvironmentVariables(char ** argv) void checkHarmfulEnvironmentVariables(char ** argv)
{ {
@ -396,6 +397,7 @@ void checkHarmfulEnvironmentVariables(char ** argv)
_exit(error); _exit(error);
} }
} }
#endif
} }
@ -422,7 +424,9 @@ int main(int argc_, char ** argv_)
/// will work only after additional call of this function. /// will work only after additional call of this function.
updatePHDRCache(); updatePHDRCache();
#ifndef DISABLE_HARMFUL_ENV_VAR_CHECK
checkHarmfulEnvironmentVariables(argv_); checkHarmfulEnvironmentVariables(argv_);
#endif
/// Reset new handler to default (that throws std::bad_alloc) /// Reset new handler to default (that throws std::bad_alloc)
/// It is needed because LLVM library clobbers it. /// It is needed because LLVM library clobbers it.

3
programs/obfuscator/Obfuscator.cpp
View File

@ -46,6 +46,7 @@
#include <boost/algorithm/string.hpp> #include <boost/algorithm/string.hpp>
#include <boost/container/flat_map.hpp> #include <boost/container/flat_map.hpp>
#include <Common/TerminalSize.h> #include <Common/TerminalSize.h>
#include <bit>
static const char * documentation = R"( static const char * documentation = R"(
@ -186,7 +187,7 @@ static UInt64 transform(UInt64 x, UInt64 seed)
if (x == 2 || x == 3) if (x == 2 || x == 3)
return x ^ (seed & 1); return x ^ (seed & 1);
size_t num_leading_zeros = __builtin_clzll(x); size_t num_leading_zeros = std::countl_zero(x);
return feistelNetwork(x, 64 - num_leading_zeros - 1, seed); return feistelNetwork(x, 64 - num_leading_zeros - 1, seed);
} }

117
programs/server/Server.cpp
View File

@ -367,7 +367,7 @@ Poco::Net::SocketAddress Server::socketBindListen(
return address; return address;
} }
std::vector<std::string> getListenHosts(const Poco::Util::AbstractConfiguration & config) Strings getListenHosts(const Poco::Util::AbstractConfiguration & config)
{ {
auto listen_hosts = DB::getMultipleValuesFromConfig(config, "", "listen_host"); auto listen_hosts = DB::getMultipleValuesFromConfig(config, "", "listen_host");
if (listen_hosts.empty()) if (listen_hosts.empty())
@ -378,6 +378,16 @@ std::vector<std::string> getListenHosts(const Poco::Util::AbstractConfiguration
return listen_hosts; return listen_hosts;
} }
Strings getInterserverListenHosts(const Poco::Util::AbstractConfiguration & config)
{
auto interserver_listen_hosts = DB::getMultipleValuesFromConfig(config, "", "interserver_listen_host");
if (!interserver_listen_hosts.empty())
return interserver_listen_hosts;
/// Use more general restriction in case of emptiness
return getListenHosts(config);
}
bool getListenTry(const Poco::Util::AbstractConfiguration & config) bool getListenTry(const Poco::Util::AbstractConfiguration & config)
{ {
bool listen_try = config.getBool("listen_try", false); bool listen_try = config.getBool("listen_try", false);
@ -627,6 +637,8 @@ int Server::main(const std::vector<std::string> & /*args*/)
MainThreadStatus::getInstance(); MainThreadStatus::getInstance();
StackTrace::setShowAddresses(config().getBool("show_addresses_in_stack_traces", true));
registerFunctions(); registerFunctions();
registerAggregateFunctions(); registerAggregateFunctions();
registerTableFunctions(); registerTableFunctions();
@ -1232,6 +1244,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
/* already_loaded = */ false); /// Reload it right now (initial loading) /* already_loaded = */ false); /// Reload it right now (initial loading)
const auto listen_hosts = getListenHosts(config()); const auto listen_hosts = getListenHosts(config());
const auto interserver_listen_hosts = getInterserverListenHosts(config());
const auto listen_try = getListenTry(config()); const auto listen_try = getListenTry(config());
if (config().has("keeper_server")) if (config().has("keeper_server"))
@ -1627,7 +1640,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
{ {
std::lock_guard lock(servers_lock); std::lock_guard lock(servers_lock);
createServers(config(), listen_hosts, listen_try, server_pool, async_metrics, servers); createServers(config(), listen_hosts, interserver_listen_hosts, listen_try, server_pool, async_metrics, servers);
if (servers.empty()) if (servers.empty())
throw Exception( throw Exception(
"No servers started (add valid listen_host and 'tcp_port' or 'http_port' to configuration file.)", "No servers started (add valid listen_host and 'tcp_port' or 'http_port' to configuration file.)",
@ -1809,7 +1822,8 @@ int Server::main(const std::vector<std::string> & /*args*/)
void Server::createServers( void Server::createServers(
Poco::Util::AbstractConfiguration & config, Poco::Util::AbstractConfiguration & config,
const std::vector<std::string> & listen_hosts, const Strings & listen_hosts,
const Strings & interserver_listen_hosts,
bool listen_try, bool listen_try,
Poco::ThreadPool & server_pool, Poco::ThreadPool & server_pool,
AsynchronousMetrics & async_metrics, AsynchronousMetrics & async_metrics,
@ -1927,51 +1941,6 @@ void Server::createServers(
#endif #endif
}); });
/// Interserver IO HTTP
port_name = "interserver_http_port";
createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter
{
Poco::Net::ServerSocket socket;
auto address = socketBindListen(config, socket, listen_host, port);
socket.setReceiveTimeout(settings.http_receive_timeout);
socket.setSendTimeout(settings.http_send_timeout);
return ProtocolServerAdapter(
listen_host,
port_name,
"replica communication (interserver): http://" + address.toString(),
std::make_unique<HTTPServer>(
context(),
createHandlerFactory(*this, async_metrics, "InterserverIOHTTPHandler-factory"),
server_pool,
socket,
http_params));
});
port_name = "interserver_https_port";
createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter
{
#if USE_SSL
Poco::Net::SecureServerSocket socket;
auto address = socketBindListen(config, socket, listen_host, port, /* secure = */ true);
socket.setReceiveTimeout(settings.http_receive_timeout);
socket.setSendTimeout(settings.http_send_timeout);
return ProtocolServerAdapter(
listen_host,
port_name,
"secure replica communication (interserver): https://" + address.toString(),
std::make_unique<HTTPServer>(
context(),
createHandlerFactory(*this, async_metrics, "InterserverIOHTTPSHandler-factory"),
server_pool,
socket,
http_params));
#else
UNUSED(port);
throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.",
ErrorCodes::SUPPORT_IS_DISABLED};
#endif
});
port_name = "mysql_port"; port_name = "mysql_port";
createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter createServer(config, listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter
{ {
@ -2030,6 +1999,55 @@ void Server::createServers(
}); });
} }
/// Now iterate over interserver_listen_hosts
for (const auto & interserver_listen_host : interserver_listen_hosts)
{
/// Interserver IO HTTP
const char * port_name = "interserver_http_port";
createServer(config, interserver_listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter
{
Poco::Net::ServerSocket socket;
auto address = socketBindListen(config, socket, interserver_listen_host, port);
socket.setReceiveTimeout(settings.http_receive_timeout);
socket.setSendTimeout(settings.http_send_timeout);
return ProtocolServerAdapter(
interserver_listen_host,
port_name,
"replica communication (interserver): http://" + address.toString(),
std::make_unique<HTTPServer>(
context(),
createHandlerFactory(*this, async_metrics, "InterserverIOHTTPHandler-factory"),
server_pool,
socket,
http_params));
});
port_name = "interserver_https_port";
createServer(config, interserver_listen_host, port_name, listen_try, start_servers, servers, [&](UInt16 port) -> ProtocolServerAdapter
{
#if USE_SSL
Poco::Net::SecureServerSocket socket;
auto address = socketBindListen(config, socket, interserver_listen_host, port, /* secure = */ true);
socket.setReceiveTimeout(settings.http_receive_timeout);
socket.setSendTimeout(settings.http_send_timeout);
return ProtocolServerAdapter(
interserver_listen_host,
port_name,
"secure replica communication (interserver): https://" + address.toString(),
std::make_unique<HTTPServer>(
context(),
createHandlerFactory(*this, async_metrics, "InterserverIOHTTPSHandler-factory"),
server_pool,
socket,
http_params));
#else
UNUSED(port);
throw Exception{"SSL support for TCP protocol is disabled because Poco library was built without NetSSL support.",
ErrorCodes::SUPPORT_IS_DISABLED};
#endif
});
}
} }
void Server::updateServers( void Server::updateServers(
@ -2041,6 +2059,7 @@ void Server::updateServers(
Poco::Logger * log = &logger(); Poco::Logger * log = &logger();
const auto listen_hosts = getListenHosts(config); const auto listen_hosts = getListenHosts(config);
const auto interserver_listen_hosts = getInterserverListenHosts(config);
const auto listen_try = getListenTry(config); const auto listen_try = getListenTry(config);
/// Remove servers once all their connections are closed /// Remove servers once all their connections are closed
@ -2073,7 +2092,7 @@ void Server::updateServers(
} }
} }
createServers(config, listen_hosts, listen_try, server_pool, async_metrics, servers, /* start_servers= */ true); createServers(config, listen_hosts, interserver_listen_hosts, listen_try, server_pool, async_metrics, servers, /* start_servers= */ true);
std::erase_if(servers, std::bind_front(check_server, "")); std::erase_if(servers, std::bind_front(check_server, ""));
} }

3
programs/server/Server.h
View File

@ -86,7 +86,8 @@ private:
void createServers( void createServers(
Poco::Util::AbstractConfiguration & config, Poco::Util::AbstractConfiguration & config,
const std::vector<std::string> & listen_hosts, const Strings & listen_hosts,
const Strings & interserver_listen_hosts,
bool listen_try, bool listen_try,
Poco::ThreadPool & server_pool, Poco::ThreadPool & server_pool,
AsynchronousMetrics & async_metrics, AsynchronousMetrics & async_metrics,

13
programs/server/config.xml
View File

@ -188,6 +188,10 @@
<listen_host>127.0.0.1</listen_host> <listen_host>127.0.0.1</listen_host>
--> -->
<!-- <interserver_listen_host>::</interserver_listen_host> -->
<!-- Listen host for communication between replicas. Used for data exchange -->
<!-- Default values - equal to listen_host -->
<!-- Don't exit if IPv6 or IPv4 networks are unavailable while trying to listen. --> <!-- Don't exit if IPv6 or IPv4 networks are unavailable while trying to listen. -->
<!-- <listen_try>0</listen_try> --> <!-- <listen_try>0</listen_try> -->
@ -1386,4 +1390,13 @@
<lru_cache_size>268435456</lru_cache_size> <lru_cache_size>268435456</lru_cache_size>
<continue_if_corrupted>true</continue_if_corrupted> <continue_if_corrupted>true</continue_if_corrupted>
</merge_tree_metadata_cache--> </merge_tree_metadata_cache-->
<!-- This allows to disable exposing addresses in stack traces for security reasons.
Please be aware that it does not improve security much, but makes debugging much harder.
The addresses that are small offsets from zero will be displayed nevertheless to show nullptr dereferences.
Regardless of this configuration, the addresses are visible in the system.stack_trace and system.trace_log tables
if the user has access to these tables.
I don't recommend to change this setting.
-->
<show_addresses_in_stack_traces>false</show_addresses_in_stack_traces>
</clickhouse> </clickhouse>

2
programs/su/CMakeLists.txt
View File

@ -1,3 +1,3 @@
set (CLICKHOUSE_SU_SOURCES clickhouse-su.cpp) set (CLICKHOUSE_SU_SOURCES clickhouse-su.cpp su.cpp)
set (CLICKHOUSE_SU_LINK PRIVATE dbms) set (CLICKHOUSE_SU_LINK PRIVATE dbms)
clickhouse_program_add(su) clickhouse_program_add(su)

147
programs/su/clickhouse-su.cpp
View File

@ -1,145 +1,2 @@
#include <Common/Exception.h> int mainEntryClickHouseSU(int argc, char ** argv);
#include <IO/ReadHelpers.h> int main(int argc_, char ** argv_) { return mainEntryClickHouseSU(argc_, argv_); }
#include <fmt/format.h>
#include <vector>
#include <sys/types.h>
#include <unistd.h>
#include <pwd.h>
#include <grp.h>
/// "su" means "set user"
/// In fact, this program can set Unix user and group.
///
/// Usage:
/// clickhouse su user[:group] args...
///
/// - will set user and, optionally, group and exec the remaining args.
/// user and group can be numeric identifiers or strings.
///
/// The motivation for this tool is very obscure and idiosyncratic. It is needed for Docker.
/// People want to run programs inside Docker with dropped privileges (less than root).
/// But the standard Linux "su" program is not suitable for usage inside Docker,
/// because it is creating pseudoterminals to avoid hijacking input from the terminal, for security,
/// but Docker is also doing something with the terminal and it is incompatible.
/// For this reason, people use alternative and less "secure" versions of "su" tools like "gosu" or "su-exec".
/// But it would be very strange to use 3rd-party software only to do two-three syscalls.
/// That's why we provide this tool.
///
/// Note: ClickHouse does not need Docker at all and works better without Docker.
/// ClickHouse has no dependencies, it is packaged and distributed in single binary.
/// There is no reason to use Docker unless you are already running all your software in Docker.
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
extern const int SYSTEM_ERROR;
}
void setUserAndGroup(std::string arg_uid, std::string arg_gid)
{
static constexpr size_t buf_size = 16384; /// Linux man page says it is enough. Nevertheless, we will check if it's not enough and throw.
std::unique_ptr<char[]> buf(new char[buf_size]);
/// Set the group first, because if we set user, the privileges will be already dropped and we will not be able to set the group later.
if (!arg_gid.empty())
{
gid_t gid = 0;
if (!tryParse(gid, arg_gid) || gid == 0)
{
group entry{};
group * result{};
if (0 != getgrnam_r(arg_gid.data(), &entry, buf.get(), buf_size, &result))
throwFromErrno(fmt::format("Cannot do 'getgrnam_r' to obtain gid from group name ({})", arg_gid), ErrorCodes::SYSTEM_ERROR);
if (!result)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Group {} is not found in the system", arg_gid);
gid = entry.gr_gid;
}
if (gid == 0 && getgid() != 0)
throw Exception("Group has id 0, but dropping privileges to gid 0 does not make sense", ErrorCodes::BAD_ARGUMENTS);
if (0 != setgid(gid))
throwFromErrno(fmt::format("Cannot do 'setgid' to user ({})", arg_gid), ErrorCodes::SYSTEM_ERROR);
}
if (!arg_uid.empty())
{
/// Is it numeric id or name?
uid_t uid = 0;
if (!tryParse(uid, arg_uid) || uid == 0)
{
passwd entry{};
passwd * result{};
if (0 != getpwnam_r(arg_uid.data(), &entry, buf.get(), buf_size, &result))
throwFromErrno(fmt::format("Cannot do 'getpwnam_r' to obtain uid from user name ({})", arg_uid), ErrorCodes::SYSTEM_ERROR);
if (!result)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "User {} is not found in the system", arg_uid);
uid = entry.pw_uid;
}
if (uid == 0 && getuid() != 0)
throw Exception("User has id 0, but dropping privileges to uid 0 does not make sense", ErrorCodes::BAD_ARGUMENTS);
if (0 != setuid(uid))
throwFromErrno(fmt::format("Cannot do 'setuid' to user ({})", arg_uid), ErrorCodes::SYSTEM_ERROR);
}
}
}
int mainEntryClickHouseSU(int argc, char ** argv)
try
{
using namespace DB;
if (argc < 3)
{
std::cout << "Usage: ./clickhouse su user:group ..." << std::endl;
exit(0);
}
std::string_view user_and_group = argv[1];
std::string user;
std::string group;
auto pos = user_and_group.find(':');
if (pos == std::string_view::npos)
{
user = user_and_group;
}
else
{
user = user_and_group.substr(0, pos);
group = user_and_group.substr(pos + 1);
}
setUserAndGroup(std::move(user), std::move(group));
std::vector<char *> new_argv;
new_argv.reserve(argc - 1);
new_argv.insert(new_argv.begin(), argv + 2, argv + argc);
new_argv.push_back(nullptr);
execvp(new_argv.front(), new_argv.data());
throwFromErrno("Cannot execvp", ErrorCodes::SYSTEM_ERROR);
}
catch (...)
{
std::cerr << DB::getCurrentExceptionMessage(false) << '\n';
return 1;
}

145
programs/su/su.cpp Normal file
View File

@ -0,0 +1,145 @@
#include <Common/Exception.h>
#include <IO/ReadHelpers.h>
#include <fmt/format.h>
#include <vector>
#include <sys/types.h>
#include <unistd.h>
#include <pwd.h>
#include <grp.h>
/// "su" means "set user"
/// In fact, this program can set Unix user and group.
///
/// Usage:
/// clickhouse su user[:group] args...
///
/// - will set user and, optionally, group and exec the remaining args.
/// user and group can be numeric identifiers or strings.
///
/// The motivation for this tool is very obscure and idiosyncratic. It is needed for Docker.
/// People want to run programs inside Docker with dropped privileges (less than root).
/// But the standard Linux "su" program is not suitable for usage inside Docker,
/// because it is creating pseudoterminals to avoid hijacking input from the terminal, for security,
/// but Docker is also doing something with the terminal and it is incompatible.
/// For this reason, people use alternative and less "secure" versions of "su" tools like "gosu" or "su-exec".
/// But it would be very strange to use 3rd-party software only to do two-three syscalls.
/// That's why we provide this tool.
///
/// Note: ClickHouse does not need Docker at all and works better without Docker.
/// ClickHouse has no dependencies, it is packaged and distributed in single binary.
/// There is no reason to use Docker unless you are already running all your software in Docker.
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
extern const int SYSTEM_ERROR;
}
void setUserAndGroup(std::string arg_uid, std::string arg_gid)
{
static constexpr size_t buf_size = 16384; /// Linux man page says it is enough. Nevertheless, we will check if it's not enough and throw.
std::unique_ptr<char[]> buf(new char[buf_size]);
/// Set the group first, because if we set user, the privileges will be already dropped and we will not be able to set the group later.
if (!arg_gid.empty())
{
gid_t gid = 0;
if (!tryParse(gid, arg_gid) || gid == 0)
{
group entry{};
group * result{};
if (0 != getgrnam_r(arg_gid.data(), &entry, buf.get(), buf_size, &result))
throwFromErrno(fmt::format("Cannot do 'getgrnam_r' to obtain gid from group name ({})", arg_gid), ErrorCodes::SYSTEM_ERROR);
if (!result)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Group {} is not found in the system", arg_gid);
gid = entry.gr_gid;
}
if (gid == 0 && getgid() != 0)
throw Exception("Group has id 0, but dropping privileges to gid 0 does not make sense", ErrorCodes::BAD_ARGUMENTS);
if (0 != setgid(gid))
throwFromErrno(fmt::format("Cannot do 'setgid' to user ({})", arg_gid), ErrorCodes::SYSTEM_ERROR);
}
if (!arg_uid.empty())
{
/// Is it numeric id or name?
uid_t uid = 0;
if (!tryParse(uid, arg_uid) || uid == 0)
{
passwd entry{};
passwd * result{};
if (0 != getpwnam_r(arg_uid.data(), &entry, buf.get(), buf_size, &result))
throwFromErrno(fmt::format("Cannot do 'getpwnam_r' to obtain uid from user name ({})", arg_uid), ErrorCodes::SYSTEM_ERROR);
if (!result)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "User {} is not found in the system", arg_uid);
uid = entry.pw_uid;
}
if (uid == 0 && getuid() != 0)
throw Exception("User has id 0, but dropping privileges to uid 0 does not make sense", ErrorCodes::BAD_ARGUMENTS);
if (0 != setuid(uid))
throwFromErrno(fmt::format("Cannot do 'setuid' to user ({})", arg_uid), ErrorCodes::SYSTEM_ERROR);
}
}
}
int mainEntryClickHouseSU(int argc, char ** argv)
try
{
using namespace DB;
if (argc < 3)
{
std::cout << "Usage: ./clickhouse su user:group ..." << std::endl;
exit(0);
}
std::string_view user_and_group = argv[1];
std::string user;
std::string group;
auto pos = user_and_group.find(':');
if (pos == std::string_view::npos)
{
user = user_and_group;
}
else
{
user = user_and_group.substr(0, pos);
group = user_and_group.substr(pos + 1);
}
setUserAndGroup(std::move(user), std::move(group));
std::vector<char *> new_argv;
new_argv.reserve(argc - 1);
new_argv.insert(new_argv.begin(), argv + 2, argv + argc);
new_argv.push_back(nullptr);
execvp(new_argv.front(), new_argv.data());
throwFromErrno("Cannot execvp", ErrorCodes::SYSTEM_ERROR);
}
catch (...)
{
std::cerr << DB::getCurrentExceptionMessage(false) << '\n';
return 1;
}

2
src/Access/examples/kerberos_init.cpp
View File

@ -25,7 +25,7 @@ int main(int argc, char ** argv)
return 0; return 0;
} }
String cache_name = ""; const char * cache_name = "";
if (argc == 4) if (argc == 4)
cache_name = argv[3]; cache_name = argv[3];

4
src/CMakeLists.txt
View File

@ -144,8 +144,8 @@ endif ()
list (APPEND clickhouse_common_io_sources ${CONFIG_BUILD}) list (APPEND clickhouse_common_io_sources ${CONFIG_BUILD})
list (APPEND clickhouse_common_io_headers ${CONFIG_VERSION} ${CONFIG_COMMON}) list (APPEND clickhouse_common_io_headers ${CONFIG_VERSION} ${CONFIG_COMMON})
list (APPEND dbms_sources Functions/IFunction.cpp Functions/FunctionFactory.cpp Functions/FunctionHelpers.cpp Functions/extractTimeZoneFromFunctionArguments.cpp Functions/replicate.cpp Functions/FunctionsLogical.cpp) list (APPEND dbms_sources Functions/IFunction.cpp Functions/FunctionFactory.cpp Functions/FunctionHelpers.cpp Functions/extractTimeZoneFromFunctionArguments.cpp Functions/FunctionsLogical.cpp)
list (APPEND dbms_headers Functions/IFunction.h Functions/FunctionFactory.h Functions/FunctionHelpers.h Functions/extractTimeZoneFromFunctionArguments.h Functions/replicate.h Functions/FunctionsLogical.h) list (APPEND dbms_headers Functions/IFunction.h Functions/FunctionFactory.h Functions/FunctionHelpers.h Functions/extractTimeZoneFromFunctionArguments.h Functions/FunctionsLogical.h)
list (APPEND dbms_sources list (APPEND dbms_sources
AggregateFunctions/IAggregateFunction.cpp AggregateFunctions/IAggregateFunction.cpp

9
src/Client/ClientBase.cpp
View File

@ -583,9 +583,14 @@ try
if (has_vertical_output_suffix) if (has_vertical_output_suffix)
current_format = "Vertical"; current_format = "Vertical";
/// It is not clear how to write progress intermixed with data with parallel formatting. bool logs_into_stdout = server_logs_file == "-";
bool extras_into_stdout = need_render_progress || logs_into_stdout;
bool select_only_into_file = select_into_file && !select_into_file_and_stdout;
/// It is not clear how to write progress and logs
/// intermixed with data with parallel formatting.
/// It may increase code complexity significantly. /// It may increase code complexity significantly.
if (!need_render_progress || (select_into_file && !select_into_file_and_stdout)) if (!extras_into_stdout || select_only_into_file)
output_format = global_context->getOutputFormatParallelIfPossible( output_format = global_context->getOutputFormatParallelIfPossible(
current_format, out_file_buf ? *out_file_buf : *out_buf, block); current_format, out_file_buf ? *out_file_buf : *out_buf, block);
else else

2
src/Columns/ColumnDecimal.cpp
View File

@ -298,7 +298,7 @@ ColumnPtr ColumnDecimal<T>::filter(const IColumn::Filter & filt, ssize_t result_
{ {
while (mask) while (mask)
{ {
size_t index = __builtin_ctzll(mask); size_t index = std::countr_zero(mask);
res_data.push_back(data_pos[index]); res_data.push_back(data_pos[index]);
#ifdef __BMI__ #ifdef __BMI__
mask = _blsr_u64(mask); mask = _blsr_u64(mask);

2
src/Columns/ColumnFixedString.cpp
View File

@ -240,7 +240,7 @@ ColumnPtr ColumnFixedString::filter(const IColumn::Filter & filt, ssize_t result
size_t res_chars_size = res->chars.size(); size_t res_chars_size = res->chars.size();
while (mask) while (mask)
{ {
size_t index = __builtin_ctzll(mask); size_t index = std::countr_zero(mask);
res->chars.resize(res_chars_size + n); res->chars.resize(res_chars_size + n);
memcpySmallAllowReadWriteOverflow15(&res->chars[res_chars_size], data_pos + index * n, n); memcpySmallAllowReadWriteOverflow15(&res->chars[res_chars_size], data_pos + index * n, n);
res_chars_size += n; res_chars_size += n;

2
src/Columns/ColumnVector.cpp
View File

@ -508,7 +508,7 @@ ColumnPtr ColumnVector<T>::filter(const IColumn::Filter & filt, ssize_t result_s
{ {
while (mask) while (mask)
{ {
size_t index = __builtin_ctzll(mask); size_t index = std::countr_zero(mask);
res_data.push_back(data_pos[index]); res_data.push_back(data_pos[index]);
#ifdef __BMI__ #ifdef __BMI__
mask = _blsr_u64(mask); mask = _blsr_u64(mask);

13
src/Columns/ColumnsCommon.cpp
View File

@ -2,13 +2,14 @@
#include <Columns/ColumnVector.h> #include <Columns/ColumnVector.h>
#include <Common/typeid_cast.h> #include <Common/typeid_cast.h>
#include <Common/HashTable/HashSet.h> #include <Common/HashTable/HashSet.h>
#include <bit>
#include "ColumnsCommon.h" #include "ColumnsCommon.h"
namespace DB namespace DB
{ {
#if defined(__SSE2__) && defined(__POPCNT__) #if defined(__SSE2__)
/// Transform 64-byte mask to 64-bit mask. /// Transform 64-byte mask to 64-bit mask.
static UInt64 toBits64(const Int8 * bytes64) static UInt64 toBits64(const Int8 * bytes64)
{ {
@ -41,11 +42,11 @@ size_t countBytesInFilter(const UInt8 * filt, size_t start, size_t end)
const Int8 * end_pos = pos + (end - start); const Int8 * end_pos = pos + (end - start);
#if defined(__SSE2__) && defined(__POPCNT__) #if defined(__SSE2__)
const Int8 * end_pos64 = pos + (end - start) / 64 * 64; const Int8 * end_pos64 = pos + (end - start) / 64 * 64;
for (; pos < end_pos64; pos += 64) for (; pos < end_pos64; pos += 64)
count += __builtin_popcountll(toBits64(pos)); count += std::popcount(toBits64(pos));
/// TODO Add duff device for tail? /// TODO Add duff device for tail?
#endif #endif
@ -74,11 +75,11 @@ size_t countBytesInFilterWithNull(const IColumn::Filter & filt, const UInt8 * nu
const Int8 * pos2 = reinterpret_cast<const Int8 *>(null_map) + start; const Int8 * pos2 = reinterpret_cast<const Int8 *>(null_map) + start;
const Int8 * end_pos = pos + (end - start); const Int8 * end_pos = pos + (end - start);
#if defined(__SSE2__) && defined(__POPCNT__) #if defined(__SSE2__)
const Int8 * end_pos64 = pos + (end - start) / 64 * 64; const Int8 * end_pos64 = pos + (end - start) / 64 * 64;
for (; pos < end_pos64; pos += 64, pos2 += 64) for (; pos < end_pos64; pos += 64, pos2 += 64)
count += __builtin_popcountll(toBits64(pos) & ~toBits64(pos2)); count += std::popcount(toBits64(pos) & ~toBits64(pos2));
/// TODO Add duff device for tail? /// TODO Add duff device for tail?
#endif #endif
@ -259,7 +260,7 @@ namespace
{ {
while (mask) while (mask)
{ {
size_t index = __builtin_ctzll(mask); size_t index = std::countr_zero(mask);
copy_array(offsets_pos + index); copy_array(offsets_pos + index);
#ifdef __BMI__ #ifdef __BMI__
mask = _blsr_u64(mask); mask = _blsr_u64(mask);

2
src/Columns/ColumnsCommon.h
View File

@ -36,7 +36,7 @@ inline UInt64 bytes64MaskToBits64Mask(const UInt8 * bytes64)
_mm256_loadu_si256(reinterpret_cast<const __m256i *>(bytes64)), zero32))) & 0xffffffff) _mm256_loadu_si256(reinterpret_cast<const __m256i *>(bytes64)), zero32))) & 0xffffffff)
| (static_cast<UInt64>(_mm256_movemask_epi8(_mm256_cmpeq_epi8( | (static_cast<UInt64>(_mm256_movemask_epi8(_mm256_cmpeq_epi8(
_mm256_loadu_si256(reinterpret_cast<const __m256i *>(bytes64+32)), zero32))) << 32); _mm256_loadu_si256(reinterpret_cast<const __m256i *>(bytes64+32)), zero32))) << 32);
#elif defined(__SSE2__) && defined(__POPCNT__) #elif defined(__SSE2__)
static const __m128i zero16 = _mm_setzero_si128(); static const __m128i zero16 = _mm_setzero_si128();
UInt64 res = UInt64 res =
(static_cast<UInt64>(_mm_movemask_epi8(_mm_cmpeq_epi8( (static_cast<UInt64>(_mm_movemask_epi8(_mm_cmpeq_epi8(

2
src/Common/CompactArray.h
View File

@ -130,7 +130,7 @@ public:
private: private:
ReadBuffer & in; ReadBuffer & in;
/// The physical location of the current cell. /// The physical location of the current cell.
Locus locus; Locus locus{};
/// The current position in the file as a cell number. /// The current position in the file as a cell number.
BucketIndex current_bucket_index = 0; BucketIndex current_bucket_index = 0;
/// The number of bytes read. /// The number of bytes read.

7
src/Common/HashTable/StringHashTable.h
View File

@ -3,6 +3,7 @@
#include <Common/HashTable/HashMap.h> #include <Common/HashTable/HashMap.h>
#include <Common/HashTable/HashTable.h> #include <Common/HashTable/HashTable.h>
#include <bit>
#include <new> #include <new>
#include <variant> #include <variant>
@ -21,17 +22,17 @@ struct StringKey24
inline StringRef ALWAYS_INLINE toStringRef(const StringKey8 & n) inline StringRef ALWAYS_INLINE toStringRef(const StringKey8 & n)
{ {
assert(n != 0); assert(n != 0);
return {reinterpret_cast<const char *>(&n), 8ul - (__builtin_clzll(n) >> 3)}; return {reinterpret_cast<const char *>(&n), 8ul - (std::countl_zero(n) >> 3)};
} }
inline StringRef ALWAYS_INLINE toStringRef(const StringKey16 & n) inline StringRef ALWAYS_INLINE toStringRef(const StringKey16 & n)
{ {
assert(n.items[1] != 0); assert(n.items[1] != 0);
return {reinterpret_cast<const char *>(&n), 16ul - (__builtin_clzll(n.items[1]) >> 3)}; return {reinterpret_cast<const char *>(&n), 16ul - (std::countl_zero(n.items[1]) >> 3)};
} }
inline StringRef ALWAYS_INLINE toStringRef(const StringKey24 & n) inline StringRef ALWAYS_INLINE toStringRef(const StringKey24 & n)
{ {
assert(n.c != 0); assert(n.c != 0);
return {reinterpret_cast<const char *>(&n), 24ul - (__builtin_clzll(n.c) >> 3)}; return {reinterpret_cast<const char *>(&n), 24ul - (std::countl_zero(n.c) >> 3)};
} }
struct StringHashTableHash struct StringHashTableHash

5
src/Common/HyperLogLogCounter.h
View File

@ -11,6 +11,7 @@
#include <IO/WriteHelpers.h> #include <IO/WriteHelpers.h>
#include <Core/Defines.h> #include <Core/Defines.h>
#include <bit>
#include <cmath> #include <cmath>
#include <cstring> #include <cstring>
@ -205,7 +206,7 @@ struct TrailingZerosCounter<UInt32>
{ {
static int apply(UInt32 val) static int apply(UInt32 val)
{ {
return __builtin_ctz(val); return std::countr_zero(val);
} }
}; };
@ -214,7 +215,7 @@ struct TrailingZerosCounter<UInt64>
{ {
static int apply(UInt64 val) static int apply(UInt64 val)
{ {
return __builtin_ctzll(val); return std::countr_zero(val);
} }
}; };

3
src/Common/IPv6ToBinary.cpp
View File

@ -5,6 +5,7 @@
#include <Common/formatIPv6.h> #include <Common/formatIPv6.h>
#include <cstring> #include <cstring>
#include <bit>
namespace DB namespace DB
@ -89,7 +90,7 @@ bool matchIPv6Subnet(const uint8_t * addr, const uint8_t * cidr_addr, UInt8 pref
if (mask) if (mask)
{ {
auto offset = __builtin_ctz(mask); auto offset = std::countr_zero(mask);
if (prefix / 8 != offset) if (prefix / 8 != offset)
return prefix / 8 < offset; return prefix / 8 < offset;

2
src/Common/SpaceSaving.h
View File

@ -78,7 +78,7 @@ private:
constexpr uint64_t nextAlphaSize(uint64_t x) constexpr uint64_t nextAlphaSize(uint64_t x)
{ {
constexpr uint64_t alpha_map_elements_per_counter = 6; constexpr uint64_t alpha_map_elements_per_counter = 6;
return 1ULL << (sizeof(uint64_t) * 8 - __builtin_clzll(x * alpha_map_elements_per_counter)); return 1ULL << (sizeof(uint64_t) * 8 - std::countl_zero(x * alpha_map_elements_per_counter));
} }
public: public:

43
src/Common/StackTrace.cpp
View File

@ -1,6 +1,5 @@
#include <Common/StackTrace.h> #include <Common/StackTrace.h>
#include <Core/Defines.h>
#include <Common/Dwarf.h> #include <Common/Dwarf.h>
#include <Common/Elf.h> #include <Common/Elf.h>
#include <Common/SymbolIndex.h> #include <Common/SymbolIndex.h>
@ -8,6 +7,7 @@
#include <base/CachedFn.h> #include <base/CachedFn.h>
#include <base/demangle.h> #include <base/demangle.h>
#include <atomic>
#include <cstring> #include <cstring>
#include <filesystem> #include <filesystem>
#include <sstream> #include <sstream>
@ -19,6 +19,32 @@
# include <libunwind.h> # include <libunwind.h>
#endif #endif
namespace
{
/// Currently this variable is set up once on server startup.
/// But we use atomic just in case, so it is possible to be modified at runtime.
std::atomic<bool> show_addresses = true;
bool shouldShowAddress(const void * addr)
{
/// If the address is less than 4096, most likely it is a nullptr dereference with offset,
/// and showing this offset is secure nevertheless.
/// NOTE: 4096 is the page size on x86 and it can be different on other systems,
/// but for the purpose of this branch, it does not matter.
if (reinterpret_cast<uintptr_t>(addr) < 4096)
return true;
return show_addresses.load(std::memory_order_relaxed);
}
}
void StackTrace::setShowAddresses(bool show)
{
show_addresses.store(show, std::memory_order_relaxed);
}
std::string signalToErrorMessage(int sig, const siginfo_t & info, [[maybe_unused]] const ucontext_t & context) std::string signalToErrorMessage(int sig, const siginfo_t & info, [[maybe_unused]] const ucontext_t & context)
{ {
std::stringstream error; // STYLE_CHECK_ALLOW_STD_STRING_STREAM std::stringstream error; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
@ -30,7 +56,7 @@ std::string signalToErrorMessage(int sig, const siginfo_t & info, [[maybe_unused
/// Print info about address and reason. /// Print info about address and reason.
if (nullptr == info.si_addr) if (nullptr == info.si_addr)
error << "Address: NULL pointer."; error << "Address: NULL pointer.";
else else if (shouldShowAddress(info.si_addr))
error << "Address: " << info.si_addr; error << "Address: " << info.si_addr;
#if defined(__x86_64__) && !defined(OS_FREEBSD) && !defined(OS_DARWIN) && !defined(__arm__) && !defined(__powerpc__) #if defined(__x86_64__) && !defined(OS_FREEBSD) && !defined(OS_DARWIN) && !defined(__arm__) && !defined(__powerpc__)
@ -372,7 +398,9 @@ static void toStringEveryLineImpl(
else else
out << "?"; out << "?";
out << " @ " << physical_addr; if (shouldShowAddress(physical_addr))
out << " @ " << physical_addr;
out << " in " << (object ? object->name : "?"); out << " in " << (object ? object->name : "?");
for (size_t j = 0; j < inline_frames.size(); ++j) for (size_t j = 0; j < inline_frames.size(); ++j)
@ -393,10 +421,13 @@ static void toStringEveryLineImpl(
for (size_t i = offset; i < size; ++i) for (size_t i = offset; i < size; ++i)
{ {
const void * addr = frame_pointers[i]; const void * addr = frame_pointers[i];
out << i << ". " << addr; if (shouldShowAddress(addr))
{
out << i << ". " << addr;
callback(out.str()); callback(out.str());
out.str({}); out.str({});
}
} }
#endif #endif
} }

5
src/Common/StackTrace.h
View File

@ -67,6 +67,11 @@ public:
void toStringEveryLine(std::function<void(const std::string &)> callback) const; void toStringEveryLine(std::function<void(const std::string &)> callback) const;
/// Displaying the addresses can be disabled for security reasons.
/// If you turn off addresses, it will be more secure, but we will be unable to help you with debugging.
/// Please note: addresses are also available in the system.stack_trace and system.trace_log tables.
static void setShowAddresses(bool show);
protected: protected:
void tryCapture(); void tryCapture();

3
src/Common/UTF8Helpers.cpp
View File

@ -2,6 +2,7 @@
#include <Common/StringUtils/StringUtils.h> #include <Common/StringUtils/StringUtils.h>
#include <widechar_width.h> #include <widechar_width.h>
#include <bit>
namespace DB namespace DB
@ -124,7 +125,7 @@ size_t computeWidthImpl(const UInt8 * data, size_t size, size_t prefix, size_t l
if (non_regular_width_mask) if (non_regular_width_mask)
{ {
auto num_regular_chars = __builtin_ctz(non_regular_width_mask); auto num_regular_chars = std::countr_zero(non_regular_width_mask);
width += num_regular_chars; width += num_regular_chars;
i += num_regular_chars; i += num_regular_chars;
break; break;

2
src/Common/UTF8Helpers.h
View File

@ -83,7 +83,7 @@ inline size_t countCodePoints(const UInt8 * data, size_t size)
const auto threshold = vdupq_n_s8(0xBF); const auto threshold = vdupq_n_s8(0xBF);
for (; data < src_end_sse; data += bytes_sse) for (; data < src_end_sse; data += bytes_sse)
res += __builtin_popcountll(get_nibble_mask(vcgtq_s8(vld1q_s8(reinterpret_cast<const int8_t *>(data)), threshold))); res += std::popcount(get_nibble_mask(vcgtq_s8(vld1q_s8(reinterpret_cast<const int8_t *>(data)), threshold)));
res >>= 2; res >>= 2;
#endif #endif

20
src/Common/ZooKeeper/ZooKeeperImpl.cpp
View File

@ -701,6 +701,7 @@ void ZooKeeper::receiveEvent()
RequestInfo request_info; RequestInfo request_info;
ZooKeeperResponsePtr response; ZooKeeperResponsePtr response;
UInt64 elapsed_ms = 0;
if (xid == PING_XID) if (xid == PING_XID)
{ {
@ -761,8 +762,8 @@ void ZooKeeper::receiveEvent()
CurrentMetrics::sub(CurrentMetrics::ZooKeeperRequest); CurrentMetrics::sub(CurrentMetrics::ZooKeeperRequest);
} }
auto elapsed_microseconds = std::chrono::duration_cast<std::chrono::microseconds>(clock::now() - request_info.time).count(); elapsed_ms = std::chrono::duration_cast<std::chrono::microseconds>(clock::now() - request_info.time).count();
ProfileEvents::increment(ProfileEvents::ZooKeeperWaitMicroseconds, elapsed_microseconds); ProfileEvents::increment(ProfileEvents::ZooKeeperWaitMicroseconds, elapsed_ms);
} }
try try
@ -812,7 +813,7 @@ void ZooKeeper::receiveEvent()
if (length != actual_length) if (length != actual_length)
throw Exception("Response length doesn't match. Expected: " + DB::toString(length) + ", actual: " + DB::toString(actual_length), Error::ZMARSHALLINGERROR); throw Exception("Response length doesn't match. Expected: " + DB::toString(length) + ", actual: " + DB::toString(actual_length), Error::ZMARSHALLINGERROR);
logOperationIfNeeded(request_info.request, response); //-V614 logOperationIfNeeded(request_info.request, response, /* finalize= */ false, elapsed_ms); //-V614
} }
catch (...) catch (...)
{ {
@ -831,7 +832,7 @@ void ZooKeeper::receiveEvent()
if (request_info.callback) if (request_info.callback)
request_info.callback(*response); request_info.callback(*response);
logOperationIfNeeded(request_info.request, response); logOperationIfNeeded(request_info.request, response, /* finalize= */ false, elapsed_ms);
} }
catch (...) catch (...)
{ {
@ -919,13 +920,14 @@ void ZooKeeper::finalize(bool error_send, bool error_receive, const String & rea
? Error::ZCONNECTIONLOSS ? Error::ZCONNECTIONLOSS
: Error::ZSESSIONEXPIRED; : Error::ZSESSIONEXPIRED;
response->xid = request_info.request->xid; response->xid = request_info.request->xid;
UInt64 elapsed_ms = std::chrono::duration_cast<std::chrono::microseconds>(clock::now() - request_info.time).count();
if (request_info.callback) if (request_info.callback)
{ {
try try
{ {
request_info.callback(*response); request_info.callback(*response);
logOperationIfNeeded(request_info.request, response, true); logOperationIfNeeded(request_info.request, response, true, elapsed_ms);
} }
catch (...) catch (...)
{ {
@ -985,7 +987,8 @@ void ZooKeeper::finalize(bool error_send, bool error_receive, const String & rea
try try
{ {
info.callback(*response); info.callback(*response);
logOperationIfNeeded(info.request, response, true); UInt64 elapsed_ms = std::chrono::duration_cast<std::chrono::microseconds>(clock::now() - info.time).count();
logOperationIfNeeded(info.request, response, true, elapsed_ms);
} }
catch (...) catch (...)
{ {
@ -1310,7 +1313,7 @@ void ZooKeeper::setZooKeeperLog(std::shared_ptr<DB::ZooKeeperLog> zk_log_)
} }
#ifdef ZOOKEEPER_LOG #ifdef ZOOKEEPER_LOG
void ZooKeeper::logOperationIfNeeded(const ZooKeeperRequestPtr & request, const ZooKeeperResponsePtr & response, bool finalize) void ZooKeeper::logOperationIfNeeded(const ZooKeeperRequestPtr & request, const ZooKeeperResponsePtr & response, bool finalize, UInt64 elapsed_ms)
{ {
auto maybe_zk_log = std::atomic_load(&zk_log); auto maybe_zk_log = std::atomic_load(&zk_log);
if (!maybe_zk_log) if (!maybe_zk_log)
@ -1348,6 +1351,7 @@ void ZooKeeper::logOperationIfNeeded(const ZooKeeperRequestPtr & request, const
elem.event_time = event_time; elem.event_time = event_time;
elem.address = socket_address; elem.address = socket_address;
elem.session_id = session_id; elem.session_id = session_id;
elem.duration_ms = elapsed_ms;
if (request) if (request)
{ {
elem.thread_id = request->thread_id; elem.thread_id = request->thread_id;
@ -1357,7 +1361,7 @@ void ZooKeeper::logOperationIfNeeded(const ZooKeeperRequestPtr & request, const
} }
} }
#else #else
void ZooKeeper::logOperationIfNeeded(const ZooKeeperRequestPtr &, const ZooKeeperResponsePtr &, bool) void ZooKeeper::logOperationIfNeeded(const ZooKeeperRequestPtr &, const ZooKeeperResponsePtr &, bool, UInt64)
{} {}
#endif #endif

2
src/Common/ZooKeeper/ZooKeeperImpl.h
View File

@ -276,7 +276,7 @@ private:
template <typename T> template <typename T>
void read(T &); void read(T &);
void logOperationIfNeeded(const ZooKeeperRequestPtr & request, const ZooKeeperResponsePtr & response = nullptr, bool finalize = false); void logOperationIfNeeded(const ZooKeeperRequestPtr & request, const ZooKeeperResponsePtr & response = nullptr, bool finalize = false, UInt64 elapsed_ms = 0);
void initApiVersion(); void initApiVersion();

1
src/Common/config.h.in
View File

@ -18,7 +18,6 @@
#cmakedefine01 USE_DATASKETCHES #cmakedefine01 USE_DATASKETCHES
#cmakedefine01 USE_YAML_CPP #cmakedefine01 USE_YAML_CPP
#cmakedefine01 CLICKHOUSE_SPLIT_BINARY
#cmakedefine01 USE_BZIP2 #cmakedefine01 USE_BZIP2
#cmakedefine01 USE_MINIZIP #cmakedefine01 USE_MINIZIP
#cmakedefine01 USE_SNAPPY #cmakedefine01 USE_SNAPPY

3
src/Common/examples/average.cpp
View File

@ -1,5 +1,6 @@
#include <iostream> #include <iostream>
#include <string> #include <string>
#include <bit>
#include <fmt/format.h> #include <fmt/format.h>
@ -561,7 +562,7 @@ int main(int argc, char ** argv)
/// Fill source data /// Fill source data
for (size_t i = 0; i < size; ++i) for (size_t i = 0; i < size; ++i)
{ {
keys[i] = __builtin_ctz(i + 1); /// Make keys to have just slightly more realistic distribution. keys[i] = std::countr_zero(i + 1); /// Make keys to have just slightly more realistic distribution.
values[i] = 1234.5; /// The distribution of values does not affect execution speed. values[i] = 1234.5; /// The distribution of values does not affect execution speed.
} }

49
src/Common/memcmpSmall.h
View File

@ -1,6 +1,7 @@
#pragma once #pragma once
#include <algorithm> #include <algorithm>
#include <bit>
#include <cstdint> #include <cstdint>
#include <Core/Defines.h> #include <Core/Defines.h>
@ -50,7 +51,7 @@ inline int memcmpSmallAllowOverflow15(const Char * a, size_t a_size, const Char
if (mask) if (mask)
{ {
offset += __builtin_ctz(mask); offset += std::countr_zero(mask);
if (offset >= min_size) if (offset >= min_size)
break; break;
@ -82,7 +83,7 @@ inline int memcmpSmallLikeZeroPaddedAllowOverflow15(const Char * a, size_t a_siz
if (mask) if (mask)
{ {
offset += __builtin_ctz(mask); offset += std::countr_zero(mask);
if (offset >= min_size) if (offset >= min_size)
break; break;
@ -123,7 +124,7 @@ inline int memcmpSmallLikeZeroPaddedAllowOverflow15(const Char * a, size_t a_siz
if (mask) if (mask)
{ {
offset += __builtin_ctz(mask); offset += std::countr_zero(mask);
if (offset >= max_size) if (offset >= max_size)
return 0; return 0;
@ -150,7 +151,7 @@ inline int memcmpSmallAllowOverflow15(const Char * a, const Char * b, size_t siz
if (mask) if (mask)
{ {
offset += __builtin_ctz(mask); offset += std::countr_zero(mask);
if (offset >= size) if (offset >= size)
return 0; return 0;
@ -180,7 +181,7 @@ inline bool memequalSmallAllowOverflow15(const Char * a, size_t a_size, const Ch
if (mask) if (mask)
{ {
offset += __builtin_ctz(mask); offset += std::countr_zero(mask);
return offset >= a_size; return offset >= a_size;
} }
} }
@ -203,7 +204,7 @@ inline int memcmpSmallMultipleOf16(const Char * a, const Char * b, size_t size)
if (mask) if (mask)
{ {
offset += __builtin_ctz(mask); offset += std::countr_zero(mask);
return detail::cmp(a[offset], b[offset]); return detail::cmp(a[offset], b[offset]);
} }
} }
@ -222,7 +223,7 @@ inline int memcmp16(const Char * a, const Char * b)
if (mask) if (mask)
{ {
auto offset = __builtin_ctz(mask); auto offset = std::countr_zero(mask);
return detail::cmp(a[offset], b[offset]); return detail::cmp(a[offset], b[offset]);
} }
@ -252,7 +253,7 @@ inline bool memoryIsZeroSmallAllowOverflow15(const void * data, size_t size)
if (mask) if (mask)
{ {
offset += __builtin_ctz(mask); offset += std::countr_zero(mask);
return offset >= size; return offset >= size;
} }
} }
@ -285,7 +286,7 @@ inline int memcmpSmallAllowOverflow15(const Char * a, size_t a_size, const Char
if (mask) if (mask)
{ {
offset += __builtin_ctz(mask); offset += std::countr_zero(mask);
if (offset >= min_size) if (offset >= min_size)
break; break;
@ -317,7 +318,7 @@ inline int memcmpSmallLikeZeroPaddedAllowOverflow15(const Char * a, size_t a_siz
if (mask) if (mask)
{ {
offset += __builtin_ctz(mask); offset += std::countr_zero(mask);
if (offset >= min_size) if (offset >= min_size)
break; break;
@ -359,7 +360,7 @@ inline int memcmpSmallLikeZeroPaddedAllowOverflow15(const Char * a, size_t a_siz
if (mask) if (mask)
{ {
offset += __builtin_ctz(mask); offset += std::countr_zero(mask);
if (offset >= max_size) if (offset >= max_size)
return 0; return 0;
@ -386,7 +387,7 @@ inline int memcmpSmallAllowOverflow15(const Char * a, const Char * b, size_t siz
if (mask) if (mask)
{ {
offset += __builtin_ctz(mask); offset += std::countr_zero(mask);
if (offset >= size) if (offset >= size)
return 0; return 0;
@ -416,7 +417,7 @@ inline bool memequalSmallAllowOverflow15(const Char * a, size_t a_size, const Ch
if (mask) if (mask)
{ {
offset += __builtin_ctz(mask); offset += std::countr_zero(mask);
return offset >= a_size; return offset >= a_size;
} }
} }
@ -439,7 +440,7 @@ inline int memcmpSmallMultipleOf16(const Char * a, const Char * b, size_t size)
if (mask) if (mask)
{ {
offset += __builtin_ctz(mask); offset += std::countr_zero(mask);
return detail::cmp(a[offset], b[offset]); return detail::cmp(a[offset], b[offset]);
} }
} }
@ -459,7 +460,7 @@ inline int memcmp16(const Char * a, const Char * b)
if (mask) if (mask)
{ {
auto offset = __builtin_ctz(mask); auto offset = std::countr_zero(mask);
return detail::cmp(a[offset], b[offset]); return detail::cmp(a[offset], b[offset]);
} }
@ -490,7 +491,7 @@ inline bool memoryIsZeroSmallAllowOverflow15(const void * data, size_t size)
if (mask) if (mask)
{ {
offset += __builtin_ctz(mask); offset += std::countr_zero(mask);
return offset >= size; return offset >= size;
} }
} }
@ -523,7 +524,7 @@ inline int memcmpSmallAllowOverflow15(const Char * a, size_t a_size, const Char
if (mask) if (mask)
{ {
offset += __builtin_ctzll(mask) >> 2; offset += std::countr_zero(mask) >> 2;
if (offset >= min_size) if (offset >= min_size)
break; break;
@ -548,7 +549,7 @@ inline int memcmpSmallLikeZeroPaddedAllowOverflow15(const Char * a, size_t a_siz
if (mask) if (mask)
{ {
offset += __builtin_ctzll(mask) >> 2; offset += std::countr_zero(mask) >> 2;
if (offset >= min_size) if (offset >= min_size)
break; break;
@ -589,7 +590,7 @@ inline int memcmpSmallLikeZeroPaddedAllowOverflow15(const Char * a, size_t a_siz
if (mask) if (mask)
{ {
offset += __builtin_ctzll(mask) >> 2; offset += std::countr_zero(mask) >> 2;
if (offset >= max_size) if (offset >= max_size)
return 0; return 0;
@ -611,7 +612,7 @@ inline int memcmpSmallAllowOverflow15(const Char * a, const Char * b, size_t siz
if (mask) if (mask)
{ {
offset += __builtin_ctzll(mask) >> 2; offset += std::countr_zero(mask) >> 2;
if (offset >= size) if (offset >= size)
return 0; return 0;
@ -637,7 +638,7 @@ inline bool memequalSmallAllowOverflow15(const Char * a, size_t a_size, const Ch
if (mask) if (mask)
{ {
offset += __builtin_ctzll(mask) >> 2; offset += std::countr_zero(mask) >> 2;
return offset >= a_size; return offset >= a_size;
} }
} }
@ -656,7 +657,7 @@ inline int memcmpSmallMultipleOf16(const Char * a, const Char * b, size_t size)
if (mask) if (mask)
{ {
offset += __builtin_ctzll(mask) >> 2; offset += std::countr_zero(mask) >> 2;
return detail::cmp(a[offset], b[offset]); return detail::cmp(a[offset], b[offset]);
} }
} }
@ -672,7 +673,7 @@ inline int memcmp16(const Char * a, const Char * b)
mask = ~mask; mask = ~mask;
if (mask) if (mask)
{ {
auto offset = __builtin_ctzll(mask) >> 2; auto offset = std::countr_zero(mask) >> 2;
return detail::cmp(a[offset], b[offset]); return detail::cmp(a[offset], b[offset]);
} }
return 0; return 0;
@ -694,7 +695,7 @@ inline bool memoryIsZeroSmallAllowOverflow15(const void * data, size_t size)
if (mask) if (mask)
{ {
offset += __builtin_ctzll(mask) >> 2; offset += std::countr_zero(mask) >> 2;
return offset >= size; return offset >= size;
} }
} }

12
src/Common/register_objects.cpp Normal file
View File

@ -0,0 +1,12 @@
#include <Common/register_objects.h>
namespace DB
{
FunctionRegisterMap & FunctionRegisterMap::instance()
{
static FunctionRegisterMap map;
return map;
}
}

33
src/Common/register_objects.h Normal file
View File

@ -0,0 +1,33 @@
#pragma once
#include <string_view>
#include <unordered_map>
namespace DB
{
class FunctionFactory;
using FunctionRegisterFunctionPtr = void (*)(::DB::FunctionFactory &);
struct FunctionRegisterMap : public std::unordered_map<std::string_view, FunctionRegisterFunctionPtr>
{
static FunctionRegisterMap & instance();
};
struct FunctionRegister
{
FunctionRegister(std::string_view name, FunctionRegisterFunctionPtr func_ptr)
{
FunctionRegisterMap::instance().emplace(std::move(name), func_ptr);
}
};
}
#define REGISTER_FUNCTION_IMPL(fn, func_name, register_name) \
void func_name(::DB::FunctionFactory & factory); \
static ::DB::FunctionRegister register_name(#fn, func_name); \
void func_name(::DB::FunctionFactory & factory)
#define REGISTER_FUNCTION(fn) REGISTER_FUNCTION_IMPL(fn, registerFunction##fn, REGISTER_FUNCTION_##fn)

5
src/Compression/CompressedReadBufferBase.cpp
View File

@ -1,5 +1,6 @@
#include "CompressedReadBufferBase.h" #include "CompressedReadBufferBase.h"
#include <bit>
#include <cstring> #include <cstring>
#include <cassert> #include <cassert>
#include <city.h> #include <city.h>
@ -93,8 +94,8 @@ static void validateChecksum(char * data, size_t size, const Checksum expected_c
} }
/// Check if the difference caused by single bit flip in stored checksum. /// Check if the difference caused by single bit flip in stored checksum.
size_t difference = __builtin_popcountll(expected_checksum.first ^ calculated_checksum.first) size_t difference = std::popcount(expected_checksum.first ^ calculated_checksum.first)
+ __builtin_popcountll(expected_checksum.second ^ calculated_checksum.second); + std::popcount(expected_checksum.second ^ calculated_checksum.second);
if (difference == 1) if (difference == 1)
{ {

3
src/Compression/CompressionCodecT64.cpp
View File

@ -8,6 +8,7 @@
#include <Parsers/ASTFunction.h> #include <Parsers/ASTFunction.h>
#include <IO/WriteHelpers.h> #include <IO/WriteHelpers.h>
#include <Core/Types.h> #include <Core/Types.h>
#include <bit>
namespace DB namespace DB
@ -413,7 +414,7 @@ UInt32 getValuableBitsNumber(UInt64 min, UInt64 max)
{ {
UInt64 diff_bits = min ^ max; UInt64 diff_bits = min ^ max;
if (diff_bits) if (diff_bits)
return 64 - __builtin_clzll(diff_bits); return 64 - std::countl_zero(diff_bits);
return 0; return 0;
} }

79
src/Core/Joins.cpp Normal file
View File

@ -0,0 +1,79 @@
#include <Core/Joins.h>
namespace DB
{
const char * toString(JoinKind kind)
{
switch (kind)
{
case JoinKind::Inner: return "INNER";
case JoinKind::Left: return "LEFT";
case JoinKind::Right: return "RIGHT";
case JoinKind::Full: return "FULL";
case JoinKind::Cross: return "CROSS";
case JoinKind::Comma: return "COMMA";
}
};
const char * toString(JoinStrictness strictness)
{
switch (strictness)
{
case JoinStrictness::Unspecified: return "UNSPECIFIED";
case JoinStrictness::RightAny: return "RIGHT_ANY";
case JoinStrictness::Any: return "ANY";
case JoinStrictness::All: return "ALL";
case JoinStrictness::Asof: return "ASOF";
case JoinStrictness::Semi: return "SEMI";
case JoinStrictness::Anti: return "ANTI";
}
}
const char * toString(JoinLocality locality)
{
switch (locality)
{
case JoinLocality::Unspecified: return "UNSPECIFIED";
case JoinLocality::Local: return "LOCAL";
case JoinLocality::Global: return "GLOBAL";
}
}
const char * toString(ASOFJoinInequality asof_join_inequality)
{
switch (asof_join_inequality)
{
case ASOFJoinInequality::None: return "NONE";
case ASOFJoinInequality::Less: return "LESS";
case ASOFJoinInequality::Greater: return "GREATER";
case ASOFJoinInequality::LessOrEquals: return "LESS_OR_EQUALS";
case ASOFJoinInequality::GreaterOrEquals: return "GREATER_OR_EQUALS";
}
}
const char * toString(JoinAlgorithm join_algorithm)
{
switch (join_algorithm)
{
case JoinAlgorithm::DEFAULT: return "DEFAULT";
case JoinAlgorithm::AUTO: return "AUTO";
case JoinAlgorithm::HASH: return "HASH";
case JoinAlgorithm::PARTIAL_MERGE: return "PARTIAL_MERGE";
case JoinAlgorithm::PREFER_PARTIAL_MERGE: return "PREFER_PARTIAL_MERGE";
case JoinAlgorithm::PARALLEL_HASH: return "PARALLEL_HASH";
case JoinAlgorithm::DIRECT: return "DIRECT";
case JoinAlgorithm::FULL_SORTING_MERGE: return "FULL_SORTING_MERGE";
}
}
const char * toString(JoinTableSide join_table_side)
{
switch (join_table_side)
{
case JoinTableSide::Left: return "LEFT";
case JoinTableSide::Right: return "RIGHT";
}
}
}

119
src/Core/Joins.h Normal file
View File

@ -0,0 +1,119 @@
#pragma once
#include <string_view>
namespace DB
{
/// Join method.
enum class JoinKind
{
Inner, /// Leave only rows that was JOINed.
Left, /// If in "right" table there is no corresponding rows, use default values instead.
Right,
Full,
Cross, /// Direct product. Strictness and condition doesn't matter.
Comma /// Same as direct product. Intended to be converted to INNER JOIN with conditions from WHERE.
};
const char * toString(JoinKind kind);
inline constexpr bool isLeft(JoinKind kind) { return kind == JoinKind::Left; }
inline constexpr bool isRight(JoinKind kind) { return kind == JoinKind::Right; }
inline constexpr bool isInner(JoinKind kind) { return kind == JoinKind::Inner; }
inline constexpr bool isFull(JoinKind kind) { return kind == JoinKind::Full; }
inline constexpr bool isCrossOrComma(JoinKind kind) { return kind == JoinKind::Comma || kind == JoinKind::Cross; }
inline constexpr bool isRightOrFull(JoinKind kind) { return kind == JoinKind::Right || kind == JoinKind::Full; }
inline constexpr bool isLeftOrFull(JoinKind kind) { return kind == JoinKind::Left || kind == JoinKind::Full; }
inline constexpr bool isInnerOrRight(JoinKind kind) { return kind == JoinKind::Inner || kind == JoinKind::Right; }
inline constexpr bool isInnerOrLeft(JoinKind kind) { return kind == JoinKind::Inner || kind == JoinKind::Left; }
/// Allows more optimal JOIN for typical cases.
enum class JoinStrictness
{
Unspecified,
RightAny, /// Old ANY JOIN. If there are many suitable rows in right table, use any from them to join.
Any, /// Semi Join with any value from filtering table. For LEFT JOIN with Any and RightAny are the same.
All, /// If there are many suitable rows to join, use all of them and replicate rows of "left" table (usual semantic of JOIN).
Asof, /// For the last JOIN column, pick the latest value
Semi, /// LEFT or RIGHT. SEMI LEFT JOIN filters left table by values exists in right table. SEMI RIGHT - otherwise.
Anti, /// LEFT or RIGHT. Same as SEMI JOIN but filter values that are NOT exists in other table.
};
const char * toString(JoinStrictness strictness);
/// Algorithm for distributed query processing.
enum class JoinLocality
{
Unspecified,
Local, /// Perform JOIN, using only data available on same servers (co-located data).
Global /// Collect and merge data from remote servers, and broadcast it to each server.
};
const char * toString(JoinLocality locality);
/// ASOF JOIN inequality type
enum class ASOFJoinInequality
{
None,
Less,
Greater,
LessOrEquals,
GreaterOrEquals,
};
const char * toString(ASOFJoinInequality asof_join_inequality);
inline constexpr ASOFJoinInequality getASOFJoinInequality(std::string_view func_name)
{
ASOFJoinInequality inequality = ASOFJoinInequality::None;
if (func_name == "less")
inequality = ASOFJoinInequality::Less;
else if (func_name == "greater")
inequality = ASOFJoinInequality::Greater;
else if (func_name == "lessOrEquals")
inequality = ASOFJoinInequality::LessOrEquals;
else if (func_name == "greaterOrEquals")
inequality = ASOFJoinInequality::GreaterOrEquals;
return inequality;
}
inline constexpr ASOFJoinInequality reverseASOFJoinInequality(ASOFJoinInequality inequality)
{
if (inequality == ASOFJoinInequality::Less)
return ASOFJoinInequality::Greater;
else if (inequality == ASOFJoinInequality::Greater)
return ASOFJoinInequality::Less;
else if (inequality == ASOFJoinInequality::LessOrEquals)
return ASOFJoinInequality::GreaterOrEquals;
else if (inequality == ASOFJoinInequality::GreaterOrEquals)
return ASOFJoinInequality::LessOrEquals;
return ASOFJoinInequality::None;
}
enum class JoinAlgorithm
{
DEFAULT = 0,
AUTO,
HASH,
PARTIAL_MERGE,
PREFER_PARTIAL_MERGE,
PARALLEL_HASH,
DIRECT,
FULL_SORTING_MERGE,
};
const char * toString(JoinAlgorithm join_algorithm);
enum class JoinTableSide
{
Left,
Right
};
const char * toString(JoinTableSide join_table_side);
}

4
src/Core/Settings.h
View File

@ -247,7 +247,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
\ \
M(Bool, join_use_nulls, false, "Use NULLs for non-joined rows of outer JOINs for types that can be inside Nullable. If false, use default value of corresponding columns data type.", IMPORTANT) \ M(Bool, join_use_nulls, false, "Use NULLs for non-joined rows of outer JOINs for types that can be inside Nullable. If false, use default value of corresponding columns data type.", IMPORTANT) \
\ \
M(JoinStrictness, join_default_strictness, JoinStrictness::ALL, "Set default strictness in JOIN query. Possible values: empty string, 'ANY', 'ALL'. If empty, query without strictness will throw exception.", 0) \ M(JoinStrictness, join_default_strictness, JoinStrictness::All, "Set default strictness in JOIN query. Possible values: empty string, 'ANY', 'ALL'. If empty, query without strictness will throw exception.", 0) \
M(Bool, any_join_distinct_right_table_keys, false, "Enable old ANY JOIN logic with many-to-one left-to-right table keys mapping for all ANY JOINs. It leads to confusing not equal results for 't1 ANY LEFT JOIN t2' and 't2 ANY RIGHT JOIN t1'. ANY RIGHT JOIN needs one-to-many keys mapping to be consistent with LEFT one.", IMPORTANT) \ M(Bool, any_join_distinct_right_table_keys, false, "Enable old ANY JOIN logic with many-to-one left-to-right table keys mapping for all ANY JOINs. It leads to confusing not equal results for 't1 ANY LEFT JOIN t2' and 't2 ANY RIGHT JOIN t1'. ANY RIGHT JOIN needs one-to-many keys mapping to be consistent with LEFT one.", IMPORTANT) \
\ \
M(UInt64, preferred_block_size_bytes, 1000000, "", 0) \ M(UInt64, preferred_block_size_bytes, 1000000, "", 0) \
@ -344,7 +344,7 @@ static constexpr UInt64 operator""_GiB(unsigned long long value)
M(UInt64, max_temporary_non_const_columns, 0, "", 0) \ M(UInt64, max_temporary_non_const_columns, 0, "", 0) \
\ \
M(UInt64, max_subquery_depth, 100, "", 0) \ M(UInt64, max_subquery_depth, 100, "", 0) \
M(UInt64, max_pipeline_depth, 1000, "", 0) \ M(UInt64, max_pipeline_depth, 10000, "", 0) \
M(UInt64, max_ast_depth, 1000, "Maximum depth of query syntax tree. Checked after parsing.", 0) \ M(UInt64, max_ast_depth, 1000, "Maximum depth of query syntax tree. Checked after parsing.", 0) \
M(UInt64, max_ast_elements, 50000, "Maximum size of query syntax tree in number of nodes. Checked after parsing.", 0) \ M(UInt64, max_ast_elements, 50000, "Maximum size of query syntax tree in number of nodes. Checked after parsing.", 0) \
M(UInt64, max_expanded_ast_elements, 500000, "Maximum size of query syntax tree in number of nodes after expansion of aliases and the asterisk.", 0) \ M(UInt64, max_expanded_ast_elements, 500000, "Maximum size of query syntax tree in number of nodes after expansion of aliases and the asterisk.", 0) \

4
src/Core/SettingsEnums.cpp
View File

@ -26,8 +26,8 @@ IMPLEMENT_SETTING_ENUM(LoadBalancing, ErrorCodes::UNKNOWN_LOAD_BALANCING,
IMPLEMENT_SETTING_ENUM(JoinStrictness, ErrorCodes::UNKNOWN_JOIN, IMPLEMENT_SETTING_ENUM(JoinStrictness, ErrorCodes::UNKNOWN_JOIN,
{{"", JoinStrictness::Unspecified}, {{"", JoinStrictness::Unspecified},
{"ALL", JoinStrictness::ALL}, {"ALL", JoinStrictness::All},
{"ANY", JoinStrictness::ANY}}) {"ANY", JoinStrictness::Any}})
IMPLEMENT_SETTING_MULTI_ENUM(JoinAlgorithm, ErrorCodes::UNKNOWN_JOIN, IMPLEMENT_SETTING_MULTI_ENUM(JoinAlgorithm, ErrorCodes::UNKNOWN_JOIN,

23
src/Core/SettingsEnums.h
View File

@ -1,12 +1,14 @@
#pragma once #pragma once
#include <Core/SettingsFields.h> #include <Core/SettingsFields.h>
#include <Core/Joins.h>
#include <QueryPipeline/SizeLimits.h> #include <QueryPipeline/SizeLimits.h>
#include <Formats/FormatSettings.h> #include <Formats/FormatSettings.h>
namespace DB namespace DB
{ {
enum class LoadBalancing enum class LoadBalancing
{ {
/// among replicas with a minimum number of errors selected randomly /// among replicas with a minimum number of errors selected randomly
@ -26,28 +28,8 @@ enum class LoadBalancing
DECLARE_SETTING_ENUM(LoadBalancing) DECLARE_SETTING_ENUM(LoadBalancing)
enum class JoinStrictness
{
Unspecified = 0, /// Query JOIN without strictness will throw Exception.
ALL, /// Query JOIN without strictness -> ALL JOIN ...
ANY, /// Query JOIN without strictness -> ANY JOIN ...
};
DECLARE_SETTING_ENUM(JoinStrictness) DECLARE_SETTING_ENUM(JoinStrictness)
enum class JoinAlgorithm
{
DEFAULT = 0,
AUTO,
HASH,
PARTIAL_MERGE,
PREFER_PARTIAL_MERGE,
PARALLEL_HASH,
DIRECT,
FULL_SORTING_MERGE,
};
DECLARE_SETTING_MULTI_ENUM(JoinAlgorithm) DECLARE_SETTING_MULTI_ENUM(JoinAlgorithm)
@ -200,4 +182,5 @@ DECLARE_SETTING_ENUM_WITH_RENAME(EnumComparingMode, FormatSettings::EnumComparin
DECLARE_SETTING_ENUM_WITH_RENAME(EscapingRule, FormatSettings::EscapingRule) DECLARE_SETTING_ENUM_WITH_RENAME(EscapingRule, FormatSettings::EscapingRule)
DECLARE_SETTING_ENUM_WITH_RENAME(MsgPackUUIDRepresentation, FormatSettings::MsgPackUUIDRepresentation) DECLARE_SETTING_ENUM_WITH_RENAME(MsgPackUUIDRepresentation, FormatSettings::MsgPackUUIDRepresentation)
} }

8
src/Core/examples/coro.cpp
View File

@ -12,7 +12,7 @@
#if defined(__clang__) #if defined(__clang__)
#include <experimental/coroutine> #include <experimental/coroutine>
namespace std namespace std // NOLINT(cert-dcl58-cpp)
{ {
using namespace experimental::coroutines_v1; using namespace experimental::coroutines_v1;
} }
@ -97,14 +97,14 @@ struct Task
static bool resumeImpl(Task *r) static bool resumeImpl(Task *r)
{ {
if (r->value) if (r->value)
return false; return false;
auto & next = r->my.promise().next; auto & next = r->my.promise().next;
if (next) if (next)
{ {
if (resumeImpl(next.promise().r)) if (resumeImpl(next.promise().r))
return true; return true;
next = {}; next = {};
} }
@ -183,7 +183,7 @@ int main()
auto t = foo("foo"); auto t = foo("foo");
std::cout << ".. started" << std::endl; std::cout << ".. started" << std::endl;
while (t.resume()) while (t.resume())
std::cout << ".. yielded" << std::endl; std::cout << ".. yielded" << std::endl;
std::cout << ".. done: " << t.res() << std::endl; std::cout << ".. done: " << t.res() << std::endl;
} }
catch (DB::Exception & e) catch (DB::Exception & e)

15
src/Dictionaries/DirectDictionary.cpp
View File

@ -171,13 +171,13 @@ ColumnUInt8::Ptr DirectDictionary<dictionary_key_type>::hasKeys(
auto requested_keys = requested_keys_extractor.extractAllKeys(); auto requested_keys = requested_keys_extractor.extractAllKeys();
size_t requested_keys_size = requested_keys.size(); size_t requested_keys_size = requested_keys.size();
HashMap<KeyType, size_t> requested_key_to_index; HashMap<KeyType, PaddedPODArray<size_t>> requested_key_to_index;
requested_key_to_index.reserve(requested_keys_size); requested_key_to_index.reserve(requested_keys_size);
for (size_t i = 0; i < requested_keys.size(); ++i) for (size_t i = 0; i < requested_keys.size(); ++i)
{ {
auto requested_key = requested_keys[i]; auto requested_key = requested_keys[i];
requested_key_to_index[requested_key] = i; requested_key_to_index[requested_key].push_back(i);
} }
auto result = ColumnUInt8::create(requested_keys_size, false); auto result = ColumnUInt8::create(requested_keys_size, false);
@ -208,10 +208,13 @@ ColumnUInt8::Ptr DirectDictionary<dictionary_key_type>::hasKeys(
const auto * it = requested_key_to_index.find(block_key); const auto * it = requested_key_to_index.find(block_key);
assert(it); assert(it);
size_t result_data_found_index = it->getMapped(); auto & result_data_found_indexes = it->getMapped();
/// block_keys_size cannot be used, due to duplicates. for (size_t result_data_found_index : result_data_found_indexes)
keys_found += !result_data[result_data_found_index]; {
result_data[result_data_found_index] = true; /// block_keys_size cannot be used, due to duplicates.
keys_found += !result_data[result_data_found_index];
result_data[result_data_found_index] = true;
}
block_keys_extractor.rollbackCurrentKey(); block_keys_extractor.rollbackCurrentKey();
} }

7
src/Formats/JSONUtils.cpp
View File

@ -529,7 +529,12 @@ namespace JSONUtils
writeObjectStart(out, 2); writeObjectStart(out, 2);
writeTitle<true>("name", out, 3); writeTitle<true>("name", out, 3);
writeDoubleQuoted(fields[i].name, out);
/// The field names are pre-escaped to be put into JSON string literal.
writeChar('"', out);
writeString(fields[i].name, out);
writeChar('"', out);
writeFieldDelimiter(out); writeFieldDelimiter(out);
writeTitle<true>("type", out, 3); writeTitle<true>("type", out, 3);
writeJSONString(fields[i].type->getName(), out, settings); writeJSONString(fields[i].type->getName(), out, settings);

57
src/Functions/CMakeLists.txt
View File

@ -6,13 +6,14 @@ add_subdirectory(divide)
include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake") include("${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake")
add_headers_and_sources(clickhouse_functions .) add_headers_and_sources(clickhouse_functions .)
list(REMOVE_ITEM clickhouse_functions_sources IFunction.cpp FunctionFactory.cpp FunctionHelpers.cpp) list(REMOVE_ITEM clickhouse_functions_sources IFunction.cpp FunctionFactory.cpp FunctionHelpers.cpp extractTimeZoneFromFunctionArguments.cpp FunctionsLogical.cpp)
list(REMOVE_ITEM clickhouse_functions_headers IFunction.h FunctionFactory.h FunctionHelpers.h) list(REMOVE_ITEM clickhouse_functions_headers IFunction.h FunctionFactory.h FunctionHelpers.h extractTimeZoneFromFunctionArguments.h FunctionsLogical.h)
add_library(clickhouse_functions ${clickhouse_functions_sources}) add_library(clickhouse_functions_obj OBJECT ${clickhouse_functions_sources})
target_link_libraries(clickhouse_functions list (APPEND OBJECT_LIBS $<TARGET_OBJECTS:clickhouse_functions_obj>)
PUBLIC
list (APPEND PUBLIC_LIBS
ch_contrib::wyhash ch_contrib::wyhash
ch_contrib::cityhash ch_contrib::cityhash
ch_contrib::farmhash ch_contrib::farmhash
@ -24,27 +25,28 @@ target_link_libraries(clickhouse_functions
ch_contrib::metrohash ch_contrib::metrohash
ch_contrib::murmurhash ch_contrib::murmurhash
ch_contrib::hashidsxx ch_contrib::hashidsxx
)
PRIVATE list (APPEND PRIVATE_LIBS
ch_contrib::zlib ch_contrib::zlib
boost::filesystem boost::filesystem
divide_impl divide_impl
) )
if (TARGET OpenSSL::Crypto) if (TARGET OpenSSL::Crypto)
target_link_libraries(clickhouse_functions PUBLIC OpenSSL::Crypto) list (APPEND PUBLIC_LIBS OpenSSL::Crypto)
endif() endif()
if (OMIT_HEAVY_DEBUG_SYMBOLS) if (OMIT_HEAVY_DEBUG_SYMBOLS)
target_compile_options(clickhouse_functions PRIVATE "-g0") target_compile_options(clickhouse_functions_obj PRIVATE "-g0")
endif() endif()
if (TARGET ch_contrib::icu) if (TARGET ch_contrib::icu)
target_link_libraries (clickhouse_functions PRIVATE ch_contrib::icu) list (APPEND PRIVATE_LIBS ch_contrib::icu)
endif () endif ()
if (TARGET ch_contrib::fastops) if (TARGET ch_contrib::fastops)
target_link_libraries (clickhouse_functions PRIVATE ch_contrib::fastops) list (APPEND PRIVATE_LIBS ch_contrib::fastops)
endif () endif ()
if (ENABLE_EXAMPLES) if (ENABLE_EXAMPLES)
@ -52,45 +54,46 @@ if (ENABLE_EXAMPLES)
endif () endif ()
if (TARGET ch_contrib::llvm) if (TARGET ch_contrib::llvm)
target_link_libraries(clickhouse_functions PRIVATE ch_contrib::llvm) list (APPEND PRIVATE_LIBS ch_contrib::llvm)
endif () endif ()
if (TARGET ch_contrib::base64) if (TARGET ch_contrib::base64)
target_link_libraries(clickhouse_functions PRIVATE ch_contrib::base64) list (APPEND PRIVATE_LIBS ch_contrib::base64)
endif() endif()
target_link_libraries(clickhouse_functions PRIVATE ch_contrib::lz4) list (APPEND PRIVATE_LIBS ch_contrib::lz4)
if (ENABLE_NLP) if (ENABLE_NLP)
target_link_libraries(clickhouse_functions PRIVATE ch_contrib::cld2) list (APPEND PRIVATE_LIBS ch_contrib::cld2)
endif() endif()
if (TARGET ch_contrib::h3) if (TARGET ch_contrib::h3)
target_link_libraries (clickhouse_functions PRIVATE ch_contrib::h3) list (APPEND PRIVATE_LIBS ch_contrib::h3)
endif() endif()
if (TARGET ch_contrib::vectorscan) if (TARGET ch_contrib::vectorscan)
target_link_libraries(clickhouse_functions PRIVATE ch_contrib::vectorscan) list (APPEND PRIVATE_LIBS ch_contrib::vectorscan)
endif() endif()
if (TARGET ch_contrib::simdjson) if (TARGET ch_contrib::simdjson)
target_link_libraries(clickhouse_functions PRIVATE ch_contrib::simdjson) list (APPEND PRIVATE_LIBS ch_contrib::simdjson)
endif() endif()
if (TARGET ch_contrib::rapidjson) if (TARGET ch_contrib::rapidjson)
target_link_libraries(clickhouse_functions PRIVATE ch_contrib::rapidjson) list (APPEND PRIVATE_LIBS ch_contrib::rapidjson)
endif() endif()
add_subdirectory(GatherUtils) add_subdirectory(GatherUtils)
target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_gatherutils) list (APPEND PRIVATE_LIBS clickhouse_functions_gatherutils)
add_subdirectory(URL) add_subdirectory(URL)
target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_url) list (APPEND OBJECT_LIBS $<TARGET_OBJECTS:clickhouse_functions_url>)
add_subdirectory(array) add_subdirectory(array)
target_link_libraries(clickhouse_functions PRIVATE clickhouse_functions_array) list (APPEND OBJECT_LIBS $<TARGET_OBJECTS:clickhouse_functions_array>)
add_subdirectory(JSONPath) add_subdirectory(JSONPath)
list (APPEND PRIVATE_LIBS clickhouse_functions_jsonpath)
# Signed integer overflow on user-provided data inside boost::geometry - ignore. # Signed integer overflow on user-provided data inside boost::geometry - ignore.
set_source_files_properties("pointInPolygon.cpp" PROPERTIES COMPILE_FLAGS -fno-sanitize=signed-integer-overflow) set_source_files_properties("pointInPolygon.cpp" PROPERTIES COMPILE_FLAGS -fno-sanitize=signed-integer-overflow)
@ -98,3 +101,15 @@ set_source_files_properties("pointInPolygon.cpp" PROPERTIES COMPILE_FLAGS -fno-s
if (ENABLE_FUZZING) if (ENABLE_FUZZING)
add_compile_definitions(FUZZING_MODE=1) add_compile_definitions(FUZZING_MODE=1)
endif () endif ()
target_link_libraries(clickhouse_functions_obj PUBLIC ${PUBLIC_LIBS} PRIVATE ${PRIVATE_LIBS})
if (USE_STATIC_LIBRARIES OR NOT SPLIT_SHARED_LIBRARIES)
# Used to forward the linking information to the final binaries such as clickhouse / unit_tests_dbms,
# since such information are lost after we convert to OBJECT target
add_library(clickhouse_functions INTERFACE)
target_link_libraries(clickhouse_functions INTERFACE ${OBJECT_LIBS} ${PUBLIC_LIBS} ${PRIVATE_LIBS})
else()
add_library(clickhouse_functions SHARED ${OBJECT_LIBS})
target_link_libraries(clickhouse_functions PUBLIC ${PUBLIC_LIBS} PRIVATE ${PRIVATE_LIBS})
endif ()

2
src/Functions/CRC.cpp
View File

@ -143,7 +143,7 @@ void registerFunctionCRCImpl(FunctionFactory & factory)
factory.registerFunction<T>(T::name, FunctionFactory::CaseInsensitive); factory.registerFunction<T>(T::name, FunctionFactory::CaseInsensitive);
} }
void registerFunctionCRC(FunctionFactory & factory) REGISTER_FUNCTION(CRC)
{ {
registerFunctionCRCImpl<FunctionCRC32ZLIB>(factory); registerFunctionCRCImpl<FunctionCRC32ZLIB>(factory);
registerFunctionCRCImpl<FunctionCRC32IEEE>(factory); registerFunctionCRCImpl<FunctionCRC32IEEE>(factory);

2
src/Functions/CastOverloadResolver.cpp
View File

@ -5,7 +5,7 @@
namespace DB namespace DB
{ {
void registerCastOverloadResolvers(FunctionFactory & factory) REGISTER_FUNCTION(CastOverloadResolvers)
{ {
factory.registerFunction<CastInternalOverloadResolver<CastType::nonAccurate>>(FunctionFactory::CaseInsensitive); factory.registerFunction<CastInternalOverloadResolver<CastType::nonAccurate>>(FunctionFactory::CaseInsensitive);
factory.registerFunction<CastInternalOverloadResolver<CastType::accurate>>(); factory.registerFunction<CastInternalOverloadResolver<CastType::accurate>>();

2
src/Functions/FunctionChar.cpp
View File

@ -113,7 +113,7 @@ private:
} }
}; };
void registerFunctionChar(FunctionFactory & factory) REGISTER_FUNCTION(Char)
{ {
factory.registerFunction<FunctionChar>(FunctionFactory::CaseInsensitive); factory.registerFunction<FunctionChar>(FunctionFactory::CaseInsensitive);
} }

2
src/Functions/FunctionFQDN.cpp
View File

@ -44,7 +44,7 @@ public:
}; };
void registerFunctionFQDN(FunctionFactory & factory) REGISTER_FUNCTION(FQDN)
{ {
factory.registerFunction<FunctionFQDN>(FunctionFactory::CaseInsensitive); factory.registerFunction<FunctionFQDN>(FunctionFactory::CaseInsensitive);
factory.registerFunction<FunctionFQDN>("fullHostName"); factory.registerFunction<FunctionFQDN>("fullHostName");

1
src/Functions/FunctionFactory.h
View File

@ -1,6 +1,7 @@
#pragma once #pragma once
#include <Interpreters/Context_fwd.h> #include <Interpreters/Context_fwd.h>
#include <Common/register_objects.h>
#include <Common/IFactoryWithAliases.h> #include <Common/IFactoryWithAliases.h>
#include <Functions/IFunction.h> #include <Functions/IFunction.h>
#include <Functions/IFunctionAdaptors.h> #include <Functions/IFunctionAdaptors.h>

83
src/Functions/FunctionFile.cpp
View File

@ -1,7 +1,10 @@
#include <Columns/ColumnNullable.h>
#include <Columns/ColumnString.h> #include <Columns/ColumnString.h>
#include <Columns/ColumnConst.h>
#include <Columns/IColumn.h> #include <Columns/IColumn.h>
#include <Functions/FunctionFactory.h> #include <Functions/FunctionFactory.h>
#include <DataTypes/DataTypeString.h> #include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypeNullable.h>
#include <IO/ReadBufferFromFile.h> #include <IO/ReadBufferFromFile.h>
#include <IO/WriteBufferFromVector.h> #include <IO/WriteBufferFromVector.h>
#include <IO/copyData.h> #include <IO/copyData.h>
@ -19,6 +22,7 @@ namespace ErrorCodes
{ {
extern const int ILLEGAL_COLUMN; extern const int ILLEGAL_COLUMN;
extern const int NOT_IMPLEMENTED; extern const int NOT_IMPLEMENTED;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int DATABASE_ACCESS_DENIED; extern const int DATABASE_ACCESS_DENIED;
} }
@ -30,21 +34,41 @@ public:
static FunctionPtr create(ContextPtr context_) { return std::make_shared<FunctionFile>(context_); } static FunctionPtr create(ContextPtr context_) { return std::make_shared<FunctionFile>(context_); }
explicit FunctionFile(ContextPtr context_) : WithContext(context_) {} explicit FunctionFile(ContextPtr context_) : WithContext(context_) {}
bool isVariadic() const override { return true; }
String getName() const override { return name; } String getName() const override { return name; }
size_t getNumberOfArguments() const override { return 1; } size_t getNumberOfArguments() const override { return 0; }
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{ {
if (arguments.empty() || arguments.size() > 2)
throw Exception(
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
"Number of arguments for function {} doesn't match: passed {}, should be 1 or 2",
getName(), toString(arguments.size()));
if (!isString(arguments[0].type)) if (!isString(arguments[0].type))
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "{} is only implemented for type String", getName()); throw Exception(ErrorCodes::NOT_IMPLEMENTED, "{} is only implemented for type String", getName());
if (arguments.size() == 2)
{
if (arguments[1].type->onlyNull())
return makeNullable(std::make_shared<DataTypeString>());
if (!isString(arguments[1].type))
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "{} only accepts String or Null as second argument", getName());
}
return std::make_shared<DataTypeString>(); return std::make_shared<DataTypeString>();
} }
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; }
bool useDefaultImplementationForNulls() const override { return false; }
bool useDefaultImplementationForConstants() const override { return true; } bool useDefaultImplementationForConstants() const override { return true; }
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
{ {
const ColumnPtr column = arguments[0].column; const ColumnPtr column = arguments[0].column;
const ColumnString * column_src = checkAndGetColumn<ColumnString>(column.get()); const ColumnString * column_src = checkAndGetColumn<ColumnString>(column.get());
@ -53,6 +77,31 @@ public:
fmt::format("Illegal column {} of argument of function {}", arguments[0].column->getName(), getName()), fmt::format("Illegal column {} of argument of function {}", arguments[0].column->getName(), getName()),
ErrorCodes::ILLEGAL_COLUMN); ErrorCodes::ILLEGAL_COLUMN);
String default_result;
ColumnUInt8::MutablePtr col_null_map_to;
ColumnUInt8::Container * vec_null_map_to [[maybe_unused]] = nullptr;
if (arguments.size() == 2)
{
if (result_type->isNullable())
{
col_null_map_to = ColumnUInt8::create(input_rows_count, false);
vec_null_map_to = &col_null_map_to->getData();
}
else
{
const auto & default_column = arguments[1].column;
const ColumnConst * default_col = checkAndGetColumn<ColumnConst>(default_column.get());
if (!default_col)
throw Exception(
"Illegal column " + arguments[1].column->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN);
default_result = default_col->getValue<String>();
}
}
auto result = ColumnString::create(); auto result = ColumnString::create();
auto & res_chars = result->getChars(); auto & res_chars = result->getChars();
auto & res_offsets = result->getOffsets(); auto & res_offsets = result->getOffsets();
@ -77,24 +126,40 @@ public:
/// Otherwise it will not allow to work with symlinks in `user_files_path` directory. /// Otherwise it will not allow to work with symlinks in `user_files_path` directory.
file_path = fs::absolute(file_path).lexically_normal(); file_path = fs::absolute(file_path).lexically_normal();
if (need_check && file_path.string().find(user_files_absolute_path_string) != 0) try
throw Exception(ErrorCodes::DATABASE_ACCESS_DENIED, "File is not inside {}", user_files_absolute_path.string()); {
if (need_check && file_path.string().find(user_files_absolute_path_string) != 0)
throw Exception(ErrorCodes::DATABASE_ACCESS_DENIED, "File is not inside {}", user_files_absolute_path.string());
ReadBufferFromFile in(file_path); ReadBufferFromFile in(file_path);
WriteBufferFromVector out(res_chars, AppendModeTag{}); WriteBufferFromVector out(res_chars, AppendModeTag{});
copyData(in, out); copyData(in, out);
out.finalize(); out.finalize();
}
catch (...)
{
if (arguments.size() == 1)
throw;
if (vec_null_map_to)
(*vec_null_map_to)[row] = true;
else
res_chars.insert(default_result.data(), default_result.data() + default_result.size());
}
res_chars.push_back(0); res_chars.push_back(0);
res_offsets[row] = res_chars.size(); res_offsets[row] = res_chars.size();
} }
if (vec_null_map_to)
return ColumnNullable::create(std::move(result), std::move(col_null_map_to));
return result; return result;
} }
}; };
void registerFunctionFile(FunctionFactory & factory) REGISTER_FUNCTION(File)
{ {
factory.registerFunction<FunctionFile>(); factory.registerFunction<FunctionFile>();
} }

2
src/Functions/FunctionHashID.cpp
View File

@ -4,7 +4,7 @@
namespace DB namespace DB
{ {
void registerFunctionHashID(FunctionFactory & factory) REGISTER_FUNCTION(HashID)
{ {
factory.registerFunction<FunctionHashID>(); factory.registerFunction<FunctionHashID>();
} }

2
src/Functions/FunctionJoinGet.cpp
View File

@ -96,7 +96,7 @@ FunctionBasePtr JoinGetOverloadResolver<or_null>::buildImpl(const ColumnsWithTyp
return std::make_unique<FunctionJoinGet<or_null>>(getContext(), table_lock, storage_join, attr_name, argument_types, return_type); return std::make_unique<FunctionJoinGet<or_null>>(getContext(), table_lock, storage_join, attr_name, argument_types, return_type);
} }
void registerFunctionJoinGet(FunctionFactory & factory) REGISTER_FUNCTION(JoinGet)
{ {
// joinGet // joinGet
factory.registerFunction<JoinGetOverloadResolver<false>>(); factory.registerFunction<JoinGetOverloadResolver<false>>();

2
src/Functions/FunctionSQLJSON.cpp
View File

@ -5,7 +5,7 @@
namespace DB namespace DB
{ {
void registerFunctionsSQLJSON(FunctionFactory & factory) REGISTER_FUNCTION(SQLJSON)
{ {
factory.registerFunction<FunctionSQLJSON<NameJSONExists, JSONExistsImpl>>(); factory.registerFunction<FunctionSQLJSON<NameJSONExists, JSONExistsImpl>>();
factory.registerFunction<FunctionSQLJSON<NameJSONQuery, JSONQueryImpl>>(); factory.registerFunction<FunctionSQLJSON<NameJSONQuery, JSONQueryImpl>>();

2
src/Functions/FunctionShowCertificate.cpp
View File

@ -4,7 +4,7 @@
namespace DB namespace DB
{ {
void registerFunctionShowCertificate(FunctionFactory & factory) REGISTER_FUNCTION(ShowCertificate)
{ {
factory.registerFunction<FunctionShowCertificate>(); factory.registerFunction<FunctionShowCertificate>();
} }

4
src/Functions/FunctionsBase58.cpp
View File

@ -3,12 +3,12 @@
namespace DB namespace DB
{ {
void registerFunctionBase58Encode(FunctionFactory & factory) REGISTER_FUNCTION(Base58Encode)
{ {
factory.registerFunction<FunctionBase58Conversion<Base58Encode>>(); factory.registerFunction<FunctionBase58Conversion<Base58Encode>>();
} }
void registerFunctionBase58Decode(FunctionFactory & factory) REGISTER_FUNCTION(Base58Decode)
{ {
factory.registerFunction<FunctionBase58Conversion<Base58Decode>>(); factory.registerFunction<FunctionBase58Conversion<Base58Decode>>();
} }

2
src/Functions/FunctionsBinaryRepresentation.cpp
View File

@ -621,7 +621,7 @@ public:
} }
}; };
void registerFunctionsBinaryRepr(FunctionFactory & factory) REGISTER_FUNCTION(BinaryRepr)
{ {
factory.registerFunction<EncodeToBinaryRepresentation<HexImpl>>(FunctionFactory::CaseInsensitive); factory.registerFunction<EncodeToBinaryRepresentation<HexImpl>>(FunctionFactory::CaseInsensitive);
factory.registerFunction<DecodeFromBinaryRepresentation<UnhexImpl>>(FunctionFactory::CaseInsensitive); factory.registerFunction<DecodeFromBinaryRepresentation<UnhexImpl>>(FunctionFactory::CaseInsensitive);

5
src/Functions/FunctionsBitToArray.cpp
View File

@ -8,6 +8,7 @@
#include <Functions/IFunction.h> #include <Functions/IFunction.h>
#include <IO/WriteBufferFromVector.h> #include <IO/WriteBufferFromVector.h>
#include <IO/WriteHelpers.h> #include <IO/WriteHelpers.h>
#include <bit>
namespace DB namespace DB
@ -285,7 +286,7 @@ public:
{ {
while (x) while (x)
{ {
result_array_values_data.push_back(getTrailingZeroBitsUnsafe(x)); result_array_values_data.push_back(std::countr_zero(x));
x &= (x - 1); x &= (x - 1);
} }
} }
@ -329,7 +330,7 @@ public:
} }
void registerFunctionsBitToArray(FunctionFactory & factory) REGISTER_FUNCTION(BitToArray)
{ {
factory.registerFunction<FunctionBitPositionsToArray>(); factory.registerFunction<FunctionBitPositionsToArray>();
factory.registerFunction<FunctionBitmaskToArray>(); factory.registerFunction<FunctionBitmaskToArray>();

2
src/Functions/FunctionsBitmap.cpp
View File

@ -7,7 +7,7 @@
namespace DB namespace DB
{ {
void registerFunctionsBitmap(FunctionFactory & factory) REGISTER_FUNCTION(Bitmap)
{ {
factory.registerFunction<FunctionBitmapBuild>(); factory.registerFunction<FunctionBitmapBuild>();
factory.registerFunction<FunctionBitmapToArray>(); factory.registerFunction<FunctionBitmapToArray>();

2
src/Functions/FunctionsCharsetClassification.cpp
View File

@ -143,7 +143,7 @@ struct NameDetectLanguageUnknown
using FunctionDetectCharset = FunctionTextClassificationString<CharsetClassificationImpl<false>, NameDetectCharset>; using FunctionDetectCharset = FunctionTextClassificationString<CharsetClassificationImpl<false>, NameDetectCharset>;
using FunctionDetectLanguageUnknown = FunctionTextClassificationString<CharsetClassificationImpl<true>, NameDetectLanguageUnknown>; using FunctionDetectLanguageUnknown = FunctionTextClassificationString<CharsetClassificationImpl<true>, NameDetectLanguageUnknown>;
void registerFunctionDetectCharset(FunctionFactory & factory) REGISTER_FUNCTION(DetectCharset)
{ {
factory.registerFunction<FunctionDetectCharset>(); factory.registerFunction<FunctionDetectCharset>();
factory.registerFunction<FunctionDetectLanguageUnknown>(); factory.registerFunction<FunctionDetectLanguageUnknown>();

2
src/Functions/FunctionsCodingIP.cpp
View File

@ -1128,7 +1128,7 @@ public:
struct NameFunctionIPv4NumToString { static constexpr auto name = "IPv4NumToString"; }; struct NameFunctionIPv4NumToString { static constexpr auto name = "IPv4NumToString"; };
struct NameFunctionIPv4NumToStringClassC { static constexpr auto name = "IPv4NumToStringClassC"; }; struct NameFunctionIPv4NumToStringClassC { static constexpr auto name = "IPv4NumToStringClassC"; };
void registerFunctionsCoding(FunctionFactory & factory) REGISTER_FUNCTION(Coding)
{ {
factory.registerFunction<FunctionCutIPv6>(); factory.registerFunction<FunctionCutIPv6>();
factory.registerFunction<FunctionIPv4ToIPv6>(); factory.registerFunction<FunctionIPv4ToIPv6>();

2
src/Functions/FunctionsCodingUUID.cpp
View File

@ -229,7 +229,7 @@ public:
} }
}; };
void registerFunctionsCodingUUID(FunctionFactory & factory) REGISTER_FUNCTION(CodingUUID)
{ {
factory.registerFunction<FunctionUUIDNumToString>(); factory.registerFunction<FunctionUUIDNumToString>();
factory.registerFunction<FunctionUUIDStringToNum>(); factory.registerFunction<FunctionUUIDStringToNum>();

10
src/Functions/FunctionsConversion.cpp
View File

@ -5,11 +5,7 @@
namespace DB namespace DB
{ {
void registerFunctionFixedString(FunctionFactory & factory); REGISTER_FUNCTION(Conversion)
void registerCastOverloadResolvers(FunctionFactory & factory);
void registerFunctionsConversion(FunctionFactory & factory)
{ {
factory.registerFunction<FunctionToUInt8>(); factory.registerFunction<FunctionToUInt8>();
factory.registerFunction<FunctionToUInt16>(); factory.registerFunction<FunctionToUInt16>();
@ -41,12 +37,8 @@ void registerFunctionsConversion(FunctionFactory & factory)
factory.registerFunction<FunctionToUUID>(); factory.registerFunction<FunctionToUUID>();
factory.registerFunction<FunctionToString>(); factory.registerFunction<FunctionToString>();
registerFunctionFixedString(factory);
factory.registerFunction<FunctionToUnixTimestamp>(); factory.registerFunction<FunctionToUnixTimestamp>();
registerCastOverloadResolvers(factory);
factory.registerFunction<FunctionToUInt8OrZero>(); factory.registerFunction<FunctionToUInt8OrZero>();
factory.registerFunction<FunctionToUInt16OrZero>(); factory.registerFunction<FunctionToUInt16OrZero>();
factory.registerFunction<FunctionToUInt32OrZero>(); factory.registerFunction<FunctionToUInt32OrZero>();

105
src/Functions/FunctionsConversion.h
View File

@ -191,27 +191,29 @@ struct ConvertImpl
vec_null_map_to = &col_null_map_to->getData(); vec_null_map_to = &col_null_map_to->getData();
} }
bool result_is_bool = isBool(result_type); if constexpr (std::is_same_v<ToDataType, DataTypeUInt8>)
for (size_t i = 0; i < input_rows_count; ++i)
{ {
if constexpr (std::is_same_v<ToDataType, DataTypeUInt8>) if (isBool(result_type))
{ {
if (result_is_bool) for (size_t i = 0; i < input_rows_count; ++i)
{ {
vec_to[i] = vec_from[i] != FromFieldType(0); vec_to[i] = vec_from[i] != FromFieldType(0);
continue;
} }
goto done;
} }
}
if constexpr (std::is_same_v<FromDataType, DataTypeUUID> != std::is_same_v<ToDataType, DataTypeUUID>) if constexpr (std::is_same_v<FromDataType, DataTypeUUID> != std::is_same_v<ToDataType, DataTypeUUID>)
{
throw Exception("Conversion between numeric types and UUID is not supported. Probably the passed UUID is unquoted", ErrorCodes::NOT_IMPLEMENTED);
}
else
{
if constexpr (IsDataTypeDecimal<FromDataType> || IsDataTypeDecimal<ToDataType>)
{ {
throw Exception("Conversion between numeric types and UUID is not supported. Probably the passed UUID is unquoted", ErrorCodes::NOT_IMPLEMENTED); if constexpr (std::is_same_v<Additions, AccurateOrNullConvertStrategyAdditions>)
}
else
{
if constexpr (IsDataTypeDecimal<FromDataType> || IsDataTypeDecimal<ToDataType>)
{ {
if constexpr (std::is_same_v<Additions, AccurateOrNullConvertStrategyAdditions>) for (size_t i = 0; i < input_rows_count; ++i)
{ {
ToFieldType result; ToFieldType result;
bool convert_result = false; bool convert_result = false;
@ -231,7 +233,10 @@ struct ConvertImpl
(*vec_null_map_to)[i] = true; (*vec_null_map_to)[i] = true;
} }
} }
else }
else
{
for (size_t i = 0; i < input_rows_count; ++i)
{ {
if constexpr (IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>) if constexpr (IsDataTypeDecimal<FromDataType> && IsDataTypeDecimal<ToDataType>)
vec_to[i] = convertDecimals<FromDataType, ToDataType>(vec_from[i], col_from->getScale(), col_to->getScale()); vec_to[i] = convertDecimals<FromDataType, ToDataType>(vec_from[i], col_from->getScale(), col_to->getScale());
@ -243,10 +248,13 @@ struct ConvertImpl
throw Exception("Unsupported data type in conversion function", ErrorCodes::CANNOT_CONVERT_TYPE); throw Exception("Unsupported data type in conversion function", ErrorCodes::CANNOT_CONVERT_TYPE);
} }
} }
else }
else
{
/// If From Data is Nan or Inf and we convert to integer type, throw exception
if constexpr (std::is_floating_point_v<FromFieldType> && !std::is_floating_point_v<ToFieldType>)
{ {
/// If From Data is Nan or Inf and we convert to integer type, throw exception for (size_t i = 0; i < input_rows_count; ++i)
if constexpr (std::is_floating_point_v<FromFieldType> && !std::is_floating_point_v<ToFieldType>)
{ {
if (!isFinite(vec_from[i])) if (!isFinite(vec_from[i]))
{ {
@ -254,15 +262,46 @@ struct ConvertImpl
{ {
vec_to[i] = 0; vec_to[i] = 0;
(*vec_null_map_to)[i] = true; (*vec_null_map_to)[i] = true;
continue;
} }
else else
throw Exception("Unexpected inf or nan to integer conversion", ErrorCodes::CANNOT_CONVERT_TYPE); throw Exception("Unexpected inf or nan to integer conversion", ErrorCodes::CANNOT_CONVERT_TYPE);
} }
} else
{
if constexpr (std::is_same_v<Additions, AccurateOrNullConvertStrategyAdditions>
|| std::is_same_v<Additions, AccurateConvertStrategyAdditions>)
{
bool convert_result = accurate::convertNumeric(vec_from[i], vec_to[i]);
if constexpr (std::is_same_v<Additions, AccurateOrNullConvertStrategyAdditions> if (!convert_result)
|| std::is_same_v<Additions, AccurateConvertStrategyAdditions>) {
if (std::is_same_v<Additions, AccurateOrNullConvertStrategyAdditions>)
{
vec_to[i] = 0;
(*vec_null_map_to)[i] = true;
}
else
{
throw Exception(
"Value in column " + named_from.column->getName() + " cannot be safely converted into type "
+ result_type->getName(),
ErrorCodes::CANNOT_CONVERT_TYPE);
}
}
}
else
{
vec_to[i] = static_cast<ToFieldType>(vec_from[i]);
}
}
}
goto done;
}
if constexpr (std::is_same_v<Additions, AccurateOrNullConvertStrategyAdditions>
|| std::is_same_v<Additions, AccurateConvertStrategyAdditions>)
{
for (size_t i = 0; i < input_rows_count; ++i)
{ {
bool convert_result = accurate::convertNumeric(vec_from[i], vec_to[i]); bool convert_result = accurate::convertNumeric(vec_from[i], vec_to[i]);
@ -282,14 +321,38 @@ struct ConvertImpl
} }
} }
} }
}
else
{
if constexpr (std::is_same_v<FromDataType, DataTypeUInt64> && std::is_same_v<ToDataType, DataTypeFloat32>)
{
/// Turns out that when ClickHouse is compiled with AVX1 or AVX2 instructions, Clang's autovectorizer produces
/// code for UInt64-to-Float23 conversion which is only ~50% as fast as scalar code. Interestingly, scalar code
/// is equally fast than code compiled for SSE4.2, so we might as well disable vectorization. This situation
/// may change with AVX512 which has a dediated instruction for that usecase (_mm512_cvtepi64_ps).
#if defined(__x86_64__)
# ifdef __clang__
# pragma clang loop vectorize(disable) interleave(disable)
# endif
#endif
for (size_t i = 0; i < input_rows_count; ++i)
{
vec_to[i] = static_cast<ToFieldType>(vec_from[i]);
}
}
else else
{ {
vec_to[i] = static_cast<ToFieldType>(vec_from[i]); for (size_t i = 0; i < input_rows_count; ++i)
{
vec_to[i] = static_cast<ToFieldType>(vec_from[i]);
}
} }
} }
} }
} }
done:
if constexpr (std::is_same_v<Additions, AccurateOrNullConvertStrategyAdditions>) if constexpr (std::is_same_v<Additions, AccurateOrNullConvertStrategyAdditions>)
return ColumnNullable::create(std::move(col_to), std::move(col_null_map_to)); return ColumnNullable::create(std::move(col_to), std::move(col_null_map_to));
else else

2
src/Functions/FunctionsEmbeddedDictionaries.cpp
View File

@ -5,7 +5,7 @@
namespace DB namespace DB
{ {
void registerFunctionsEmbeddedDictionaries(FunctionFactory & factory) REGISTER_FUNCTION(EmbeddedDictionaries)
{ {
factory.registerFunction<FunctionRegionToCity>(); factory.registerFunction<FunctionRegionToCity>();
factory.registerFunction<FunctionRegionToArea>(); factory.registerFunction<FunctionRegionToArea>();

2
src/Functions/FunctionsExternalDictionaries.cpp
View File

@ -5,7 +5,7 @@
namespace DB namespace DB
{ {
void registerFunctionsExternalDictionaries(FunctionFactory & factory) REGISTER_FUNCTION(ExternalDictionaries)
{ {
factory.registerFunction<FunctionDictHas>(); factory.registerFunction<FunctionDictHas>();
factory.registerFunction<FunctionDictGetUInt8>(); factory.registerFunction<FunctionDictGetUInt8>();

2
src/Functions/FunctionsHashing.cpp
View File

@ -6,7 +6,7 @@
namespace DB namespace DB
{ {
void registerFunctionsHashing(FunctionFactory & factory) REGISTER_FUNCTION(Hashing)
{ {
#if USE_SSL #if USE_SSL
factory.registerFunction<FunctionMD4>(); factory.registerFunction<FunctionMD4>();

2
src/Functions/FunctionsJSON.cpp
View File

@ -1443,7 +1443,7 @@ public:
} }
}; };
void registerFunctionsJSON(FunctionFactory & factory) REGISTER_FUNCTION(JSON)
{ {
factory.registerFunction<JSONOverloadResolver<NameJSONHas, JSONHasImpl>>(); factory.registerFunction<JSONOverloadResolver<NameJSONHas, JSONHasImpl>>();
factory.registerFunction<JSONOverloadResolver<NameIsValidJSON, IsValidJSONImpl>>(); factory.registerFunction<JSONOverloadResolver<NameIsValidJSON, IsValidJSONImpl>>();

2
src/Functions/FunctionsLanguageClassification.cpp
View File

@ -221,7 +221,7 @@ struct NameDetectLanguage
using FunctionDetectLanguage = FunctionTextClassificationString<FunctionDetectLanguageImpl, NameDetectLanguage>; using FunctionDetectLanguage = FunctionTextClassificationString<FunctionDetectLanguageImpl, NameDetectLanguage>;
void registerFunctionsDetectLanguage(FunctionFactory & factory) REGISTER_FUNCTION(DetectLanguage)
{ {
factory.registerFunction<FunctionDetectLanguage>(); factory.registerFunction<FunctionDetectLanguage>();
factory.registerFunction<FunctionDetectLanguageMixed>(); factory.registerFunction<FunctionDetectLanguageMixed>();

2
src/Functions/FunctionsLogical.cpp
View File

@ -22,7 +22,7 @@
namespace DB namespace DB
{ {
void registerFunctionsLogical(FunctionFactory & factory) REGISTER_FUNCTION(Logical)
{ {
factory.registerFunction<FunctionAnd>(); factory.registerFunction<FunctionAnd>();
factory.registerFunction<FunctionOr>(); factory.registerFunction<FunctionOr>();

2
src/Functions/FunctionsProgrammingClassification.cpp
View File

@ -112,7 +112,7 @@ struct NameDetectProgrammingLanguage
using FunctionDetectProgrammingLanguage = FunctionTextClassificationString<FunctionDetectProgrammingLanguageImpl, NameDetectProgrammingLanguage>; using FunctionDetectProgrammingLanguage = FunctionTextClassificationString<FunctionDetectProgrammingLanguageImpl, NameDetectProgrammingLanguage>;
void registerFunctionDetectProgrammingLanguage(FunctionFactory & factory) REGISTER_FUNCTION(DetectProgrammingLanguage)
{ {
factory.registerFunction<FunctionDetectProgrammingLanguage>(); factory.registerFunction<FunctionDetectProgrammingLanguage>();
} }

2
src/Functions/FunctionsRound.cpp
View File

@ -5,7 +5,7 @@
namespace DB namespace DB
{ {
void registerFunctionsRound(FunctionFactory & factory) REGISTER_FUNCTION(Round)
{ {
factory.registerFunction<FunctionRound>("round", FunctionFactory::CaseInsensitive); factory.registerFunction<FunctionRound>("round", FunctionFactory::CaseInsensitive);
factory.registerFunction<FunctionRoundBankers>("roundBankers", FunctionFactory::CaseSensitive); factory.registerFunction<FunctionRoundBankers>("roundBankers", FunctionFactory::CaseSensitive);

2
src/Functions/FunctionsStringArray.cpp
View File

@ -27,7 +27,7 @@ DataTypePtr FunctionArrayStringConcat::getReturnTypeImpl(const DataTypes & argum
return std::make_shared<DataTypeString>(); return std::make_shared<DataTypeString>();
} }
void registerFunctionsStringArray(FunctionFactory & factory) REGISTER_FUNCTION(StringArray)
{ {
factory.registerFunction<FunctionExtractAll>(); factory.registerFunction<FunctionExtractAll>();
factory.registerFunction<FunctionAlphaTokens>(); factory.registerFunction<FunctionAlphaTokens>();

2
src/Functions/FunctionsStringHash.cpp
View File

@ -743,7 +743,7 @@ using FunctionWordShingleMinHashArgUTF8
using FunctionWordShingleMinHashArgCaseInsensitiveUTF8 using FunctionWordShingleMinHashArgCaseInsensitiveUTF8
= FunctionsStringHash<MinHashImpl<true, false, true>, NameWordShingleMinHashArgCaseInsensitiveUTF8, false, true>; = FunctionsStringHash<MinHashImpl<true, false, true>, NameWordShingleMinHashArgCaseInsensitiveUTF8, false, true>;
void registerFunctionsStringHash(FunctionFactory & factory) REGISTER_FUNCTION(StringHash)
{ {
factory.registerFunction<FunctionNgramSimHash>(); factory.registerFunction<FunctionNgramSimHash>();
factory.registerFunction<FunctionNgramSimHashCaseInsensitive>(); factory.registerFunction<FunctionNgramSimHashCaseInsensitive>();

2
src/Functions/FunctionsStringSimilarity.cpp
View File

@ -530,7 +530,7 @@ using FunctionNgramSearchUTF8 = FunctionsStringSimilarity<NgramDistanceImpl<3, U
using FunctionNgramSearchCaseInsensitiveUTF8 = FunctionsStringSimilarity<NgramDistanceImpl<3, UInt32, true, true, false>, NameNgramSearchUTF8CaseInsensitive>; using FunctionNgramSearchCaseInsensitiveUTF8 = FunctionsStringSimilarity<NgramDistanceImpl<3, UInt32, true, true, false>, NameNgramSearchUTF8CaseInsensitive>;
void registerFunctionsStringSimilarity(FunctionFactory & factory) REGISTER_FUNCTION(StringSimilarity)
{ {
factory.registerFunction<FunctionNgramDistance>(); factory.registerFunction<FunctionNgramDistance>();
factory.registerFunction<FunctionNgramDistanceCaseInsensitive>(); factory.registerFunction<FunctionNgramDistanceCaseInsensitive>();

Some files were not shown because too many files have changed in this diff Show More