thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 08:40:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into ncb/hostname-system-log-tables

Browse Source
This commit is contained in:
Bharat Nallan Chakravarthy 2023-11-26 16:24:18 -08:00
commit 6454687fa6
410 changed files with 8016 additions and 3068 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored
View File

@ -354,3 +354,6 @@
[submodule "contrib/aklomp-base64"]
path = contrib/aklomp-base64
url = https://github.com/aklomp/base64.git
[submodule "contrib/pocketfft"]
path = contrib/pocketfft
url = https://github.com/mreineck/pocketfft.git

13
CMakeLists.txt
View File

@ -21,8 +21,11 @@ include (cmake/clang_tidy.cmake)
include (cmake/git.cmake)
include (cmake/utils.cmake)
# This is needed to set up the CMAKE_INSTALL_BINDIR variable.
include (GNUInstallDirs)
# Ignore export() since we don't use it,
# but it gets broken with a global targets via link_libraries()
# but it gets broken with global targets via link_libraries()
macro (export)
endmacro ()
@ -460,14 +463,6 @@ endif ()
message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE}")
include (GNUInstallDirs)
# When testing for memory leaks with Valgrind, don't link tcmalloc or jemalloc.
if (TARGET global-group)
install (EXPORT global DESTINATION cmake)
endif ()
add_subdirectory (contrib EXCLUDE_FROM_ALL)
if (NOT ENABLE_JEMALLOC)

6
base/glibc-compatibility/CMakeLists.txt
View File

@ -35,12 +35,6 @@ if (GLIBC_COMPATIBILITY)
target_link_libraries(global-libs INTERFACE glibc-compatibility ${MEMCPY_LIBRARY})
install(
TARGETS glibc-compatibility ${MEMCPY_LIBRARY}
EXPORT global
ARCHIVE DESTINATION lib
)
message (STATUS "Some symbols from glibc will be replaced for compatibility")
elseif (CLICKHOUSE_OFFICIAL_BUILD)

1
base/harmful/CMakeLists.txt
View File

@ -1,2 +1 @@
add_library(harmful harmful.c)
install(TARGETS harmful EXPORT global ARCHIVE DESTINATION lib)

6
cmake/ccache.cmake
View File

@ -9,10 +9,10 @@ if (CMAKE_CXX_COMPILER_LAUNCHER MATCHES "ccache" OR CMAKE_C_COMPILER_LAUNCHER MA
return()
endif()
set(COMPILER_CACHE "auto" CACHE STRING "Speedup re-compilations using the caching tools; valid options are 'auto' (ccache, then sccache), 'ccache', 'sccache', or 'disabled'")
set(COMPILER_CACHE "auto" CACHE STRING "Speedup re-compilations using the caching tools; valid options are 'auto' (sccache, then ccache), 'ccache', 'sccache', or 'disabled'")
if(COMPILER_CACHE STREQUAL "auto")
find_program (CCACHE_EXECUTABLE NAMES ccache sccache)
find_program (CCACHE_EXECUTABLE NAMES sccache ccache)
elseif (COMPILER_CACHE STREQUAL "ccache")
find_program (CCACHE_EXECUTABLE ccache)
elseif(COMPILER_CACHE STREQUAL "sccache")
@ -21,7 +21,7 @@ elseif(COMPILER_CACHE STREQUAL "disabled")
message(STATUS "Using *ccache: no (disabled via configuration)")
return()
else()
message(${RECONFIGURE_MESSAGE_LEVEL} "The COMPILER_CACHE must be one of (auto|ccache|sccache|disabled), value: '${COMPILER_CACHE}'")
message(${RECONFIGURE_MESSAGE_LEVEL} "The COMPILER_CACHE must be one of (auto|sccache|ccache|disabled), value: '${COMPILER_CACHE}'")
endif()

6
cmake/darwin/default_libs.cmake
View File

@ -22,9 +22,3 @@ link_libraries(global-group)
target_link_libraries(global-group INTERFACE
$<TARGET_PROPERTY:global-libs,INTERFACE_LINK_LIBRARIES>
)
# FIXME: remove when all contribs will get custom cmake lists
install(
TARGETS global-group global-libs
EXPORT global
)

6
cmake/freebsd/default_libs.cmake
View File

@ -25,9 +25,3 @@ link_libraries(global-group)
target_link_libraries(global-group INTERFACE
$<TARGET_PROPERTY:global-libs,INTERFACE_LINK_LIBRARIES>
)
# FIXME: remove when all contribs will get custom cmake lists
install(
TARGETS global-group global-libs
EXPORT global
)

4
cmake/fuzzer.cmake
View File

@ -4,8 +4,8 @@ if (FUZZER)
# NOTE: Eldar Zaitov decided to name it "libfuzzer" instead of "fuzzer" to keep in mind another possible fuzzer backends.
# NOTE: no-link means that all the targets are built with instrumentation for fuzzer, but only some of them
# (tests) have entry point for fuzzer and it's not checked.
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=fuzzer-no-link")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=fuzzer-no-link")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${SAN_FLAGS} -fsanitize=fuzzer-no-link -DFUZZER=1")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${SAN_FLAGS} -fsanitize=fuzzer-no-link -DFUZZER=1")
# NOTE: oss-fuzz can change LIB_FUZZING_ENGINE variable
if (NOT LIB_FUZZING_ENGINE)

6
cmake/linux/default_libs.cmake
View File

@ -50,9 +50,3 @@ target_link_libraries(global-group INTERFACE
$<TARGET_PROPERTY:global-libs,INTERFACE_LINK_LIBRARIES>
-Wl,--end-group
)
# FIXME: remove when all contribs will get custom cmake lists
install(
TARGETS global-group global-libs
EXPORT global
)

1
contrib/CMakeLists.txt vendored
View File

@ -44,6 +44,7 @@ else ()
endif ()
add_contrib (miniselect-cmake miniselect)
add_contrib (pdqsort-cmake pdqsort)
add_contrib (pocketfft-cmake pocketfft)
add_contrib (crc32-vpmsum-cmake crc32-vpmsum)
add_contrib (sparsehash-c11-cmake sparsehash-c11)
add_contrib (abseil-cpp-cmake abseil-cpp)

2
contrib/grpc vendored

@ -1 +1 @@
Subproject commit 740e3dfd97301a52ad8165b65285bcc149d9e817
Subproject commit 77b2737a709d43d8c6895e3f03ca62b00bd9201c

2
contrib/libunwind vendored

@ -1 +1 @@
Subproject commit 30cc1d3fd3655a5cfa0ab112fe320fb9fc0a8344
Subproject commit 40d8eadf96b127d9b22d53ce7a4fc52aaedea965

10
contrib/libunwind-cmake/CMakeLists.txt
View File

@ -20,15 +20,7 @@ set(LIBUNWIND_ASM_SOURCES
"${LIBUNWIND_SOURCE_DIR}/src/UnwindRegistersRestore.S"
"${LIBUNWIND_SOURCE_DIR}/src/UnwindRegistersSave.S")
# CMake doesn't pass the correct architecture for Apple prior to CMake 3.19 [1]
# Workaround these two issues by compiling as C.
#
# [1]: https://gitlab.kitware.com/cmake/cmake/-/issues/20771
if (APPLE AND CMAKE_VERSION VERSION_LESS 3.19)
set_source_files_properties(${LIBUNWIND_ASM_SOURCES} PROPERTIES LANGUAGE C)
else()
enable_language(ASM)
endif()
enable_language(ASM)
set(LIBUNWIND_SOURCES
${LIBUNWIND_CXX_SOURCES}

3
contrib/llvm-project-cmake/CMakeLists.txt
View File

@ -61,6 +61,9 @@ set (REQUIRED_LLVM_LIBRARIES
LLVMDemangle
)
# Skip useless "install" instructions from CMake:
set (LLVM_INSTALL_TOOLCHAIN_ONLY 1 CACHE INTERNAL "")
if (ARCH_AMD64)
set (LLVM_TARGETS_TO_BUILD "X86" CACHE INTERNAL "")
list(APPEND REQUIRED_LLVM_LIBRARIES LLVMX86Info LLVMX86Desc LLVMX86CodeGen)

1
contrib/pocketfft vendored Submodule

@ -0,0 +1 @@
Subproject commit 9efd4da52cf8d28d14531d14e43ad9d913807546

10
contrib/pocketfft-cmake/CMakeLists.txt Normal file
View File

@ -0,0 +1,10 @@
option (ENABLE_POCKETFFT "Enable pocketfft" ${ENABLE_LIBRARIES})
if (NOT ENABLE_POCKETFFT)
message(STATUS "Not using pocketfft")
return()
endif()
add_library(_pocketfft INTERFACE)
target_include_directories(_pocketfft INTERFACE ${ClickHouse_SOURCE_DIR}/contrib/pocketfft)
add_library(ch_contrib::pocketfft ALIAS _pocketfft)

14
contrib/update-submodules.sh vendored
View File

@ -9,4 +9,16 @@ cd $GIT_DIR
contrib/sparse-checkout/setup-sparse-checkout.sh
git submodule init
git submodule sync
git config --file .gitmodules --get-regexp .*path | sed 's/[^ ]* //' | xargs -I _ --max-procs 64 git submodule update --depth=1 --single-branch _
# NOTE: do not use --remote for `git submodule update`[1] command, since the submodule references to the specific commit SHA1 in the subproject.
# It may cause unexpected behavior. Instead you need to commit a new SHA1 for a submodule.
#
# [1] - https://git-scm.com/book/en/v2/Git-Tools-Submodules
git config --file .gitmodules --get-regexp '.*path' | sed 's/[^ ]* //' | xargs -I _ --max-procs 64 git submodule update --depth=1 --single-branch _
# We don't want to depend on any third-party CMake files.
# To check it, find and delete them.
grep -o -P '"contrib/[^"]+"' .gitmodules |
grep -v -P 'contrib/(llvm-project|google-protobuf|grpc|abseil-cpp|corrosion)' |
xargs -I@ find @ \
-'(' -name 'CMakeLists.txt' -or -name '*.cmake' -')' -and -not -name '*.h.cmake' \
-delete

2
docker/keeper/Dockerfile
View File

@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \
# lts / testing / prestable / etc
ARG REPO_CHANNEL="stable"
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
ARG VERSION="23.10.4.25"
ARG VERSION="23.10.5.20"
ARG PACKAGES="clickhouse-keeper"
# user/group precreated explicitly with fixed uid/gid on purpose.

9
docker/packager/binary/build.sh
View File

@ -34,15 +34,6 @@ cd /build/build_docker
rm -f CMakeCache.txt
# We don't want to depend on any third-party CMake files.
# To check it, find and delete them.
grep -o -P '"contrib/[^"]+"' ../.gitmodules |
grep -v -P 'llvm-project|google-protobuf|grpc|abseil-cpp|corrosion' |
xargs -I@ find ../@ -'(' -name 'CMakeLists.txt' -or -name '*.cmake' -')' -and -not -name '*.h.cmake' |
xargs rm
if [ -n "$MAKE_DEB" ]; then
rm -rf /build/packages/root
# NOTE: this is for backward compatibility with previous releases,

8
docker/server/.dockerignore
View File

@ -1,8 +0,0 @@
# post / preinstall scripts (not needed, we do it in Dockerfile)
alpine-root/install/*
# docs (looks useless)
alpine-root/usr/share/doc/*
# packages, etc. (used by alpine-build.sh)
tgz-packages/*

2
docker/server/Dockerfile.alpine
View File

@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \
# lts / testing / prestable / etc
ARG REPO_CHANNEL="stable"
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
ARG VERSION="23.10.4.25"
ARG VERSION="23.10.5.20"
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
# user/group precreated explicitly with fixed uid/gid on purpose.

2
docker/server/Dockerfile.ubuntu
View File

@ -30,7 +30,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
ARG REPO_CHANNEL="stable"
ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
ARG VERSION="23.10.4.25"
ARG VERSION="23.10.5.20"
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
# set non-empty deb_location_url url to create a docker image

22
docker/test/base/setup_export_logs.sh
View File

@ -126,6 +126,9 @@ function setup_logs_replication
# It's doesn't make sense to try creating tables if SYNC fails
echo "SYSTEM SYNC DATABASE REPLICA default" | clickhouse-client "${CONNECTION_ARGS[@]}" || return 0
debug_or_sanitizer_build=$(clickhouse-client -q "WITH ((SELECT value FROM system.build_options WHERE name='BUILD_TYPE') AS build, (SELECT value FROM system.build_options WHERE name='CXX_FLAGS') as flags) SELECT build='Debug' OR flags LIKE '%fsanitize%'")
echo "Build is debug or sanitizer: $debug_or_sanitizer_build"
# For each system log table:
echo 'Create %_log tables'
clickhouse-client --query "SHOW TABLES FROM system LIKE '%\\_log'" | while read -r table
@ -133,7 +136,14 @@ function setup_logs_replication
if [[ "$table" = "trace_log" ]]
then
EXTRA_COLUMNS_FOR_TABLE="${EXTRA_COLUMNS_TRACE_LOG}"
EXTRA_COLUMNS_EXPRESSION_FOR_TABLE="${EXTRA_COLUMNS_EXPRESSION_TRACE_LOG}"
# Do not try to resolve stack traces in case of debug/sanitizers
# build, since it is too slow (flushing of trace_log can take ~1min
# with such MV attached)
if [[ "$debug_or_sanitizer_build" = 1 ]]; then
EXTRA_COLUMNS_EXPRESSION_FOR_TABLE="${EXTRA_COLUMNS_EXPRESSION}"
else
EXTRA_COLUMNS_EXPRESSION_FOR_TABLE="${EXTRA_COLUMNS_EXPRESSION_TRACE_LOG}"
fi
else
EXTRA_COLUMNS_FOR_TABLE="${EXTRA_COLUMNS}"
EXTRA_COLUMNS_EXPRESSION_FOR_TABLE="${EXTRA_COLUMNS_EXPRESSION}"
@ -182,3 +192,13 @@ function setup_logs_replication
" || continue
done
)
function stop_logs_replication
{
echo "Detach all logs replication"
clickhouse-client --query "select database||'.'||table from system.tables where database = 'system' and (table like '%_sender' or table like '%_watcher')" | {
tee /dev/stderr
} | {
xargs -n1 -r -i clickhouse-client --query "drop table {}"
}
}

2
docker/test/fuzzer/generate-test-j2.py
View File

@ -3,6 +3,7 @@
from argparse import ArgumentParser
import os
import jinja2
import itertools
def removesuffix(text, suffix):
@ -47,6 +48,7 @@ def main(args):
loader=jinja2.FileSystemLoader(suite_dir),
keep_trailing_newline=True,
)
j2env.globals.update(product=itertools.product)
test_names = os.listdir(suite_dir)
for test_name in test_names:

5
docker/test/fuzzer/query-fuzzer-tweaks-users.xml
View File

@ -23,11 +23,6 @@
<max>10G</max>
</max_memory_usage>
<!-- Analyzer is unstable, not ready for testing. -->
<allow_experimental_analyzer>
<readonly/>
</allow_experimental_analyzer>
<table_function_remote_max_addresses>
<max>200</max>
</table_function_remote_max_addresses>

15
docker/test/fuzzer/run-fuzzer.sh
View File

@ -212,11 +212,11 @@ quit
gdb -batch -command script.gdb -p $server_pid &
sleep 5
# gdb will send SIGSTOP, spend some time loading debug info and then send SIGCONT, wait for it (up to send_timeout, 300s)
# gdb will send SIGSTOP, spend some time loading debug info, and then send SIGCONT, wait for it (up to send_timeout, 300s)
time clickhouse-client --query "SELECT 'Connected to clickhouse-server after attaching gdb'" ||:
# Check connectivity after we attach gdb, because it might cause the server
# to freeze and the fuzzer will fail. In debug build it can take a lot of time.
# to freeze, and the fuzzer will fail. In debug build, it can take a lot of time.
for _ in {1..180}
do
if clickhouse-client --query "select 1"
@ -226,14 +226,15 @@ quit
sleep 1
done
kill -0 $server_pid # This checks that it is our server that is started and not some other one
echo 'Server started and responded'
echo 'Server started and responded.'
setup_logs_replication
# SC2012: Use find instead of ls to better handle non-alphanumeric filenames. They are all alphanumeric.
# SC2046: Quote this to prevent word splitting. Actually I need word splitting.
# SC2046: Quote this to prevent word splitting. Actually, I need word splitting.
# shellcheck disable=SC2012,SC2046
timeout -s TERM --preserve-status 30m clickhouse-client \
--max_memory_usage_in_client=1000000000 \
--receive_timeout=10 \
--receive_data_timeout_ms=10000 \
--stacktrace \
@ -253,10 +254,10 @@ quit
wait "$fuzzer_pid" || fuzzer_exit_code=$?
echo "Fuzzer exit code is $fuzzer_exit_code"
# If the server dies, most often the fuzzer returns code 210: connetion
# If the server dies, most often the fuzzer returns Code 210: Connetion
# refused, and sometimes also code 32: attempt to read after eof. For
# simplicity, check again whether the server is accepting connections, using
# clickhouse-client. We don't check for existence of server process, because
# simplicity, check again whether the server is accepting connections using
# clickhouse-client. We don't check for the existence of the server process, because
# the process is still present while the server is terminating and not
# accepting the connections anymore.

6
docker/test/integration/runner/compose/docker_compose_rabbitmq.yml
View File

@ -6,9 +6,13 @@ services:
hostname: rabbitmq1
expose:
- ${RABBITMQ_PORT:-5672}
- ${RABBITMQ_SECURE_PORT:-5671}
volumes:
- type: ${RABBITMQ_LOGS_FS:-tmpfs}
source: ${RABBITMQ_LOGS:-}
target: /rabbitmq_logs/
- "${RABBITMQ_COOKIE_FILE}:/var/lib/rabbitmq/.erlang.cookie"
- /misc/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf
- /misc/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf
- /misc/rabbitmq/ca-cert.pem:/etc/rabbitmq/ca-cert.pem
- /misc/rabbitmq/server-cert.pem:/etc/rabbitmq/server-cert.pem
- /misc/rabbitmq/server-key.pem:/etc/rabbitmq/server-key.pem

8
docker/test/integration/runner/misc/rabbitmq.conf
View File

@ -1,8 +0,0 @@
loopback_users.guest = false
listeners.tcp.default = 5672
default_pass = clickhouse
default_user = root
management.tcp.port = 15672
log.file = /rabbitmq_logs/rabbit.log
log.file.level = debug

32
docker/test/integration/runner/misc/rabbitmq/ca-cert.pem Normal file
View File

@ -0,0 +1,32 @@
-----BEGIN CERTIFICATE-----
MIIFhTCCA22gAwIBAgIUWhfjFfbwannH3KIqITDtgcvSItMwDQYJKoZIhvcNAQEL
BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjMxMTE0
MTgyODI2WhcNMzMxMTExMTgyODI2WjBSMQswCQYDVQQGEwJSVTETMBEGA1UECAwK
U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQsw
CQYDVQQDDAJjYTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAJfJegdC
gavNGYzSdva+5QMxGvqyLwZzjophMeyEzlW/Di4KFGPho+fVlVMB/EwaTRoBRLEu
SQusQwoFg71mGvUTOpgHzlsUz4vcVVFOiL4bJdzCWQKzdC8M8rUFoks9FMboVeSx
jhAnKAm/NpCLpm9VYnRjEq2KEbJp7VkPAHgZEXR7VABwCFvmDcztrfcWfmXxm6IH
o+AkF/nqdphLu7Q1yDQiF8Q8TuszuhqgQ7/1PrRcaSADrF15jJjQb05sILpGCT3e
lxJYId5RF0+fgTIqy03bAKB53+8V8cAkowI4rvPTmcFXhcG3rkDO6lyZixHhlpKi
PmXEzHh0kfsRjzkNBP0CKqPnu3D2iymROiPAH2cteaYe6jdD2HIjuVLk/TjX1ZFy
DlZCrJIwj0l8A2xAfLq8Gw5RSr0a9k5TiMD5nZtfd12Vd0K82vO32vmcjO2Igddc
VWccDDwUY/ZWV3uznkusOBrB8wba3ZsXA5hjJzs0KlTvQKPjX0y4lFMmZGbelwjt
pR5dRNLi5XTdMPzV0mAnvJhDTFEmME19Bh6AEsjuAz3gHUdwNTbSxUS3mF/hTL9k
v2wh5udUAOwqD1uEzqPJyG4JCJQozIDOEEZVixWqQ60b9wUHN8meqO4y9fxTdmHW
Vo5BAF1xEJhJJb0QY/O6GahPtWqb/Mr1rtPJAgMBAAGjUzBRMB0GA1UdDgQWBBSw
fQcOabXwX/v9F1hd2cmuIug56jAfBgNVHSMEGDAWgBSwfQcOabXwX/v9F1hd2cmu
Iug56jAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQAms8y6RVxl
mKSUbsU8JscYwOzcRUQJWETeIr4rtZvMHH+3vkdBU0yKxGpEm7U8J3+5oVTYPhbs
11ZAL+DvIZ6gT6pjDvECyVox1OkjNogz843fTMbNqjuuehjSKXwpMTy5/kmT2aLj
//nBi5UX1xo3RQ9vtmBwzZ3VFK99DFXraDOPS/yk43WV2uqdWsXCNvyEyCHmM1IB
9FQe2EFcO6s4/N+TarhIZ8Udhj5bl8d4eDd1yEckmTD4aHJBgMII2uEwrAxR5CT1
tCqUKutvNrkXI5PIULvmy+Lwm7PJAC7grPtUHK6anSugpljd7bFj18fHH9APiC45
Ou4OOK1BUZogCEo7rD36UlanxQO0GEzgDCVEoEdoe0WRdc6T9b4fM8vpQqwBdf9t
nkPB8oLCKerqqYwCiMuWm4BcRmExA7ypIkUCcluGO9/kTmdps3NqOvET9oLTjXuA
z5TPmaK5a3poKLoxBfv6WfRTgisOnMNTsjL1R8+xuhEn5hSlE2r3wAi8Cys9Z9PV
LhTj0SRTXILd2NW3lO8QfO0pGdjgk90GqkyUY9YjuiMVPvdUAFQsHm+0GEZEXjOD
Bw7tLSJQ4IKhfactg/Puxd15ahcWAxeelyED+w/zVGdHYblqbvfdtiGj370KVhoj
DL5HkdPa0IhTPqMBnmoVQ4C/WzKofXBjQQ==
-----END CERTIFICATE-----

10
docker/test/integration/runner/misc/rabbitmq/generate_certs.sh Executable file
View File

@ -0,0 +1,10 @@
#!/bin/bash
# 1. Generate CA's private key and self-signed certificate
openssl req -newkey rsa:4096 -x509 -days 3650 -nodes -batch -keyout ca-key.pem -out ca-cert.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=ca"
# 2. Generate server's private key and certificate signing request (CSR)
openssl req -newkey rsa:4096 -nodes -batch -keyout server-key.pem -out server-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=server"
# 3. Use CA's private key to sign server's CSR and get back the signed certificate
openssl x509 -req -days 3650 -in server-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -extfile server-ext.cnf -out server-cert.pem

15
docker/test/integration/runner/misc/rabbitmq/rabbitmq.conf Normal file
View File

@ -0,0 +1,15 @@
loopback_users.guest = false
listeners.tcp.default = 5672
default_pass = clickhouse
default_user = root
management.tcp.port = 15672
log.file = /rabbitmq_logs/rabbit.log
log.file.level = debug
listeners.ssl.default = 5671
ssl_options.verify = verify_none
ssl_options.fail_if_no_peer_cert = false
ssl_options.cacertfile = /etc/rabbitmq/ca-cert.pem
ssl_options.certfile = /etc/rabbitmq/server-cert.pem
ssl_options.keyfile = /etc/rabbitmq/server-key.pem

33
docker/test/integration/runner/misc/rabbitmq/server-cert.pem Normal file
View File

@ -0,0 +1,33 @@
-----BEGIN CERTIFICATE-----
MIIFpTCCA42gAwIBAgIUJvQslezZO09XgFGQCxOM6orIsWowDQYJKoZIhvcNAQEL
BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjMxMTE0
MTgyODI5WhcNMzMxMTExMTgyODI5WjBWMQswCQYDVQQGEwJSVTETMBEGA1UECAwK
U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQ8w
DQYDVQQDDAZzZXJ2ZXIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCe
o/K71WdKpVpdDvhaZy6wBVhFlu7j7DhfTSYvcPpAJfExmzO8JK3vh5/yGyAO1t79
gAjqyXLMCZKw7ajM2rez9YnGYqaFi70BlTcU2KQ8LbFEYRc3cYNDmmWIKBpwpSri
We5SQrRLnDXqAn6T8FG5ejQ/t+1IUMrtZENB4lp8fBmEOJb5yr1TE++6EhiDBQho
cLDWWWP8b55kyZhqP/VgmId4lvboGMRKxbiRJ6/SPr/i/pteBD8jTYfbJr6ceXov
/p5yxIp61z5ry1anU7W3B8jTl/gj7SqtFdSnRajZ0DGJJAUKpiiJSCSlp5YB5Ub2
eBBMHmdA5R1MuiU9TOA35nUW5wkhEOJXnBR/WCsYioVmn/+5dm6JPYiwp/TefYnr
x9iLbb/Tyx7MnXzeyvKg781SwmnvS6Blhtr0zhAW9szZz8cVHPBqFs6PzGs/5mwE
C+tM3Zp85aHd28nIT4NQLHdMDwVmGwmPdy4uavtYWMDhsuIyEU8hCZymiHhPnuHU
VbmfZ8GOTIzUgQAvZb0fL1Xow2Tf6XuARnvuU9weRttg9jSOqPuUENRsFXv0mU8M
EpQjrxry88Wfz7bBEjN5JHC16PB/Nu7zTGJ4/slThbxNv0bIONzvTBPbXrKnxw7Z
d9WhGJI+LQxRqLTynQe6yzDwIuW9LRdBNTp7CtQRwQIDAQABo28wbTArBgNVHREE
JDAigiBpbnRlZ3JhdGlvbi10ZXN0cy5jbGlja2hvdXNlLmNvbTAdBgNVHQ4EFgQU
54GvBUYWvMADpTz/zglwMlaJuskwHwYDVR0jBBgwFoAUsH0HDmm18F/7/RdYXdnJ
riLoOeowDQYJKoZIhvcNAQELBQADggIBADfNH6O6ay+xg0XmV6sR0n4j6PwL9Cnc
VjuCmHQbpFXfMvgCdfHvbtT0Y/pG7IoeKmrrm0JPvKa2E9Ht0j6ZnowQ2m9mJk8U
5Fd/PbC1I4KgVCw6HRSOcwqANJxOGe7RyN9PTZZ8fxzmzIR3FiQ2bXfr+LaotZOK
aVS8F8xCOzoMvL9LFls2YpEn20p/1EATIf2MFX3j9vKfcJVOyDJV4i5BMImStFLM
g3sdC96de/59yxt9khM0PNucU1ldNFs/kZVEcNSwGOAIgQEPwULJtDY+ZSWeROpX
EpWndN6zQsv1pdNvLtXsDXfi4YoH9QVaA/k4aFFJ08CjSZfMYmwyPOGsf/wqT65i
ADID2yb1A/FIIe/fM+d2gXHBVFBDmydJ1JCdCoYrEJgfWj1LO/0jLi34ZZ17Hu7F
D33fLARF9nlLzlUiWjcQlOjNoCM48AgG/3wHk4eiSfc/3PIJDuDGDa0NdtDeKKhH
XkP2ll4cMUH6EQ9KO1jHPmf5RokX4QJgH+ofO4U5XQFwc3lOyJzEQnED+wame7do
R7TE4F/OXhxLqA6DFkzXe89/kSCoAF9bjzmUn/ilrg8NXKKgprgHg4DJHgvCQVVC
34ab7Xj7msUm4D9vI+GAeUbUqnqCaWxDF6vCMT0Qq7iSVDxa/SV8TX8Vp2Zh+PSh
4m23Did+KjLq
-----END CERTIFICATE-----

1
docker/test/integration/runner/misc/rabbitmq/server-ext.cnf Normal file
View File

@ -0,0 +1 @@
subjectAltName=DNS:integration-tests.clickhouse.com

52
docker/test/integration/runner/misc/rabbitmq/server-key.pem Normal file
View File

@ -0,0 +1,52 @@
-----BEGIN PRIVATE KEY-----
MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCeo/K71WdKpVpd
DvhaZy6wBVhFlu7j7DhfTSYvcPpAJfExmzO8JK3vh5/yGyAO1t79gAjqyXLMCZKw
7ajM2rez9YnGYqaFi70BlTcU2KQ8LbFEYRc3cYNDmmWIKBpwpSriWe5SQrRLnDXq
An6T8FG5ejQ/t+1IUMrtZENB4lp8fBmEOJb5yr1TE++6EhiDBQhocLDWWWP8b55k
yZhqP/VgmId4lvboGMRKxbiRJ6/SPr/i/pteBD8jTYfbJr6ceXov/p5yxIp61z5r
y1anU7W3B8jTl/gj7SqtFdSnRajZ0DGJJAUKpiiJSCSlp5YB5Ub2eBBMHmdA5R1M
uiU9TOA35nUW5wkhEOJXnBR/WCsYioVmn/+5dm6JPYiwp/TefYnrx9iLbb/Tyx7M
nXzeyvKg781SwmnvS6Blhtr0zhAW9szZz8cVHPBqFs6PzGs/5mwEC+tM3Zp85aHd
28nIT4NQLHdMDwVmGwmPdy4uavtYWMDhsuIyEU8hCZymiHhPnuHUVbmfZ8GOTIzU
gQAvZb0fL1Xow2Tf6XuARnvuU9weRttg9jSOqPuUENRsFXv0mU8MEpQjrxry88Wf
z7bBEjN5JHC16PB/Nu7zTGJ4/slThbxNv0bIONzvTBPbXrKnxw7Zd9WhGJI+LQxR
qLTynQe6yzDwIuW9LRdBNTp7CtQRwQIDAQABAoICAA0lev0T3z5xW36wueYL/PN7
TehebKeYsMc9BngR/bsJKea5fN0PkRZzf865brusFMifLp3+WbQM6wocd8uaKHUS
WPuGu1P/04bpDap9lYajJriK7ziaAI2+osFYyXAiT954I2bPvk8xv8oHsOOjm7Iq
LWBGZrSCdX6cu3IfRu5f/mFVqzVCFtRmp4wc6ckZxquZAx6QQ9fsjAzAJBBSAoyh
t0BICmgLfWDQ582no0tiBdbS0J9G7NCJIUQI/uzKqFSH3iuWm/84DSUzsZemOT3U
uFDInDil885qK7g87pQ2S5SY1o4eXOebgeX0cFrx3CKaqocUUewv0HDGUEW3NDFs
KhUvlJZIFgk6bMend16U6kfRCUsjLA22Rfxzanl53cGVywCeIMirnLYuEu0TsxyK
CblBvyhcpjrGi7FQskzR+J9LpZPnmtn6TAb7JCAALRVHcAGKhGeh613SjPUfkWb0
KpDps08x8MWGEAALuHbOK0nMLFm+PuMt7+krqCeJET+XM44GT+6ZstrDv0RufxUN
+pkLW7AsVZoXcFvaOWjuyBvX/f6UHCSfueo0mB3H80WoftDIfdhM+AI7/oBTYCBx
Z8BtW+g7Eq3pOUg/Um7S7Z2bybBWE14kpi95gRf3upEYPqHJUpJPdu20lk24iAt9
LCXF4AjZBIdAuyJrYOJBAoIBAQDd/Bm14WvmBOablGLn6hmohi6M75D+/eQanlg9
eJhXJUVd8FzOTjKi70EHWvkqswenNDbe/WGtImqG+9G+N/ol2qhi5xVSQ2XQmcVQ
U+k15Bzm9xKM0OqsStFvRgP1Cy6Ms3/jxr5JEEwUepmjvWTDGTlhTQASA/D7Uh2q
5HpPiHEVm4g5eTAYWeAbI6cGwVS0L4y6xkFGde37Kh2P8ZodWB+d3fglVu4Ok9Nf
wE2f8MK2ewQ0SbF/Nj2WjlVomvOvOJG/2CDLuiH/vc4YUvLAm8pNwvsmgtSh1Okt
E/HfXegrlPPEgw6owqoQFt+aGUITgEhiwEVAcYS0pXzzkQX5AoIBAQC28wJ8ueKr
fINpJM2pSc7WRDFduP5yGsRreSLBXLKMbvOlIVb3PaWp11Cg3+X5O90bPXYJ9mBI
WGR0g14/VD8edxs2D5TUZcP4/vKXGHaWRY9Z4A3jVpjzAxAaviNDHJ08tLXEMXZQ
lbA7dX8z6lpoQfwnPzjBwB01mVegwXPeIwIIfT/FmAiGzvSnAMXBGSGWRRdzof0M
/vPFbgllcQmM4AnEGcErCgFRpwcssO87T2jnvf6QVE5JCcnUcGIli1ThxCU9TRZM
5s6R7Nvk3/UjwcpRcqMtnGpTT2QXSnRwvWUfM+bKTwaxz4PjqKpgIc11kwJAjlxk
4CxYf1mDGLwJAoIBAGFJRTNS8ejDKRXyOE6PaGNVOz2FGLTILJoF34JBQfKfYQFE
gEfiOYry9Dr3AdBW2fnLhmi//3jTZoB2CHwnKDhC1h1STSPaadq8KZ+ExuZZbNlE
WxrfzJlpyNPNiZpxJht/54K57Vc0D0PCX2dFb82ZVm5wQqGinJBocpwcugX1NCpW
GaOmmw9xBCigvWjWffriA/kvPhhVQtEaqg4Vwoctwd18FG645Gf7HV4Pd3WrHIrA
6xzHV0T7To6XHpNTpYybbDT50ZW3o4LjellqsPz8yfK+izdbizjJiM+6t/w+uauw
Ag2Tqm8HsWSPwbtVaoIFbLPqs+8EUTaieFp+qnECggEAVuaTdd9uFfrtCNKchh8z
CoAV2uj2pAim6E3//k0j2qURQozVnFdCC6zk9aWkvYB8BGZrXUwUbAjgnp+P8xD3
cmctG77G+STls66WWMMcAUFFWHGe5y/JMxVvXuSWJ1i+L4m/FVRRWPHhZjznkSdu
jjtZpOLY+N9igIU4JHn/qbKDUrj7w8X1tuMzPuiVBqYDWDe1bg2x/6xS6qLb/71z
xeDdgrKhGOqFud1XARmCaW/M6tdKxg/lp7fokOpZFHBcf2kGL1ogj6LK2HHj+ZGQ
Bc4VZh7H9/BmaPA7IP0S1kKAeBPVOp/TFD737Pm/BC7KQ2DzHusAZEI/jkHfqO/k
0QKCAQEAuiYLn9iLgk4uQO9oaSBGWKrJsR2L2dqI7IWU0X9xJlsQrJKcEeWg4LXt
djLsz0HrxZV/c+Pnh79hmFlBoEmH+hz32D/xd+/qrwwAcMkHAwMbznJu0IIuW2O9
Uzma++7SvVmr9H0DkUwXFP3jn1A2n3uuI4czqtQ8N7GiH0UAWR5CsIP7azHvZTSj
s4Fzf8rTE6pNqVgQXjrVbI9H/h0uPP4alJbhnPba9mgB1cGmfBEnPkKgYNqSZse+
95G2TlcK74sKBUSdBKqYBZ4ZUeTXV974Nva9guE9vzDQt1Cj6k0HWISVPUshPzIh
qrdHdxcM6yhA0Z0Gu6zj+Zsy4lU8gA==
-----END PRIVATE KEY-----

3
docker/test/stateless/run.sh
View File

@ -217,6 +217,9 @@ ls -la /
clickhouse-client -q "system flush logs" ||:
# stop logs replication to make it possible to dump logs tables via clickhouse-local
stop_logs_replication
# Stop server so we can safely read data with clickhouse-local.
# Why do we read data with clickhouse-local?
# Because it's the simplest way to read it when server has crashed.

15
docker/test/stateless/stress_tests.lib
View File

@ -140,21 +140,6 @@ EOL
-->
<core_path>$PWD</core_path>
</clickhouse>
EOL
# Analyzer is not yet ready for testing
cat > /etc/clickhouse-server/users.d/no_analyzer.xml <<EOL
<clickhouse>
<profiles>
<default>
<constraints>
<allow_experimental_analyzer>
<readonly/>
</allow_experimental_analyzer>
</constraints>
</default>
</profiles>
</clickhouse>
EOL
}

2
docker/test/upgrade/run.sh
View File

@ -78,6 +78,7 @@ remove_keeper_config "create_if_not_exists" "[01]"
rm /etc/clickhouse-server/config.d/merge_tree.xml
rm /etc/clickhouse-server/config.d/enable_wait_for_shutdown_replicated_tables.xml
rm /etc/clickhouse-server/users.d/nonconst_timezone.xml
rm /etc/clickhouse-server/users.d/s3_cache_new.xml
start
stop
@ -114,6 +115,7 @@ sudo chgrp clickhouse /etc/clickhouse-server/config.d/s3_storage_policy_by_defau
rm /etc/clickhouse-server/config.d/merge_tree.xml
rm /etc/clickhouse-server/config.d/enable_wait_for_shutdown_replicated_tables.xml
rm /etc/clickhouse-server/users.d/nonconst_timezone.xml
rm /etc/clickhouse-server/users.d/s3_cache_new.xml
start

28
docs/changelogs/v23.10.5.20-stable.md Normal file
View File

@ -0,0 +1,28 @@
---
sidebar_position: 1
sidebar_label: 2023
---
# 2023 Changelog
### ClickHouse release v23.10.5.20-stable (e84001e5c61) FIXME as compared to v23.10.4.25-stable (330fd687d41)
#### Improvement
* Backported in [#56924](https://github.com/ClickHouse/ClickHouse/issues/56924): There was a potential vulnerability in previous ClickHouse versions: if a user has connected and unsuccessfully tried to authenticate with the "interserver secret" method, the server didn't terminate the connection immediately but continued to receive and ignore the leftover packets from the client. While these packets are ignored, they are still parsed, and if they use a compression method with another known vulnerability, it will lead to exploitation of it without authentication. This issue was found with [ClickHouse Bug Bounty Program](https://github.com/ClickHouse/ClickHouse/issues/38986) by https://twitter.com/malacupa. [#56794](https://github.com/ClickHouse/ClickHouse/pull/56794) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
#### Build/Testing/Packaging Improvement
* Backported in [#57023](https://github.com/ClickHouse/ClickHouse/issues/57023): There was an attempt to have the proper listing in [#44311](https://github.com/ClickHouse/ClickHouse/issues/44311), but the fix itself was in the wrong place, so it's still broken. See an [example](https://github.com/ClickHouse/ClickHouse/actions/runs/6897342568/job/18781001022#step:8:25). [#56989](https://github.com/ClickHouse/ClickHouse/pull/56989) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
#### Bug Fix (user-visible misbehavior in an official stable release)
* Fix ON CLUSTER queries without database on initial node [#56484](https://github.com/ClickHouse/ClickHouse/pull/56484) ([Nikolay Degterinsky](https://github.com/evillique)).
* Fix buffer overflow in Gorilla codec [#57107](https://github.com/ClickHouse/ClickHouse/pull/57107) ([Nikolay Degterinsky](https://github.com/evillique)).
* Close interserver connection on any exception before authentication [#57142](https://github.com/ClickHouse/ClickHouse/pull/57142) ([Antonio Andelic](https://github.com/antonio2368)).
#### NOT FOR CHANGELOG / INSIGNIFICANT
* Fix client suggestions for user without grants [#56234](https://github.com/ClickHouse/ClickHouse/pull/56234) ([Nikolay Degterinsky](https://github.com/evillique)).
* Fix pygithub [#56778](https://github.com/ClickHouse/ClickHouse/pull/56778) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Avoid dependencies with no fixed versions [#56914](https://github.com/ClickHouse/ClickHouse/pull/56914) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Tiny improvement security [#57171](https://github.com/ClickHouse/ClickHouse/pull/57171) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).

26
docs/changelogs/v23.3.18.15-lts.md Normal file
View File

@ -0,0 +1,26 @@
---
sidebar_position: 1
sidebar_label: 2023
---
# 2023 Changelog
### ClickHouse release v23.3.18.15-lts (7228475d77a) FIXME as compared to v23.3.17.13-lts (e867d59020f)
#### Improvement
* Backported in [#56928](https://github.com/ClickHouse/ClickHouse/issues/56928): There was a potential vulnerability in previous ClickHouse versions: if a user has connected and unsuccessfully tried to authenticate with the "interserver secret" method, the server didn't terminate the connection immediately but continued to receive and ignore the leftover packets from the client. While these packets are ignored, they are still parsed, and if they use a compression method with another known vulnerability, it will lead to exploitation of it without authentication. This issue was found with [ClickHouse Bug Bounty Program](https://github.com/ClickHouse/ClickHouse/issues/38986) by https://twitter.com/malacupa. [#56794](https://github.com/ClickHouse/ClickHouse/pull/56794) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
#### Build/Testing/Packaging Improvement
* Backported in [#57019](https://github.com/ClickHouse/ClickHouse/issues/57019): There was an attempt to have the proper listing in [#44311](https://github.com/ClickHouse/ClickHouse/issues/44311), but the fix itself was in the wrong place, so it's still broken. See an [example](https://github.com/ClickHouse/ClickHouse/actions/runs/6897342568/job/18781001022#step:8:25). [#56989](https://github.com/ClickHouse/ClickHouse/pull/56989) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
#### Bug Fix (user-visible misbehavior in an official stable release)
* Fix buffer overflow in Gorilla codec [#57107](https://github.com/ClickHouse/ClickHouse/pull/57107) ([Nikolay Degterinsky](https://github.com/evillique)).
* Close interserver connection on any exception before authentication [#57142](https://github.com/ClickHouse/ClickHouse/pull/57142) ([Antonio Andelic](https://github.com/antonio2368)).
#### NOT FOR CHANGELOG / INSIGNIFICANT
* Fix pygithub [#56778](https://github.com/ClickHouse/ClickHouse/pull/56778) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Avoid dependencies with no fixed versions [#56914](https://github.com/ClickHouse/ClickHouse/pull/56914) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Tiny improvement security [#57171](https://github.com/ClickHouse/ClickHouse/pull/57171) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).

28
docs/changelogs/v23.8.8.20-lts.md Normal file
View File

@ -0,0 +1,28 @@
---
sidebar_position: 1
sidebar_label: 2023
---
# 2023 Changelog
### ClickHouse release v23.8.8.20-lts (5e012a03bf2) FIXME as compared to v23.8.7.24-lts (812b95e14ba)
#### Improvement
* Backported in [#56509](https://github.com/ClickHouse/ClickHouse/issues/56509): Allow backup of materialized view with dropped inner table instead of failing the backup. [#56387](https://github.com/ClickHouse/ClickHouse/pull/56387) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Backported in [#56929](https://github.com/ClickHouse/ClickHouse/issues/56929): There was a potential vulnerability in previous ClickHouse versions: if a user has connected and unsuccessfully tried to authenticate with the "interserver secret" method, the server didn't terminate the connection immediately but continued to receive and ignore the leftover packets from the client. While these packets are ignored, they are still parsed, and if they use a compression method with another known vulnerability, it will lead to exploitation of it without authentication. This issue was found with [ClickHouse Bug Bounty Program](https://github.com/ClickHouse/ClickHouse/issues/38986) by https://twitter.com/malacupa. [#56794](https://github.com/ClickHouse/ClickHouse/pull/56794) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
#### Build/Testing/Packaging Improvement
* Backported in [#57020](https://github.com/ClickHouse/ClickHouse/issues/57020): There was an attempt to have the proper listing in [#44311](https://github.com/ClickHouse/ClickHouse/issues/44311), but the fix itself was in the wrong place, so it's still broken. See an [example](https://github.com/ClickHouse/ClickHouse/actions/runs/6897342568/job/18781001022#step:8:25). [#56989](https://github.com/ClickHouse/ClickHouse/pull/56989) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
#### Bug Fix (user-visible misbehavior in an official stable release)
* Fix ON CLUSTER queries without database on initial node [#56484](https://github.com/ClickHouse/ClickHouse/pull/56484) ([Nikolay Degterinsky](https://github.com/evillique)).
* Fix buffer overflow in Gorilla codec [#57107](https://github.com/ClickHouse/ClickHouse/pull/57107) ([Nikolay Degterinsky](https://github.com/evillique)).
* Close interserver connection on any exception before authentication [#57142](https://github.com/ClickHouse/ClickHouse/pull/57142) ([Antonio Andelic](https://github.com/antonio2368)).
#### NOT FOR CHANGELOG / INSIGNIFICANT
* Fix pygithub [#56778](https://github.com/ClickHouse/ClickHouse/pull/56778) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Avoid dependencies with no fixed versions [#56914](https://github.com/ClickHouse/ClickHouse/pull/56914) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Tiny improvement security [#57171](https://github.com/ClickHouse/ClickHouse/pull/57171) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).

28
docs/changelogs/v23.9.6.20-stable.md Normal file
View File

@ -0,0 +1,28 @@
---
sidebar_position: 1
sidebar_label: 2023
---
# 2023 Changelog
### ClickHouse release v23.9.6.20-stable (cf7e84bb8cf) FIXME as compared to v23.9.5.29-stable (f8554c1a1ff)
#### Improvement
* Backported in [#56930](https://github.com/ClickHouse/ClickHouse/issues/56930): There was a potential vulnerability in previous ClickHouse versions: if a user has connected and unsuccessfully tried to authenticate with the "interserver secret" method, the server didn't terminate the connection immediately but continued to receive and ignore the leftover packets from the client. While these packets are ignored, they are still parsed, and if they use a compression method with another known vulnerability, it will lead to exploitation of it without authentication. This issue was found with [ClickHouse Bug Bounty Program](https://github.com/ClickHouse/ClickHouse/issues/38986) by https://twitter.com/malacupa. [#56794](https://github.com/ClickHouse/ClickHouse/pull/56794) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
#### Build/Testing/Packaging Improvement
* Backported in [#57022](https://github.com/ClickHouse/ClickHouse/issues/57022): There was an attempt to have the proper listing in [#44311](https://github.com/ClickHouse/ClickHouse/issues/44311), but the fix itself was in the wrong place, so it's still broken. See an [example](https://github.com/ClickHouse/ClickHouse/actions/runs/6897342568/job/18781001022#step:8:25). [#56989](https://github.com/ClickHouse/ClickHouse/pull/56989) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
#### Bug Fix (user-visible misbehavior in an official stable release)
* Fix ON CLUSTER queries without database on initial node [#56484](https://github.com/ClickHouse/ClickHouse/pull/56484) ([Nikolay Degterinsky](https://github.com/evillique)).
* Fix buffer overflow in Gorilla codec [#57107](https://github.com/ClickHouse/ClickHouse/pull/57107) ([Nikolay Degterinsky](https://github.com/evillique)).
* Close interserver connection on any exception before authentication [#57142](https://github.com/ClickHouse/ClickHouse/pull/57142) ([Antonio Andelic](https://github.com/antonio2368)).
#### NOT FOR CHANGELOG / INSIGNIFICANT
* Fix client suggestions for user without grants [#56234](https://github.com/ClickHouse/ClickHouse/pull/56234) ([Nikolay Degterinsky](https://github.com/evillique)).
* Fix pygithub [#56778](https://github.com/ClickHouse/ClickHouse/pull/56778) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Avoid dependencies with no fixed versions [#56914](https://github.com/ClickHouse/ClickHouse/pull/56914) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Tiny improvement security [#57171](https://github.com/ClickHouse/ClickHouse/pull/57171) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).

5
docs/en/engines/database-engines/materialized-mysql.md
View File

@ -7,7 +7,10 @@ sidebar_position: 70
# [experimental] MaterializedMySQL
:::note
This is an experimental feature that should not be used in production.
This database engine is experimental. To use it, set `allow_experimental_database_materialized_mysql` to 1 in your configuration files or by using the `SET` command:
```sql
SET allow_experimental_database_materialized_mysql=1
```
:::
Creates a ClickHouse database with all the tables existing in MySQL, and all the data in those tables. The ClickHouse server works as MySQL replica. It reads `binlog` and performs DDL and DML queries.

4
docs/en/engines/database-engines/materialized-postgresql.md
View File

@ -8,7 +8,7 @@ sidebar_position: 60
Creates a ClickHouse database with tables from PostgreSQL database. Firstly, database with engine `MaterializedPostgreSQL` creates a snapshot of PostgreSQL database and loads required tables. Required tables can include any subset of tables from any subset of schemas from specified database. Along with the snapshot database engine acquires LSN and once initial dump of tables is performed - it starts pulling updates from WAL. After database is created, newly added tables to PostgreSQL database are not automatically added to replication. They have to be added manually with `ATTACH TABLE db.table` query.
Replication is implemented with PostgreSQL Logical Replication Protocol, which does not allow to replicate DDL, but allows to know whether replication breaking changes happened (column type changes, adding/removing columns). Such changes are detected and according tables stop receiving updates. In this case you should use `ATTACH`/ `DETACH` queries to reload table completely. If DDL does not break replication (for example, renaming a column) table will still receive updates (insertion is done by position).
Replication is implemented with PostgreSQL Logical Replication Protocol, which does not allow to replicate DDL, but allows to know whether replication breaking changes happened (column type changes, adding/removing columns). Such changes are detected and according tables stop receiving updates. In this case you should use `ATTACH`/ `DETACH PERMANENTLY` queries to reload table completely. If DDL does not break replication (for example, renaming a column) table will still receive updates (insertion is done by position).
:::note
This database engine is experimental. To use it, set `allow_experimental_database_materialized_postgresql` to 1 in your configuration files or by using the `SET` command:
@ -63,7 +63,7 @@ Before version 22.1, adding a table to replication left a non-removed temporary
It is possible to remove specific tables from replication:
``` sql
DETACH TABLE postgres_database.table_to_remove;
DETACH TABLE postgres_database.table_to_remove PERMANENTLY;
```
## PostgreSQL schema {#schema}

8
docs/en/engines/table-engines/integrations/materialized-postgresql.md
View File

@ -8,6 +8,14 @@ sidebar_label: MaterializedPostgreSQL
Creates ClickHouse table with an initial data dump of PostgreSQL table and starts replication process, i.e. executes background job to apply new changes as they happen on PostgreSQL table in the remote PostgreSQL database.
:::note
This table engine is experimental. To use it, set `allow_experimental_materialized_postgresql_table` to 1 in your configuration files or by using the `SET` command:
```sql
SET allow_experimental_materialized_postgresql_table=1
```
:::
If more than one table is required, it is highly recommended to use the [MaterializedPostgreSQL](../../../engines/database-engines/materialized-postgresql.md) database engine instead of the table engine and use the `materialized_postgresql_tables_list` setting, which specifies the tables to be replicated (will also be possible to add database `schema`). It will be much better in terms of CPU, fewer connections and fewer replication slots inside the remote PostgreSQL database.
## Creating a Table {#creating-a-table}

12
docs/en/engines/table-engines/mergetree-family/mergetree.md
View File

@ -504,8 +504,8 @@ Indexes of type `set` can be utilized by all functions. The other index types ar
| Function (operator) / Index | primary key | minmax | ngrambf_v1 | tokenbf_v1 | bloom_filter | inverted |
|------------------------------------------------------------------------------------------------------------|-------------|--------|------------|------------|--------------|----------|
| [equals (=, ==)](/docs/en/sql-reference/functions/comparison-functions.md/#function-equals) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ |
| [notEquals(!=, &lt;&gt;)](/docs/en/sql-reference/functions/comparison-functions.md/#function-notequals) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ |
| [equals (=, ==)](/docs/en/sql-reference/functions/comparison-functions.md/#equals) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ |
| [notEquals(!=, &lt;&gt;)](/docs/en/sql-reference/functions/comparison-functions.md/#notequals) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ |
| [like](/docs/en/sql-reference/functions/string-search-functions.md/#function-like) | ✔ | ✔ | ✔ | ✔ | ✗ | ✔ |
| [notLike](/docs/en/sql-reference/functions/string-search-functions.md/#function-notlike) | ✔ | ✔ | ✔ | ✔ | ✗ | ✔ |
| [startsWith](/docs/en/sql-reference/functions/string-functions.md/#startswith) | ✔ | ✔ | ✔ | ✔ | ✗ | ✔ |
@ -513,10 +513,10 @@ Indexes of type `set` can be utilized by all functions. The other index types ar
| [multiSearchAny](/docs/en/sql-reference/functions/string-search-functions.md/#function-multisearchany) | ✗ | ✗ | ✔ | ✗ | ✗ | ✔ |
| [in](/docs/en/sql-reference/functions/in-functions#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ |
| [notIn](/docs/en/sql-reference/functions/in-functions#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ | ✔ |
| [less (<)](/docs/en/sql-reference/functions/comparison-functions.md/#function-less) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ |
| [greater (>)](/docs/en/sql-reference/functions/comparison-functions.md/#function-greater) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ |
| [lessOrEquals (<=)](/docs/en/sql-reference/functions/comparison-functions.md/#function-lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ |
| [greaterOrEquals (>=)](/docs/en/sql-reference/functions/comparison-functions.md/#function-greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ |
| [less (<)](/docs/en/sql-reference/functions/comparison-functions.md/#less) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ |
| [greater (>)](/docs/en/sql-reference/functions/comparison-functions.md/#greater) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ |
| [lessOrEquals (<=)](/docs/en/sql-reference/functions/comparison-functions.md/#lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ |
| [greaterOrEquals (>=)](/docs/en/sql-reference/functions/comparison-functions.md/#greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ |
| [empty](/docs/en/sql-reference/functions/array-functions#function-empty) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ |
| [notEmpty](/docs/en/sql-reference/functions/array-functions#function-notempty) | ✔ | ✔ | ✗ | ✗ | ✗ | ✗ |
| [has](/docs/en/sql-reference/functions/array-functions#function-has) | ✗ | ✗ | ✔ | ✔ | ✔ | ✔ |

34
docs/en/operations/server-configuration-parameters/settings.md
View File

@ -74,7 +74,7 @@ The maximum number of threads that will be used for fetching data parts from ano
Type: UInt64
Default: 8
Default: 16
## background_merges_mutations_concurrency_ratio
@ -136,7 +136,7 @@ The maximum number of threads that will be used for constantly executing some li
Type: UInt64
Default: 128
Default: 512
## backup_threads
@ -963,11 +963,9 @@ Lazy loading of dictionaries.
If `true`, then each dictionary is loaded on the first use. If the loading is failed, the function that was using the dictionary throws an exception.
If `false`, then the server starts loading all dictionaries at startup.
Dictionaries are loaded in background.
The server doesn't wait at startup until all the dictionaries finish their loading
(exception: if `wait_dictionaries_load_at_startup` is set to `true` - see below).
When a dictionary is used in a query for the first time then the query waits until the dictionary is loaded if it's not loaded yet.
If `false`, then the server loads all dictionaries at startup.
The server will wait at startup until all the dictionaries finish their loading before receiving any connections
(exception: if `wait_dictionaries_load_at_startup` is set to `false` - see below).
The default is `true`.
@ -2397,20 +2395,24 @@ Path to the file that contains:
## wait_dictionaries_load_at_startup {#wait_dictionaries_load_at_startup}
If `false`, then the server will not wait at startup until all the dictionaries finish their loading.
This allows to start ClickHouse faster.
This setting allows to specify behavior if `dictionaries_lazy_load` is `false`.
(If `dictionaries_lazy_load` is `true` this setting doesn't affect anything.)
If `true`, then the server will wait at startup until all the dictionaries finish their loading (successfully or not)
before listening to any connections.
This can make ClickHouse start slowly, however after that some queries can be executed faster
(because they won't have to wait for the used dictionaries to be load).
If `wait_dictionaries_load_at_startup` is `false`, then the server
will start loading all the dictionaries at startup and it will receive connections in parallel with that loading.
When a dictionary is used in a query for the first time then the query will wait until the dictionary is loaded if it's not loaded yet.
Setting `wait_dictionaries_load_at_startup` to `false` can make ClickHouse start faster, however some queries can be executed slower
(because they will have to wait for some dictionaries to be loaded).
The default is `false`.
If `wait_dictionaries_load_at_startup` is `true`, then the server will wait at startup
until all the dictionaries finish their loading (successfully or not) before receiving any connections.
The default is `true`.
**Example**
``` xml
<wait_dictionaries_load_at_startup>false</wait_dictionaries_load_at_startup>
<wait_dictionaries_load_at_startup>true</wait_dictionaries_load_at_startup>
```
## zookeeper {#server-settings_zookeeper}
@ -2740,7 +2742,7 @@ ClickHouse will use it to form the proxy URI using the following template: `{pro
<proxy_cache_time>10</proxy_cache_time>
</resolver>
</http>
<https>
<resolver>
<endpoint>http://resolver:8080/hostname</endpoint>

237
docs/en/operations/settings/settings.md
View File

@ -4805,6 +4805,243 @@ a Tuple(
If a table has a space-filling curve in its index, e.g. `ORDER BY mortonEncode(x, y)`, and the query has conditions on its arguments, e.g. `x >= 10 AND x <= 20 AND y >= 20 AND y <= 30`, use the space-filling curve for index analysis.
## query_plan_enable_optimizations {#query_plan_enable_optimizations}
Toggles query optimization at the query plan level.
:::note
This is an expert-level setting which should only be used for debugging by developers. The setting may change in future in backward-incompatible ways or be removed.
:::
Possible values:
- 0 - Disable all optimizations at the query plan level
- 1 - Enable optimizations at the query plan level (but individual optimizations may still be disabled via their individual settings)
Default value: `1`.
## query_plan_max_optimizations_to_apply
Limits the total number of optimizations applied to query plan, see setting [query_plan_enable_optimizations](#query_plan_enable_optimizations).
Useful to avoid long optimization times for complex queries.
If the actual number of optimizations exceeds this setting, an exception is thrown.
:::note
This is an expert-level setting which should only be used for debugging by developers. The setting may change in future in backward-incompatible ways or be removed.
:::
Type: [UInt64](../../sql-reference/data-types/int-uint.md).
Default value: '10000'
## query_plan_lift_up_array_join
Toggles a query-plan-level optimization which moves ARRAY JOINs up in the execution plan.
Only takes effect if setting [query_plan_enable_optimizations](#query_plan_enable_optimizations) is 1.
:::note
This is an expert-level setting which should only be used for debugging by developers. The setting may change in future in backward-incompatible ways or be removed.
:::
Possible values:
- 0 - Disable
- 1 - Enable
Default value: `1`.
## query_plan_push_down_limit
Toggles a query-plan-level optimization which moves LIMITs down in the execution plan.
Only takes effect if setting [query_plan_enable_optimizations](#query_plan_enable_optimizations) is 1.
:::note
This is an expert-level setting which should only be used for debugging by developers. The setting may change in future in backward-incompatible ways or be removed.
:::
Possible values:
- 0 - Disable
- 1 - Enable
Default value: `1`.
## query_plan_split_filter
:::note
This is an expert-level setting which should only be used for debugging by developers. The setting may change in future in backward-incompatible ways or be removed.
:::
Toggles a query-plan-level optimization which splits filters into expressions.
Only takes effect if setting [query_plan_enable_optimizations](#query_plan_enable_optimizations) is 1.
Possible values:
- 0 - Disable
- 1 - Enable
Default value: `1`.
## query_plan_merge_expressions
Toggles a query-plan-level optimization which merges consecutive filters.
Only takes effect if setting [query_plan_enable_optimizations](#query_plan_enable_optimizations) is 1.
:::note
This is an expert-level setting which should only be used for debugging by developers. The setting may change in future in backward-incompatible ways or be removed.
:::
Possible values:
- 0 - Disable
- 1 - Enable
Default value: `1`.
## query_plan_filter_push_down
Toggles a query-plan-level optimization which moves filters down in the execution plan.
Only takes effect if setting [query_plan_enable_optimizations](#query_plan_enable_optimizations) is 1.
:::note
This is an expert-level setting which should only be used for debugging by developers. The setting may change in future in backward-incompatible ways or be removed.
:::
Possible values:
- 0 - Disable
- 1 - Enable
Default value: `1`.
## query_plan_execute_functions_after_sorting
Toggles a query-plan-level optimization which moves expressions after sorting steps.
Only takes effect if setting [query_plan_enable_optimizations](#query_plan_enable_optimizations) is 1.
:::note
This is an expert-level setting which should only be used for debugging by developers. The setting may change in future in backward-incompatible ways or be removed.
:::
Possible values:
- 0 - Disable
- 1 - Enable
Default value: `1`.
## query_plan_reuse_storage_ordering_for_window_functions
Toggles a query-plan-level optimization which uses storage sorting when sorting for window functions.
Only takes effect if setting [query_plan_enable_optimizations](#query_plan_enable_optimizations) is 1.
:::note
This is an expert-level setting which should only be used for debugging by developers. The setting may change in future in backward-incompatible ways or be removed.
:::
Possible values:
- 0 - Disable
- 1 - Enable
Default value: `1`.
## query_plan_lift_up_union
Toggles a query-plan-level optimization which moves larger subtrees of the query plan into union to enable further optimizations.
Only takes effect if setting [query_plan_enable_optimizations](#query_plan_enable_optimizations) is 1.
:::note
This is an expert-level setting which should only be used for debugging by developers. The setting may change in future in backward-incompatible ways or be removed.
:::
Possible values:
- 0 - Disable
- 1 - Enable
Default value: `1`.
## query_plan_distinct_in_order
Toggles the distinct in-order optimization query-plan-level optimization.
Only takes effect if setting [query_plan_enable_optimizations](#query_plan_enable_optimizations) is 1.
:::note
This is an expert-level setting which should only be used for debugging by developers. The setting may change in future in backward-incompatible ways or be removed.
:::
Possible values:
- 0 - Disable
- 1 - Enable
Default value: `1`.
## query_plan_read_in_order
Toggles the read in-order optimization query-plan-level optimization.
Only takes effect if setting [query_plan_enable_optimizations](#query_plan_enable_optimizations) is 1.
:::note
This is an expert-level setting which should only be used for debugging by developers. The setting may change in future in backward-incompatible ways or be removed.
:::
Possible values:
- 0 - Disable
- 1 - Enable
Default value: `1`.
## query_plan_aggregation_in_order
Toggles the aggregation in-order query-plan-level optimization.
Only takes effect if setting [query_plan_enable_optimizations](#query_plan_enable_optimizations) is 1.
:::note
This is an expert-level setting which should only be used for debugging by developers. The setting may change in future in backward-incompatible ways or be removed.
:::
Possible values:
- 0 - Disable
- 1 - Enable
Default value: `0`.
## query_plan_remove_redundant_sorting
Toggles a query-plan-level optimization which removes redundant sorting steps, e.g. in subqueries.
Only takes effect if setting [query_plan_enable_optimizations](#query_plan_enable_optimizations) is 1.
:::note
This is an expert-level setting which should only be used for debugging by developers. The setting may change in future in backward-incompatible ways or be removed.
:::
Possible values:
- 0 - Disable
- 1 - Enable
Default value: `1`.
## query_plan_remove_redundant_distinct
Toggles a query-plan-level optimization which removes redundant DISTINCT steps.
Only takes effect if setting [query_plan_enable_optimizations](#query_plan_enable_optimizations) is 1.
:::note
This is an expert-level setting which should only be used for debugging by developers. The setting may change in future in backward-incompatible ways or be removed.
:::
Possible values:
- 0 - Disable
- 1 - Enable
Default value: `1`.
## dictionary_use_async_executor {#dictionary_use_async_executor}
Execute a pipeline for reading dictionary source in several threads. It's supported only by dictionaries with local CLICKHOUSE source.

59
docs/en/operations/system-tables/blob_storage_log.md Normal file
View File

@ -0,0 +1,59 @@
---
slug: /en/operations/system-tables/blob_storage_log
---
# blob_storage_log
Contains logging entries with information about various blob storage operations such as uploads and deletes.
Columns:
- `event_date` ([Date](../../sql-reference/data-types/date.md)) — Date of the event.
- `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Time of the event.
- `event_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Time of the event with microseconds precision.
- `event_type` ([Enum8](../../sql-reference/data-types/enum.md)) — Type of the event. Possible values:
- `'Upload'`
- `'Delete'`
- `'MultiPartUploadCreate'`
- `'MultiPartUploadWrite'`
- `'MultiPartUploadComplete'`
- `'MultiPartUploadAbort'`
- `query_id` ([String](../../sql-reference/data-types/string.md)) — Identifier of the query associated with the event, if any.
- `thread_id` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Identifier of the thread performing the operation.
- `thread_name` ([String](../../sql-reference/data-types/string.md)) — Name of the thread performing the operation.
- `disk_name` ([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md)) — Name of the associated disk.
- `bucket` ([String](../../sql-reference/data-types/string.md)) — Name of the bucket.
- `remote_path` ([String](../../sql-reference/data-types/string.md)) — Path to the remote resource.
- `local_path` ([String](../../sql-reference/data-types/string.md)) — Path to the metadata file on the local system, which references the remote resource.
- `data_size` ([UInt32](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Size of the data involved in the upload event.
- `error` ([String](../../sql-reference/data-types/string.md)) — Error message associated with the event, if any.
**Example**
Suppose a blob storage operation uploads a file, and an event is logged:
```sql
SELECT * FROM system.blob_storage_log WHERE query_id = '7afe0450-504d-4e4b-9a80-cd9826047972' ORDER BY event_date, event_time_microseconds \G
```
```text
Row 1:
──────
event_date: 2023-10-31
event_time: 2023-10-31 16:03:40
event_time_microseconds: 2023-10-31 16:03:40.481437
event_type: Upload
query_id: 7afe0450-504d-4e4b-9a80-cd9826047972
thread_id: 2381740
disk_name: disk_s3
bucket: bucket1
remote_path: rrr/kxo/tbnqtrghgtnxkzgtcrlutwuslgawe
local_path: store/654/6549e8b3-d753-4447-8047-d462df6e6dbe/tmp_insert_all_1_1_0/checksums.txt
data_size: 259
error:
```
In this example, upload operation was associated with the `INSERT` query with ID `7afe0450-504d-4e4b-9a80-cd9826047972`. The local metadata file `store/654/6549e8b3-d753-4447-8047-d462df6e6dbe/tmp_insert_all_1_1_0/checksums.txt` refers to remote path `rrr/kxo/tbnqtrghgtnxkzgtcrlutwuslgawe` in bucket `bucket1` on disk `disk_s3`, with a size of 259 bytes.
**See Also**
- [External Disks for Storing Data](../../operations/storing-data.md)

68
docs/en/operations/system-tables/dashboards.md Normal file
View File

@ -0,0 +1,68 @@
---
slug: /en/operations/system-tables/dashboards
---
# dashboards
Contains queries used by `/dashboard` page accessible though [HTTP interface](/docs/en/interfaces/http.md).
This table can be useful for monitoring and troubleshooting. The table contains a row for every chart in a dashboard.
:::note
`/dashboard` page can render queries not only from `system.dashboards`, but from any table with the same schema.
This can be useful to create custom dashboards.
:::
Example:
``` sql
SELECT *
FROM system.dashboards
WHERE title ILIKE '%CPU%'
```
``` text
Row 1:
──────
dashboard: overview
title: CPU Usage (cores)
query: SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_OSCPUVirtualTimeMicroseconds) / 1000000
FROM system.metric_log
WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}
GROUP BY t
ORDER BY t WITH FILL STEP {rounding:UInt32}
Row 2:
──────
dashboard: overview
title: CPU Wait
query: SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_OSCPUWaitMicroseconds) / 1000000
FROM system.metric_log
WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}
GROUP BY t
ORDER BY t WITH FILL STEP {rounding:UInt32}
Row 3:
──────
dashboard: overview
title: OS CPU Usage (Userspace)
query: SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(value)
FROM system.asynchronous_metric_log
WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} AND metric = 'OSUserTimeNormalized'
GROUP BY t
ORDER BY t WITH FILL STEP {rounding:UInt32}
Row 4:
──────
dashboard: overview
title: OS CPU Usage (Kernel)
query: SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(value)
FROM system.asynchronous_metric_log
WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} AND metric = 'OSSystemTimeNormalized'
GROUP BY t
ORDER BY t WITH FILL STEP {rounding:UInt32}
```
Columns:
- `dashboard` (`String`) - The dashboard name.
- `title` (`String`) - The title of a chart.
- `query` (`String`) - The query to obtain data to be displayed.

1
docs/en/operations/system-tables/databases.md
View File

@ -14,6 +14,7 @@ Columns:
- `uuid` ([UUID](../../sql-reference/data-types/uuid.md)) — Database UUID.
- `comment` ([String](../../sql-reference/data-types/enum.md)) — Database comment.
- `engine_full` ([String](../../sql-reference/data-types/enum.md)) — Parameters of the database engine.
- `database` ([String](../../sql-reference/data-types/string.md)) – Alias for `name`.
The `name` column from this system table is used for implementing the `SHOW DATABASES` query.

12
docs/en/sql-reference/functions/comparison-functions.md
View File

@ -20,7 +20,7 @@ Strings are compared byte-by-byte. Note that this may lead to unexpected results
A string S1 which has another string S2 as prefix is considered longer than S2.
## equals, `=`, `==` operators
## equals, `=`, `==` operators {#equals}
**Syntax**
@ -32,7 +32,7 @@ Alias:
- `a = b` (operator)
- `a == b` (operator)
## notEquals, `!=`, `<>` operators
## notEquals, `!=`, `<>` operators {#notequals}
**Syntax**
@ -44,7 +44,7 @@ Alias:
- `a != b` (operator)
- `a <> b` (operator)
## less, `<` operator
## less, `<` operator {#less}
**Syntax**
@ -55,7 +55,7 @@ less(a, b)
Alias:
- `a < b` (operator)
## greater, `>` operator
## greater, `>` operator {#greater}
**Syntax**
@ -66,7 +66,7 @@ greater(a, b)
Alias:
- `a > b` (operator)
## lessOrEquals, `<=` operator
## lessOrEquals, `<=` operator {#lessorequals}
**Syntax**
@ -77,7 +77,7 @@ lessOrEquals(a, b)
Alias:
- `a <= b` (operator)
## greaterOrEquals, `>=` operator
## greaterOrEquals, `>=` operator {#greaterorequals}
**Syntax**

166
docs/en/sql-reference/functions/math-functions.md
View File

@ -6,11 +6,9 @@ sidebar_label: Mathematical
# Mathematical Functions
All the functions return a Float64 number. Results are generally as close to the actual result as possible, but in some cases less precise than the machine-representable number.
## e
Returns e.
Returns e ([Euler's constant](https://en.wikipedia.org/wiki/Euler%27s_constant))
**Syntax**
@ -18,15 +16,22 @@ Returns e.
e()
```
**Returned value**
Type: [Float64](../../sql-reference/data-types/float.md).
## pi
Returns π.
Returns π ([Pi](https://en.wikipedia.org/wiki/Pi)).
**Syntax**
```sql
pi()
```
**Returned value**
Type: [Float64](../../sql-reference/data-types/float.md).
## exp
@ -38,6 +43,14 @@ Returns e to the power of the given argument.
exp(x)
```
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
**Returned value**
Type: [Float*](../../sql-reference/data-types/float.md).
## log
Returns the natural logarithm of the argument.
@ -50,6 +63,14 @@ log(x)
Alias: `ln(x)`
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
**Returned value**
Type: [Float*](../../sql-reference/data-types/float.md).
## exp2
Returns 2 to the power of the given argument
@ -60,6 +81,14 @@ Returns 2 to the power of the given argument
exp2(x)
```
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
**Returned value**
Type: [Float*](../../sql-reference/data-types/float.md).
## intExp2
Like `exp` but returns a UInt64.
@ -80,6 +109,14 @@ Returns the binary logarithm of the argument.
log2(x)
```
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
**Returned value**
Type: [Float*](../../sql-reference/data-types/float.md).
## exp10
Returns 10 to the power of the given argument.
@ -90,6 +127,14 @@ Returns 10 to the power of the given argument.
exp10(x)
```
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
**Returned value**
Type: [Float*](../../sql-reference/data-types/float.md).
## intExp10
Like `exp10` but returns a UInt64.
@ -110,6 +155,14 @@ Returns the decimal logarithm of the argument.
log10(x)
```
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
**Returned value**
Type: [Float*](../../sql-reference/data-types/float.md).
## sqrt
Returns the square root of the argument.
@ -118,6 +171,14 @@ Returns the square root of the argument.
sqrt(x)
```
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
**Returned value**
Type: [Float*](../../sql-reference/data-types/float.md).
## cbrt
Returns the cubic root of the argument.
@ -126,6 +187,14 @@ Returns the cubic root of the argument.
cbrt(x)
```
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
**Returned value**
Type: [Float*](../../sql-reference/data-types/float.md).
## erf
If `x` is non-negative, then `erf(x / σ√2)` is the probability that a random variable having a normal distribution with standard deviation `σ` takes the value that is separated from the expected value by more than `x`.
@ -136,6 +205,14 @@ If `x` is non-negative, then `erf(x / σ√2)` is the probability that a random
erf(x)
```
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
**Returned value**
Type: [Float*](../../sql-reference/data-types/float.md).
**Example**
(three sigma rule)
@ -160,6 +237,14 @@ Returns a number close to `1 - erf(x)` without loss of precision for large ‘x
erfc(x)
```
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
**Returned value**
Type: [Float*](../../sql-reference/data-types/float.md).
## lgamma
Returns the logarithm of the gamma function.
@ -170,6 +255,14 @@ Returns the logarithm of the gamma function.
lgamma(x)
```
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
**Returned value**
Type: [Float*](../../sql-reference/data-types/float.md).
## tgamma
Returns the gamma function.
@ -180,6 +273,14 @@ Returns the gamma function.
gamma(x)
```
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
**Returned value**
Type: [Float*](../../sql-reference/data-types/float.md).
## sin
Returns the sine of the argument
@ -190,6 +291,14 @@ Returns the sine of the argument
sin(x)
```
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
**Returned value**
Type: [Float*](../../sql-reference/data-types/float.md).
## cos
Returns the cosine of the argument.
@ -200,6 +309,14 @@ Returns the cosine of the argument.
cos(x)
```
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
**Returned value**
Type: [Float*](../../sql-reference/data-types/float.md).
## tan
Returns the tangent of the argument.
@ -210,6 +327,14 @@ Returns the tangent of the argument.
tan(x)
```
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
**Returned value**
Type: [Float*](../../sql-reference/data-types/float.md).
## asin
Returns the arc sine of the argument.
@ -220,6 +345,14 @@ Returns the arc sine of the argument.
asin(x)
```
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
**Returned value**
Type: [Float*](../../sql-reference/data-types/float.md).
## acos
Returns the arc cosine of the argument.
@ -230,6 +363,14 @@ Returns the arc cosine of the argument.
acos(x)
```
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
**Returned value**
Type: [Float*](../../sql-reference/data-types/float.md).
## atan
Returns the arc tangent of the argument.
@ -240,6 +381,14 @@ Returns the arc tangent of the argument.
atan(x)
```
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
**Returned value**
Type: [Float*](../../sql-reference/data-types/float.md).
## pow
Returns `x` to the power of `y`.
@ -252,6 +401,15 @@ pow(x, y)
Alias: `power(x, y)`
**Arguments**
- `x` - [(U)Int8/16/32/64](../../sql-reference/data-types/int-uint.md) or [Float*](../../sql-reference/data-types/float.md)
- `y` - [(U)Int8/16/32/64](../../sql-reference/data-types/int-uint.md) or [Float*](../../sql-reference/data-types/float.md)
**Returned value**
Type: [Float64](../../sql-reference/data-types/float.md).
## cosh
Returns the [hyperbolic cosine](https://in.mathworks.com/help/matlab/ref/cosh.html) of the argument.

2
docs/en/sql-reference/functions/other-functions.md
View File

@ -1556,7 +1556,7 @@ initializeAggregation (aggregate_function, arg1, arg2, ..., argN)
- Result of aggregation for every row passed to the function.
The return type is the same as the return type of function, that `initializeAgregation` takes as first argument.
The return type is the same as the return type of function, that `initializeAggregation` takes as first argument.
**Example**

2
docs/en/sql-reference/functions/string-functions.md
View File

@ -439,7 +439,7 @@ concat(s1, s2, ...)
**Arguments**
At least two values of arbitrary type.
At least one value of arbitrary type.
Arguments which are not of types [String](../../sql-reference/data-types/string.md) or [FixedString](../../sql-reference/data-types/fixedstring.md) are converted to strings using their default serialization. As this decreases performance, it is not recommended to use non-String/FixedString arguments.

47
docs/en/sql-reference/functions/time-series-functions.md Normal file
View File

@ -0,0 +1,47 @@
---
slug: /en/sql-reference/functions/time-series-functions
sidebar_position: 172
sidebar_label: Time Series
---
# Time Series Functions
Below functions are used for time series analysis.
## seriesPeriodDetectFFT
Finds the period of the given time series data using FFT
Detect Period in time series data using FFT.
FFT - Fast Fourier transform (https://en.wikipedia.org/wiki/Fast_Fourier_transform)
**Syntax**
``` sql
seriesPeriodDetectFFT(series);
```
**Arguments**
- `series` - An array of numeric values
**Returned value**
- A real value equal to the period of time series
Type: [Float64](../../sql-reference/data-types/float.md).
**Examples**
Query:
``` sql
SELECT seriesPeriodDetectFFT([1, 4, 6, 1, 4, 6, 1, 4, 6, 1, 4, 6, 1, 4, 6, 1, 4, 6, 1, 4, 6]) AS print_0;
```
Result:
``` text
┌───────────print_0──────┐
│ 3 │
└────────────────────────┘
```

4
docs/en/sql-reference/table-functions/gcs.md
View File

@ -9,6 +9,10 @@ keywords: [gcs, bucket]
Provides a table-like interface to `SELECT` and `INSERT` data from [Google Cloud Storage](https://cloud.google.com/storage/). Requires the [`Storage Object User` IAM role](https://cloud.google.com/storage/docs/access-control/iam-roles).
This is an alias of the [s3 table function](../../sql-reference/table-functions/s3.md).
If you have multiple replicas in your cluster, you can use the [s3Cluster function](../../sql-reference/table-functions/s3Cluster.md) (which works with GCS) instead to parallelize inserts.
**Syntax**
``` sql

4
docs/en/sql-reference/table-functions/s3.md
View File

@ -9,6 +9,10 @@ keywords: [s3, gcs, bucket]
Provides a table-like interface to select/insert files in [Amazon S3](https://aws.amazon.com/s3/) and [Google Cloud Storage](https://cloud.google.com/storage/). This table function is similar to the [hdfs function](../../sql-reference/table-functions/hdfs.md), but provides S3-specific features.
If you have multiple replicas in your cluster, you can use the [s3Cluster function](../../sql-reference/table-functions/s3Cluster.md) instead to parallelize inserts.
When using the `s3 table function` with [`INSERT INTO...SELECT`](../../sql-reference/statements/insert-into#inserting-the-results-of-select), data is read and inserted in a streaming fashion. Only a few blocks of data reside in memory while the blocks are continuously read from S3 and pushed into the destination table.
**Syntax**
``` sql

2
docs/en/sql-reference/table-functions/s3Cluster.md
View File

@ -5,7 +5,7 @@ sidebar_label: s3Cluster
title: "s3Cluster Table Function"
---
Allows processing files from [Amazon S3](https://aws.amazon.com/s3/) in parallel from many nodes in a specified cluster. On initiator it creates a connection to all nodes in the cluster, discloses asterisks in S3 file path, and dispatches each file dynamically. On the worker node it asks the initiator about the next task to process and processes it. This is repeated until all tasks are finished.
Allows processing files from [Amazon S3](https://aws.amazon.com/s3/) and Google Cloud Storage [Google Cloud Storage](https://cloud.google.com/storage/) in parallel from many nodes in a specified cluster. On initiator it creates a connection to all nodes in the cluster, discloses asterisks in S3 file path, and dispatches each file dynamically. On the worker node it asks the initiator about the next task to process and processes it. This is repeated until all tasks are finished.
**Syntax**

14
docs/ru/engines/table-engines/mergetree-family/mergetree.md
View File

@ -337,7 +337,7 @@ SELECT count() FROM table WHERE u64 * i32 == 10 AND u64 * length(s) >= 1234
Поддерживаемые типы данных: `Int*`, `UInt*`, `Float*`, `Enum`, `Date`, `DateTime`, `String`, `FixedString`.
Фильтром могут пользоваться функции: [equals](../../../sql-reference/functions/comparison-functions.md), [notEquals](../../../sql-reference/functions/comparison-functions.md), [in](../../../sql-reference/functions/in-functions.md), [notIn](../../../sql-reference/functions/in-functions.md), [has](../../../sql-reference/functions/array-functions.md#hasarr-elem), [hasAny](../../../sql-reference/functions/array-functions.md#hasany), [hasAll](../../../sql-reference/functions/array-functions.md#hasall).
Фильтром могут пользоваться функции: [equals](../../../sql-reference/functions/comparison-functions.md#equals), [notEquals](../../../sql-reference/functions/comparison-functions.md#notequals), [in](../../../sql-reference/functions/in-functions.md), [notIn](../../../sql-reference/functions/in-functions.md), [has](../../../sql-reference/functions/array-functions.md#hasarr-elem), [hasAny](../../../sql-reference/functions/array-functions.md#hasany), [hasAll](../../../sql-reference/functions/array-functions.md#hasall).
**Примеры**
@ -354,8 +354,8 @@ INDEX b (u64 * length(str), i32 + f64 * 100, date, str) TYPE set(100) GRANULARIT
| Функция (оператор) / Индекс | primary key | minmax | ngrambf_v1 | tokenbf_v1 | bloom_filter |
|------------------------------------------------------------------------------------------------------------|-------------|--------|-------------|-------------|---------------|
| [equals (=, ==)](../../../sql-reference/functions/comparison-functions.md#function-equals) | ✔ | ✔ | ✔ | ✔ | ✔ |
| [notEquals(!=, &lt;&gt;)](../../../sql-reference/functions/comparison-functions.md#function-notequals) | ✔ | ✔ | ✔ | ✔ | ✔ |
| [equals (=, ==)](../../../sql-reference/functions/comparison-functions.md#equals) | ✔ | ✔ | ✔ | ✔ | ✔ |
| [notEquals(!=, &lt;&gt;)](../../../sql-reference/functions/comparison-functions.md#notequals) | ✔ | ✔ | ✔ | ✔ | ✔ |
| [like](../../../sql-reference/functions/string-search-functions.md#function-like) | ✔ | ✔ | ✔ | ✔ | ✗ |
| [notLike](../../../sql-reference/functions/string-search-functions.md#function-notlike) | ✔ | ✔ | ✔ | ✔ | ✗ |
| [startsWith](../../../sql-reference/functions/string-functions.md#startswith) | ✔ | ✔ | ✔ | ✔ | ✗ |
@ -363,10 +363,10 @@ INDEX b (u64 * length(str), i32 + f64 * 100, date, str) TYPE set(100) GRANULARIT
| [multiSearchAny](../../../sql-reference/functions/string-search-functions.md#function-multisearchany) | ✗ | ✗ | ✔ | ✗ | ✗ |
| [in](../../../sql-reference/functions/in-functions.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ |
| [notIn](../../../sql-reference/functions/in-functions.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ |
| [less (\<)](../../../sql-reference/functions/comparison-functions.md#function-less) | ✔ | ✔ | ✗ | ✗ | ✗ |
| [greater (\>)](../../../sql-reference/functions/comparison-functions.md#function-greater) | ✔ | ✔ | ✗ | ✗ | ✗ |
| [lessOrEquals (\<=)](../../../sql-reference/functions/comparison-functions.md#function-lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ |
| [greaterOrEquals (\>=)](../../../sql-reference/functions/comparison-functions.md#function-greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ |
| [less (\<)](../../../sql-reference/functions/comparison-functions.md#less) | ✔ | ✔ | ✗ | ✗ | ✗ |
| [greater (\>)](../../../sql-reference/functions/comparison-functions.md#greater) | ✔ | ✔ | ✗ | ✗ | ✗ |
| [lessOrEquals (\<=)](../../../sql-reference/functions/comparison-functions.md#lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ |
| [greaterOrEquals (\>=)](../../../sql-reference/functions/comparison-functions.md#greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ |
| [empty](../../../sql-reference/functions/array-functions.md#function-empty) | ✔ | ✔ | ✗ | ✗ | ✗ |
| [notEmpty](../../../sql-reference/functions/array-functions.md#function-notempty) | ✔ | ✔ | ✗ | ✗ | ✗ |
| hasToken | ✗ | ✗ | ✗ | ✔ | ✗ |

32
docs/ru/operations/server-configuration-parameters/settings.md
View File

@ -275,12 +275,11 @@ ClickHouse проверяет условия для `min_part_size` и `min_part
Отложенная загрузка словарей.
Если `true`, то каждый словарь создаётся при первом использовании. Если словарь не удалось создать, то вызов функции, использующей словарь, сгенерирует исключение.
Если `true`, то каждый словарь загружается при первом использовании. Если словарь не удалось загрузить, то вызов функции, использующей словарь, сгенерирует исключение.
Если `false`, сервер начнет загрузку всех словарей на старте сервера.
Словари загружаются в фоне. Сервер не ждет на старте, пока словари закончат загружаться
(исключение: если `wait_dictionaries_load_at_startup` установлена в `true` - см. ниже).
Когда словарь используется в запросе первый раз, этот запрос будет ждать окончания загрузки словаря, если он еще не загрузился.
Если `false`, все словари будут загружаться на старте сервера.
Сервер будет ждать на старте окончания загрузки всех словарей перед началом обработки соединений
(исключение: если `wait_dictionaries_load_at_startup` установлена в `false` - см. ниже).
По умолчанию - `true`.
@ -995,7 +994,7 @@ ClickHouse использует потоки из глобального пул
- Положительное целое число.
Значение по умолчанию: 128.
Значение по умолчанию: 512.
## background_fetches_pool_size {#background_fetches_pool_size}
@ -1005,7 +1004,7 @@ ClickHouse использует потоки из глобального пул
- Положительное целое число.
Значение по умолчанию: 8.
Значение по умолчанию: 16.
## background_distributed_schedule_pool_size {#background_distributed_schedule_pool_size}
@ -1722,20 +1721,23 @@ TCP порт для защищённого обмена данными с кли
## wait_dictionaries_load_at_startup {#wait_dictionaries_load_at_startup}
Если `false`, то сервер не будет ждать на старте, пока словари закончат загружаться.
Это позволяет ClickHouse стартовать быстрее.
Эта настройка позволяет указать поведение если `dictionaries_lazy_load` установлено в `false`.
(Если `dictionaries_lazy_load` установлено в `true`, то эта настройка ни на что не влияет.)
Если `true`, то ClickHouse будет ждать на старте до окончания загрузки всех словарей (успешно или нет)
перед тем, как начать принимать соединения.
Это может привести к медленному старту ClickHouse, однако после этого некоторые запросы могут выполняться быстрее
(потому что им не придется ждать окончания загрузки используемых словарей).
Если `wait_dictionaries_load_at_startup` установлено в `false`, то сервер начнет загрузку всех словарей на старте
и будет обрабатывать соединения, не дожидаясь окончания загрузки словарей.
Когда словарь первый раз используется в запросе, запрос будет ждать окончания загрузки этого словаря, если он еще не загрузился.
Установка `wait_dictionaries_load_at_startup` в `false` может помочь ClickHouse стартовать быстрее, однако некоторые запросы могут выполняться медленее (потому что они будут ждать окончания загрузки используемых в них словарей).
По умолчанию - `false`.
Если `wait_dictionaries_load_at_startup` установлено в `true`, то сервер будет ждать окончания загрузки всех словарей на старте
до начала обработки соединений.
По умолчанию - `true`.
**Пример**
``` xml
<wait_dictionaries_load_at_startup>false</wait_dictionaries_load_at_startup>
<wait_dictionaries_load_at_startup>true</wait_dictionaries_load_at_startup>
```
## zookeeper {#server-settings_zookeeper}

12
docs/ru/sql-reference/functions/comparison-functions.md
View File

@ -23,14 +23,14 @@ sidebar_label: "Функции сравнения"
Замечание. До версии 1.1.54134 сравнение знаковых и беззнаковых целых чисел производилось также, как в C++. То есть, вы могли получить неверный результат в таких случаях: SELECT 9223372036854775807 \> -1. С версии 1.1.54134 поведение изменилось и стало математически корректным.
## equals, оператор a = b и a == b {#function-equals}
## equals, оператор a = b и a == b {#equals}
## notEquals, оператор a != b и a `<>` b {#function-notequals}
## notEquals, оператор a != b и a `<>` b {#notequals}
## less, оператор `<` {#function-less}
## less, оператор `<` {#less}
## greater, оператор `>` {#function-greater}
## greater, оператор `>` {#greater}
## lessOrEquals, оператор `<=` {#function-lessorequals}
## lessOrEquals, оператор `<=` {#lessorequals}
## greaterOrEquals, оператор `>=` {#function-greaterorequals}
## greaterOrEquals, оператор `>=` {#greaterorequals}

2
docs/ru/sql-reference/table-functions/numbers.md
View File

@ -7,7 +7,7 @@ sidebar_label: numbers
# numbers {#numbers}
`numbers(N)` - возвращает таблицу с единственным столбцом `number` (UInt64), содержащим натуральные числа от `0` до `N-1`.
`numbers(N, M)` - возвращает таблицу с единственным столбцом `number` (UInt64), содержащим натуральные числа от `N` to `(N + M - 1)`.
`numbers(N, M)` - возвращает таблицу с единственным столбцом `number` (UInt64), содержащим натуральные числа от `N` до `(N + M - 1)`.
Так же как и таблица `system.numbers` может использоваться для тестов и генерации последовательных значений. Функция `numbers(N, M)` работает более эффективно, чем выборка из `system.numbers`.

14
docs/zh/engines/table-engines/mergetree-family/mergetree.md
View File

@ -66,7 +66,7 @@ ORDER BY expr
- `PARTITION BY` — [分区键](custom-partitioning-key.md) ，可选项。
大多数情况下，不需要分使用区键。即使需要使用，也不需要使用比月更细粒度的分区键。分区不会加快查询（这与 ORDER BY 表达式不同）。永远也别使用过细粒度的分区键。不要使用客户端指定分区标识符或分区字段名称来对数据进行分区（而是将分区字段标识或名称作为 ORDER BY 表达式的第一列来指定分区）。
大多数情况下，不需要使用分区键。即使需要使用，也不需要使用比月更细粒度的分区键。分区不会加快查询（这与 ORDER BY 表达式不同）。永远也别使用过细粒度的分区键。不要使用客户端指定分区标识符或分区字段名称来对数据进行分区（而是将分区字段标识或名称作为 ORDER BY 表达式的第一列来指定分区）。
要按月分区，可以使用表达式 `toYYYYMM(date_column)` ，这里的 `date_column` 是一个 [Date](../../../engines/table-engines/mergetree-family/mergetree.md) 类型的列。分区名的格式会是 `"YYYYMM"` 。
@ -349,8 +349,8 @@ WHERE 子句中的条件可以包含对某列数据进行运算的函数表达
| 函数 (操作符) / 索引 | primary key | minmax | ngrambf_v1 | tokenbf_v1 | bloom_filter |
| ------------------------------------------------------------ | ----------- | ------ | ---------- | ---------- | ------------ |
| [equals (=, ==)](../../../sql-reference/functions/comparison-functions.md#function-equals) | ✔ | ✔ | ✔ | ✔ | ✔ |
| [notEquals(!=, &lt;&gt;)](../../../sql-reference/functions/comparison-functions.md#function-notequals) | ✔ | ✔ | ✔ | ✔ | ✔ |
| [equals (=, ==)](../../../sql-reference/functions/comparison-functions.md#equals) | ✔ | ✔ | ✔ | ✔ | ✔ |
| [notEquals(!=, &lt;&gt;)](../../../sql-reference/functions/comparison-functions.md#notequals) | ✔ | ✔ | ✔ | ✔ | ✔ |
| [like](../../../sql-reference/functions/string-search-functions.md#function-like) | ✔ | ✔ | ✔ | ✔ | ✔ |
| [notLike](../../../sql-reference/functions/string-search-functions.md#function-notlike) | ✔ | ✔ | ✗ | ✗ | ✗ |
| [startsWith](../../../sql-reference/functions/string-functions.md#startswith) | ✔ | ✔ | ✔ | ✔ | ✗ |
@ -358,10 +358,10 @@ WHERE 子句中的条件可以包含对某列数据进行运算的函数表达
| [multiSearchAny](../../../sql-reference/functions/string-search-functions.md#function-multisearchany) | ✗ | ✗ | ✔ | ✗ | ✗ |
| [in](../../../sql-reference/functions/in-functions.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ |
| [notIn](../../../sql-reference/functions/in-functions.md#in-functions) | ✔ | ✔ | ✔ | ✔ | ✔ |
| [less (\<)](../../../sql-reference/functions/comparison-functions.md#function-less) | ✔ | ✔ | ✗ | ✗ | ✗ |
| [greater (\>)](../../../sql-reference/functions/comparison-functions.md#function-greater) | ✔ | ✔ | ✗ | ✗ | ✗ |
| [lessOrEquals (\<=)](../../../sql-reference/functions/comparison-functions.md#function-lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ |
| [greaterOrEquals (\>=)](../../../sql-reference/functions/comparison-functions.md#function-greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ |
| [less (\<)](../../../sql-reference/functions/comparison-functions.md#less) | ✔ | ✔ | ✗ | ✗ | ✗ |
| [greater (\>)](../../../sql-reference/functions/comparison-functions.md#greater) | ✔ | ✔ | ✗ | ✗ | ✗ |
| [lessOrEquals (\<=)](../../../sql-reference/functions/comparison-functions.md#lessorequals) | ✔ | ✔ | ✗ | ✗ | ✗ |
| [greaterOrEquals (\>=)](../../../sql-reference/functions/comparison-functions.md#greaterorequals) | ✔ | ✔ | ✗ | ✗ | ✗ |
| [empty](../../../sql-reference/functions/array-functions.md#function-empty) | ✔ | ✔ | ✗ | ✗ | ✗ |
| [notEmpty](../../../sql-reference/functions/array-functions.md#function-notempty) | ✔ | ✔ | ✗ | ✗ | ✗ |
| hasToken | ✗ | ✗ | ✗ | ✔ | ✗ |

12
docs/zh/sql-reference/functions/comparison-functions.md
View File

@ -21,14 +21,14 @@ sidebar_label: 比较函数
字符串按字节进行比较。较短的字符串小于以其开头并且至少包含一个字符的所有字符串。
## 等于，a=b和a==b 运算符 {#equals-a-b-and-a-b-operator}
## 等于，a=b和a==b 运算符 {#equals}
## 不等于，a!=b和a&lt;&gt;b 运算符 {#notequals-a-operator-b-and-a-b}
## 不等于，a!=b和a&lt;&gt;b 运算符 {#notequals}
## 少, &lt; 运算符 {#less-operator}
## 少, &lt; 运算符 {#less}
## 大于, &gt; 运算符 {#greater-operator}
## 大于, &gt; 运算符 {#greater}
## 小于等于, &lt;= 运算符 {#lessorequals-operator}
## 小于等于, &lt;= 运算符 {#lessorequals}
## 大于等于, &gt;= 运算符 {#greaterorequals-operator}
## 大于等于, &gt;= 运算符 {#greaterorequals}

2
packages/clickhouse-server.yaml
View File

@ -52,8 +52,6 @@ contents:
dst: /lib/systemd/system/clickhouse-server.service
- src: root/usr/bin/clickhouse-copier
dst: /usr/bin/clickhouse-copier
- src: root/usr/bin/clickhouse-report
dst: /usr/bin/clickhouse-report
- src: root/usr/bin/clickhouse-server
dst: /usr/bin/clickhouse-server
# clickhouse-keeper part

5
programs/CMakeLists.txt
View File

@ -63,8 +63,6 @@ option (ENABLE_CLICKHOUSE_SU "A tool similar to 'su'" ${ENABLE_CLICKHOUSE_ALL})
option (ENABLE_CLICKHOUSE_DISKS "A tool to manage disks" ${ENABLE_CLICKHOUSE_ALL})
option (ENABLE_CLICKHOUSE_REPORT "A tiny tool to collect a clickhouse-server state" ${ENABLE_CLICKHOUSE_ALL})
if (NOT ENABLE_NURAFT)
# RECONFIGURE_MESSAGE_LEVEL should not be used here,
# since ENABLE_NURAFT is set to OFF for FreeBSD and Darwin.
@ -390,9 +388,6 @@ if (ENABLE_CLICKHOUSE_SU)
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/clickhouse-su" DESTINATION ${CMAKE_INSTALL_BINDIR} COMPONENT clickhouse)
list(APPEND CLICKHOUSE_BUNDLE clickhouse-su)
endif ()
if (ENABLE_CLICKHOUSE_REPORT)
include(${ClickHouse_SOURCE_DIR}/utils/report/CMakeLists.txt)
endif ()
if (ENABLE_CLICKHOUSE_KEEPER)
if (NOT BUILD_STANDALONE_KEEPER AND CREATE_KEEPER_SYMLINK)

4
programs/client/Client.cpp
View File

@ -325,7 +325,7 @@ try
processConfig();
adjustSettings();
initTtyBuffer(toProgressOption(config().getString("progress", "default")));
initTTYBuffer(toProgressOption(config().getString("progress", "default")));
{
// All that just to set DB::CurrentThread::get().getGlobalContext()
@ -1238,7 +1238,6 @@ void Client::processConfig()
global_context->setCurrentQueryId(query_id);
}
print_stack_trace = config().getBool("stacktrace", false);
logging_initialized = true;
if (config().has("multiquery"))
is_multiquery = true;
@ -1460,7 +1459,6 @@ int mainEntryClickHouseClient(int argc, char ** argv)
DB::Client client;
// Initialize command line options
client.init(argc, argv);
/// Initialize config file
return client.run();
}
catch (const DB::Exception & e)

4
programs/copier/ClusterCopier.cpp
View File

@ -1408,7 +1408,7 @@ TaskStatus ClusterCopier::processPartitionPieceTaskImpl(
/// 3) Create helping table on the whole destination cluster
auto & settings_push = task_cluster->settings_push;
auto connection = task_table.cluster_push->getAnyShardInfo().pool->get(timeouts, &settings_push, true);
auto connection = task_table.cluster_push->getAnyShardInfo().pool->get(timeouts, settings_push, true);
String create_query = getRemoteCreateTable(task_shard.task_table.table_push, *connection, settings_push);
ParserCreateQuery parser_create_query;
@ -1786,7 +1786,7 @@ String ClusterCopier::getRemoteCreateTable(const DatabaseAndTableName & table, C
ASTPtr ClusterCopier::getCreateTableForPullShard(const ConnectionTimeouts & timeouts, TaskShard & task_shard)
{
/// Fetch and parse (possibly) new definition
auto connection_entry = task_shard.info.pool->get(timeouts, &task_cluster->settings_pull, true);
auto connection_entry = task_shard.info.pool->get(timeouts, task_cluster->settings_pull, true);
String create_query_pull_str = getRemoteCreateTable(
task_shard.task_table.table_pull,
*connection_entry,

2
programs/disks/CommandCopy.cpp
View File

@ -57,7 +57,7 @@ public:
String relative_path_from = validatePathAndGetAsRelative(path_from);
String relative_path_to = validatePathAndGetAsRelative(path_to);
disk_from->copyDirectoryContent(relative_path_from, disk_to, relative_path_to, /* read_settings= */ {}, /* write_settings= */ {});
disk_from->copyDirectoryContent(relative_path_from, disk_to, relative_path_to, /* read_settings= */ {}, /* write_settings= */ {}, /* cancellation_hook= */ {});
}
};
}

3
programs/keeper/Keeper.cpp
View File

@ -556,7 +556,8 @@ catch (...)
{
/// Poco does not provide stacktrace.
tryLogCurrentException("Application");
throw;
auto code = getCurrentExceptionCode();
return code ? code : -1;
}

21
programs/local/LocalServer.cpp
View File

@ -495,7 +495,7 @@ try
processConfig();
adjustSettings();
initTtyBuffer(toProgressOption(config().getString("progress", "default")));
initTTYBuffer(toProgressOption(config().getString("progress", "default")));
applyCmdSettings(global_context);
@ -563,9 +563,6 @@ catch (...)
void LocalServer::updateLoggerLevel(const String & logs_level)
{
if (!logging_initialized)
return;
config().setString("logger.level", logs_level);
updateLevels(config(), logger());
}
@ -607,21 +604,13 @@ void LocalServer::processConfig()
Poco::AutoPtr<OwnPatternFormatter> pf = new OwnPatternFormatter;
Poco::AutoPtr<OwnFormattingChannel> log = new OwnFormattingChannel(pf, new Poco::SimpleFileChannel(server_logs_file));
Poco::Logger::root().setChannel(log);
logging_initialized = true;
}
else if (logging || is_interactive)
{
config().setString("logger", "logger");
auto log_level_default = is_interactive && !logging ? "none" : level;
config().setString("logger.level", config().getString("log-level", config().getString("send_logs_level", log_level_default)));
buildLoggers(config(), logger(), "clickhouse-local");
logging_initialized = true;
}
else
{
Poco::Logger::root().setLevel("none");
Poco::Logger::root().setChannel(Poco::AutoPtr<Poco::NullChannel>(new Poco::NullChannel()));
logging_initialized = false;
config().setString("logger", "logger");
auto log_level_default = logging ? level : "fatal";
config().setString("logger.level", config().getString("log-level", config().getString("send_logs_level", log_level_default)));
buildLoggers(config(), logger(), "clickhouse-local");
}
shared_context = Context::createShared();

7
programs/server/Server.cpp
View File

@ -1163,6 +1163,8 @@ try
CompiledExpressionCacheFactory::instance().init(compiled_expression_cache_max_size_in_bytes, compiled_expression_cache_max_elements);
#endif
NamedCollectionUtils::loadIfNot();
/// Initialize main config reloader.
std::string include_from_path = config().getString("include_from", "/etc/metrika.xml");
@ -1823,7 +1825,7 @@ try
{
global_context->loadOrReloadDictionaries(config());
if (config().getBool("wait_dictionaries_load_at_startup", false))
if (!config().getBool("dictionaries_lazy_load", true) && config().getBool("wait_dictionaries_load_at_startup", true))
global_context->waitForDictionariesLoad();
}
catch (...)
@ -1970,7 +1972,8 @@ catch (...)
{
/// Poco does not provide stacktrace.
tryLogCurrentException("Application");
throw;
auto code = getCurrentExceptionCode();
return code ? code : -1;
}
std::unique_ptr<TCPProtocolStackFactory> Server::buildProtocolStackFromConfig(

33
programs/server/config.xml
View File

@ -830,13 +830,13 @@
And also (and which is more important), the initial_user will
be used as current user for the query.
Right now the protocol is pretty simple and it only takes into account:
Right now the protocol is pretty simple, and it only takes into account:
- cluster name
- query
Also it will be nice if the following will be implemented:
- source hostname (see interserver_http_host), but then it will depends from DNS,
it can use IP address instead, but then the you need to get correct on the initiator node.
Also, it will be nice if the following will be implemented:
- source hostname (see interserver_http_host), but then it will depend on DNS,
it can use IP address instead, but then you need to get correct on the initiator node.
- target hostname / ip address (same notes as for source hostname)
- time-based security tokens
-->
@ -1248,6 +1248,25 @@
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
</backup_log>
<!-- Storage S3Queue log.
-->
<s3queue_log>
<database>system</database>
<table>s3queue_log</table>
<partition_by>toYYYYMM(event_date)</partition_by>
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
</s3queue_log>
<!-- Blob storage object operations log.
-->
<blob_storage_log>
<database>system</database>
<table>blob_storage_log</table>
<partition_by>toYYYYMM(event_date)</partition_by>
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
<ttl>event_date + INTERVAL 30 DAY</ttl>
</blob_storage_log>
<!-- <top_level_domains_path>/var/lib/clickhouse/top_level_domains/</top_level_domains_path> -->
<!-- Custom TLD lists.
Format: <name>/path/to/file</name>
@ -1272,10 +1291,10 @@
<dictionaries_lazy_load>true</dictionaries_lazy_load>
<!-- Wait at startup until all the dictionaries finish their loading (successfully or not)
before listening to connections. Setting this to 1 can make ClickHouse start slowly,
however some queries can be executed faster (because it won't have to wait for the used dictionaries to be load).
before receiving any connections. Affects dictionaries only if "dictionaries_lazy_load" is false.
Setting this to false can make ClickHouse start faster, however some queries can be executed slower.
-->
<wait_dictionaries_load_at_startup>false</wait_dictionaries_load_at_startup>
<wait_dictionaries_load_at_startup>true</wait_dictionaries_load_at_startup>
<!-- Configuration of user defined executable functions -->
<user_defined_executable_functions_config>*_function.*ml</user_defined_executable_functions_config>

390
programs/server/dashboard.html
View File

@ -14,7 +14,7 @@
--moving-shadow-color: rgba(0, 0, 0, 0.5);
--input-shadow-color: rgba(0, 255, 0, 1);
--error-color: red;
--auth-error-color: white;
--global-error-color: white;
--legend-background: rgba(255, 255, 255, 0.75);
--title-color: #666;
--text-color: black;
@ -202,6 +202,10 @@
margin-right: 0.25rem;
}
#chart-params .param {
width: 6%;
}
input {
font-family: Liberation Sans, DejaVu Sans, sans-serif, Noto Color Emoji, Apple Color Emoji, Segoe UI Emoji;
outline: none;
@ -240,14 +244,13 @@
font-weight: bold;
user-select: none;
cursor: pointer;
margin-bottom: 1rem;
}
#run:hover {
filter: contrast(125%);
}
#add, #reload, #edit {
#add, #reload, #edit, #search {
padding: 0.25rem 0.5rem;
text-align: center;
font-weight: bold;
@ -264,16 +267,22 @@
height: 3ex;
}
#add:hover, #reload:hover, #edit:hover {
#add:hover, #reload:hover, #edit:hover, #search:hover {
background: var(--button-background-color);
}
#auth-error {
#search-query {
float: right;
width: 36%;
}
#global-error {
align-self: center;
width: 60%;
padding: .5rem;
color: var(--auth-error-color);
color: var(--global-error-color);
display: flex;
flex-flow: row nowrap;
@ -417,7 +426,7 @@
}
#mass-editor-message {
color: var(--auth-error-color);
color: var(--global-error-color);
}
#charts > div:only-child .display-only-if-more-than-one-chart {
@ -445,10 +454,11 @@
<input id="edit" type="button" value="✎" style="display: none;">
<input id="add" type="button" value="Add chart" style="display: none;">
<input id="reload" type="button" value="Reload">
<span id="search-span" class="nowrap" style="display: none;"><input id="search" type="button" value="🔎" title="Run query to obtain list of charts from ClickHouse"><input id="search-query" name="search" type="text" spellcheck="false"></span>
<div id="chart-params"></div>
</div>
</form>
<div id="auth-error"></div>
<div id="global-error"></div>
</div>
<div id="charts"></div>
<div id="mass-editor">
@ -501,151 +511,11 @@ const errorMessages = [
}
]
/// This is just a demo configuration of the dashboard.
let queries = [
{
"title": "Queries/second",
"query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_Query)
FROM system.metric_log
WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}
GROUP BY t
ORDER BY t WITH FILL STEP {rounding:UInt32}`
},
{
"title": "CPU Usage (cores)",
"query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_OSCPUVirtualTimeMicroseconds) / 1000000
FROM system.metric_log
WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}
GROUP BY t
ORDER BY t WITH FILL STEP {rounding:UInt32}`
},
{
"title": "Queries Running",
"query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(CurrentMetric_Query)
FROM system.metric_log
WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}
GROUP BY t
ORDER BY t WITH FILL STEP {rounding:UInt32}`
},
{
"title": "Merges Running",
"query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(CurrentMetric_Merge)
FROM system.metric_log
WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}
GROUP BY t
ORDER BY t WITH FILL STEP {rounding:UInt32}`
},
{
"title": "Selected Bytes/second",
"query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_SelectedBytes)
FROM system.metric_log
WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}
GROUP BY t
ORDER BY t WITH FILL STEP {rounding:UInt32}`
},
{
"title": "IO Wait",
"query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_OSIOWaitMicroseconds) / 1000000
FROM system.metric_log
WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}
GROUP BY t
ORDER BY t WITH FILL STEP {rounding:UInt32}`
},
{
"title": "CPU Wait",
"query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_OSCPUWaitMicroseconds) / 1000000
FROM system.metric_log
WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}
GROUP BY t
ORDER BY t WITH FILL STEP {rounding:UInt32}`
},
{
"title": "OS CPU Usage (Userspace)",
"query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(value)
FROM system.asynchronous_metric_log
WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}
AND metric = 'OSUserTimeNormalized'
GROUP BY t
ORDER BY t WITH FILL STEP {rounding:UInt32}`
},
{
"title": "OS CPU Usage (Kernel)",
"query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(value)
FROM system.asynchronous_metric_log
WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}
AND metric = 'OSSystemTimeNormalized'
GROUP BY t
ORDER BY t WITH FILL STEP {rounding:UInt32}`
},
{
"title": "Read From Disk",
"query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_OSReadBytes)
FROM system.metric_log
WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}
GROUP BY t
ORDER BY t WITH FILL STEP {rounding:UInt32}`
},
{
"title": "Read From Filesystem",
"query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_OSReadChars)
FROM system.metric_log
WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}
GROUP BY t
ORDER BY t WITH FILL STEP {rounding:UInt32}`
},
{
"title": "Memory (tracked)",
"query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(CurrentMetric_MemoryTracking)
FROM system.metric_log
WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}
GROUP BY t
ORDER BY t WITH FILL STEP {rounding:UInt32}`
},
{
"title": "Load Average (15 minutes)",
"query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(value)
FROM system.asynchronous_metric_log
WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}
AND metric = 'LoadAverage15'
GROUP BY t
ORDER BY t WITH FILL STEP {rounding:UInt32}`
},
{
"title": "Selected Rows/second",
"query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_SelectedRows)
FROM system.metric_log
WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}
GROUP BY t
ORDER BY t WITH FILL STEP {rounding:UInt32}`
},
{
"title": "Inserted Rows/second",
"query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_InsertedRows)
FROM system.metric_log
WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}
GROUP BY t
ORDER BY t WITH FILL STEP {rounding:UInt32}`
},
{
"title": "Total MergeTree Parts",
"query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(value)
FROM system.asynchronous_metric_log
WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}
AND metric = 'TotalPartsOfMergeTreeTables'
GROUP BY t
ORDER BY t WITH FILL STEP {rounding:UInt32}`
},
{
"title": "Max Parts For Partition",
"query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, max(value)
FROM system.asynchronous_metric_log
WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}
AND metric = 'MaxPartCountForPartition'
GROUP BY t
ORDER BY t WITH FILL STEP {rounding:UInt32}`
}
];
/// Query to fill `queries` list for the dashboard
let search_query = `SELECT title, query FROM system.dashboards WHERE dashboard = 'overview'`;
let customized = false;
let queries = [];
/// Query parameters with predefined default values.
/// All other parameters will be automatically found in the queries.
@ -683,7 +553,7 @@ function findParamsInQuery(query, new_params) {
}
function findParamsInQueries() {
let new_params = {}
let new_params = {};
queries.forEach(q => findParamsInQuery(q.query, new_params));
params = new_params;
}
@ -790,9 +660,10 @@ function insertChart(i) {
title_text.data = '';
findParamsInQuery(q.query, params);
buildParams();
refreshCustomized(true);
saveState();
const idx = getCurrentIndex();
draw(idx, chart, getParamsForURL(), q.query);
saveState();
}
query_editor_confirm.addEventListener('click', editConfirm);
@ -940,6 +811,7 @@ function insertChart(i) {
findParamsInQueries();
buildParams();
resize();
refreshCustomized(true);
saveState();
});
@ -958,7 +830,7 @@ function insertChart(i) {
charts.appendChild(chart);
return {chart: chart, textarea: query_editor_textarea};
};
}
document.getElementById('add').addEventListener('click', e => {
queries.push({ title: '', query: '' });
@ -972,9 +844,12 @@ document.getElementById('add').addEventListener('click', e => {
});
document.getElementById('reload').addEventListener('click', e => {
reloadAll();
reloadAll(false);
});
document.getElementById('search').addEventListener('click', e => {
reloadAll(true);
});
let mass_editor_active = false;
@ -1002,8 +877,9 @@ function massEditorApplyChanges() {
({params, queries} = JSON.parse(editor.value));
hideMassEditor();
regenerate();
drawAll();
refreshCustomized(true);
saveState();
drawAll();
}
document.getElementById('edit').addEventListener('click', e => {
@ -1085,13 +961,8 @@ function legendAsTooltipPlugin({ className, style = { background: "var(--legend-
};
}
async function draw(idx, chart, url_params, query) {
if (plots[idx]) {
plots[idx].destroy();
plots[idx] = null;
}
host = document.getElementById('url').value;
async function doFetch(query, url_params = '') {
host = document.getElementById('url').value || host;
user = document.getElementById('user').value;
password = document.getElementById('password').value;
@ -1135,6 +1006,17 @@ async function draw(idx, chart, url_params, query) {
}
}
return {data, error};
}
async function draw(idx, chart, url_params, query) {
if (plots[idx]) {
plots[idx].destroy();
plots[idx] = null;
}
let {data, error} = await doFetch(query, url_params);
if (!error) {
if (!Array.isArray(data)) {
error = "Query should return an array.";
@ -1151,7 +1033,7 @@ async function draw(idx, chart, url_params, query) {
let title_div = chart.querySelector('.title');
if (error) {
error_div.firstChild.data = error;
title_div.style.display = 'none';
title_div.style.display = 'none';
error_div.style.display = 'block';
return false;
} else {
@ -1194,29 +1076,29 @@ async function draw(idx, chart, url_params, query) {
/// Set title
const title = queries[idx] && queries[idx].title ? queries[idx].title.replaceAll(/\{(\w+)\}/g, (_, name) => params[name] ) : '';
chart.querySelector('.title').firstChild.data = title;
return true
return true;
}
function showAuthError(message) {
function showError(message) {
const charts = document.getElementById('charts');
charts.style.height = '0px';
charts.style.opacity = '0';
document.getElementById('add').style.display = 'none';
document.getElementById('edit').style.display = 'none';
const authError = document.getElementById('auth-error');
authError.textContent = message;
authError.style.display = 'flex';
const error = document.getElementById('global-error');
error.textContent = message;
error.style.display = 'flex';
}
function hideAuthError() {
function hideError() {
const charts = document.getElementById('charts');
charts.style.height = 'auto';
charts.style.opacity = '1';
const authError = document.getElementById('auth-error');
authError.textContent = '';
authError.style.display = 'none';
const error = document.getElementById('global-error');
error.textContent = '';
error.style.display = 'none';
}
let firstLoad = true;
@ -1226,12 +1108,12 @@ async function drawAll() {
const chartsArray = document.getElementsByClassName('chart');
if (!firstLoad) {
hideAuthError();
hideError();
}
await Promise.all([...Array(queries.length)].map(async (_, i) => {
return draw(i, chartsArray[i], params, queries[i].query).catch((e) => {
if (!firstLoad) {
showAuthError(e.message);
showError(e.message);
}
return false;
});
@ -1239,20 +1121,20 @@ async function drawAll() {
if (firstLoad) {
firstLoad = false;
} else {
enableReloadButton();
enableRunButton();
enableButtons();
}
if (results.includes(true)) {
const element = document.querySelector('.inputs');
element.classList.remove('unconnected');
document.getElementById('add').style.display = 'inline-block';
document.getElementById('edit').style.display = 'inline-block';
}
document.getElementById('search-span').style.display = '';
}
else {
const charts = document.getElementById('charts')
charts.style.height = '0px';
}
})
});
}
function resize() {
@ -1266,58 +1148,108 @@ function resize() {
new ResizeObserver(resize).observe(document.body);
function disableReloadButton() {
const reloadButton = document.getElementById('reload')
reloadButton.value = 'Reloading…'
reloadButton.disabled = true
reloadButton.classList.add('disabled')
function disableButtons() {
const reloadButton = document.getElementById('reload');
reloadButton.value = 'Reloading…';
reloadButton.disabled = true;
reloadButton.classList.add('disabled');
const runButton = document.getElementById('run');
runButton.value = 'Reloading…';
runButton.disabled = true;
runButton.classList.add('disabled');
const searchButton = document.getElementById('search');
searchButton.value = '…';
searchButton.disabled = true;
searchButton.classList.add('disabled');
}
function disableRunButton() {
const runButton = document.getElementById('run')
runButton.value = 'Reloading…'
runButton.disabled = true
runButton.classList.add('disabled')
function enableButtons() {
const reloadButton = document.getElementById('reload');
reloadButton.value = 'Reload';
reloadButton.disabled = false;
reloadButton.classList.remove('disabled');
const runButton = document.getElementById('run');
runButton.value = 'Ok';
runButton.disabled = false;
runButton.classList.remove('disabled');
const searchButton = document.getElementById('search');
searchButton.value = '🔎';
searchButton.disabled = false;
searchButton.classList.remove('disabled');
}
function enableReloadButton() {
const reloadButton = document.getElementById('reload')
reloadButton.value = 'Reload'
reloadButton.disabled = false
reloadButton.classList.remove('disabled')
}
function enableRunButton() {
const runButton = document.getElementById('run')
runButton.value = 'Ok'
runButton.disabled = false
runButton.classList.remove('disabled')
}
function reloadAll() {
updateParams();
drawAll();
saveState();
disableReloadButton();
disableRunButton();
async function reloadAll(do_search) {
disableButtons();
try {
updateParams();
if (do_search) {
search_query = document.getElementById('search-query').value;
queries = [];
refreshCustomized(false);
}
saveState();
if (do_search) {
await searchQueries();
}
await drawAll();
} catch (e) {
showError(e.toString());
}
enableButtons();
}
document.getElementById('params').onsubmit = function(event) {
reloadAll();
let do_search = document.activeElement === document.getElementById('search-query');
reloadAll(do_search);
event.preventDefault();
}
function saveState() {
const state = { host: host, user: user, queries: queries, params: params };
const state = { host, user, queries, params, search_query, customized };
history.pushState(state, '',
window.location.pathname + (window.location.search || '') + '#' + btoa(JSON.stringify(state)));
}
async function searchQueries() {
let {data, error} = await doFetch(search_query);
if (error) {
throw new Error(error);
}
if (!Array.isArray(data)) {
throw new Error("Search query should return an array.");
} else if (data.length == 0) {
throw new Error("Search query returned empty result.");
} else if (data.length != 2) {
throw new Error("Search query should return exactly two columns: title and query.");
} else if (!Array.isArray(data[0]) || !Array.isArray(data[1]) || data[0].length != data[1].length) {
throw new Error("Wrong data format of the search query.");
}
for (let i = 0; i < data[0].length; i++) {
queries.push({title: data[0][i], query: data[1][i]});
}
regenerate();
}
function refreshCustomized(value) {
if (value !== undefined) {
customized = value;
}
document.getElementById('search-span').style.opacity = customized ? 0.5 : 1.0;
}
function regenerate() {
document.getElementById('url').value = host;
document.getElementById('user').value = user;
document.getElementById('password').value = password;
document.getElementById('search-query').value = search_query;
refreshCustomized();
findParamsInQueries();
buildParams();
@ -1336,7 +1268,7 @@ function regenerate() {
window.onpopstate = function(event) {
if (!event.state) { return; }
({host, user, queries, params} = event.state);
({host, user, queries, params, search_query, customized} = event.state);
regenerate();
drawAll();
@ -1344,19 +1276,35 @@ window.onpopstate = function(event) {
if (window.location.hash) {
try {
({host, user, queries, params} = JSON.parse(atob(window.location.hash.substring(1))));
let search_query_, customized_;
({host, user, queries, params, search_query_, customized_} = JSON.parse(atob(window.location.hash.substring(1))));
// For compatibility with old URLs' hashes
search_query = search_query_ !== undefined ? search_query_ : search_query;
customized = customized_ !== undefined ? customized_ : true;
} catch {}
}
regenerate();
let new_theme = window.localStorage.getItem('theme');
if (new_theme && new_theme != theme) {
setTheme(new_theme);
} else {
drawAll();
async function start() {
try {
if (queries.length == 0) {
await searchQueries();
} else {
regenerate();
}
saveState();
let new_theme = window.localStorage.getItem('theme');
if (new_theme && new_theme != theme) {
setTheme(new_theme);
} else {
drawAll();
}
} catch (e) {
showError(e.toString());
}
}
start();
</script>
</body>
</html>

4
rust/skim/Cargo.toml
View File

@ -23,3 +23,7 @@ debug = true
inherits = "release"
# We use LTO here as well to slightly decrease binary size
lto = true
[patch.crates-io]
# Ref: https://github.com/lotabout/tuikit/pull/51
tuikit = { git = "https://github.com/azat-rust/tuikit.git", rev = "e1994c0e03ff02c49cf1471f0cc3cbf185ce0104" }

7
src/Analyzer/FunctionNode.cpp
View File

@ -209,15 +209,17 @@ ASTPtr FunctionNode::toASTImpl(const ConvertToASTOptions & options) const
function_ast->kind = ASTFunction::Kind::WINDOW_FUNCTION;
}
const auto & arguments = getArguments();
auto new_options = options;
const auto & argument_nodes = arguments.getNodes();
/// To avoid surrounding constants with several internal casts.
if (function_name == "_CAST" && (*getArguments().begin())->getNodeType() == QueryTreeNodeType::CONSTANT)
if (function_name == "_CAST" && !argument_nodes.empty() && argument_nodes[0]->getNodeType() == QueryTreeNodeType::CONSTANT)
new_options.add_cast_for_constants = false;
/// Avoid cast for `IN tuple(...)` expression.
/// Tuples colud be quite big, and adding a type may significantly increase query size.
/// It should be safe because set type for `column IN tuple` is deduced from `column` type.
if (isNameOfInFunction(function_name) && (*(++getArguments().begin()))->getNodeType() == QueryTreeNodeType::CONSTANT)
if (isNameOfInFunction(function_name) && argument_nodes.size() > 1 && argument_nodes[1]->getNodeType() == QueryTreeNodeType::CONSTANT)
new_options.add_cast_for_constants = false;
const auto & parameters = getParameters();
@ -227,7 +229,6 @@ ASTPtr FunctionNode::toASTImpl(const ConvertToASTOptions & options) const
function_ast->parameters = function_ast->children.back();
}
const auto & arguments = getArguments();
function_ast->children.push_back(arguments.toAST(new_options));
function_ast->arguments = function_ast->children.back();

217
src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp
View File

@ -5,11 +5,214 @@
#include <Analyzer/InDepthQueryTreeVisitor.h>
#include <Analyzer/FunctionNode.h>
#include <Analyzer/ConstantNode.h>
#include <Analyzer/JoinNode.h>
#include <Analyzer/HashUtils.h>
#include <Analyzer/Utils.h>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
}
/// Visitor that optimizes logical expressions _only_ in JOIN ON section
class JoinOnLogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithContext<JoinOnLogicalExpressionOptimizerVisitor>
{
public:
using Base = InDepthQueryTreeVisitorWithContext<JoinOnLogicalExpressionOptimizerVisitor>;
explicit JoinOnLogicalExpressionOptimizerVisitor(ContextPtr context)
: Base(std::move(context))
{}
void enterImpl(QueryTreeNodePtr & node)
{
auto * function_node = node->as<FunctionNode>();
if (!function_node)
return;
if (function_node->getFunctionName() == "or")
{
bool is_argument_type_changed = tryOptimizeIsNotDistinctOrIsNull(node, getContext());
if (is_argument_type_changed)
need_rerun_resolve = true;
return;
}
}
void leaveImpl(QueryTreeNodePtr & node)
{
if (!need_rerun_resolve)
return;
if (auto * function_node = node->as<FunctionNode>())
rerunFunctionResolve(function_node, getContext());
}
private:
bool need_rerun_resolve = false;
/// Returns true if type of some operand is changed and parent function needs to be re-resolved
static bool tryOptimizeIsNotDistinctOrIsNull(QueryTreeNodePtr & node, const ContextPtr & context)
{
auto & function_node = node->as<FunctionNode &>();
chassert(function_node.getFunctionName() == "or");
QueryTreeNodes or_operands;
or_operands.reserve(function_node.getArguments().getNodes().size());
/// Indices of `equals` or `isNotDistinctFrom` functions in the vector above
std::vector<size_t> equals_functions_indices;
/** Map from `isNull` argument to indices of operands that contains that `isNull` functions
* `a = b OR (a IS NULL AND b IS NULL) OR (a IS NULL AND c IS NULL)`
* will be mapped to
* {
* a => [(a IS NULL AND b IS NULL), (a IS NULL AND c IS NULL)]
* b => [(a IS NULL AND b IS NULL)]
* c => [(a IS NULL AND c IS NULL)]
* }
* Then for each a <=> b we can find all operands that contains both a IS NULL and b IS NULL
*/
QueryTreeNodePtrWithHashMap<std::vector<size_t>> is_null_argument_to_indices;
for (const auto & argument : function_node.getArguments())
{
or_operands.push_back(argument);
auto * argument_function = argument->as<FunctionNode>();
if (!argument_function)
continue;
const auto & func_name = argument_function->getFunctionName();
if (func_name == "equals" || func_name == "isNotDistinctFrom")
{
equals_functions_indices.push_back(or_operands.size() - 1);
}
else if (func_name == "and")
{
for (const auto & and_argument : argument_function->getArguments().getNodes())
{
auto * and_argument_function = and_argument->as<FunctionNode>();
if (and_argument_function && and_argument_function->getFunctionName() == "isNull")
{
const auto & is_null_argument = and_argument_function->getArguments().getNodes()[0];
is_null_argument_to_indices[is_null_argument].push_back(or_operands.size() - 1);
}
}
}
}
/// OR operands that are changed to and needs to be re-resolved
std::unordered_set<size_t> arguments_to_reresolve;
for (size_t equals_function_idx : equals_functions_indices)
{
auto * equals_function = or_operands[equals_function_idx]->as<FunctionNode>();
/// For a <=> b we are looking for expressions containing both `a IS NULL` and `b IS NULL` combined with AND
const auto & argument_nodes = equals_function->getArguments().getNodes();
const auto & lhs_is_null_parents = is_null_argument_to_indices[argument_nodes[0]];
const auto & rhs_is_null_parents = is_null_argument_to_indices[argument_nodes[1]];
std::unordered_set<size_t> operands_to_optimize;
std::set_intersection(lhs_is_null_parents.begin(), lhs_is_null_parents.end(),
rhs_is_null_parents.begin(), rhs_is_null_parents.end(),
std::inserter(operands_to_optimize, operands_to_optimize.begin()));
/// If we have `a = b OR (a IS NULL AND b IS NULL)` we can optimize it to `a <=> b`
if (!operands_to_optimize.empty() && equals_function->getFunctionName() == "equals")
arguments_to_reresolve.insert(equals_function_idx);
for (size_t to_optimize_idx : operands_to_optimize)
{
/// We are looking for operand `a IS NULL AND b IS NULL AND ...`
auto * operand_to_optimize = or_operands[to_optimize_idx]->as<FunctionNode>();
/// Remove `a IS NULL` and `b IS NULL` arguments from AND
QueryTreeNodes new_arguments;
for (const auto & and_argument : operand_to_optimize->getArguments().getNodes())
{
bool to_eliminate = false;
const auto * and_argument_function = and_argument->as<FunctionNode>();
if (and_argument_function && and_argument_function->getFunctionName() == "isNull")
{
const auto & is_null_argument = and_argument_function->getArguments().getNodes()[0];
to_eliminate = (is_null_argument->isEqual(*argument_nodes[0]) || is_null_argument->isEqual(*argument_nodes[1]));
}
if (to_eliminate)
arguments_to_reresolve.insert(to_optimize_idx);
else
new_arguments.emplace_back(and_argument);
}
/// If less than two arguments left, we will remove or replace the whole AND below
operand_to_optimize->getArguments().getNodes() = std::move(new_arguments);
}
}
if (arguments_to_reresolve.empty())
/// Nothing have been changed
return false;
auto and_function_resolver = FunctionFactory::instance().get("and", context);
auto strict_equals_function_resolver = FunctionFactory::instance().get("isNotDistinctFrom", context);
bool need_reresolve = false;
QueryTreeNodes new_or_operands;
for (size_t i = 0; i < or_operands.size(); ++i)
{
if (arguments_to_reresolve.contains(i))
{
auto * function = or_operands[i]->as<FunctionNode>();
if (function->getFunctionName() == "equals")
{
/// We should replace `a = b` with `a <=> b` because we removed checks for IS NULL
need_reresolve |= function->getResultType()->isNullable();
function->resolveAsFunction(strict_equals_function_resolver);
new_or_operands.emplace_back(std::move(or_operands[i]));
}
else if (function->getFunctionName() == "and")
{
const auto & and_arguments = function->getArguments().getNodes();
if (and_arguments.size() > 1)
{
function->resolveAsFunction(and_function_resolver);
new_or_operands.emplace_back(std::move(or_operands[i]));
}
else if (and_arguments.size() == 1)
{
/// Replace AND with a single argument with the argument itself
new_or_operands.emplace_back(and_arguments[0]);
}
}
else
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected function name: '{}'", function->getFunctionName());
}
else
{
new_or_operands.emplace_back(std::move(or_operands[i]));
}
}
if (new_or_operands.size() == 1)
{
node = std::move(new_or_operands[0]);
return need_reresolve;
}
/// Rebuild OR function
auto or_function_resolver = FunctionFactory::instance().get("or", context);
function_node.getArguments().getNodes() = std::move(new_or_operands);
function_node.resolveAsFunction(or_function_resolver);
return need_reresolve;
}
};
class LogicalExpressionOptimizerVisitor : public InDepthQueryTreeVisitorWithContext<LogicalExpressionOptimizerVisitor>
{
public:
@ -21,6 +224,17 @@ public:
void enterImpl(QueryTreeNodePtr & node)
{
if (auto * join_node = node->as<JoinNode>())
{
/// Operator <=> is not supported outside of JOIN ON section
if (join_node->hasJoinExpression())
{
JoinOnLogicalExpressionOptimizerVisitor join_on_visitor(getContext());
join_on_visitor.visit(join_node->getJoinExpression());
}
return;
}
auto * function_node = node->as<FunctionNode>();
if (!function_node)
@ -38,6 +252,7 @@ public:
return;
}
}
private:
void tryReplaceAndEqualsChainsWithConstant(QueryTreeNodePtr & node)
{
@ -224,7 +439,7 @@ private:
}
/// otherwise add a stub 0 to make OR correct
or_operands.push_back(std::make_shared<ConstantNode>(static_cast<UInt8>(0)));
or_operands.push_back(std::make_shared<ConstantNode>(static_cast<UInt8>(0), function_node.getResultType()));
}
auto or_function_resolver = FunctionFactory::instance().get("or", getContext());

11
src/Analyzer/Passes/LogicalExpressionOptimizerPass.h
View File

@ -67,6 +67,17 @@ namespace DB
* FROM TABLE
* WHERE a = 1 AND b = 'test';
* -------------------------------
*
* 5. Remove unnecessary IS NULL checks in JOIN ON clause
* - equality check with explicit IS NULL check replaced with <=> operator
* -------------------------------
* SELECT * FROM t1 JOIN t2 ON a = b OR (a IS NULL AND b IS NULL)
* SELECT * FROM t1 JOIN t2 ON a <=> b OR (a IS NULL AND b IS NULL)
*
* will be transformed into
*
* SELECT * FROM t1 JOIN t2 ON a <=> b
* -------------------------------
*/
class LogicalExpressionOptimizerPass final : public IQueryTreePass

39
src/Backups/BackupIO_S3.cpp
View File

@ -127,6 +127,9 @@ BackupReaderS3::BackupReaderS3(
request_settings.max_single_read_retries = context_->getSettingsRef().s3_max_single_read_retries; // FIXME: Avoid taking value for endpoint
request_settings.allow_native_copy = allow_s3_native_copy;
client = makeS3Client(s3_uri_, access_key_id_, secret_access_key_, s3_settings, context_);
if (auto blob_storage_system_log = context_->getBlobStorageLog())
blob_storage_log = std::make_shared<BlobStorageLogWriter>(blob_storage_system_log);
}
BackupReaderS3::~BackupReaderS3() = default;
@ -178,6 +181,7 @@ void BackupReaderS3::copyFileToDisk(const String & path_in_backup, size_t file_s
/* dest_key= */ blob_path[0],
s3_settings.request_settings,
read_settings,
blob_storage_log,
object_attributes,
threadPoolCallbackRunner<void>(getBackupsIOThreadPool().get(), "BackupReaderS3"),
/* for_disk_s3= */ true);
@ -214,6 +218,12 @@ BackupWriterS3::BackupWriterS3(
request_settings.allow_native_copy = allow_s3_native_copy;
request_settings.setStorageClassName(storage_class_name);
client = makeS3Client(s3_uri_, access_key_id_, secret_access_key_, s3_settings, context_);
if (auto blob_storage_system_log = context_->getBlobStorageLog())
{
blob_storage_log = std::make_shared<BlobStorageLogWriter>(blob_storage_system_log);
if (context_->hasQueryContext())
blob_storage_log->query_id = context_->getQueryContext()->getCurrentQueryId();
}
}
void BackupWriterS3::copyFileFromDisk(const String & path_in_backup, DiskPtr src_disk, const String & src_path,
@ -239,6 +249,7 @@ void BackupWriterS3::copyFileFromDisk(const String & path_in_backup, DiskPtr src
fs::path(s3_uri.key) / path_in_backup,
s3_settings.request_settings,
read_settings,
blob_storage_log,
{},
threadPoolCallbackRunner<void>(getBackupsIOThreadPool().get(), "BackupWriterS3"));
return; /// copied!
@ -262,13 +273,15 @@ void BackupWriterS3::copyFile(const String & destination, const String & source,
fs::path(s3_uri.key) / destination,
s3_settings.request_settings,
read_settings,
blob_storage_log,
{},
threadPoolCallbackRunner<void>(getBackupsIOThreadPool().get(), "BackupWriterS3"));
}
void BackupWriterS3::copyDataToFile(const String & path_in_backup, const CreateReadBufferFunction & create_read_buffer, UInt64 start_pos, UInt64 length)
{
copyDataToS3File(create_read_buffer, start_pos, length, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup, s3_settings.request_settings, {},
copyDataToS3File(create_read_buffer, start_pos, length, client, s3_uri.bucket, fs::path(s3_uri.key) / path_in_backup,
s3_settings.request_settings, blob_storage_log, {},
threadPoolCallbackRunner<void>(getBackupsIOThreadPool().get(), "BackupWriterS3"));
}
@ -302,6 +315,7 @@ std::unique_ptr<WriteBuffer> BackupWriterS3::writeFile(const String & file_name)
fs::path(s3_uri.key) / file_name,
DBMS_DEFAULT_BUFFER_SIZE,
s3_settings.request_settings,
blob_storage_log,
std::nullopt,
threadPoolCallbackRunner<void>(getBackupsIOThreadPool().get(), "BackupWriterS3"),
write_settings);
@ -311,8 +325,19 @@ void BackupWriterS3::removeFile(const String & file_name)
{
S3::DeleteObjectRequest request;
request.SetBucket(s3_uri.bucket);
request.SetKey(fs::path(s3_uri.key) / file_name);
auto key = fs::path(s3_uri.key) / file_name;
request.SetKey(key);
auto outcome = client->DeleteObject(request);
if (blob_storage_log)
{
blob_storage_log->addEvent(
BlobStorageLogElement::EventType::Delete,
s3_uri.bucket, key, /* local_path */ "", /* data_size */ 0,
outcome.IsSuccess() ? nullptr : &outcome.GetError());
}
if (!outcome.IsSuccess() && !isNotFoundError(outcome.GetError().GetErrorType()))
throw S3Exception(outcome.GetError().GetMessage(), outcome.GetError().GetErrorType());
}
@ -371,6 +396,16 @@ void BackupWriterS3::removeFilesBatch(const Strings & file_names)
request.SetDelete(delkeys);
auto outcome = client->DeleteObjects(request);
if (blob_storage_log)
{
const auto * outcome_error = outcome.IsSuccess() ? nullptr : &outcome.GetError();
auto time_now = std::chrono::system_clock::now();
for (const auto & obj : current_chunk)
blob_storage_log->addEvent(BlobStorageLogElement::EventType::Delete, s3_uri.bucket, obj.GetKey(),
/* local_path */ "", /* data_size */ 0, outcome_error, time_now);
}
if (!outcome.IsSuccess() && !isNotFoundError(outcome.GetError().GetErrorType()))
throw S3Exception(outcome.GetError().GetMessage(), outcome.GetError().GetErrorType());
}

6
src/Backups/BackupIO_S3.h
View File

@ -8,7 +8,7 @@
#include <IO/S3Common.h>
#include <Storages/StorageS3Settings.h>
#include <Interpreters/Context_fwd.h>
#include <IO/S3/BlobStorageLogWriter.h>
namespace DB
{
@ -32,6 +32,8 @@ private:
const DataSourceDescription data_source_description;
S3Settings s3_settings;
std::shared_ptr<S3::Client> client;
BlobStorageLogWriterPtr blob_storage_log;
};
@ -63,6 +65,8 @@ private:
S3Settings s3_settings;
std::shared_ptr<S3::Client> client;
std::optional<bool> supports_batch_delete;
BlobStorageLogWriterPtr blob_storage_log;
};
}

4
src/CMakeLists.txt
View File

@ -435,6 +435,10 @@ dbms_target_link_libraries(PRIVATE ch_contrib::zstd)
target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::zstd)
target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::xz)
if (TARGET ch_contrib::pocketfft)
target_link_libraries(clickhouse_common_io PUBLIC ch_contrib::pocketfft)
endif ()
if (TARGET ch_contrib::icu)
dbms_target_link_libraries (PRIVATE ch_contrib::icu)
endif ()

10
src/Client/ClientBase.cpp
View File

@ -722,7 +722,7 @@ void ClientBase::adjustSettings()
global_context->setSettings(settings);
}
void ClientBase::initTtyBuffer(ProgressOption progress)
void ClientBase::initTTYBuffer(ProgressOption progress)
{
if (tty_buf)
return;
@ -2566,6 +2566,14 @@ bool ClientBase::processMultiQueryFromFile(const String & file_name)
ReadBufferFromFile in(file_name);
readStringUntilEOF(queries_from_file, in);
if (!global_context->getSettings().log_comment.changed)
{
Settings settings = global_context->getSettings();
/// NOTE: cannot use even weakly_canonical() since it fails for /dev/stdin due to resolving of "pipe:[X]"
settings.log_comment = fs::absolute(fs::path(file_name));
global_context->setSettings(settings);
}
return executeMultiQuery(queries_from_file);
}

4
src/Client/ClientBase.h
View File

@ -185,7 +185,7 @@ protected:
/// Adjust some settings after command line options and config had been processed.
void adjustSettings();
void initTtyBuffer(ProgressOption progress);
void initTTYBuffer(ProgressOption progress);
/// Should be one of the first, to be destroyed the last,
/// since other members can use them.
@ -321,8 +321,6 @@ protected:
bool allow_merge_tree_settings = false;
bool cancelled = false;
bool logging_initialized = false;
};
}

6
src/Client/ConnectionEstablisher.cpp
View File

@ -24,7 +24,7 @@ namespace ErrorCodes
ConnectionEstablisher::ConnectionEstablisher(
IConnectionPool * pool_,
const ConnectionTimeouts * timeouts_,
const Settings * settings_,
const Settings & settings_,
Poco::Logger * log_,
const QualifiedTableName * table_to_check_)
: pool(pool_), timeouts(timeouts_), settings(settings_), log(log_), table_to_check(table_to_check_), is_finished(false)
@ -72,7 +72,7 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std::
ProfileEvents::increment(ProfileEvents::DistributedConnectionUsable);
result.is_usable = true;
UInt64 max_allowed_delay = settings ? UInt64(settings->max_replica_delay_for_distributed_queries) : 0;
const UInt64 max_allowed_delay = settings.max_replica_delay_for_distributed_queries;
if (!max_allowed_delay)
{
result.is_up_to_date = true;
@ -113,7 +113,7 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std::
ConnectionEstablisherAsync::ConnectionEstablisherAsync(
IConnectionPool * pool_,
const ConnectionTimeouts * timeouts_,
const Settings * settings_,
const Settings & settings_,
Poco::Logger * log_,
const QualifiedTableName * table_to_check_)
: AsyncTaskExecutor(std::make_unique<Task>(*this)), connection_establisher(pool_, timeouts_, settings_, log_, table_to_check_)

14
src/Client/ConnectionEstablisher.h
View File

@ -22,7 +22,7 @@ public:
ConnectionEstablisher(IConnectionPool * pool_,
const ConnectionTimeouts * timeouts_,
const Settings * settings_,
const Settings & settings_,
Poco::Logger * log,
const QualifiedTableName * table_to_check = nullptr);
@ -37,13 +37,12 @@ public:
private:
IConnectionPool * pool;
const ConnectionTimeouts * timeouts;
const Settings * settings;
const Settings & settings;
Poco::Logger * log;
const QualifiedTableName * table_to_check;
bool is_finished;
AsyncCallback async_callback = {};
};
#if defined(OS_LINUX)
@ -61,7 +60,7 @@ public:
ConnectionEstablisherAsync(IConnectionPool * pool_,
const ConnectionTimeouts * timeouts_,
const Settings * settings_,
const Settings & settings_,
Poco::Logger * log_,
const QualifiedTableName * table_to_check_ = nullptr);
@ -72,7 +71,7 @@ public:
/// Check if the process of connection establishing was finished.
/// The process is considered finished if connection is ready,
/// some exception occurred or timeout exceeded.
bool isFinished() { return is_finished; }
bool isFinished() const { return is_finished; }
TryResult getResult() const { return result; }
const std::string & getFailMessage() const { return fail_message; }
@ -87,7 +86,10 @@ private:
struct Task : public AsyncTask
{
Task(ConnectionEstablisherAsync & connection_establisher_async_) : connection_establisher_async(connection_establisher_async_) {}
explicit Task(ConnectionEstablisherAsync & connection_establisher_async_)
: connection_establisher_async(connection_establisher_async_)
{
}
ConnectionEstablisherAsync & connection_establisher_async;

18
src/Client/ConnectionPool.h
View File

@ -30,9 +30,10 @@ public:
virtual ~IConnectionPool() = default;
/// Selects the connection to work.
virtual Entry get(const ConnectionTimeouts & timeouts) = 0;
/// If force_connected is false, the client must manually ensure that returned connection is good.
virtual Entry get(const ConnectionTimeouts & timeouts, /// NOLINT
const Settings * settings = nullptr,
const Settings & settings,
bool force_connected = true) = 0;
virtual Priority getPriority() const { return Priority{1}; }
@ -79,15 +80,18 @@ public:
{
}
Entry get(const ConnectionTimeouts & timeouts) override
{
Entry entry = Base::get(-1);
entry->forceConnected(timeouts);
return entry;
}
Entry get(const ConnectionTimeouts & timeouts, /// NOLINT
const Settings * settings = nullptr,
const Settings & settings,
bool force_connected = true) override
{
Entry entry;
if (settings)
entry = Base::get(settings->connection_pool_max_wait_ms.totalMilliseconds());
else
entry = Base::get(-1);
Entry entry = Base::get(settings.connection_pool_max_wait_ms.totalMilliseconds());
if (force_connected)
entry->forceConnected(timeouts);

63
src/Client/ConnectionPoolWithFailover.cpp
View File

@ -44,8 +44,19 @@ ConnectionPoolWithFailover::ConnectionPoolWithFailover(
}
}
IConnectionPool::Entry ConnectionPoolWithFailover::get(const ConnectionTimeouts & timeouts)
{
Settings settings;
settings.load_balancing = get_priority_load_balancing.load_balancing;
settings.load_balancing_first_offset = 0;
settings.distributed_replica_max_ignored_errors = 0;
settings.fallback_to_stale_replicas_for_distributed_queries = true;
return get(timeouts, settings, true);
}
IConnectionPool::Entry ConnectionPoolWithFailover::get(const ConnectionTimeouts & timeouts,
const Settings * settings,
const Settings & settings,
bool /*force_connected*/)
{
if (nested_pools.empty())
@ -57,18 +68,13 @@ IConnectionPool::Entry ConnectionPoolWithFailover::get(const ConnectionTimeouts
return tryGetEntry(pool, timeouts, fail_message, settings, {});
};
size_t offset = 0;
LoadBalancing load_balancing = get_priority_load_balancing.load_balancing;
if (settings)
{
offset = settings->load_balancing_first_offset % nested_pools.size();
load_balancing = LoadBalancing(settings->load_balancing);
}
const size_t offset = settings.load_balancing_first_offset % nested_pools.size();
const LoadBalancing load_balancing = settings.load_balancing;
GetPriorityFunc get_priority = get_priority_load_balancing.getPriorityFunc(load_balancing, offset, nested_pools.size());
UInt64 max_ignored_errors = settings ? settings->distributed_replica_max_ignored_errors.value : 0;
bool fallback_to_stale_replicas = settings ? settings->fallback_to_stale_replicas_for_distributed_queries.value : true;
const UInt64 max_ignored_errors = settings.distributed_replica_max_ignored_errors;
const bool fallback_to_stale_replicas = settings.fallback_to_stale_replicas_for_distributed_queries;
return Base::get(max_ignored_errors, fallback_to_stale_replicas, try_get_entry, get_priority);
}
@ -113,7 +119,7 @@ ConnectionPoolWithFailover::Status ConnectionPoolWithFailover::getStatus() const
}
std::vector<IConnectionPool::Entry> ConnectionPoolWithFailover::getMany(const ConnectionTimeouts & timeouts,
const Settings * settings,
const Settings & settings,
PoolMode pool_mode,
AsyncCallback async_callback,
std::optional<bool> skip_unavailable_endpoints)
@ -134,7 +140,7 @@ std::vector<IConnectionPool::Entry> ConnectionPoolWithFailover::getMany(const Co
std::vector<ConnectionPoolWithFailover::TryResult> ConnectionPoolWithFailover::getManyForTableFunction(
const ConnectionTimeouts & timeouts,
const Settings * settings,
const Settings & settings,
PoolMode pool_mode)
{
TryGetEntryFunc try_get_entry = [&](NestedPool & pool, std::string & fail_message)
@ -147,7 +153,7 @@ std::vector<ConnectionPoolWithFailover::TryResult> ConnectionPoolWithFailover::g
std::vector<ConnectionPoolWithFailover::TryResult> ConnectionPoolWithFailover::getManyChecked(
const ConnectionTimeouts & timeouts,
const Settings * settings, PoolMode pool_mode,
const Settings & settings, PoolMode pool_mode,
const QualifiedTableName & table_to_check,
AsyncCallback async_callback,
std::optional<bool> skip_unavailable_endpoints)
@ -160,21 +166,16 @@ std::vector<ConnectionPoolWithFailover::TryResult> ConnectionPoolWithFailover::g
return getManyImpl(settings, pool_mode, try_get_entry, skip_unavailable_endpoints);
}
ConnectionPoolWithFailover::Base::GetPriorityFunc ConnectionPoolWithFailover::makeGetPriorityFunc(const Settings * settings)
ConnectionPoolWithFailover::Base::GetPriorityFunc ConnectionPoolWithFailover::makeGetPriorityFunc(const Settings & settings)
{
size_t offset = 0;
LoadBalancing load_balancing = get_priority_load_balancing.load_balancing;
if (settings)
{
offset = settings->load_balancing_first_offset % nested_pools.size();
load_balancing = LoadBalancing(settings->load_balancing);
}
const size_t offset = settings.load_balancing_first_offset % nested_pools.size();
const LoadBalancing load_balancing = LoadBalancing(settings.load_balancing);
return get_priority_load_balancing.getPriorityFunc(load_balancing, offset, nested_pools.size());
}
std::vector<ConnectionPoolWithFailover::TryResult> ConnectionPoolWithFailover::getManyImpl(
const Settings * settings,
const Settings & settings,
PoolMode pool_mode,
const TryGetEntryFunc & try_get_entry,
std::optional<bool> skip_unavailable_endpoints)
@ -184,13 +185,11 @@ std::vector<ConnectionPoolWithFailover::TryResult> ConnectionPoolWithFailover::g
"Cannot get connection from ConnectionPoolWithFailover cause nested pools are empty");
if (!skip_unavailable_endpoints.has_value())
skip_unavailable_endpoints = (settings && settings->skip_unavailable_shards);
skip_unavailable_endpoints = settings.skip_unavailable_shards;
size_t min_entries = skip_unavailable_endpoints.value() ? 0 : 1;
size_t max_tries = (settings ?
size_t{settings->connections_with_failover_max_tries} :
size_t{DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES});
size_t max_tries = settings.connections_with_failover_max_tries;
size_t max_entries;
if (pool_mode == PoolMode::GET_ALL)
{
@ -200,14 +199,14 @@ std::vector<ConnectionPoolWithFailover::TryResult> ConnectionPoolWithFailover::g
else if (pool_mode == PoolMode::GET_ONE)
max_entries = 1;
else if (pool_mode == PoolMode::GET_MANY)
max_entries = settings ? size_t(settings->max_parallel_replicas) : 1;
max_entries = settings.max_parallel_replicas;
else
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Unknown pool allocation mode");
GetPriorityFunc get_priority = makeGetPriorityFunc(settings);
UInt64 max_ignored_errors = settings ? settings->distributed_replica_max_ignored_errors.value : 0;
bool fallback_to_stale_replicas = settings ? settings->fallback_to_stale_replicas_for_distributed_queries.value : true;
UInt64 max_ignored_errors = settings.distributed_replica_max_ignored_errors.value;
bool fallback_to_stale_replicas = settings.fallback_to_stale_replicas_for_distributed_queries.value;
return Base::getMany(min_entries, max_entries, max_tries,
max_ignored_errors, fallback_to_stale_replicas,
@ -219,7 +218,7 @@ ConnectionPoolWithFailover::tryGetEntry(
IConnectionPool & pool,
const ConnectionTimeouts & timeouts,
std::string & fail_message,
const Settings * settings,
const Settings & settings,
const QualifiedTableName * table_to_check,
[[maybe_unused]] AsyncCallback async_callback)
{
@ -252,10 +251,10 @@ ConnectionPoolWithFailover::tryGetEntry(
return result;
}
std::vector<ConnectionPoolWithFailover::Base::ShuffledPool> ConnectionPoolWithFailover::getShuffledPools(const Settings * settings)
std::vector<ConnectionPoolWithFailover::Base::ShuffledPool> ConnectionPoolWithFailover::getShuffledPools(const Settings & settings)
{
GetPriorityFunc get_priority = makeGetPriorityFunc(settings);
UInt64 max_ignored_errors = settings ? settings->distributed_replica_max_ignored_errors.value : 0;
UInt64 max_ignored_errors = settings.distributed_replica_max_ignored_errors.value;
return Base::getShuffledPools(max_ignored_errors, get_priority);
}

17
src/Client/ConnectionPoolWithFailover.h
View File

@ -44,8 +44,9 @@ public:
using Entry = IConnectionPool::Entry;
/** Allocates connection to work. */
Entry get(const ConnectionTimeouts & timeouts) override;
Entry get(const ConnectionTimeouts & timeouts,
const Settings * settings,
const Settings & settings,
bool force_connected) override; /// From IConnectionPool
Priority getPriority() const override; /// From IConnectionPool
@ -54,13 +55,13 @@ public:
* Connections provide access to different replicas of one shard.
*/
std::vector<Entry> getMany(const ConnectionTimeouts & timeouts,
const Settings * settings, PoolMode pool_mode,
const Settings & settings, PoolMode pool_mode,
AsyncCallback async_callback = {},
std::optional<bool> skip_unavailable_endpoints = std::nullopt);
/// The same as getMany(), but return std::vector<TryResult>.
std::vector<TryResult> getManyForTableFunction(const ConnectionTimeouts & timeouts,
const Settings * settings, PoolMode pool_mode);
const Settings & settings, PoolMode pool_mode);
using Base = PoolWithFailoverBase<IConnectionPool>;
using TryResult = Base::TryResult;
@ -69,7 +70,7 @@ public:
/// Delay threshold is taken from settings.
std::vector<TryResult> getManyChecked(
const ConnectionTimeouts & timeouts,
const Settings * settings,
const Settings & settings,
PoolMode pool_mode,
const QualifiedTableName & table_to_check,
AsyncCallback async_callback = {},
@ -86,7 +87,7 @@ public:
using Status = std::vector<NestedPoolStatus>;
Status getStatus() const;
std::vector<Base::ShuffledPool> getShuffledPools(const Settings * settings);
std::vector<Base::ShuffledPool> getShuffledPools(const Settings & settings);
size_t getMaxErrorCup() const { return Base::max_error_cap; }
@ -98,7 +99,7 @@ public:
private:
/// Get the values of relevant settings and call Base::getMany()
std::vector<TryResult> getManyImpl(
const Settings * settings,
const Settings & settings,
PoolMode pool_mode,
const TryGetEntryFunc & try_get_entry,
std::optional<bool> skip_unavailable_endpoints = std::nullopt);
@ -110,11 +111,11 @@ private:
IConnectionPool & pool,
const ConnectionTimeouts & timeouts,
std::string & fail_message,
const Settings * settings,
const Settings & settings,
const QualifiedTableName * table_to_check = nullptr,
AsyncCallback async_callback = {});
GetPriorityFunc makeGetPriorityFunc(const Settings * settings);
GetPriorityFunc makeGetPriorityFunc(const Settings & settings);
GetPriorityForLoadBalancing get_priority_load_balancing;
};

10
src/Client/HedgedConnections.cpp
View File

@ -29,7 +29,15 @@ HedgedConnections::HedgedConnections(
PoolMode pool_mode,
std::shared_ptr<QualifiedTableName> table_to_check_,
AsyncCallback async_callback)
: hedged_connections_factory(pool_, &context_->getSettingsRef(), timeouts_, table_to_check_)
: hedged_connections_factory(
pool_,
context_->getSettingsRef(),
timeouts_,
context_->getSettingsRef().connections_with_failover_max_tries.value,
context_->getSettingsRef().fallback_to_stale_replicas_for_distributed_queries.value,
context_->getSettingsRef().max_parallel_replicas.value,
context_->getSettingsRef().skip_unavailable_shards.value,
table_to_check_)
, context(std::move(context_))
, settings(context->getSettingsRef())
, throttler(throttler_)

28
src/Client/HedgedConnectionsFactory.cpp
View File

@ -23,19 +23,25 @@ namespace ErrorCodes
HedgedConnectionsFactory::HedgedConnectionsFactory(
const ConnectionPoolWithFailoverPtr & pool_,
const Settings * settings_,
const Settings & settings_,
const ConnectionTimeouts & timeouts_,
UInt64 max_tries_,
bool fallback_to_stale_replicas_,
UInt64 max_parallel_replicas_,
bool skip_unavailable_shards_,
std::shared_ptr<QualifiedTableName> table_to_check_)
: pool(pool_), settings(settings_), timeouts(timeouts_), table_to_check(table_to_check_), log(&Poco::Logger::get("HedgedConnectionsFactory"))
: pool(pool_)
, timeouts(timeouts_)
, table_to_check(table_to_check_)
, log(&Poco::Logger::get("HedgedConnectionsFactory"))
, max_tries(max_tries_)
, fallback_to_stale_replicas(fallback_to_stale_replicas_)
, max_parallel_replicas(max_parallel_replicas_)
, skip_unavailable_shards(skip_unavailable_shards_)
{
shuffled_pools = pool->getShuffledPools(settings);
shuffled_pools = pool->getShuffledPools(settings_);
for (auto shuffled_pool : shuffled_pools)
replicas.emplace_back(std::make_unique<ConnectionEstablisherAsync>(shuffled_pool.pool, &timeouts, settings, log, table_to_check.get()));
max_tries
= (settings ? size_t{settings->connections_with_failover_max_tries} : size_t{DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES});
fallback_to_stale_replicas = settings && settings->fallback_to_stale_replicas_for_distributed_queries;
replicas.emplace_back(std::make_unique<ConnectionEstablisherAsync>(shuffled_pool.pool, &timeouts, settings_, log, table_to_check.get()));
}
HedgedConnectionsFactory::~HedgedConnectionsFactory()
@ -55,7 +61,7 @@ HedgedConnectionsFactory::~HedgedConnectionsFactory()
std::vector<Connection *> HedgedConnectionsFactory::getManyConnections(PoolMode pool_mode, AsyncCallback async_callback)
{
size_t min_entries = (settings && settings->skip_unavailable_shards) ? 0 : 1;
size_t min_entries = skip_unavailable_shards ? 0 : 1;
size_t max_entries = 1;
switch (pool_mode)
@ -73,7 +79,7 @@ std::vector<Connection *> HedgedConnectionsFactory::getManyConnections(PoolMode
}
case PoolMode::GET_MANY:
{
max_entries = settings ? size_t(settings->max_parallel_replicas) : 1;
max_entries = max_parallel_replicas;
break;
}
}

21
src/Client/HedgedConnectionsFactory.h
View File

@ -45,10 +45,15 @@ public:
bool is_ready = false;
};
HedgedConnectionsFactory(const ConnectionPoolWithFailoverPtr & pool_,
const Settings * settings_,
const ConnectionTimeouts & timeouts_,
std::shared_ptr<QualifiedTableName> table_to_check_ = nullptr);
HedgedConnectionsFactory(
const ConnectionPoolWithFailoverPtr & pool_,
const Settings & settings_,
const ConnectionTimeouts & timeouts_,
UInt64 max_tries_,
bool fallback_to_stale_replicas_,
UInt64 max_parallel_replicas_,
bool skip_unavailable_shards_,
std::shared_ptr<QualifiedTableName> table_to_check_ = nullptr);
/// Create and return active connections according to pool_mode.
std::vector<Connection *> getManyConnections(PoolMode pool_mode, AsyncCallback async_callback = {});
@ -109,7 +114,6 @@ private:
bool isTwoLevelAggregationIncompatible(Connection * connection);
const ConnectionPoolWithFailoverPtr pool;
const Settings * settings;
const ConnectionTimeouts timeouts;
std::vector<ShuffledPool> shuffled_pools;
@ -127,13 +131,13 @@ private:
std::shared_ptr<QualifiedTableName> table_to_check;
int last_used_index = -1;
bool fallback_to_stale_replicas;
Epoll epoll;
Poco::Logger * log;
std::string fail_messages;
/// The maximum number of attempts to connect to replicas.
size_t max_tries;
const size_t max_tries;
const bool fallback_to_stale_replicas;
/// Total number of established connections.
size_t entries_count = 0;
/// The number of established connections that are usable.
@ -152,6 +156,9 @@ private:
/// The number of requested in startNewConnection replicas (it's needed for
/// checking the number of requested replicas that are still in process).
size_t requested_connections_count = 0;
const size_t max_parallel_replicas = 0;
const bool skip_unavailable_shards = 0;
};
}

11
src/Client/ReplxxLineReader.cpp
View File

@ -447,6 +447,17 @@ ReplxxLineReader::ReplxxLineReader(
uint32_t reverse_search = Replxx::KEY::control('R');
return rx.invoke(Replxx::ACTION::HISTORY_INCREMENTAL_SEARCH, reverse_search);
});
/// Change cursor style for overwrite mode to blinking (see console_codes(5))
rx.bind_key(Replxx::KEY::INSERT, [this](char32_t)
{
overwrite_mode = !overwrite_mode;
if (overwrite_mode)
rx.print("%s", "\033[5 q");
else
rx.print("%s", "\033[0 q");
return rx.invoke(Replxx::ACTION::TOGGLE_OVERWRITE_MODE, 0);
});
}
ReplxxLineReader::~ReplxxLineReader()

1
src/Client/ReplxxLineReader.h
View File

@ -41,6 +41,7 @@ private:
bool bracketed_paste_enabled = false;
std::string editor;
bool overwrite_mode = false;
};
}

3
src/Columns/ColumnString.h
View File

@ -11,7 +11,6 @@
#include <Common/memcmpSmall.h>
#include <Common/assert_cast.h>
#include <Core/Field.h>
#include <Common/Arena.h>
class Collator;
@ -20,6 +19,8 @@ class Collator;
namespace DB
{
class Arena;
/** Column for String values.
*/
class ColumnString final : public COWHelper<IColumn, ColumnString>

137
src/Columns/IColumnDummy.cpp Normal file
View File

@ -0,0 +1,137 @@
#include <Common/Arena.h>
#include <Columns/IColumnDummy.h>
#include <Columns/ColumnsCommon.h>
namespace DB
{
namespace ErrorCodes
{
extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
extern const int NOT_IMPLEMENTED;
}
Field IColumnDummy::operator[](size_t) const
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get value from {}", getName());
}
void IColumnDummy::get(size_t, Field &) const
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get value from {}", getName());
}
void IColumnDummy::insert(const Field &)
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot insert element into {}", getName());
}
bool IColumnDummy::isDefaultAt(size_t) const
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "isDefaultAt is not implemented for {}", getName());
}
StringRef IColumnDummy::serializeValueIntoArena(size_t /*n*/, Arena & arena, char const *& begin, const UInt8 *) const
{
/// Has to put one useless byte into Arena, because serialization into zero number of bytes is ambiguous.
char * res = arena.allocContinue(1, begin);
*res = 0;
return { res, 1 };
}
const char * IColumnDummy::deserializeAndInsertFromArena(const char * pos)
{
++s;
return pos + 1;
}
const char * IColumnDummy::skipSerializedInArena(const char * pos) const
{
return pos;
}
ColumnPtr IColumnDummy::filter(const Filter & filt, ssize_t /*result_size_hint*/) const
{
size_t bytes = countBytesInFilter(filt);
return cloneDummy(bytes);
}
void IColumnDummy::expand(const IColumn::Filter & mask, bool inverted)
{
size_t bytes = countBytesInFilter(mask);
if (inverted)
bytes = mask.size() - bytes;
s = bytes;
}
ColumnPtr IColumnDummy::permute(const Permutation & perm, size_t limit) const
{
if (s != perm.size())
throw Exception(ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH, "Size of permutation doesn't match size of column.");
return cloneDummy(limit ? std::min(s, limit) : s);
}
ColumnPtr IColumnDummy::index(const IColumn & indexes, size_t limit) const
{
if (indexes.size() < limit)
throw Exception(ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH, "Size of indexes is less than required.");
return cloneDummy(limit ? limit : s);
}
void IColumnDummy::getPermutation(IColumn::PermutationSortDirection /*direction*/, IColumn::PermutationSortStability /*stability*/,
size_t /*limit*/, int /*nan_direction_hint*/, Permutation & res) const
{
res.resize(s);
for (size_t i = 0; i < s; ++i)
res[i] = i;
}
ColumnPtr IColumnDummy::replicate(const Offsets & offsets) const
{
if (s != offsets.size())
throw Exception(ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH, "Size of offsets doesn't match size of column.");
return cloneDummy(offsets.back());
}
MutableColumns IColumnDummy::scatter(ColumnIndex num_columns, const Selector & selector) const
{
if (s != selector.size())
throw Exception(ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH, "Size of selector doesn't match size of column.");
std::vector<size_t> counts(num_columns);
for (auto idx : selector)
++counts[idx];
MutableColumns res(num_columns);
for (size_t i = 0; i < num_columns; ++i)
res[i] = cloneResized(counts[i]);
return res;
}
double IColumnDummy::getRatioOfDefaultRows(double) const
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getRatioOfDefaultRows is not supported for {}", getName());
}
UInt64 IColumnDummy::getNumberOfDefaultRows() const
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getNumberOfDefaultRows is not supported for {}", getName());
}
void IColumnDummy::getIndicesOfNonDefaultRows(Offsets &, size_t, size_t) const
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getIndicesOfNonDefaultRows is not supported for {}", getName());
}
void IColumnDummy::gather(ColumnGathererStream &)
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method gather is not supported for {}", getName());
}
}

124
src/Columns/IColumnDummy.h
View File

@ -1,21 +1,12 @@
#pragma once
#include <Common/Arena.h>
#include <Common/PODArray.h>
#include <Columns/IColumn.h>
#include <Columns/ColumnsCommon.h>
#include <Core/Field.h>
namespace DB
{
namespace ErrorCodes
{
extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
extern const int NOT_IMPLEMENTED;
}
class Arena;
/** Base class for columns-constants that contain a value that is not in the `Field`.
* Not a full-fledged column and is used in a special way.
@ -42,10 +33,10 @@ public:
bool hasEqualValues() const override { return true; }
Field operator[](size_t) const override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get value from {}", getName()); }
void get(size_t, Field &) const override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get value from {}", getName()); }
void insert(const Field &) override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot insert element into {}", getName()); }
bool isDefaultAt(size_t) const override { throw Exception(ErrorCodes::NOT_IMPLEMENTED, "isDefaultAt is not implemented for {}", getName()); }
Field operator[](size_t) const override;
void get(size_t, Field &) const override;
void insert(const Field &) override;
bool isDefaultAt(size_t) const override;
StringRef getDataAt(size_t) const override
{
@ -57,24 +48,9 @@ public:
++s;
}
StringRef serializeValueIntoArena(size_t /*n*/, Arena & arena, char const *& begin, const UInt8 *) const override
{
/// Has to put one useless byte into Arena, because serialization into zero number of bytes is ambiguous.
char * res = arena.allocContinue(1, begin);
*res = 0;
return { res, 1 };
}
const char * deserializeAndInsertFromArena(const char * pos) override
{
++s;
return pos + 1;
}
const char * skipSerializedInArena(const char * pos) const override
{
return pos;
}
StringRef serializeValueIntoArena(size_t /*n*/, Arena & arena, char const *& begin, const UInt8 *) const override;
const char * deserializeAndInsertFromArena(const char * pos) override;
const char * skipSerializedInArena(const char * pos) const override;
void updateHashWithValue(size_t /*n*/, SipHash & /*hash*/) const override
{
@ -98,90 +74,30 @@ public:
s += length;
}
ColumnPtr filter(const Filter & filt, ssize_t /*result_size_hint*/) const override
{
size_t bytes = countBytesInFilter(filt);
return cloneDummy(bytes);
}
ColumnPtr filter(const Filter & filt, ssize_t /*result_size_hint*/) const override;
void expand(const IColumn::Filter & mask, bool inverted) override
{
size_t bytes = countBytesInFilter(mask);
if (inverted)
bytes = mask.size() - bytes;
s = bytes;
}
void expand(const IColumn::Filter & mask, bool inverted) override;
ColumnPtr permute(const Permutation & perm, size_t limit) const override
{
if (s != perm.size())
throw Exception(ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH, "Size of permutation doesn't match size of column.");
ColumnPtr permute(const Permutation & perm, size_t limit) const override;
return cloneDummy(limit ? std::min(s, limit) : s);
}
ColumnPtr index(const IColumn & indexes, size_t limit) const override
{
if (indexes.size() < limit)
throw Exception(ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH, "Size of indexes is less than required.");
return cloneDummy(limit ? limit : s);
}
ColumnPtr index(const IColumn & indexes, size_t limit) const override;
void getPermutation(IColumn::PermutationSortDirection /*direction*/, IColumn::PermutationSortStability /*stability*/,
size_t /*limit*/, int /*nan_direction_hint*/, Permutation & res) const override
{
res.resize(s);
for (size_t i = 0; i < s; ++i)
res[i] = i;
}
size_t /*limit*/, int /*nan_direction_hint*/, Permutation & res) const override;
void updatePermutation(IColumn::PermutationSortDirection /*direction*/, IColumn::PermutationSortStability /*stability*/,
size_t, int, Permutation &, EqualRanges&) const override {}
ColumnPtr replicate(const Offsets & offsets) const override
size_t, int, Permutation &, EqualRanges&) const override
{
if (s != offsets.size())
throw Exception(ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH, "Size of offsets doesn't match size of column.");
return cloneDummy(offsets.back());
}
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override
{
if (s != selector.size())
throw Exception(ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH, "Size of selector doesn't match size of column.");
ColumnPtr replicate(const Offsets & offsets) const override;
std::vector<size_t> counts(num_columns);
for (auto idx : selector)
++counts[idx];
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
MutableColumns res(num_columns);
for (size_t i = 0; i < num_columns; ++i)
res[i] = cloneResized(counts[i]);
return res;
}
double getRatioOfDefaultRows(double) const override
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getRatioOfDefaultRows is not supported for {}", getName());
}
UInt64 getNumberOfDefaultRows() const override
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getNumberOfDefaultRows is not supported for {}", getName());
}
void getIndicesOfNonDefaultRows(Offsets &, size_t, size_t) const override
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getIndicesOfNonDefaultRows is not supported for {}", getName());
}
void gather(ColumnGathererStream &) override
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method gather is not supported for {}", getName());
}
double getRatioOfDefaultRows(double) const override;
UInt64 getNumberOfDefaultRows() const override;
void getIndicesOfNonDefaultRows(Offsets &, size_t, size_t) const override;
void gather(ColumnGathererStream &) override;
void getExtremes(Field &, Field &) const override
{

119
src/Common/Arena.h
View File

@ -5,13 +5,14 @@
#include <vector>
#include <boost/noncopyable.hpp>
#include <Core/Defines.h>
#if __has_include(<sanitizer/asan_interface.h>) && defined(ADDRESS_SANITIZER)
# include <sanitizer/asan_interface.h>
#endif
#include <Common/memcpySmall.h>
#include <Common/ProfileEvents.h>
#include <Common/Allocator.h>
#if __has_include(<sanitizer/asan_interface.h>) && defined(ADDRESS_SANITIZER)
# include <sanitizer/asan_interface.h>
#endif
namespace ProfileEvents
{
@ -39,13 +40,36 @@ private:
/// Contiguous MemoryChunk of memory and pointer to free space inside it. Member of single-linked list.
struct alignas(16) MemoryChunk : private Allocator<false> /// empty base optimization
{
char * begin;
char * pos;
char * end; /// does not include padding.
char * begin = nullptr;
char * pos = nullptr;
char * end = nullptr; /// does not include padding.
MemoryChunk * prev;
std::unique_ptr<MemoryChunk> prev;
MemoryChunk(size_t size_, MemoryChunk * prev_)
MemoryChunk()
{
}
void swap(MemoryChunk & other)
{
std::swap(begin, other.begin);
std::swap(pos, other.pos);
std::swap(end, other.end);
prev.swap(other.prev);
}
MemoryChunk(MemoryChunk && other)
{
*this = std::move(other);
}
MemoryChunk & operator=(MemoryChunk && other)
{
swap(other);
return *this;
}
MemoryChunk(size_t size_)
{
ProfileEvents::increment(ProfileEvents::ArenaAllocChunks);
ProfileEvents::increment(ProfileEvents::ArenaAllocBytes, size_);
@ -53,13 +77,15 @@ private:
begin = reinterpret_cast<char *>(Allocator<false>::alloc(size_));
pos = begin;
end = begin + size_ - pad_right;
prev = prev_;
ASAN_POISON_MEMORY_REGION(begin, size_);
}
~MemoryChunk()
{
if (empty())
return;
/// We must unpoison the memory before returning to the allocator,
/// because the allocator might not have asan integration, and the
/// memory would stay poisoned forever. If the allocator supports
@ -67,21 +93,21 @@ private:
ASAN_UNPOISON_MEMORY_REGION(begin, size());
Allocator<false>::free(begin, size());
delete prev;
}
bool empty() const { return begin == end;}
size_t size() const { return end + pad_right - begin; }
size_t remaining() const { return end - pos; }
};
size_t initial_size;
size_t growth_factor;
size_t linear_growth_threshold;
/// Last contiguous MemoryChunk of memory.
MemoryChunk * head;
size_t allocated_bytes;
size_t used_bytes;
MemoryChunk head;
size_t allocated_bytes = 0;
size_t used_bytes = 0;
size_t page_size;
static size_t roundUpToPageSize(size_t s, size_t page_size)
@ -95,9 +121,13 @@ private:
{
size_t size_after_grow = 0;
if (head->size() < linear_growth_threshold)
if (head.empty())
{
size_after_grow = std::max(min_next_size, head->size() * growth_factor);
size_after_grow = std::max(min_next_size, initial_size);
}
else if (head.size() < linear_growth_threshold)
{
size_after_grow = std::max(min_next_size, head.size() * growth_factor);
}
else
{
@ -119,8 +149,18 @@ private:
/// Add next contiguous MemoryChunk of memory with size not less than specified.
void NO_INLINE addMemoryChunk(size_t min_size)
{
head = new MemoryChunk(nextSize(min_size + pad_right), head);
allocated_bytes += head->size();
size_t next_size = nextSize(min_size + pad_right);
if (head.empty())
{
head = MemoryChunk(next_size);
}
else
{
auto chunk = std::make_unique<MemoryChunk>(next_size);
head.swap(*chunk);
head.prev = std::move(chunk);
}
allocated_bytes += head.size();
}
friend class ArenaAllocator;
@ -128,29 +168,23 @@ private:
public:
explicit Arena(size_t initial_size_ = 4096, size_t growth_factor_ = 2, size_t linear_growth_threshold_ = 128 * 1024 * 1024)
: growth_factor(growth_factor_)
: initial_size(initial_size_)
, growth_factor(growth_factor_)
, linear_growth_threshold(linear_growth_threshold_)
, head(new MemoryChunk(initial_size_, nullptr))
, allocated_bytes(head->size())
, used_bytes(0)
, page_size(static_cast<size_t>(::getPageSize()))
{
}
~Arena()
{
delete head;
}
/// Get piece of memory, without alignment.
/// Note: we expect it will return a non-nullptr even if the size is zero.
char * alloc(size_t size)
{
used_bytes += size;
if (unlikely(static_cast<std::ptrdiff_t>(size) > head->end - head->pos))
if (unlikely(head.empty() || static_cast<std::ptrdiff_t>(size) > head.end - head.pos))
addMemoryChunk(size);
char * res = head->pos;
head->pos += size;
char * res = head.pos;
head.pos += size;
ASAN_UNPOISON_MEMORY_REGION(res, size + pad_right);
return res;
}
@ -161,14 +195,14 @@ public:
used_bytes += size;
do
{
void * head_pos = head->pos;
size_t space = head->end - head->pos;
void * head_pos = head.pos;
size_t space = head.end - head.pos;
auto * res = static_cast<char *>(std::align(alignment, size, head_pos, space));
if (res)
{
head->pos = static_cast<char *>(head_pos);
head->pos += size;
head.pos = static_cast<char *>(head_pos);
head.pos += size;
ASAN_UNPOISON_MEMORY_REGION(res, size + pad_right);
return res;
}
@ -191,9 +225,9 @@ public:
void * rollback(size_t size)
{
used_bytes -= size;
head->pos -= size;
ASAN_POISON_MEMORY_REGION(head->pos, size + pad_right);
return head->pos;
head.pos -= size;
ASAN_POISON_MEMORY_REGION(head.pos, size + pad_right);
return head.pos;
}
/** Begin or expand a contiguous range of memory.
@ -234,10 +268,10 @@ public:
// This method only works for extending the last allocation. For lack of
// original size, check a weaker condition: that 'begin' is at least in
// the current MemoryChunk.
assert(range_start >= head->begin);
assert(range_start < head->end);
assert(range_start >= head.begin);
assert(range_start < head.end);
if (head->pos + additional_bytes <= head->end)
if (head.pos + additional_bytes <= head.end)
{
// The new size fits into the last MemoryChunk, so just alloc the
// additional size. We can alloc without alignment here, because it
@ -254,7 +288,7 @@ public:
// solved not by complicating this method, but by rethinking the
// approach to memory management for aggregate function states, so that
// we can provide a proper realloc().
const size_t existing_bytes = head->pos - range_start;
const size_t existing_bytes = head.pos - range_start;
const size_t new_bytes = existing_bytes + additional_bytes;
const char * old_range = range_start;
@ -317,12 +351,11 @@ public:
/// yourself having to use this method, probably you're doing something wrong.
size_t remainingSpaceInCurrentMemoryChunk() const
{
return head->remaining();
return head.remaining();
}
};
using ArenaPtr = std::shared_ptr<Arena>;
using Arenas = std::vector<ArenaPtr>;
}

4
src/Common/ArenaAllocator.h
View File

@ -20,7 +20,7 @@ public:
char const * data = reinterpret_cast<char *>(buf);
// Invariant should be maintained: new_size > old_size
if (data + old_size == arena->head->pos)
if (data + old_size == arena->head.pos)
{
// Consecutive optimization
arena->allocContinue(new_size - old_size, data);
@ -59,7 +59,7 @@ public:
{
char const * data = reinterpret_cast<char *>(buf);
if (data + old_size == arena->head->pos)
if (data + old_size == arena->head.pos)
{
arena->allocContinue(new_size - old_size, data, alignment);
return reinterpret_cast<void *>(const_cast<char *>(data));

4
src/Common/GetPriorityForLoadBalancing.h
View File

@ -8,8 +8,8 @@ namespace DB
class GetPriorityForLoadBalancing
{
public:
GetPriorityForLoadBalancing(LoadBalancing load_balancing_) : load_balancing(load_balancing_) {}
GetPriorityForLoadBalancing(){}
explicit GetPriorityForLoadBalancing(LoadBalancing load_balancing_) : load_balancing(load_balancing_) {}
GetPriorityForLoadBalancing() = default;
bool operator == (const GetPriorityForLoadBalancing & other) const
{

2
src/Common/ProfileEvents.cpp
View File

@ -395,6 +395,8 @@ The server successfully detected this situation and will download merged part fr
M(WriteBufferFromS3WaitInflightLimitMicroseconds, "Time spent on waiting while some of the current requests are done when its number reached the limit defined by s3_max_inflight_parts_for_one_file.") \
M(QueryMemoryLimitExceeded, "Number of times when memory limit exceeded for query.") \
\
M(CachedReadBufferReadFromCacheHits, "Number of times the read from filesystem cache hit the cache.") \
M(CachedReadBufferReadFromCacheMisses, "Number of times the read from filesystem cache miss the cache.") \
M(CachedReadBufferReadFromSourceMicroseconds, "Time reading from filesystem cache source (from remote filesystem, etc)") \
M(CachedReadBufferReadFromCacheMicroseconds, "Time reading from filesystem cache") \
M(CachedReadBufferReadFromSourceBytes, "Bytes read from filesystem cache source (from remote fs, etc)") \

Some files were not shown because too many files have changed in this diff Show More