mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-22 07:31:57 +00:00
Merge branch 'master' into impl-libfuzzer-3
This commit is contained in:
Yakov Olkhovskiy
commit
ed232f2675
2
.github/ISSUE_TEMPLATE/20_feature-request.md
vendored
2
.github/ISSUE_TEMPLATE/20_feature-request.md
vendored
@ -15,7 +15,7 @@ assignees: ''
|
|||||||
|
|
||||||
**Use case**
|
**Use case**
|
||||||
|
|
||||||
> A clear and concise description of what is the intended usage scenario is.
|
> A clear and concise description of what the intended usage scenario is.
|
||||||
|
|
||||||
**Describe the solution you'd like**
|
**Describe the solution you'd like**
|
||||||
|
|
||||||
|
|||||||
3
.github/workflows/pull_request.yml
vendored
3
.github/workflows/pull_request.yml
vendored
@ -33,9 +33,6 @@ jobs:
|
|||||||
filter: tree:0
|
filter: tree:0
|
||||||
- name: Debug Info
|
- name: Debug Info
|
||||||
uses: ./.github/actions/debug
|
uses: ./.github/actions/debug
|
||||||
- name: Cancel previous Sync PR workflow
|
|
||||||
run: |
|
|
||||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --cancel-previous-run
|
|
||||||
- name: Set pending Sync status
|
- name: Set pending Sync status
|
||||||
run: |
|
run: |
|
||||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --set-pending-status
|
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --set-pending-status
|
||||||
|
|||||||
@ -545,7 +545,7 @@ endif()
|
|||||||
|
|
||||||
if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO"
|
if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO"
|
||||||
AND NOT SANITIZE AND NOT SANITIZE_COVERAGE AND NOT ENABLE_FUZZING
|
AND NOT SANITIZE AND NOT SANITIZE_COVERAGE AND NOT ENABLE_FUZZING
|
||||||
AND OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64))
|
AND OMIT_HEAVY_DEBUG_SYMBOLS AND OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64))
|
||||||
set(CHECK_LARGE_OBJECT_SIZES_DEFAULT ON)
|
set(CHECK_LARGE_OBJECT_SIZES_DEFAULT ON)
|
||||||
else ()
|
else ()
|
||||||
set(CHECK_LARGE_OBJECT_SIZES_DEFAULT OFF)
|
set(CHECK_LARGE_OBJECT_SIZES_DEFAULT OFF)
|
||||||
|
|||||||
@ -11,6 +11,9 @@ if (GLIBC_COMPATIBILITY)
|
|||||||
if (ARCH_AARCH64)
|
if (ARCH_AARCH64)
|
||||||
list (APPEND glibc_compatibility_sources musl/aarch64/syscall.s musl/aarch64/longjmp.s)
|
list (APPEND glibc_compatibility_sources musl/aarch64/syscall.s musl/aarch64/longjmp.s)
|
||||||
set (musl_arch_include_dir musl/aarch64)
|
set (musl_arch_include_dir musl/aarch64)
|
||||||
|
# Disable getauxval in aarch64. ARM glibc minimum requirement for the project is 2.18 and getauxval is present
|
||||||
|
# in 2.16. Having a custom one introduces issues with sanitizers
|
||||||
|
list (REMOVE_ITEM glibc_compatibility_sources musl/getauxval.c)
|
||||||
elseif (ARCH_AMD64)
|
elseif (ARCH_AMD64)
|
||||||
list (APPEND glibc_compatibility_sources musl/x86_64/syscall.s musl/x86_64/longjmp.s)
|
list (APPEND glibc_compatibility_sources musl/x86_64/syscall.s musl/x86_64/longjmp.s)
|
||||||
set (musl_arch_include_dir musl/x86_64)
|
set (musl_arch_include_dir musl/x86_64)
|
||||||
@ -18,7 +21,7 @@ if (GLIBC_COMPATIBILITY)
|
|||||||
message (FATAL_ERROR "glibc_compatibility can only be used on x86_64 or aarch64.")
|
message (FATAL_ERROR "glibc_compatibility can only be used on x86_64 or aarch64.")
|
||||||
endif ()
|
endif ()
|
||||||
|
|
||||||
if (SANITIZE STREQUAL thread)
|
if (SANITIZE STREQUAL thread AND ARCH_AMD64)
|
||||||
# Disable TSAN instrumentation that conflicts with re-exec due to high ASLR entropy using getauxval
|
# Disable TSAN instrumentation that conflicts with re-exec due to high ASLR entropy using getauxval
|
||||||
# See longer comment in __auxv_init_procfs
|
# See longer comment in __auxv_init_procfs
|
||||||
# In the case of tsan we need to make sure getauxval is not instrumented as that would introduce tsan
|
# In the case of tsan we need to make sure getauxval is not instrumented as that would introduce tsan
|
||||||
|
|||||||
@ -11,6 +11,38 @@ option (ARCH_NATIVE "Add -march=native compiler flag. This makes your binaries n
|
|||||||
if (ARCH_NATIVE)
|
if (ARCH_NATIVE)
|
||||||
set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=native")
|
set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=native")
|
||||||
|
|
||||||
|
# Populate the ENABLE_ option flags. This is required for the build of some third-party dependencies, specifically snappy, which
|
||||||
|
# (somewhat weirdly) expects the relative SNAPPY_HAVE_ preprocessor variables to be populated, in addition to the microarchitecture
|
||||||
|
# feature flags being enabled in the compiler. This fixes the ARCH_NATIVE flag by automatically populating the ENABLE_ option flags
|
||||||
|
# according to the current CPU's capabilities, detected using clang.
|
||||||
|
if (ARCH_AMD64)
|
||||||
|
execute_process(
|
||||||
|
COMMAND sh -c "clang -E - -march=native -###"
|
||||||
|
INPUT_FILE /dev/null
|
||||||
|
OUTPUT_QUIET
|
||||||
|
ERROR_VARIABLE TEST_FEATURE_RESULT)
|
||||||
|
|
||||||
|
macro(TEST_AMD64_FEATURE TEST_FEATURE_RESULT feat flag)
|
||||||
|
if (${TEST_FEATURE_RESULT} MATCHES "\"\\+${feat}\"")
|
||||||
|
set(${flag} ON)
|
||||||
|
else ()
|
||||||
|
set(${flag} OFF)
|
||||||
|
endif ()
|
||||||
|
endmacro()
|
||||||
|
|
||||||
|
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} ssse3 ENABLE_SSSE3)
|
||||||
|
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} sse4.1 ENABLE_SSE41)
|
||||||
|
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} sse4.2 ENABLE_SSE42)
|
||||||
|
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} vpclmulqdq ENABLE_PCLMULQDQ)
|
||||||
|
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} popcnt ENABLE_POPCNT)
|
||||||
|
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} avx ENABLE_AVX)
|
||||||
|
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} avx2 ENABLE_AVX2)
|
||||||
|
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} avx512f ENABLE_AVX512)
|
||||||
|
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} avx512vbmi ENABLE_AVX512_VBMI)
|
||||||
|
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} bmi ENABLE_BMI)
|
||||||
|
TEST_AMD64_FEATURE (${TEST_FEATURE_RESULT} bmi2 ENABLE_BMI2)
|
||||||
|
endif ()
|
||||||
|
|
||||||
elseif (ARCH_AARCH64)
|
elseif (ARCH_AARCH64)
|
||||||
# ARM publishes almost every year a new revision of it's ISA [1]. Each version comes with new mandatory and optional features from
|
# ARM publishes almost every year a new revision of it's ISA [1]. Each version comes with new mandatory and optional features from
|
||||||
# which CPU vendors can pick and choose. This creates a lot of variability ... We provide two build "profiles", one for maximum
|
# which CPU vendors can pick and choose. This creates a lot of variability ... We provide two build "profiles", one for maximum
|
||||||
|
|||||||
@ -1,4 +1,21 @@
|
|||||||
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_DEBUG=0") # More checks in debug build.
|
if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG")
|
||||||
|
# Enable libcxx debug mode: https://releases.llvm.org/15.0.0/projects/libcxx/docs/DesignDocs/DebugMode.html
|
||||||
|
# The docs say the debug mode violates complexity guarantees, so do this only for Debug builds.
|
||||||
|
# set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_ENABLE_DEBUG_MODE=1")
|
||||||
|
# ^^ Crashes the database upon startup, needs investigation.
|
||||||
|
# Besides that, the implementation looks like a poor man's MSAN specific to libcxx. Since CI tests MSAN
|
||||||
|
# anyways, we can keep the debug mode disabled.
|
||||||
|
|
||||||
|
# Libcxx also provides extra assertions:
|
||||||
|
# --> https://releases.llvm.org/15.0.0/projects/libcxx/docs/UsingLibcxx.html#assertions-mode
|
||||||
|
# These look orthogonal to the debug mode but the debug mode enables them implicitly:
|
||||||
|
# --> https://github.com/llvm/llvm-project/blob/release/15.x/libcxx/include/__assert#L29
|
||||||
|
# They are cheap and straightforward, so enable them in debug builds:
|
||||||
|
set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_LIBCPP_ENABLE_ASSERTIONS=1")
|
||||||
|
|
||||||
|
# TODO Once we upgrade to LLVM 18+, reconsider all of the above as they introduced "hardening modes":
|
||||||
|
# https://libcxx.llvm.org/Hardening.html
|
||||||
|
endif ()
|
||||||
|
|
||||||
add_subdirectory(contrib/libcxxabi-cmake)
|
add_subdirectory(contrib/libcxxabi-cmake)
|
||||||
add_subdirectory(contrib/libcxx-cmake)
|
add_subdirectory(contrib/libcxx-cmake)
|
||||||
|
|||||||
@ -1,6 +1,9 @@
|
|||||||
set(ABSL_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/abseil-cpp")
|
set(ABSL_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/abseil-cpp")
|
||||||
set(ABSL_COMMON_INCLUDE_DIRS "${ABSL_ROOT_DIR}")
|
set(ABSL_COMMON_INCLUDE_DIRS "${ABSL_ROOT_DIR}")
|
||||||
|
|
||||||
|
# To avoid errors "'X' does not refer to a value" while using `offsetof` function.
|
||||||
|
set(CMAKE_CXX_STANDARD 17)
|
||||||
|
|
||||||
# This is a minimized version of the function definition in CMake/AbseilHelpers.cmake
|
# This is a minimized version of the function definition in CMake/AbseilHelpers.cmake
|
||||||
|
|
||||||
#
|
#
|
||||||
|
|||||||
@ -5,6 +5,9 @@ if(NOT ENABLE_PROTOBUF)
|
|||||||
return()
|
return()
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
|
# To avoid errors "'X' does not refer to a value" while using `offsetof` function.
|
||||||
|
set(CMAKE_CXX_STANDARD 17)
|
||||||
|
|
||||||
set(Protobuf_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/google-protobuf/src")
|
set(Protobuf_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/google-protobuf/src")
|
||||||
if(OS_FREEBSD AND SANITIZE STREQUAL "address")
|
if(OS_FREEBSD AND SANITIZE STREQUAL "address")
|
||||||
# ../contrib/protobuf/src/google/protobuf/arena_impl.h:45:10: fatal error: 'sanitizer/asan_interface.h' file not found
|
# ../contrib/protobuf/src/google/protobuf/arena_impl.h:45:10: fatal error: 'sanitizer/asan_interface.h' file not found
|
||||||
|
|||||||
@ -6,6 +6,8 @@ if(NOT ENABLE_GRPC)
|
|||||||
return()
|
return()
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
|
set(CMAKE_CXX_STANDARD 17)
|
||||||
|
|
||||||
set(_gRPC_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/grpc")
|
set(_gRPC_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/grpc")
|
||||||
set(_gRPC_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/grpc")
|
set(_gRPC_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/grpc")
|
||||||
|
|
||||||
|
|||||||
@ -22,7 +22,7 @@
|
|||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
# We want to use C++23, but GRPC is not ready
|
# We want to use C++23, but GRPC is not ready
|
||||||
set (CMAKE_CXX_STANDARD 20)
|
set (CMAKE_CXX_STANDARD 17)
|
||||||
|
|
||||||
set(_gRPC_ZLIB_INCLUDE_DIR "")
|
set(_gRPC_ZLIB_INCLUDE_DIR "")
|
||||||
set(_gRPC_ZLIB_LIBRARIES ch_contrib::zlib)
|
set(_gRPC_ZLIB_LIBRARIES ch_contrib::zlib)
|
||||||
|
|||||||
2
contrib/libhdfs3
vendored
2
contrib/libhdfs3
vendored
@ -1 +1 @@
|
|||||||
Subproject commit 0d04201c45359f0d0701fb1e8297d25eff7cfecf
|
Subproject commit de6f1e0750aa3670a603cbfeddf5df3de1097687
|
||||||
5
docker/test/integration/runner/misc/openldap/initialized.sh
Executable file
5
docker/test/integration/runner/misc/openldap/initialized.sh
Executable file
@ -0,0 +1,5 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# workaround for https://github.com/bitnami/containers/issues/73310
|
||||||
|
touch /tmp/.openldap-initialized
|
||||||
@ -1,4 +1,4 @@
|
|||||||
aiohttp==3.9.5
|
aiohttp==3.10.2
|
||||||
aiosignal==1.3.1
|
aiosignal==1.3.1
|
||||||
astroid==3.1.0
|
astroid==3.1.0
|
||||||
async-timeout==4.0.3
|
async-timeout==4.0.3
|
||||||
@ -6,12 +6,12 @@ attrs==23.2.0
|
|||||||
black==24.4.2
|
black==24.4.2
|
||||||
boto3==1.34.131
|
boto3==1.34.131
|
||||||
botocore==1.34.131
|
botocore==1.34.131
|
||||||
certifi==2024.6.2
|
certifi==2024.07.04
|
||||||
cffi==1.16.0
|
cffi==1.16.0
|
||||||
charset-normalizer==3.3.2
|
charset-normalizer==3.3.2
|
||||||
click==8.1.7
|
click==8.1.7
|
||||||
codespell==2.2.1
|
codespell==2.2.1
|
||||||
cryptography==42.0.8
|
cryptography==43.0.1
|
||||||
Deprecated==1.2.14
|
Deprecated==1.2.14
|
||||||
dill==0.3.8
|
dill==0.3.8
|
||||||
flake8==4.0.1
|
flake8==4.0.1
|
||||||
@ -19,7 +19,6 @@ frozenlist==1.4.1
|
|||||||
idna==3.7
|
idna==3.7
|
||||||
isort==5.13.2
|
isort==5.13.2
|
||||||
jmespath==1.0.1
|
jmespath==1.0.1
|
||||||
jwt==1.3.1
|
|
||||||
mccabe==0.6.1
|
mccabe==0.6.1
|
||||||
multidict==6.0.5
|
multidict==6.0.5
|
||||||
mypy==1.8.0
|
mypy==1.8.0
|
||||||
@ -27,13 +26,12 @@ mypy-extensions==1.0.0
|
|||||||
packaging==24.1
|
packaging==24.1
|
||||||
pathspec==0.9.0
|
pathspec==0.9.0
|
||||||
pip==24.1.1
|
pip==24.1.1
|
||||||
pipdeptree==2.23.0
|
|
||||||
platformdirs==4.2.2
|
platformdirs==4.2.2
|
||||||
pycodestyle==2.8.0
|
pycodestyle==2.8.0
|
||||||
pycparser==2.22
|
pycparser==2.22
|
||||||
pyflakes==2.4.0
|
pyflakes==2.4.0
|
||||||
PyGithub==2.3.0
|
PyGithub==2.3.0
|
||||||
PyJWT==2.8.0
|
PyJWT==2.9.0
|
||||||
pylint==3.1.0
|
pylint==3.1.0
|
||||||
PyNaCl==1.5.0
|
PyNaCl==1.5.0
|
||||||
python-dateutil==2.9.0.post0
|
python-dateutil==2.9.0.post0
|
||||||
@ -42,7 +40,7 @@ PyYAML==6.0.1
|
|||||||
rapidfuzz==3.9.3
|
rapidfuzz==3.9.3
|
||||||
requests==2.32.3
|
requests==2.32.3
|
||||||
s3transfer==0.10.1
|
s3transfer==0.10.1
|
||||||
setuptools==59.6.0
|
setuptools==70.0.0
|
||||||
six==1.16.0
|
six==1.16.0
|
||||||
thefuzz==0.22.1
|
thefuzz==0.22.1
|
||||||
tomli==2.0.1
|
tomli==2.0.1
|
||||||
@ -52,7 +50,7 @@ types-requests==2.32.0.20240622
|
|||||||
typing_extensions==4.12.2
|
typing_extensions==4.12.2
|
||||||
unidiff==0.7.5
|
unidiff==0.7.5
|
||||||
urllib3==2.2.2
|
urllib3==2.2.2
|
||||||
wheel==0.37.1
|
wheel==0.38.1
|
||||||
wrapt==1.16.0
|
wrapt==1.16.0
|
||||||
yamllint==1.26.3
|
yamllint==1.26.3
|
||||||
yarl==1.9.4
|
yarl==1.9.4
|
||||||
|
|||||||
@ -196,7 +196,6 @@ When writing docs, you can use prepared templates. Copy the code of a template a
|
|||||||
Templates:
|
Templates:
|
||||||
|
|
||||||
- [Function](_description_templates/template-function.md)
|
- [Function](_description_templates/template-function.md)
|
||||||
- [Setting](_description_templates/template-setting.md)
|
|
||||||
- [Server Setting](_description_templates/template-server-setting.md)
|
- [Server Setting](_description_templates/template-server-setting.md)
|
||||||
- [Database or Table engine](_description_templates/template-engine.md)
|
- [Database or Table engine](_description_templates/template-engine.md)
|
||||||
- [System table](_description_templates/template-system-table.md)
|
- [System table](_description_templates/template-system-table.md)
|
||||||
|
|||||||
@ -1,27 +0,0 @@
|
|||||||
## setting_name {#setting_name}
|
|
||||||
|
|
||||||
Description.
|
|
||||||
|
|
||||||
For the switch setting, use the typical phrase: “Enables or disables something ...”.
|
|
||||||
|
|
||||||
Possible values:
|
|
||||||
|
|
||||||
*For switcher setting:*
|
|
||||||
|
|
||||||
- 0 — Disabled.
|
|
||||||
- 1 — Enabled.
|
|
||||||
|
|
||||||
*For another setting (typical phrases):*
|
|
||||||
|
|
||||||
- Positive integer.
|
|
||||||
- 0 — Disabled or unlimited or something else.
|
|
||||||
|
|
||||||
Default value: `value`.
|
|
||||||
|
|
||||||
**Additional Info** (Optional)
|
|
||||||
|
|
||||||
The name of an additional section can be any, for example, **Usage**.
|
|
||||||
|
|
||||||
**See Also** (Optional)
|
|
||||||
|
|
||||||
- [link](#)
|
|
||||||
@ -1,11 +0,0 @@
|
|||||||
sudo apt-get install -y apt-transport-https ca-certificates dirmngr
|
|
||||||
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754
|
|
||||||
|
|
||||||
echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee \
|
|
||||||
/etc/apt/sources.list.d/clickhouse.list
|
|
||||||
sudo apt-get update
|
|
||||||
|
|
||||||
sudo apt-get install -y clickhouse-server clickhouse-client
|
|
||||||
|
|
||||||
sudo service clickhouse-server start
|
|
||||||
clickhouse-client # or "clickhouse-client --password" if you've set up a password.
|
|
||||||
@ -1,6 +0,0 @@
|
|||||||
sudo yum install -y yum-utils
|
|
||||||
sudo yum-config-manager --add-repo https://packages.clickhouse.com/rpm/clickhouse.repo
|
|
||||||
sudo yum install -y clickhouse-server clickhouse-client
|
|
||||||
|
|
||||||
sudo /etc/init.d/clickhouse-server start
|
|
||||||
clickhouse-client # or "clickhouse-client --password" if you set up a password.
|
|
||||||
@ -1,32 +0,0 @@
|
|||||||
LATEST_VERSION=$(curl -s https://packages.clickhouse.com/tgz/stable/ | \
|
|
||||||
grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | sort -V -r | head -n 1)
|
|
||||||
export LATEST_VERSION
|
|
||||||
|
|
||||||
case $(uname -m) in
|
|
||||||
x86_64) ARCH=amd64 ;;
|
|
||||||
aarch64) ARCH=arm64 ;;
|
|
||||||
*) echo "Unknown architecture $(uname -m)"; exit 1 ;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
for PKG in clickhouse-common-static clickhouse-common-static-dbg clickhouse-server clickhouse-client
|
|
||||||
do
|
|
||||||
curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION-${ARCH}.tgz" \
|
|
||||||
|| curl -fO "https://packages.clickhouse.com/tgz/stable/$PKG-$LATEST_VERSION.tgz"
|
|
||||||
done
|
|
||||||
|
|
||||||
tar -xzvf "clickhouse-common-static-$LATEST_VERSION-${ARCH}.tgz" \
|
|
||||||
|| tar -xzvf "clickhouse-common-static-$LATEST_VERSION.tgz"
|
|
||||||
sudo "clickhouse-common-static-$LATEST_VERSION/install/doinst.sh"
|
|
||||||
|
|
||||||
tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION-${ARCH}.tgz" \
|
|
||||||
|| tar -xzvf "clickhouse-common-static-dbg-$LATEST_VERSION.tgz"
|
|
||||||
sudo "clickhouse-common-static-dbg-$LATEST_VERSION/install/doinst.sh"
|
|
||||||
|
|
||||||
tar -xzvf "clickhouse-server-$LATEST_VERSION-${ARCH}.tgz" \
|
|
||||||
|| tar -xzvf "clickhouse-server-$LATEST_VERSION.tgz"
|
|
||||||
sudo "clickhouse-server-$LATEST_VERSION/install/doinst.sh" configure
|
|
||||||
sudo /etc/init.d/clickhouse-server start
|
|
||||||
|
|
||||||
tar -xzvf "clickhouse-client-$LATEST_VERSION-${ARCH}.tgz" \
|
|
||||||
|| tar -xzvf "clickhouse-client-$LATEST_VERSION.tgz"
|
|
||||||
sudo "clickhouse-client-$LATEST_VERSION/install/doinst.sh"
|
|
||||||
@ -14,7 +14,12 @@ then
|
|||||||
HAS_SSE42=$(grep sse4_2 /proc/cpuinfo)
|
HAS_SSE42=$(grep sse4_2 /proc/cpuinfo)
|
||||||
if [ "${HAS_SSE42}" ]
|
if [ "${HAS_SSE42}" ]
|
||||||
then
|
then
|
||||||
|
if ldd --version 2>&1 | grep -q musl
|
||||||
|
then
|
||||||
|
DIR="amd64musl"
|
||||||
|
else
|
||||||
DIR="amd64"
|
DIR="amd64"
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
DIR="amd64compat"
|
DIR="amd64compat"
|
||||||
fi
|
fi
|
||||||
|
|||||||
@ -82,7 +82,7 @@ cd ./utils/check-style
|
|||||||
# Check duplicate includes
|
# Check duplicate includes
|
||||||
./check-duplicate-includes.sh
|
./check-duplicate-includes.sh
|
||||||
|
|
||||||
# Check c++ formatiing
|
# Check c++ formatting
|
||||||
./check-style
|
./check-style
|
||||||
|
|
||||||
# Check python formatting with black
|
# Check python formatting with black
|
||||||
|
|||||||
@ -7,315 +7,5 @@ sidebar_position: 70
|
|||||||
# [experimental] MaterializedMySQL
|
# [experimental] MaterializedMySQL
|
||||||
|
|
||||||
:::note
|
:::note
|
||||||
This database engine is experimental. To use it, set `allow_experimental_database_materialized_mysql` to 1 in your configuration files or by using the `SET` command:
|
This database engine is obsolete and cannot be used.
|
||||||
```sql
|
|
||||||
SET allow_experimental_database_materialized_mysql=1
|
|
||||||
```
|
|
||||||
:::
|
:::
|
||||||
|
|
||||||
Creates a ClickHouse database with all the tables existing in MySQL, and all the data in those tables. The ClickHouse server works as MySQL replica. It reads `binlog` and performs DDL and DML queries.
|
|
||||||
|
|
||||||
## Creating a Database {#creating-a-database}
|
|
||||||
|
|
||||||
``` sql
|
|
||||||
CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster]
|
|
||||||
ENGINE = MaterializedMySQL('host:port', ['database' | database], 'user', 'password') [SETTINGS ...]
|
|
||||||
[TABLE OVERRIDE table1 (...), TABLE OVERRIDE table2 (...)]
|
|
||||||
```
|
|
||||||
|
|
||||||
**Engine Parameters**
|
|
||||||
|
|
||||||
- `host:port` — MySQL server endpoint.
|
|
||||||
- `database` — MySQL database name.
|
|
||||||
- `user` — MySQL user.
|
|
||||||
- `password` — User password.
|
|
||||||
|
|
||||||
## Engine Settings
|
|
||||||
|
|
||||||
### max_rows_in_buffer
|
|
||||||
|
|
||||||
`max_rows_in_buffer` — Maximum number of rows that data is allowed to cache in memory (for single table and the cache data unable to query). When this number is exceeded, the data will be materialized. Default: `65 505`.
|
|
||||||
|
|
||||||
### max_bytes_in_buffer
|
|
||||||
|
|
||||||
`max_bytes_in_buffer` — Maximum number of bytes that data is allowed to cache in memory (for single table and the cache data unable to query). When this number is exceeded, the data will be materialized. Default: `1 048 576`.
|
|
||||||
|
|
||||||
### max_flush_data_time
|
|
||||||
|
|
||||||
`max_flush_data_time` — Maximum number of milliseconds that data is allowed to cache in memory (for database and the cache data unable to query). When this time is exceeded, the data will be materialized. Default: `1000`.
|
|
||||||
|
|
||||||
### max_wait_time_when_mysql_unavailable
|
|
||||||
|
|
||||||
`max_wait_time_when_mysql_unavailable` — Retry interval when MySQL is not available (milliseconds). Negative value disables retry. Default: `1000`.
|
|
||||||
|
|
||||||
### allows_query_when_mysql_lost
|
|
||||||
`allows_query_when_mysql_lost` — Allows to query a materialized table when MySQL is lost. Default: `0` (`false`).
|
|
||||||
|
|
||||||
### allow_startup_database_without_connection_to_mysql
|
|
||||||
`allow_startup_database_without_connection_to_mysql` — Allow to create and attach database without available connection to MySQL. Default: `0` (`false`).
|
|
||||||
|
|
||||||
### materialized_mysql_tables_list
|
|
||||||
|
|
||||||
`materialized_mysql_tables_list` — a comma-separated list of mysql database tables, which will be replicated by MaterializedMySQL database engine. Default value: empty list — means whole tables will be replicated.
|
|
||||||
|
|
||||||
```sql
|
|
||||||
CREATE DATABASE mysql ENGINE = MaterializedMySQL('localhost:3306', 'db', 'user', '***')
|
|
||||||
SETTINGS
|
|
||||||
allows_query_when_mysql_lost=true,
|
|
||||||
max_wait_time_when_mysql_unavailable=10000;
|
|
||||||
```
|
|
||||||
|
|
||||||
## Settings on MySQL-server Side
|
|
||||||
|
|
||||||
For the correct work of `MaterializedMySQL`, there are few mandatory `MySQL`-side configuration settings that must be set:
|
|
||||||
|
|
||||||
### default_authentication_plugin
|
|
||||||
|
|
||||||
`default_authentication_plugin = mysql_native_password` since `MaterializedMySQL` can only authorize with this method.
|
|
||||||
|
|
||||||
### gtid_mode
|
|
||||||
|
|
||||||
`gtid_mode = on` since GTID based logging is a mandatory for providing correct `MaterializedMySQL` replication.
|
|
||||||
|
|
||||||
:::note
|
|
||||||
While turning on `gtid_mode` you should also specify `enforce_gtid_consistency = on`.
|
|
||||||
:::
|
|
||||||
|
|
||||||
## Virtual Columns {#virtual-columns}
|
|
||||||
|
|
||||||
When working with the `MaterializedMySQL` database engine, [ReplacingMergeTree](/docs/en/engines/table-engines/mergetree-family/replacingmergetree.md) tables are used with virtual `_sign` and `_version` columns.
|
|
||||||
|
|
||||||
### \_version
|
|
||||||
|
|
||||||
`_version` — Transaction counter. Type [UInt64](/docs/en/sql-reference/data-types/int-uint.md).
|
|
||||||
|
|
||||||
### \_sign
|
|
||||||
|
|
||||||
`_sign` — Deletion mark. Type [Int8](/docs/en/sql-reference/data-types/int-uint.md). Possible values:
|
|
||||||
- `1` — Row is not deleted,
|
|
||||||
- `-1` — Row is deleted.
|
|
||||||
|
|
||||||
## Data Types Support {#data_types-support}
|
|
||||||
|
|
||||||
| MySQL | ClickHouse |
|
|
||||||
|-------------------------|--------------------------------------------------------------|
|
|
||||||
| TINY | [Int8](/docs/en/sql-reference/data-types/int-uint.md) |
|
|
||||||
| SHORT | [Int16](/docs/en/sql-reference/data-types/int-uint.md) |
|
|
||||||
| INT24 | [Int32](/docs/en/sql-reference/data-types/int-uint.md) |
|
|
||||||
| LONG | [UInt32](/docs/en/sql-reference/data-types/int-uint.md) |
|
|
||||||
| LONGLONG | [UInt64](/docs/en/sql-reference/data-types/int-uint.md) |
|
|
||||||
| FLOAT | [Float32](/docs/en/sql-reference/data-types/float.md) |
|
|
||||||
| DOUBLE | [Float64](/docs/en/sql-reference/data-types/float.md) |
|
|
||||||
| DECIMAL, NEWDECIMAL | [Decimal](/docs/en/sql-reference/data-types/decimal.md) |
|
|
||||||
| DATE, NEWDATE | [Date](/docs/en/sql-reference/data-types/date.md) |
|
|
||||||
| DATETIME, TIMESTAMP | [DateTime](/docs/en/sql-reference/data-types/datetime.md) |
|
|
||||||
| DATETIME2, TIMESTAMP2 | [DateTime64](/docs/en/sql-reference/data-types/datetime64.md) |
|
|
||||||
| YEAR | [UInt16](/docs/en/sql-reference/data-types/int-uint.md) |
|
|
||||||
| TIME | [Int64](/docs/en/sql-reference/data-types/int-uint.md) |
|
|
||||||
| ENUM | [Enum](/docs/en/sql-reference/data-types/enum.md) |
|
|
||||||
| STRING | [String](/docs/en/sql-reference/data-types/string.md) |
|
|
||||||
| VARCHAR, VAR_STRING | [String](/docs/en/sql-reference/data-types/string.md) |
|
|
||||||
| BLOB | [String](/docs/en/sql-reference/data-types/string.md) |
|
|
||||||
| GEOMETRY | [String](/docs/en/sql-reference/data-types/string.md) |
|
|
||||||
| BINARY | [FixedString](/docs/en/sql-reference/data-types/fixedstring.md) |
|
|
||||||
| BIT | [UInt64](/docs/en/sql-reference/data-types/int-uint.md) |
|
|
||||||
| SET | [UInt64](/docs/en/sql-reference/data-types/int-uint.md) |
|
|
||||||
|
|
||||||
[Nullable](/docs/en/sql-reference/data-types/nullable.md) is supported.
|
|
||||||
|
|
||||||
The data of TIME type in MySQL is converted to microseconds in ClickHouse.
|
|
||||||
|
|
||||||
Other types are not supported. If MySQL table contains a column of such type, ClickHouse throws an exception and stops replication.
|
|
||||||
|
|
||||||
## Specifics and Recommendations {#specifics-and-recommendations}
|
|
||||||
|
|
||||||
### Compatibility Restrictions {#compatibility-restrictions}
|
|
||||||
|
|
||||||
Apart of the data types limitations there are few restrictions comparing to `MySQL` databases, that should be resolved before replication will be possible:
|
|
||||||
|
|
||||||
- Each table in `MySQL` should contain `PRIMARY KEY`.
|
|
||||||
|
|
||||||
- Replication for tables, those are containing rows with `ENUM` field values out of range (specified in `ENUM` signature) will not work.
|
|
||||||
|
|
||||||
### DDL Queries {#ddl-queries}
|
|
||||||
|
|
||||||
MySQL DDL queries are converted into the corresponding ClickHouse DDL queries ([ALTER](/docs/en/sql-reference/statements/alter/index.md), [CREATE](/docs/en/sql-reference/statements/create/index.md), [DROP](/docs/en/sql-reference/statements/drop.md), [RENAME](/docs/en/sql-reference/statements/rename.md)). If ClickHouse cannot parse some DDL query, the query is ignored.
|
|
||||||
|
|
||||||
### Data Replication {#data-replication}
|
|
||||||
|
|
||||||
`MaterializedMySQL` does not support direct `INSERT`, `DELETE` and `UPDATE` queries. However, they are supported in terms of data replication:
|
|
||||||
|
|
||||||
- MySQL `INSERT` query is converted into `INSERT` with `_sign=1`.
|
|
||||||
|
|
||||||
- MySQL `DELETE` query is converted into `INSERT` with `_sign=-1`.
|
|
||||||
|
|
||||||
- MySQL `UPDATE` query is converted into `INSERT` with `_sign=-1` and `INSERT` with `_sign=1` if the primary key has been changed, or
|
|
||||||
`INSERT` with `_sign=1` if not.
|
|
||||||
|
|
||||||
### Selecting from MaterializedMySQL Tables {#select}
|
|
||||||
|
|
||||||
`SELECT` query from `MaterializedMySQL` tables has some specifics:
|
|
||||||
|
|
||||||
- If `_version` is not specified in the `SELECT` query, the
|
|
||||||
[FINAL](/docs/en/sql-reference/statements/select/from.md/#select-from-final) modifier is used, so only rows with
|
|
||||||
`MAX(_version)` are returned for each primary key value.
|
|
||||||
|
|
||||||
- If `_sign` is not specified in the `SELECT` query, `WHERE _sign=1` is used by default. So the deleted rows are not
|
|
||||||
included into the result set.
|
|
||||||
|
|
||||||
- The result includes columns comments in case they exist in MySQL database tables.
|
|
||||||
|
|
||||||
### Index Conversion {#index-conversion}
|
|
||||||
|
|
||||||
MySQL `PRIMARY KEY` and `INDEX` clauses are converted into `ORDER BY` tuples in ClickHouse tables.
|
|
||||||
|
|
||||||
ClickHouse has only one physical order, which is determined by `ORDER BY` clause. To create a new physical order, use
|
|
||||||
[materialized views](/docs/en/sql-reference/statements/create/view.md/#materialized).
|
|
||||||
|
|
||||||
**Notes**
|
|
||||||
|
|
||||||
- Rows with `_sign=-1` are not deleted physically from the tables.
|
|
||||||
- Cascade `UPDATE/DELETE` queries are not supported by the `MaterializedMySQL` engine, as they are not visible in the
|
|
||||||
MySQL binlog.
|
|
||||||
- Replication can be easily broken.
|
|
||||||
- Manual operations on database and tables are forbidden.
|
|
||||||
- `MaterializedMySQL` is affected by the [optimize_on_insert](/docs/en/operations/settings/settings.md/#optimize-on-insert)
|
|
||||||
setting. Data is merged in the corresponding table in the `MaterializedMySQL` database when a table in the MySQL
|
|
||||||
server changes.
|
|
||||||
|
|
||||||
### Table Overrides {#table-overrides}
|
|
||||||
|
|
||||||
Table overrides can be used to customize the ClickHouse DDL queries, allowing you to make schema optimizations for your
|
|
||||||
application. This is especially useful for controlling partitioning, which is important for the overall performance of
|
|
||||||
MaterializedMySQL.
|
|
||||||
|
|
||||||
These are the schema conversion manipulations you can do with table overrides for MaterializedMySQL:
|
|
||||||
|
|
||||||
* Modify column type. Must be compatible with the original type, or replication will fail. For example,
|
|
||||||
you can modify a UInt32 column to UInt64, but you can not modify a String column to Array(String).
|
|
||||||
* Modify [column TTL](/docs/en/engines/table-engines/mergetree-family/mergetree.md/#mergetree-column-ttl).
|
|
||||||
* Modify [column compression codec](/docs/en/sql-reference/statements/create/table.md/#codecs).
|
|
||||||
* Add [ALIAS columns](/docs/en/sql-reference/statements/create/table.md/#alias).
|
|
||||||
* Add [skipping indexes](/docs/en/engines/table-engines/mergetree-family/mergetree.md/#table_engine-mergetree-data_skipping-indexes). Note that you need to enable `use_skip_indexes_if_final` setting to make them work (MaterializedMySQL is using `SELECT ... FINAL` by default)
|
|
||||||
* Add [projections](/docs/en/engines/table-engines/mergetree-family/mergetree.md/#projections). Note that projection optimizations are
|
|
||||||
disabled when using `SELECT ... FINAL` (which MaterializedMySQL does by default), so their utility is limited here.
|
|
||||||
`INDEX ... TYPE hypothesis` as [described in the v21.12 blog post]](https://clickhouse.com/blog/en/2021/clickhouse-v21.12-released/)
|
|
||||||
may be more useful in this case.
|
|
||||||
* Modify [PARTITION BY](/docs/en/engines/table-engines/mergetree-family/custom-partitioning-key/)
|
|
||||||
* Modify [ORDER BY](/docs/en/engines/table-engines/mergetree-family/mergetree.md/#mergetree-query-clauses)
|
|
||||||
* Modify [PRIMARY KEY](/docs/en/engines/table-engines/mergetree-family/mergetree.md/#mergetree-query-clauses)
|
|
||||||
* Add [SAMPLE BY](/docs/en/engines/table-engines/mergetree-family/mergetree.md/#mergetree-query-clauses)
|
|
||||||
* Add [table TTL](/docs/en/engines/table-engines/mergetree-family/mergetree.md/#mergetree-query-clauses)
|
|
||||||
|
|
||||||
```sql
|
|
||||||
CREATE DATABASE db_name ENGINE = MaterializedMySQL(...)
|
|
||||||
[SETTINGS ...]
|
|
||||||
[TABLE OVERRIDE table_name (
|
|
||||||
[COLUMNS (
|
|
||||||
[col_name [datatype] [ALIAS expr] [CODEC(...)] [TTL expr], ...]
|
|
||||||
[INDEX index_name expr TYPE indextype[(...)] GRANULARITY val, ...]
|
|
||||||
[PROJECTION projection_name (SELECT <COLUMN LIST EXPR> [GROUP BY] [ORDER BY]), ...]
|
|
||||||
)]
|
|
||||||
[ORDER BY expr]
|
|
||||||
[PRIMARY KEY expr]
|
|
||||||
[PARTITION BY expr]
|
|
||||||
[SAMPLE BY expr]
|
|
||||||
[TTL expr]
|
|
||||||
), ...]
|
|
||||||
```
|
|
||||||
|
|
||||||
Example:
|
|
||||||
|
|
||||||
```sql
|
|
||||||
CREATE DATABASE db_name ENGINE = MaterializedMySQL(...)
|
|
||||||
TABLE OVERRIDE table1 (
|
|
||||||
COLUMNS (
|
|
||||||
userid UUID,
|
|
||||||
category LowCardinality(String),
|
|
||||||
timestamp DateTime CODEC(Delta, Default)
|
|
||||||
)
|
|
||||||
PARTITION BY toYear(timestamp)
|
|
||||||
),
|
|
||||||
TABLE OVERRIDE table2 (
|
|
||||||
COLUMNS (
|
|
||||||
client_ip String TTL created + INTERVAL 72 HOUR
|
|
||||||
)
|
|
||||||
SAMPLE BY ip_hash
|
|
||||||
)
|
|
||||||
```
|
|
||||||
|
|
||||||
The `COLUMNS` list is sparse; existing columns are modified as specified, extra ALIAS columns are added. It is not
|
|
||||||
possible to add ordinary or MATERIALIZED columns. Modified columns with a different type must be assignable from the
|
|
||||||
original type. There is currently no validation of this or similar issues when the `CREATE DATABASE` query executes, so
|
|
||||||
extra care needs to be taken.
|
|
||||||
|
|
||||||
You may specify overrides for tables that do not exist yet.
|
|
||||||
|
|
||||||
:::important
|
|
||||||
It is easy to break replication with table overrides if not used with care. For example:
|
|
||||||
|
|
||||||
* If an ALIAS column is added with a table override, and a column with the same name is later added to the source
|
|
||||||
MySQL table, the converted ALTER TABLE query in ClickHouse will fail and replication stops.
|
|
||||||
* It is currently possible to add overrides that reference nullable columns where not-nullable are required, such as in
|
|
||||||
`ORDER BY` or `PARTITION BY`. This will cause CREATE TABLE queries that will fail, also causing replication to stop.
|
|
||||||
:::
|
|
||||||
|
|
||||||
## Examples of Use {#examples-of-use}
|
|
||||||
|
|
||||||
Queries in MySQL:
|
|
||||||
|
|
||||||
``` sql
|
|
||||||
mysql> CREATE DATABASE db;
|
|
||||||
mysql> CREATE TABLE db.test (a INT PRIMARY KEY, b INT);
|
|
||||||
mysql> INSERT INTO db.test VALUES (1, 11), (2, 22);
|
|
||||||
mysql> DELETE FROM db.test WHERE a=1;
|
|
||||||
mysql> ALTER TABLE db.test ADD COLUMN c VARCHAR(16);
|
|
||||||
mysql> UPDATE db.test SET c='Wow!', b=222;
|
|
||||||
mysql> SELECT * FROM test;
|
|
||||||
```
|
|
||||||
|
|
||||||
```text
|
|
||||||
┌─a─┬───b─┬─c────┐
|
|
||||||
│ 2 │ 222 │ Wow! │
|
|
||||||
└───┴─────┴──────┘
|
|
||||||
```
|
|
||||||
|
|
||||||
Database in ClickHouse, exchanging data with the MySQL server:
|
|
||||||
|
|
||||||
The database and the table created:
|
|
||||||
|
|
||||||
``` sql
|
|
||||||
CREATE DATABASE mysql ENGINE = MaterializedMySQL('localhost:3306', 'db', 'user', '***');
|
|
||||||
SHOW TABLES FROM mysql;
|
|
||||||
```
|
|
||||||
|
|
||||||
``` text
|
|
||||||
┌─name─┐
|
|
||||||
│ test │
|
|
||||||
└──────┘
|
|
||||||
```
|
|
||||||
|
|
||||||
After inserting data:
|
|
||||||
|
|
||||||
``` sql
|
|
||||||
SELECT * FROM mysql.test;
|
|
||||||
```
|
|
||||||
|
|
||||||
``` text
|
|
||||||
┌─a─┬──b─┐
|
|
||||||
│ 1 │ 11 │
|
|
||||||
│ 2 │ 22 │
|
|
||||||
└───┴────┘
|
|
||||||
```
|
|
||||||
|
|
||||||
After deleting data, adding the column and updating:
|
|
||||||
|
|
||||||
``` sql
|
|
||||||
SELECT * FROM mysql.test;
|
|
||||||
```
|
|
||||||
|
|
||||||
``` text
|
|
||||||
┌─a─┬───b─┬─c────┐
|
|
||||||
│ 2 │ 222 │ Wow! │
|
|
||||||
└───┴─────┴──────┘
|
|
||||||
```
|
|
||||||
|
|||||||
@ -63,7 +63,34 @@ Currently there are 3 ways to authenticate:
|
|||||||
- `SAS Token` - Can be used by providing an `endpoint`, `connection_string` or `storage_account_url`. It is identified by presence of '?' in the url.
|
- `SAS Token` - Can be used by providing an `endpoint`, `connection_string` or `storage_account_url`. It is identified by presence of '?' in the url.
|
||||||
- `Workload Identity` - Can be used by providing an `endpoint` or `storage_account_url`. If `use_workload_identity` parameter is set in config, ([workload identity](https://github.com/Azure/azure-sdk-for-cpp/tree/main/sdk/identity/azure-identity#authenticate-azure-hosted-applications)) is used for authentication.
|
- `Workload Identity` - Can be used by providing an `endpoint` or `storage_account_url`. If `use_workload_identity` parameter is set in config, ([workload identity](https://github.com/Azure/azure-sdk-for-cpp/tree/main/sdk/identity/azure-identity#authenticate-azure-hosted-applications)) is used for authentication.
|
||||||
|
|
||||||
|
### Data cache {#data-cache}
|
||||||
|
|
||||||
|
`Azure` table engine supports data caching on local disk.
|
||||||
|
See filesystem cache configuration options and usage in this [section](/docs/en/operations/storing-data.md/#using-local-cache).
|
||||||
|
Caching is made depending on the path and ETag of the storage object, so clickhouse will not read a stale cache version.
|
||||||
|
|
||||||
|
To enable caching use a setting `filesystem_cache_name = '<name>'` and `enable_filesystem_cache = 1`.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT *
|
||||||
|
FROM azureBlobStorage('DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite1:10000/devstoreaccount1/;', 'test_container', 'test_table', 'CSV')
|
||||||
|
SETTINGS filesystem_cache_name = 'cache_for_azure', enable_filesystem_cache = 1;
|
||||||
|
```
|
||||||
|
|
||||||
|
1. add the following section to clickhouse configuration file:
|
||||||
|
|
||||||
|
``` xml
|
||||||
|
<clickhouse>
|
||||||
|
<filesystem_caches>
|
||||||
|
<cache_for_azure>
|
||||||
|
<path>path to cache directory</path>
|
||||||
|
<max_size>10Gi</max_size>
|
||||||
|
</cache_for_azure>
|
||||||
|
</filesystem_caches>
|
||||||
|
</clickhouse>
|
||||||
|
```
|
||||||
|
|
||||||
|
2. reuse cache configuration (and therefore cache storage) from clickhouse `storage_configuration` section, [described here](/docs/en/operations/storing-data.md/#using-local-cache)
|
||||||
|
|
||||||
## See also
|
## See also
|
||||||
|
|
||||||
|
|||||||
@ -48,6 +48,10 @@ Using named collections:
|
|||||||
CREATE TABLE deltalake ENGINE=DeltaLake(deltalake_conf, filename = 'test_table')
|
CREATE TABLE deltalake ENGINE=DeltaLake(deltalake_conf, filename = 'test_table')
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Data cache {#data-cache}
|
||||||
|
|
||||||
|
`Iceberg` table engine and table function support data caching same as `S3`, `AzureBlobStorage`, `HDFS` storages. See [here](../../../engines/table-engines/integrations/s3.md#data-cache).
|
||||||
|
|
||||||
## See also
|
## See also
|
||||||
|
|
||||||
- [deltaLake table function](../../../sql-reference/table-functions/deltalake.md)
|
- [deltaLake table function](../../../sql-reference/table-functions/deltalake.md)
|
||||||
|
|||||||
@ -6,7 +6,7 @@ sidebar_label: Iceberg
|
|||||||
|
|
||||||
# Iceberg Table Engine
|
# Iceberg Table Engine
|
||||||
|
|
||||||
This engine provides a read-only integration with existing Apache [Iceberg](https://iceberg.apache.org/) tables in Amazon S3, Azure and locally stored tables.
|
This engine provides a read-only integration with existing Apache [Iceberg](https://iceberg.apache.org/) tables in Amazon S3, Azure, HDFS and locally stored tables.
|
||||||
|
|
||||||
## Create Table
|
## Create Table
|
||||||
|
|
||||||
@ -19,13 +19,16 @@ CREATE TABLE iceberg_table_s3
|
|||||||
CREATE TABLE iceberg_table_azure
|
CREATE TABLE iceberg_table_azure
|
||||||
ENGINE = IcebergAzure(connection_string|storage_account_url, container_name, blobpath, [account_name, account_key, format, compression])
|
ENGINE = IcebergAzure(connection_string|storage_account_url, container_name, blobpath, [account_name, account_key, format, compression])
|
||||||
|
|
||||||
|
CREATE TABLE iceberg_table_hdfs
|
||||||
|
ENGINE = IcebergHDFS(path_to_table, [,format] [,compression_method])
|
||||||
|
|
||||||
CREATE TABLE iceberg_table_local
|
CREATE TABLE iceberg_table_local
|
||||||
ENGINE = IcebergLocal(path_to_table, [,format] [,compression_method])
|
ENGINE = IcebergLocal(path_to_table, [,format] [,compression_method])
|
||||||
```
|
```
|
||||||
|
|
||||||
**Engine arguments**
|
**Engine arguments**
|
||||||
|
|
||||||
Description of the arguments coincides with description of arguments in engines `S3`, `AzureBlobStorage` and `File` correspondingly.
|
Description of the arguments coincides with description of arguments in engines `S3`, `AzureBlobStorage`, `HDFS` and `File` correspondingly.
|
||||||
`format` stands for the format of data files in the Iceberg table.
|
`format` stands for the format of data files in the Iceberg table.
|
||||||
|
|
||||||
Engine parameters can be specified using [Named Collections](../../../operations/named-collections.md)
|
Engine parameters can be specified using [Named Collections](../../../operations/named-collections.md)
|
||||||
@ -60,6 +63,10 @@ CREATE TABLE iceberg_table ENGINE=IcebergS3(iceberg_conf, filename = 'test_table
|
|||||||
|
|
||||||
Table engine `Iceberg` is an alias to `IcebergS3` now.
|
Table engine `Iceberg` is an alias to `IcebergS3` now.
|
||||||
|
|
||||||
|
### Data cache {#data-cache}
|
||||||
|
|
||||||
|
`Iceberg` table engine and table function support data caching same as `S3`, `AzureBlobStorage`, `HDFS` storages. See [here](../../../engines/table-engines/integrations/s3.md#data-cache).
|
||||||
|
|
||||||
## See also
|
## See also
|
||||||
|
|
||||||
- [iceberg table function](/docs/en/sql-reference/table-functions/iceberg.md)
|
- [iceberg table function](/docs/en/sql-reference/table-functions/iceberg.md)
|
||||||
|
|||||||
@ -9,7 +9,7 @@ sidebar_label: MongoDB
|
|||||||
MongoDB engine is read-only table engine which allows to read data from remote [MongoDB](https://www.mongodb.com/) collection.
|
MongoDB engine is read-only table engine which allows to read data from remote [MongoDB](https://www.mongodb.com/) collection.
|
||||||
|
|
||||||
Only MongoDB v3.6+ servers are supported.
|
Only MongoDB v3.6+ servers are supported.
|
||||||
[Seed list(`mongodb**+srv**`)](https://www.mongodb.com/docs/manual/reference/glossary/#std-term-seed-list) is not yet supported.
|
[Seed list(`mongodb+srv`)](https://www.mongodb.com/docs/manual/reference/glossary/#std-term-seed-list) is not yet supported.
|
||||||
|
|
||||||
:::note
|
:::note
|
||||||
If you're facing troubles, please report the issue, and try to use [the legacy implementation](../../../operations/server-configuration-parameters/settings.md#use_legacy_mongodb_integration).
|
If you're facing troubles, please report the issue, and try to use [the legacy implementation](../../../operations/server-configuration-parameters/settings.md#use_legacy_mongodb_integration).
|
||||||
|
|||||||
@ -4,12 +4,8 @@ sidebar_position: 138
|
|||||||
sidebar_label: MySQL
|
sidebar_label: MySQL
|
||||||
---
|
---
|
||||||
|
|
||||||
import CloudAvailableBadge from '@theme/badges/CloudAvailableBadge';
|
|
||||||
|
|
||||||
# MySQL Table Engine
|
# MySQL Table Engine
|
||||||
|
|
||||||
<CloudAvailableBadge />
|
|
||||||
|
|
||||||
The MySQL engine allows you to perform `SELECT` and `INSERT` queries on data that is stored on a remote MySQL server.
|
The MySQL engine allows you to perform `SELECT` and `INSERT` queries on data that is stored on a remote MySQL server.
|
||||||
|
|
||||||
## Creating a Table {#creating-a-table}
|
## Creating a Table {#creating-a-table}
|
||||||
|
|||||||
@ -26,6 +26,7 @@ SELECT * FROM s3_engine_table LIMIT 2;
|
|||||||
│ two │ 2 │
|
│ two │ 2 │
|
||||||
└──────┴───────┘
|
└──────┴───────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
## Create Table {#creating-a-table}
|
## Create Table {#creating-a-table}
|
||||||
|
|
||||||
``` sql
|
``` sql
|
||||||
@ -43,6 +44,37 @@ CREATE TABLE s3_engine_table (name String, value UInt32)
|
|||||||
- `aws_access_key_id`, `aws_secret_access_key` - Long-term credentials for the [AWS](https://aws.amazon.com/) account user. You can use these to authenticate your requests. Parameter is optional. If credentials are not specified, they are used from the configuration file. For more information see [Using S3 for Data Storage](../mergetree-family/mergetree.md#table_engine-mergetree-s3).
|
- `aws_access_key_id`, `aws_secret_access_key` - Long-term credentials for the [AWS](https://aws.amazon.com/) account user. You can use these to authenticate your requests. Parameter is optional. If credentials are not specified, they are used from the configuration file. For more information see [Using S3 for Data Storage](../mergetree-family/mergetree.md#table_engine-mergetree-s3).
|
||||||
- `compression` — Compression type. Supported values: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. Parameter is optional. By default, it will auto-detect compression by file extension.
|
- `compression` — Compression type. Supported values: `none`, `gzip/gz`, `brotli/br`, `xz/LZMA`, `zstd/zst`. Parameter is optional. By default, it will auto-detect compression by file extension.
|
||||||
|
|
||||||
|
### Data cache {#data-cache}
|
||||||
|
|
||||||
|
`S3` table engine supports data caching on local disk.
|
||||||
|
See filesystem cache configuration options and usage in this [section](/docs/en/operations/storing-data.md/#using-local-cache).
|
||||||
|
Caching is made depending on the path and ETag of the storage object, so clickhouse will not read a stale cache version.
|
||||||
|
|
||||||
|
To enable caching use a setting `filesystem_cache_name = '<name>'` and `enable_filesystem_cache = 1`.
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT *
|
||||||
|
FROM s3('http://minio:10000/clickhouse//test_3.csv', 'minioadmin', 'minioadminpassword', 'CSV')
|
||||||
|
SETTINGS filesystem_cache_name = 'cache_for_s3', enable_filesystem_cache = 1;
|
||||||
|
```
|
||||||
|
|
||||||
|
There are two ways to define cache in configuration file.
|
||||||
|
|
||||||
|
1. add the following section to clickhouse configuration file:
|
||||||
|
|
||||||
|
``` xml
|
||||||
|
<clickhouse>
|
||||||
|
<filesystem_caches>
|
||||||
|
<cache_for_s3>
|
||||||
|
<path>path to cache directory</path>
|
||||||
|
<max_size>10Gi</max_size>
|
||||||
|
</cache_for_s3>
|
||||||
|
</filesystem_caches>
|
||||||
|
</clickhouse>
|
||||||
|
```
|
||||||
|
|
||||||
|
2. reuse cache configuration (and therefore cache storage) from clickhouse `storage_configuration` section, [described here](/docs/en/operations/storing-data.md/#using-local-cache)
|
||||||
|
|
||||||
### PARTITION BY
|
### PARTITION BY
|
||||||
|
|
||||||
`PARTITION BY` — Optional. In most cases you don't need a partition key, and if it is needed you generally don't need a partition key more granular than by month. Partitioning does not speed up queries (in contrast to the ORDER BY expression). You should never use too granular partitioning. Don't partition your data by client identifiers or names (instead, make client identifier or name the first column in the ORDER BY expression).
|
`PARTITION BY` — Optional. In most cases you don't need a partition key, and if it is needed you generally don't need a partition key more granular than by month. Partitioning does not speed up queries (in contrast to the ORDER BY expression). You should never use too granular partitioning. Don't partition your data by client identifiers or names (instead, make client identifier or name the first column in the ORDER BY expression).
|
||||||
|
|||||||
@ -374,15 +374,15 @@ Users can create [UDF](/docs/en/sql-reference/statements/create/function.md) to
|
|||||||
```sql
|
```sql
|
||||||
CREATE FUNCTION bfEstimateFunctions [ON CLUSTER cluster]
|
CREATE FUNCTION bfEstimateFunctions [ON CLUSTER cluster]
|
||||||
AS
|
AS
|
||||||
(total_nubmer_of_all_grams, size_of_bloom_filter_in_bits) -> round((size_of_bloom_filter_in_bits / total_nubmer_of_all_grams) * log(2));
|
(total_number_of_all_grams, size_of_bloom_filter_in_bits) -> round((size_of_bloom_filter_in_bits / total_number_of_all_grams) * log(2));
|
||||||
|
|
||||||
CREATE FUNCTION bfEstimateBmSize [ON CLUSTER cluster]
|
CREATE FUNCTION bfEstimateBmSize [ON CLUSTER cluster]
|
||||||
AS
|
AS
|
||||||
(total_nubmer_of_all_grams, probability_of_false_positives) -> ceil((total_nubmer_of_all_grams * log(probability_of_false_positives)) / log(1 / pow(2, log(2))));
|
(total_number_of_all_grams, probability_of_false_positives) -> ceil((total_number_of_all_grams * log(probability_of_false_positives)) / log(1 / pow(2, log(2))));
|
||||||
|
|
||||||
CREATE FUNCTION bfEstimateFalsePositive [ON CLUSTER cluster]
|
CREATE FUNCTION bfEstimateFalsePositive [ON CLUSTER cluster]
|
||||||
AS
|
AS
|
||||||
(total_nubmer_of_all_grams, number_of_hash_functions, size_of_bloom_filter_in_bytes) -> pow(1 - exp(-number_of_hash_functions/ (size_of_bloom_filter_in_bytes / total_nubmer_of_all_grams)), number_of_hash_functions);
|
(total_number_of_all_grams, number_of_hash_functions, size_of_bloom_filter_in_bytes) -> pow(1 - exp(-number_of_hash_functions/ (size_of_bloom_filter_in_bytes / total_number_of_all_grams)), number_of_hash_functions);
|
||||||
|
|
||||||
CREATE FUNCTION bfEstimateGramNumber [ON CLUSTER cluster]
|
CREATE FUNCTION bfEstimateGramNumber [ON CLUSTER cluster]
|
||||||
AS
|
AS
|
||||||
|
|||||||
@ -35,7 +35,7 @@ Engine parameters:
|
|||||||
|
|
||||||
- `root_path` - ZooKeeper path where the `table_name` will be stored.
|
- `root_path` - ZooKeeper path where the `table_name` will be stored.
|
||||||
This path should not contain the prefix defined by `<keeper_map_path_prefix>` config because the prefix will be automatically appended to the `root_path`.
|
This path should not contain the prefix defined by `<keeper_map_path_prefix>` config because the prefix will be automatically appended to the `root_path`.
|
||||||
Additionally, format of `auxiliary_zookeper_cluster_name:/some/path` is also supported where `auxiliary_zookeper_cluster` is a ZooKeeper cluster defined inside `<auxiliary_zookeepers>` config.
|
Additionally, format of `auxiliary_zookeeper_cluster_name:/some/path` is also supported where `auxiliary_zookeeper_cluster` is a ZooKeeper cluster defined inside `<auxiliary_zookeepers>` config.
|
||||||
By default, ZooKeeper cluster defined inside `<zookeeper>` config is used.
|
By default, ZooKeeper cluster defined inside `<zookeeper>` config is used.
|
||||||
- `keys_limit` - number of keys allowed inside the table.
|
- `keys_limit` - number of keys allowed inside the table.
|
||||||
This limit is a soft limit and it can be possible that more keys will end up in the table for some edge cases.
|
This limit is a soft limit and it can be possible that more keys will end up in the table for some edge cases.
|
||||||
|
|||||||
@ -877,7 +877,7 @@ INSERT INTO json_as_object (json) FORMAT JSONAsObject {"any json stucture":1}
|
|||||||
SELECT time, json FROM json_as_object FORMAT JSONEachRow
|
SELECT time, json FROM json_as_object FORMAT JSONEachRow
|
||||||
```
|
```
|
||||||
|
|
||||||
```resonse
|
```response
|
||||||
{"time":"2024-09-16 12:18:10","json":{}}
|
{"time":"2024-09-16 12:18:10","json":{}}
|
||||||
{"time":"2024-09-16 12:18:13","json":{"any json stucture":"1"}}
|
{"time":"2024-09-16 12:18:13","json":{"any json stucture":"1"}}
|
||||||
{"time":"2024-09-16 12:18:08","json":{"foo":{"bar":{"x":"y"},"baz":"1"}}}
|
{"time":"2024-09-16 12:18:08","json":{"foo":{"bar":{"x":"y"},"baz":"1"}}}
|
||||||
|
|||||||
@ -509,7 +509,7 @@ DESC format(JSONEachRow, $$
|
|||||||
{"value" : "424242424242"}
|
{"value" : "424242424242"}
|
||||||
$$)
|
$$)
|
||||||
```
|
```
|
||||||
```reponse
|
```response
|
||||||
┌─name──┬─type────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
|
┌─name──┬─type────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
|
||||||
│ value │ Nullable(Int64) │ │ │ │ │ │
|
│ value │ Nullable(Int64) │ │ │ │ │ │
|
||||||
└───────┴─────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
|
└───────┴─────────────────┴──────────────┴────────────────────┴─────────┴──────────────────┴────────────────┘
|
||||||
@ -910,9 +910,9 @@ This setting is disabled by default.
|
|||||||
|
|
||||||
```sql
|
```sql
|
||||||
SET input_format_json_try_infer_numbers_from_strings = 1;
|
SET input_format_json_try_infer_numbers_from_strings = 1;
|
||||||
DESC format(CSV, '"42","42.42"');
|
DESC format(CSV, '42,42.42');
|
||||||
```
|
```
|
||||||
```reponse
|
```response
|
||||||
┌─name─┬─type──────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
|
┌─name─┬─type──────────────┬─default_type─┬─default_expression─┬─comment─┬─codec_expression─┬─ttl_expression─┐
|
||||||
│ c1 │ Nullable(Int64) │ │ │ │ │ │
|
│ c1 │ Nullable(Int64) │ │ │ │ │ │
|
||||||
│ c2 │ Nullable(Float64) │ │ │ │ │ │
|
│ c2 │ Nullable(Float64) │ │ │ │ │ │
|
||||||
|
|||||||
@ -31,6 +31,10 @@ The table must be enabled in the server configuration, see the `opentelemetry_sp
|
|||||||
|
|
||||||
The tags or attributes are saved as two parallel arrays, containing the keys and values. Use [ARRAY JOIN](../sql-reference/statements/select/array-join.md) to work with them.
|
The tags or attributes are saved as two parallel arrays, containing the keys and values. Use [ARRAY JOIN](../sql-reference/statements/select/array-join.md) to work with them.
|
||||||
|
|
||||||
|
## Log-query-settings
|
||||||
|
|
||||||
|
ClickHouse allows you to log changes to query settings during query execution. When enabled, any modifications made to query settings will be recorded in the OpenTelemetry span log. This feature is particularly useful in production environments for tracking configuration changes that may affect query performance.
|
||||||
|
|
||||||
## Integration with monitoring systems
|
## Integration with monitoring systems
|
||||||
|
|
||||||
At the moment, there is no ready tool that can export the tracing data from ClickHouse to a monitoring system.
|
At the moment, there is no ready tool that can export the tracing data from ClickHouse to a monitoring system.
|
||||||
|
|||||||
@ -2,7 +2,6 @@
|
|||||||
title: "Settings Overview"
|
title: "Settings Overview"
|
||||||
sidebar_position: 1
|
sidebar_position: 1
|
||||||
slug: /en/operations/settings/
|
slug: /en/operations/settings/
|
||||||
pagination_next: en/operations/settings/settings
|
|
||||||
---
|
---
|
||||||
|
|
||||||
# Settings Overview
|
# Settings Overview
|
||||||
|
|||||||
@ -1,176 +0,0 @@
|
|||||||
# The MySQL Binlog Client
|
|
||||||
|
|
||||||
The MySQL Binlog Client provides a mechanism in ClickHouse to share the binlog from a MySQL instance among multiple [MaterializedMySQL](../../engines/database-engines/materialized-mysql.md) databases. This avoids consuming unnecessary bandwidth and CPU when replicating more than one schema/database.
|
|
||||||
|
|
||||||
The implementation is resilient against crashes and disk issues. The executed GTID sets of the binlog itself and the consuming databases have persisted only after the data they describe has been safely persisted as well. The implementation also tolerates re-doing aborted operations (at-least-once delivery).
|
|
||||||
|
|
||||||
# Settings
|
|
||||||
|
|
||||||
## use_binlog_client
|
|
||||||
|
|
||||||
Forces to reuse existing MySQL binlog connection or creates new one if does not exist. The connection is defined by `user:pass@host:port`.
|
|
||||||
|
|
||||||
Default value: 0
|
|
||||||
|
|
||||||
**Example**
|
|
||||||
|
|
||||||
```sql
|
|
||||||
-- create MaterializedMySQL databases that read the events from the binlog client
|
|
||||||
CREATE DATABASE db1 ENGINE = MaterializedMySQL('host:port', 'db1', 'user', 'password') SETTINGS use_binlog_client=1
|
|
||||||
CREATE DATABASE db2 ENGINE = MaterializedMySQL('host:port', 'db2', 'user', 'password') SETTINGS use_binlog_client=1
|
|
||||||
CREATE DATABASE db3 ENGINE = MaterializedMySQL('host:port', 'db3', 'user2', 'password2') SETTINGS use_binlog_client=1
|
|
||||||
```
|
|
||||||
|
|
||||||
Databases `db1` and `db2` will use the same binlog connection, since they use the same `user:pass@host:port`. Database `db3` will use separate binlog connection.
|
|
||||||
|
|
||||||
## max_bytes_in_binlog_queue
|
|
||||||
|
|
||||||
Defines the limit of bytes in the events binlog queue. If bytes in the queue increases this limit, it will stop reading new events from MySQL until the space for new events will be freed. This introduces the memory limits. Very high value could consume all available memory. Very low value could make the databases to wait for new events.
|
|
||||||
|
|
||||||
Default value: 67108864
|
|
||||||
|
|
||||||
**Example**
|
|
||||||
|
|
||||||
```sql
|
|
||||||
CREATE DATABASE db1 ENGINE = MaterializedMySQL('host:port', 'db1', 'user', 'password') SETTINGS use_binlog_client=1, max_bytes_in_binlog_queue=33554432
|
|
||||||
CREATE DATABASE db2 ENGINE = MaterializedMySQL('host:port', 'db2', 'user', 'password') SETTINGS use_binlog_client=1
|
|
||||||
```
|
|
||||||
|
|
||||||
If database `db1` is unable to consume binlog events fast enough and the size of the events queue exceeds `33554432` bytes, reading of new events from MySQL is postponed until `db1`
|
|
||||||
consumes the events and releases some space.
|
|
||||||
|
|
||||||
NOTE: This will impact to `db2`, and it will be waiting for new events too, since they share the same connection.
|
|
||||||
|
|
||||||
## max_milliseconds_to_wait_in_binlog_queue
|
|
||||||
|
|
||||||
Defines the max milliseconds to wait when `max_bytes_in_binlog_queue` exceeded. After that it will detach the database from current binlog connection and will retry establish new one to prevent other databases to wait for this database.
|
|
||||||
|
|
||||||
Default value: 10000
|
|
||||||
|
|
||||||
**Example**
|
|
||||||
|
|
||||||
```sql
|
|
||||||
CREATE DATABASE db1 ENGINE = MaterializedMySQL('host:port', 'db1', 'user', 'password') SETTINGS use_binlog_client=1, max_bytes_in_binlog_queue=33554432, max_milliseconds_to_wait_in_binlog_queue=1000
|
|
||||||
CREATE DATABASE db2 ENGINE = MaterializedMySQL('host:port', 'db2', 'user', 'password') SETTINGS use_binlog_client=1
|
|
||||||
```
|
|
||||||
|
|
||||||
If the event queue of database `db1` is full, the binlog connection will be waiting in `1000`ms and if the database is not able to consume the events, it will be detached from the connection to create another one.
|
|
||||||
|
|
||||||
NOTE: If the database `db1` has been detached from the shared connection and created new one, after the binlog connections for `db1` and `db2` have the same positions they will be merged to one. And `db1` and `db2` will use the same connection again.
|
|
||||||
|
|
||||||
## max_bytes_in_binlog_dispatcher_buffer
|
|
||||||
|
|
||||||
Defines the max bytes in the binlog dispatcher's buffer before it is flushed to attached binlog. The events from MySQL binlog connection are buffered before sending to attached databases. It increases the events throughput from the binlog to databases.
|
|
||||||
|
|
||||||
Default value: 1048576
|
|
||||||
|
|
||||||
## max_flush_milliseconds_in_binlog_dispatcher
|
|
||||||
|
|
||||||
Defines the max milliseconds in the binlog dispatcher's buffer to wait before it is flushed to attached binlog. If there are no events received from MySQL binlog connection for a while, after some time buffered events should be sent to the attached databases.
|
|
||||||
|
|
||||||
Default value: 1000
|
|
||||||
|
|
||||||
# Design
|
|
||||||
|
|
||||||
## The Binlog Events Dispatcher
|
|
||||||
|
|
||||||
Currently each MaterializedMySQL database opens its own connection to MySQL to subscribe to binlog events. There is a need to have only one connection and _dispatch_ the binlog events to all databases that replicate from the same MySQL instance.
|
|
||||||
|
|
||||||
## Each MaterializedMySQL Database Has Its Own Event Queue
|
|
||||||
|
|
||||||
To prevent slowing down other instances there should be an _event queue_ per MaterializedMySQL database to handle the events independently of the speed of other instances. The dispatcher reads an event from the binlog, and sends it to every MaterializedMySQL database that needs it. Each database handles its events in separate threads.
|
|
||||||
|
|
||||||
## Catching up
|
|
||||||
|
|
||||||
If several databases have the same binlog position, they can use the same dispatcher. If a newly created database (or one that has been detached for some time) requests events that have been already processed, we need to create another communication _channel_ to the binlog. We do this by creating another temporary dispatcher for such databases. When the new dispatcher _catches up with_ the old one, the new/temporary dispatcher is not needed anymore and all databases getting events from this dispatcher can be moved to the old one.
|
|
||||||
|
|
||||||
## Memory Limit
|
|
||||||
|
|
||||||
There is a _memory limit_ to control event queue memory consumption per MySQL Client. If a database is not able to handle events fast enough, and the event queue is getting full, we have the following options:
|
|
||||||
|
|
||||||
1. The dispatcher is blocked until the slowest database frees up space for new events. All other databases are waiting for the slowest one. (Preferred)
|
|
||||||
2. The dispatcher is _never_ blocked, but suspends incremental sync for the slow database and continues dispatching events to remained databases.
|
|
||||||
|
|
||||||
## Performance
|
|
||||||
|
|
||||||
A lot of CPU can be saved by not processing every event in every database. The binlog contains events for all databases, it is wasteful to distribute row events to a database that it will not process it, especially if there are a lot of databases. This requires some sort of per-database binlog filtering and buffering.
|
|
||||||
|
|
||||||
Currently all events are sent to all MaterializedMySQL databases but parsing the event which consumes CPU is up to the database.
|
|
||||||
|
|
||||||
# Detailed Design
|
|
||||||
|
|
||||||
1. If a client (e.g. database) wants to read a stream of the events from MySQL binlog, it creates a connection to remote binlog by host/user/password and _executed GTID set_ params.
|
|
||||||
2. If another client wants to read the events from the binlog but for different _executed GTID set_, it is **not** possible to reuse existing connection to MySQL, then need to create another connection to the same remote binlog. (_This is how it is implemented today_).
|
|
||||||
3. When these 2 connections get the same binlog positions, they read the same events. It is logical to drop duplicate connection and move all its users out. And now one connection dispatches binlog events to several clients. Obviously only connections to the same binlog should be merged.
|
|
||||||
|
|
||||||
## Classes
|
|
||||||
|
|
||||||
1. One connection can send (or dispatch) events to several clients and might be called `BinlogEventsDispatcher`.
|
|
||||||
2. Several dispatchers grouped by _user:password@host:port_ in `BinlogClient`. Since they point to the same binlog.
|
|
||||||
3. The clients should communicate only with public API from `BinlogClient`. The result of using `BinlogClient` is an object that implements `IBinlog` to read events from. This implementation of `IBinlog` must be compatible with old implementation `MySQLFlavor` -> when replacing old implementation by new one, the behavior must not be changed.
|
|
||||||
|
|
||||||
## SQL
|
|
||||||
|
|
||||||
```sql
|
|
||||||
-- create MaterializedMySQL databases that read the events from the binlog client
|
|
||||||
CREATE DATABASE db1_client1 ENGINE = MaterializedMySQL('host:port', 'db', 'user', 'password') SETTINGS use_binlog_client=1, max_bytes_in_binlog_queue=1024;
|
|
||||||
CREATE DATABASE db2_client1 ENGINE = MaterializedMySQL('host:port', 'db', 'user', 'password') SETTINGS use_binlog_client=1;
|
|
||||||
CREATE DATABASE db3_client1 ENGINE = MaterializedMySQL('host:port', 'db2', 'user', 'password') SETTINGS use_binlog_client=1;
|
|
||||||
CREATE DATABASE db4_client2 ENGINE = MaterializedMySQL('host2:port', 'db', 'user', 'password') SETTINGS use_binlog_client=1;
|
|
||||||
CREATE DATABASE db5_client3 ENGINE = MaterializedMySQL('host:port', 'db', 'user1', 'password') SETTINGS use_binlog_client=1;
|
|
||||||
CREATE DATABASE db6_old ENGINE = MaterializedMySQL('host:port', 'db', 'user1', 'password') SETTINGS use_binlog_client=0;
|
|
||||||
```
|
|
||||||
|
|
||||||
Databases `db1_client1`, `db2_client1` and `db3_client1` share one instance of `BinlogClient` since they have the same params. `BinlogClient` will create 3 connections to MySQL server thus 3 instances of `BinlogEventsDispatcher`, but if these connections would have the same binlog position, they should be merged to one connection. Means all clients will be moved to one dispatcher and others will be closed. Databases `db4_client2` and `db5_client3` would use 2 different independent `BinlogClient` instances. Database `db6_old` will use old implementation. NOTE: By default `use_binlog_client` is disabled. Setting `max_bytes_in_binlog_queue` defines the max allowed bytes in the binlog queue. By default, it is `1073741824` bytes. If number of bytes exceeds this limit, the dispatching will be stopped until the space will be freed for new events.
|
|
||||||
|
|
||||||
## Binlog Table Structure
|
|
||||||
|
|
||||||
To see the status of the all `BinlogClient` instances there is `system.mysql_binlogs` system table. It shows the list of all created and _alive_ `IBinlog` instances with information about its `BinlogEventsDispatcher` and `BinlogClient`.
|
|
||||||
|
|
||||||
Example:
|
|
||||||
|
|
||||||
```
|
|
||||||
SELECT * FROM system.mysql_binlogs FORMAT Vertical
|
|
||||||
Row 1:
|
|
||||||
──────
|
|
||||||
binlog_client_name: root@127.0.0.1:3306
|
|
||||||
name: test_Clickhouse1
|
|
||||||
mysql_binlog_name: binlog.001154
|
|
||||||
mysql_binlog_pos: 7142294
|
|
||||||
mysql_binlog_timestamp: 1660082447
|
|
||||||
mysql_binlog_executed_gtid_set: a9d88f83-c14e-11ec-bb36-244bfedf7766:1-30523304
|
|
||||||
dispatcher_name: Applier
|
|
||||||
dispatcher_mysql_binlog_name: binlog.001154
|
|
||||||
dispatcher_mysql_binlog_pos: 7142294
|
|
||||||
dispatcher_mysql_binlog_timestamp: 1660082447
|
|
||||||
dispatcher_mysql_binlog_executed_gtid_set: a9d88f83-c14e-11ec-bb36-244bfedf7766:1-30523304
|
|
||||||
size: 0
|
|
||||||
bytes: 0
|
|
||||||
max_bytes: 0
|
|
||||||
```
|
|
||||||
|
|
||||||
### Tests
|
|
||||||
|
|
||||||
Unit tests:
|
|
||||||
|
|
||||||
```
|
|
||||||
$ ./unit_tests_dbms --gtest_filter=MySQLBinlog.*
|
|
||||||
```
|
|
||||||
|
|
||||||
Integration tests:
|
|
||||||
|
|
||||||
```
|
|
||||||
$ pytest -s -vv test_materialized_mysql_database/test.py::test_binlog_client
|
|
||||||
```
|
|
||||||
|
|
||||||
Dumps events from the file
|
|
||||||
|
|
||||||
```
|
|
||||||
$ ./utils/check-mysql-binlog/check-mysql-binlog --binlog binlog.001392
|
|
||||||
```
|
|
||||||
|
|
||||||
Dumps events from the server
|
|
||||||
|
|
||||||
```
|
|
||||||
$ ./utils/check-mysql-binlog/check-mysql-binlog --host 127.0.0.1 --port 3306 --user root --password pass --gtid a9d88f83-c14e-11ec-bb36-244bfedf7766:1-30462856
|
|
||||||
```
|
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -11,6 +11,9 @@ Columns:
|
|||||||
- `table` ([String](../../sql-reference/data-types/string.md)) — Table name.
|
- `table` ([String](../../sql-reference/data-types/string.md)) — Table name.
|
||||||
- `name` ([String](../../sql-reference/data-types/string.md)) — Projection name.
|
- `name` ([String](../../sql-reference/data-types/string.md)) — Projection name.
|
||||||
- `type` ([Enum](../../sql-reference/data-types/enum.md)) — Projection type ('Normal' = 0, 'Aggregate' = 1).
|
- `type` ([Enum](../../sql-reference/data-types/enum.md)) — Projection type ('Normal' = 0, 'Aggregate' = 1).
|
||||||
|
- `total_rows`, ([UInt64](../../sql-reference/data-types/int-uint.md)) — Total number of rows.
|
||||||
|
- `data_compressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The size of compressed data, in bytes.
|
||||||
|
- `data_uncompressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The size of decompressed data, in bytes.
|
||||||
- `sorting_key` ([Array(String)](../../sql-reference/data-types/array.md)) — Projection sorting key.
|
- `sorting_key` ([Array(String)](../../sql-reference/data-types/array.md)) — Projection sorting key.
|
||||||
- `query` ([String](../../sql-reference/data-types/string.md)) — Projection query.
|
- `query` ([String](../../sql-reference/data-types/string.md)) — Projection query.
|
||||||
|
|
||||||
@ -27,6 +30,9 @@ database: default
|
|||||||
table: landing
|
table: landing
|
||||||
name: improved_sorting_key
|
name: improved_sorting_key
|
||||||
type: Normal
|
type: Normal
|
||||||
|
total_rows: 1000
|
||||||
|
data_compressed_bytes: 8081
|
||||||
|
data_uncompressed_bytes: 12890
|
||||||
sorting_key: ['user_id','date']
|
sorting_key: ['user_id','date']
|
||||||
query: SELECT * ORDER BY user_id, date
|
query: SELECT * ORDER BY user_id, date
|
||||||
|
|
||||||
@ -34,8 +40,11 @@ Row 2:
|
|||||||
──────
|
──────
|
||||||
database: default
|
database: default
|
||||||
table: landing
|
table: landing
|
||||||
name: agg_no_key
|
name: agg
|
||||||
type: Aggregate
|
type: Aggregate
|
||||||
sorting_key: []
|
total_rows: 2
|
||||||
query: SELECT count()
|
data_compressed_bytes: 82
|
||||||
|
data_uncompressed_bytes: 32
|
||||||
|
sorting_key: ['user_id']
|
||||||
|
query: SELECT user_id, max(date) AS max_date GROUP BY user_id
|
||||||
```
|
```
|
||||||
|
|||||||
@ -177,6 +177,26 @@ When you are ready to insert your files into ClickHouse, startup a ClickHouse se
|
|||||||
:::
|
:::
|
||||||
|
|
||||||
|
|
||||||
|
## Format Conversions
|
||||||
|
|
||||||
|
You can use `clickhouse-local` for converting data between different formats. Example:
|
||||||
|
|
||||||
|
``` bash
|
||||||
|
$ clickhouse-local --input-format JSONLines --output-format CSV --query "SELECT * FROM table" < data.json > data.csv
|
||||||
|
```
|
||||||
|
|
||||||
|
Formats are auto-detected from file extensions:
|
||||||
|
|
||||||
|
``` bash
|
||||||
|
$ clickhouse-local --query "SELECT * FROM table" < data.json > data.csv
|
||||||
|
```
|
||||||
|
|
||||||
|
As a shortcut, you can write it using the `--copy` argument:
|
||||||
|
``` bash
|
||||||
|
$ clickhouse-local --copy < data.json > data.csv
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
## Usage {#usage}
|
## Usage {#usage}
|
||||||
|
|
||||||
By default `clickhouse-local` has access to data of a ClickHouse server on the same host, and it does not depend on the server's configuration. It also supports loading server configuration using `--config-file` argument. For temporary data, a unique temporary data directory is created by default.
|
By default `clickhouse-local` has access to data of a ClickHouse server on the same host, and it does not depend on the server's configuration. It also supports loading server configuration using `--config-file` argument. For temporary data, a unique temporary data directory is created by default.
|
||||||
|
|||||||
@ -124,7 +124,7 @@ Converts an aggregate function for tables into an aggregate function for arrays
|
|||||||
## -Distinct
|
## -Distinct
|
||||||
|
|
||||||
Every unique combination of arguments will be aggregated only once. Repeating values are ignored.
|
Every unique combination of arguments will be aggregated only once. Repeating values are ignored.
|
||||||
Examples: `sum(DISTINCT x)`, `groupArray(DISTINCT x)`, `corrStableDistinct(DISTINCT x, y)` and so on.
|
Examples: `sum(DISTINCT x)` (or `sumDistinct(x)`), `groupArray(DISTINCT x)` (or `groupArrayDistinct(x)`), `corrStable(DISTINCT x, y)` (or `corrStableDistinct(x, y)`) and so on.
|
||||||
|
|
||||||
## -OrDefault
|
## -OrDefault
|
||||||
|
|
||||||
|
|||||||
@ -261,9 +261,10 @@ windowFunnel(window, [mode, [mode, ... ]])(timestamp, cond1, cond2, ..., condN)
|
|||||||
|
|
||||||
- `window` — Length of the sliding window, it is the time interval between the first and the last condition. The unit of `window` depends on the `timestamp` itself and varies. Determined using the expression `timestamp of cond1 <= timestamp of cond2 <= ... <= timestamp of condN <= timestamp of cond1 + window`.
|
- `window` — Length of the sliding window, it is the time interval between the first and the last condition. The unit of `window` depends on the `timestamp` itself and varies. Determined using the expression `timestamp of cond1 <= timestamp of cond2 <= ... <= timestamp of condN <= timestamp of cond1 + window`.
|
||||||
- `mode` — It is an optional argument. One or more modes can be set.
|
- `mode` — It is an optional argument. One or more modes can be set.
|
||||||
- `'strict_deduplication'` — If the same condition holds for the sequence of events, then such repeating event interrupts further processing.
|
- `'strict_deduplication'` — If the same condition holds for the sequence of events, then such repeating event interrupts further processing. Note: it may work unexpectedly if several conditions hold for the same event.
|
||||||
- `'strict_order'` — Don't allow interventions of other events. E.g. in the case of `A->B->D->C`, it stops finding `A->B->C` at the `D` and the max event level is 2.
|
- `'strict_order'` — Don't allow interventions of other events. E.g. in the case of `A->B->D->C`, it stops finding `A->B->C` at the `D` and the max event level is 2.
|
||||||
- `'strict_increase'` — Apply conditions only to events with strictly increasing timestamps.
|
- `'strict_increase'` — Apply conditions only to events with strictly increasing timestamps.
|
||||||
|
- `'strict_once'` — Count each event only once in the chain even if it meets the condition several times
|
||||||
|
|
||||||
**Returned value**
|
**Returned value**
|
||||||
|
|
||||||
|
|||||||
@ -1,190 +0,0 @@
|
|||||||
---
|
|
||||||
slug: /en/sql-reference/ansi
|
|
||||||
sidebar_position: 40
|
|
||||||
sidebar_label: ANSI Compatibility
|
|
||||||
title: "ANSI SQL Compatibility of ClickHouse SQL Dialect"
|
|
||||||
---
|
|
||||||
|
|
||||||
:::note
|
|
||||||
This article relies on Table 38, “Feature taxonomy and definition for mandatory features”, Annex F of [ISO/IEC CD 9075-2:2011](https://www.iso.org/obp/ui/#iso:std:iso-iec:9075:-2:ed-4:v1:en:sec:8).
|
|
||||||
:::
|
|
||||||
|
|
||||||
## Differences in Behaviour
|
|
||||||
|
|
||||||
The following table lists cases when query feature works in ClickHouse, but behaves not as specified in ANSI SQL.
|
|
||||||
|
|
||||||
| Feature ID | Feature Name | Difference |
|
|
||||||
|------------|-----------------------------|-----------------------------------------------------------------------------------------------------------|
|
|
||||||
| E011 | Numeric data types | Numeric literal with period is interpreted as approximate (`Float64`) instead of exact (`Decimal`) |
|
|
||||||
| E051-05 | Select items can be renamed | Item renames have a wider visibility scope than just the SELECT result |
|
|
||||||
| E141-01 | NOT NULL constraints | `NOT NULL` is implied for table columns by default |
|
|
||||||
| E011-04 | Arithmetic operators | ClickHouse overflows instead of checked arithmetic and changes the result data type based on custom rules |
|
|
||||||
|
|
||||||
## Feature Status
|
|
||||||
|
|
||||||
| Feature ID | Feature Name | Status | Comment |
|
|
||||||
|------------|--------------------------------------------------------------------------------------------------------------------------|----------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|
||||||
| **E011** | **Numeric data types** | <span class="text-warning">Partial</span> | |
|
|
||||||
| E011-01 | INTEGER and SMALLINT data types | <span class="text-success">Yes</span> | |
|
|
||||||
| E011-02 | REAL, DOUBLE PRECISION and FLOAT data types data types | <span class="text-success">Yes</span> | |
|
|
||||||
| E011-03 | DECIMAL and NUMERIC data types | <span class="text-success">Yes</span> | |
|
|
||||||
| E011-04 | Arithmetic operators | <span class="text-success">Yes</span> | |
|
|
||||||
| E011-05 | Numeric comparison | <span class="text-success">Yes</span> | |
|
|
||||||
| E011-06 | Implicit casting among the numeric data types | <span class="text-danger">No</span> | ANSI SQL allows arbitrary implicit cast between numeric types, while ClickHouse relies on functions having multiple overloads instead of implicit cast |
|
|
||||||
| **E021** | **Character string types** | <span class="text-warning">Partial</span> | |
|
|
||||||
| E021-01 | CHARACTER data type | <span class="text-success">Yes</span> | |
|
|
||||||
| E021-02 | CHARACTER VARYING data type | <span class="text-success">Yes</span> | |
|
|
||||||
| E021-03 | Character literals | <span class="text-success">Yes</span> | |
|
|
||||||
| E021-04 | CHARACTER_LENGTH function | <span class="text-warning">Partial</span> | No `USING` clause |
|
|
||||||
| E021-05 | OCTET_LENGTH function | <span class="text-danger">No</span> | `LENGTH` behaves similarly |
|
|
||||||
| E021-06 | SUBSTRING | <span class="text-warning">Partial</span> | No support for `SIMILAR` and `ESCAPE` clauses, no `SUBSTRING_REGEX` variant |
|
|
||||||
| E021-07 | Character concatenation | <span class="text-warning">Partial</span> | No `COLLATE` clause |
|
|
||||||
| E021-08 | UPPER and LOWER functions | <span class="text-success">Yes</span> | |
|
|
||||||
| E021-09 | TRIM function | <span class="text-success">Yes</span> | |
|
|
||||||
| E021-10 | Implicit casting among the fixed-length and variable-length character string types | <span class="text-warning">Partial</span> | ANSI SQL allows arbitrary implicit cast between string types, while ClickHouse relies on functions having multiple overloads instead of implicit cast |
|
|
||||||
| E021-11 | POSITION function | <span class="text-warning">Partial</span> | No support for `IN` and `USING` clauses, no `POSITION_REGEX` variant |
|
|
||||||
| E021-12 | Character comparison | <span class="text-success">Yes</span> | |
|
|
||||||
| **E031** | **Identifiers** | <span class="text-warning">Partial</span>| |
|
|
||||||
| E031-01 | Delimited identifiers | <span class="text-warning">Partial</span> | Unicode literal support is limited |
|
|
||||||
| E031-02 | Lower case identifiers | <span class="text-success">Yes</span> | |
|
|
||||||
| E031-03 | Trailing underscore | <span class="text-success">Yes</span> | |
|
|
||||||
| **E051** | **Basic query specification** | <span class="text-warning">Partial</span>| |
|
|
||||||
| E051-01 | SELECT DISTINCT | <span class="text-success">Yes</span> | |
|
|
||||||
| E051-02 | GROUP BY clause | <span class="text-success">Yes</span> | |
|
|
||||||
| E051-04 | GROUP BY can contain columns not in `<select list>` | <span class="text-success">Yes</span> | |
|
|
||||||
| E051-05 | Select items can be renamed | <span class="text-success">Yes</span> | |
|
|
||||||
| E051-06 | HAVING clause | <span class="text-success">Yes</span> | |
|
|
||||||
| E051-07 | Qualified \* in select list | <span class="text-success">Yes</span> | |
|
|
||||||
| E051-08 | Correlation name in the FROM clause | <span class="text-success">Yes</span> | |
|
|
||||||
| E051-09 | Rename columns in the FROM clause | <span class="text-danger">No</span> | |
|
|
||||||
| **E061** | **Basic predicates and search conditions** | <span class="text-warning">Partial</span> | |
|
|
||||||
| E061-01 | Comparison predicate | <span class="text-success">Yes</span> | |
|
|
||||||
| E061-02 | BETWEEN predicate | <span class="text-warning">Partial</span> | No `SYMMETRIC` and `ASYMMETRIC` clause |
|
|
||||||
| E061-03 | IN predicate with list of values | <span class="text-success">Yes</span> | |
|
|
||||||
| E061-04 | LIKE predicate | <span class="text-success">Yes</span> | |
|
|
||||||
| E061-05 | LIKE predicate: ESCAPE clause | <span class="text-danger">No</span> | |
|
|
||||||
| E061-06 | NULL predicate | <span class="text-success">Yes</span> | |
|
|
||||||
| E061-07 | Quantified comparison predicate | <span class="text-danger">No</span> | |
|
|
||||||
| E061-08 | EXISTS predicate | <span class="text-danger">No</span> | |
|
|
||||||
| E061-09 | Subqueries in comparison predicate | <span class="text-success">Yes</span> | |
|
|
||||||
| E061-11 | Subqueries in IN predicate | <span class="text-success">Yes</span> | |
|
|
||||||
| E061-12 | Subqueries in quantified comparison predicate | <span class="text-danger">No</span> | |
|
|
||||||
| E061-13 | Correlated subqueries | <span class="text-danger">No</span> | |
|
|
||||||
| E061-14 | Search condition | <span class="text-success">Yes</span> | |
|
|
||||||
| **E071** | **Basic query expressions** | <span class="text-warning">Partial</span> | |
|
|
||||||
| E071-01 | UNION DISTINCT table operator | <span class="text-success">Yes</span> | |
|
|
||||||
| E071-02 | UNION ALL table operator | <span class="text-success">Yes</span> | |
|
|
||||||
| E071-03 | EXCEPT DISTINCT table operator | <span class="text-danger">No</span> | |
|
|
||||||
| E071-05 | Columns combined via table operators need not have exactly the same data type | <span class="text-success">Yes</span> | |
|
|
||||||
| E071-06 | Table operators in subqueries | <span class="text-success">Yes</span> | |
|
|
||||||
| **E081** | **Basic privileges** | <span class="text-success">Yes</span> |
|
|
||||||
| E081-01 | SELECT privilege at the table level | <span class="text-success">Yes</span> |
|
|
||||||
| E081-02 | DELETE privilege | |
|
|
||||||
| E081-03 | INSERT privilege at the table level | <span class="text-success">Yes</span> |
|
|
||||||
| E081-04 | UPDATE privilege at the table level | <span class="text-success">Yes</span> |
|
|
||||||
| E081-05 | UPDATE privilege at the column level | |
|
|
||||||
| E081-06 | REFERENCES privilege at the table level | | |
|
|
||||||
| E081-07 | REFERENCES privilege at the column level | | |
|
|
||||||
| E081-08 | WITH GRANT OPTION | <span class="text-success">Yes</span> | |
|
|
||||||
| E081-09 | USAGE privilege | | |
|
|
||||||
| E081-10 | EXECUTE privilege | | |
|
|
||||||
| **E091** | **Set functions** |<span class="text-success">Yes</span> |
|
|
||||||
| E091-01 | AVG | <span class="text-success">Yes</span> | |
|
|
||||||
| E091-02 | COUNT | <span class="text-success">Yes</span> | |
|
|
||||||
| E091-03 | MAX | <span class="text-success">Yes</span> | |
|
|
||||||
| E091-04 | MIN | <span class="text-success">Yes</span> | |
|
|
||||||
| E091-05 | SUM | <span class="text-success">Yes</span> | |
|
|
||||||
| E091-06 | ALL quantifier | <span class="text-success">Yes</span> | |
|
|
||||||
| E091-07 | DISTINCT quantifier | <span class="text-success">Yes</span> | Not all aggregate functions supported |
|
|
||||||
| **E101** | **Basic data manipulation** | <span class="text-warning">Partial</span> | |
|
|
||||||
| E101-01 | INSERT statement | <span class="text-success">Yes</span> | Note: primary key in ClickHouse does not imply the `UNIQUE` constraint |
|
|
||||||
| E101-03 | Searched UPDATE statement | <span class="text-warning">Partial</span> | There’s an `ALTER UPDATE` statement for batch data modification |
|
|
||||||
| E101-04 | Searched DELETE statement | <span class="text-warning">Partial</span> | There’s an `ALTER DELETE` statement for batch data removal |
|
|
||||||
| **E111** | **Single row SELECT statement** | <span class="text-danger">No</span> | |
|
|
||||||
| **E121** | **Basic cursor support** | <span class="text-danger">No</span> | |
|
|
||||||
| E121-01 | DECLARE CURSOR | <span class="text-danger">No</span> | |
|
|
||||||
| E121-02 | ORDER BY columns need not be in select list | <span class="text-success">Yes</span> | |
|
|
||||||
| E121-03 | Value expressions in ORDER BY clause | <span class="text-success">Yes</span> | |
|
|
||||||
| E121-04 | OPEN statement | <span class="text-danger">No</span> | |
|
|
||||||
| E121-06 | Positioned UPDATE statement | <span class="text-danger">No</span> | |
|
|
||||||
| E121-07 | Positioned DELETE statement | <span class="text-danger">No</span> | |
|
|
||||||
| E121-08 | CLOSE statement | <span class="text-danger">No</span> | |
|
|
||||||
| E121-10 | FETCH statement: implicit NEXT | <span class="text-danger">No</span> | |
|
|
||||||
| E121-17 | WITH HOLD cursors | <span class="text-danger">No</span> | |
|
|
||||||
| **E131** | **Null value support (nulls in lieu of values)** | <span class="text-success">Yes</span> | Some restrictions apply |
|
|
||||||
| **E141** | **Basic integrity constraints** | <span class="text-warning">Partial</span> | |
|
|
||||||
| E141-01 | NOT NULL constraints | <span class="text-success">Yes</span> | Note: `NOT NULL` is implied for table columns by default |
|
|
||||||
| E141-02 | UNIQUE constraint of NOT NULL columns | <span class="text-danger">No</span> | |
|
|
||||||
| E141-03 | PRIMARY KEY constraints | <span class="text-warning">Partial</span> | |
|
|
||||||
| E141-04 | Basic FOREIGN KEY constraint with the NO ACTION default for both referential delete action and referential update action | <span class="text-danger">No</span> | |
|
|
||||||
| E141-06 | CHECK constraint | <span class="text-success">Yes</span> | |
|
|
||||||
| E141-07 | Column defaults | <span class="text-success">Yes</span> | |
|
|
||||||
| E141-08 | NOT NULL inferred on PRIMARY KEY | <span class="text-success">Yes</span> | |
|
|
||||||
| E141-10 | Names in a foreign key can be specified in any order | <span class="text-danger">No</span> | |
|
|
||||||
| **E151** | **Transaction support** | <span class="text-danger">No</span> | |
|
|
||||||
| E151-01 | COMMIT statement | <span class="text-danger">No</span> | |
|
|
||||||
| E151-02 | ROLLBACK statement | <span class="text-danger">No</span> | |
|
|
||||||
| **E152** | **Basic SET TRANSACTION statement** | <span class="text-danger">No</span> | |
|
|
||||||
| E152-01 | SET TRANSACTION statement: ISOLATION LEVEL SERIALIZABLE clause | <span class="text-danger">No</span> | |
|
|
||||||
| E152-02 | SET TRANSACTION statement: READ ONLY and READ WRITE clauses | <span class="text-danger">No</span> | |
|
|
||||||
| **E153** | **Updatable queries with subqueries** | <span class="text-success">Yes</span> | |
|
|
||||||
| **E161** | **SQL comments using leading double minus** | <span class="text-success">Yes</span> | |
|
|
||||||
| **E171** | **SQLSTATE support** | <span class="text-danger">No</span> | |
|
|
||||||
| **E182** | **Host language binding** | <span class="text-danger">No</span> | |
|
|
||||||
| **F031** | **Basic schema manipulation** | <span class="text-warning">Partial</span>| |
|
|
||||||
| F031-01 | CREATE TABLE statement to create persistent base tables | <span class="text-warning">Partial</span> | No `SYSTEM VERSIONING`, `ON COMMIT`, `GLOBAL`, `LOCAL`, `PRESERVE`, `DELETE`, `REF IS`, `WITH OPTIONS`, `UNDER`, `LIKE`, `PERIOD FOR` clauses and no support for user resolved data types |
|
|
||||||
| F031-02 | CREATE VIEW statement | <span class="text-warning">Partial</span> | No `RECURSIVE`, `CHECK`, `UNDER`, `WITH OPTIONS` clauses and no support for user resolved data types |
|
|
||||||
| F031-03 | GRANT statement | <span class="text-success">Yes</span> | |
|
|
||||||
| F031-04 | ALTER TABLE statement: ADD COLUMN clause | <span class="text-success">Yes</span> | No support for `GENERATED` clause and system time period |
|
|
||||||
| F031-13 | DROP TABLE statement: RESTRICT clause | <span class="text-danger">No</span> | |
|
|
||||||
| F031-16 | DROP VIEW statement: RESTRICT clause | <span class="text-danger">No</span> | |
|
|
||||||
| F031-19 | REVOKE statement: RESTRICT clause | <span class="text-danger">No</span> | |
|
|
||||||
| **F041** | **Basic joined table** | <span class="text-warning">Partial</span> | |
|
|
||||||
| F041-01 | Inner join (but not necessarily the INNER keyword) | <span class="text-success">Yes</span> | |
|
|
||||||
| F041-02 | INNER keyword | <span class="text-success">Yes</span> | |
|
|
||||||
| F041-03 | LEFT OUTER JOIN | <span class="text-success">Yes</span> | |
|
|
||||||
| F041-04 | RIGHT OUTER JOIN | <span class="text-success">Yes</span> | |
|
|
||||||
| F041-05 | Outer joins can be nested | <span class="text-success">Yes</span> | |
|
|
||||||
| F041-07 | The inner table in a left or right outer join can also be used in an inner join | <span class="text-success">Yes</span> | |
|
|
||||||
| F041-08 | All comparison operators are supported (rather than just =) | <span class="text-danger">No</span> | |
|
|
||||||
| **F051** | **Basic date and time** | <span class="text-warning">Partial</span> | |
|
|
||||||
| F051-01 | DATE data type (including support of DATE literal) | <span class="text-success">Yes</span> | |
|
|
||||||
| F051-02 | TIME data type (including support of TIME literal) with fractional seconds precision of at least 0 | <span class="text-danger">No</span> | |
|
|
||||||
| F051-03 | TIMESTAMP data type (including support of TIMESTAMP literal) with fractional seconds precision of at least 0 and 6 | <span class="text-success">Yes</span> | |
|
|
||||||
| F051-04 | Comparison predicate on DATE, TIME, and TIMESTAMP data types | <span class="text-success">Yes</span> | |
|
|
||||||
| F051-05 | Explicit CAST between datetime types and character string types | <span class="text-success">Yes</span> | |
|
|
||||||
| F051-06 | CURRENT_DATE | <span class="text-danger">No</span> | `today()` is similar |
|
|
||||||
| F051-07 | LOCALTIME | <span class="text-danger">No</span> | `now()` is similar |
|
|
||||||
| F051-08 | LOCALTIMESTAMP | <span class="text-danger">No</span> | |
|
|
||||||
| **F081** | **UNION and EXCEPT in views** | <span class="text-warning">Partial</span> | |
|
|
||||||
| **F131** | **Grouped operations** | <span class="text-warning">Partial</span> | |
|
|
||||||
| F131-01 | WHERE, GROUP BY, and HAVING clauses supported in queries with grouped views | <span class="text-success">Yes</span> | |
|
|
||||||
| F131-02 | Multiple tables supported in queries with grouped views | <span class="text-success">Yes</span> | |
|
|
||||||
| F131-03 | Set functions supported in queries with grouped views | <span class="text-success">Yes</span> | |
|
|
||||||
| F131-04 | Subqueries with GROUP BY and HAVING clauses and grouped views | <span class="text-success">Yes</span> | |
|
|
||||||
| F131-05 | Single row SELECT with GROUP BY and HAVING clauses and grouped views | <span class="text-danger">No</span> | |
|
|
||||||
| **F181** | **Multiple module support** | <span class="text-danger">No</span> | |
|
|
||||||
| **F201** | **CAST function** | <span class="text-success">Yes</span> | |
|
|
||||||
| **F221** | **Explicit defaults** | <span class="text-danger">No</span> | |
|
|
||||||
| **F261** | **CASE expression** | <span class="text-success">Yes</span> | |
|
|
||||||
| F261-01 | Simple CASE | <span class="text-success">Yes</span> | |
|
|
||||||
| F261-02 | Searched CASE | <span class="text-success">Yes</span> | |
|
|
||||||
| F261-03 | NULLIF | <span class="text-success">Yes</span> | |
|
|
||||||
| F261-04 | COALESCE | <span class="text-success">Yes</span> | |
|
|
||||||
| **F311** | **Schema definition statement** | <span class="text-warning">Partial</span> | |
|
|
||||||
| F311-01 | CREATE SCHEMA | <span class="text-warning">Partial</span> | See CREATE DATABASE |
|
|
||||||
| F311-02 | CREATE TABLE for persistent base tables | <span class="text-success">Yes</span> | |
|
|
||||||
| F311-03 | CREATE VIEW | <span class="text-success">Yes</span> | |
|
|
||||||
| F311-04 | CREATE VIEW: WITH CHECK OPTION | <span class="text-danger">No</span> | |
|
|
||||||
| F311-05 | GRANT statement | <span class="text-success">Yes</span> | |
|
|
||||||
| **F471** | **Scalar subquery values** | <span class="text-success">Yes</span> | |
|
|
||||||
| **F481** | **Expanded NULL predicate** | <span class="text-success">Yes</span> | |
|
|
||||||
| **F812** | **Basic flagging** | <span class="text-danger">No</span> | |
|
|
||||||
| **S011** | **Distinct data types** | | |
|
|
||||||
| **T321** | **Basic SQL-invoked routines** | <span class="text-danger">No</span> | |
|
|
||||||
| T321-01 | User-defined functions with no overloading | <span class="text-danger">No</span> | |
|
|
||||||
| T321-02 | User-defined stored procedures with no overloading | <span class="text-danger">No</span> | |
|
|
||||||
| T321-03 | Function invocation | <span class="text-danger">No</span> | |
|
|
||||||
| T321-04 | CALL statement | <span class="text-danger">No</span> | |
|
|
||||||
| T321-05 | RETURN statement | <span class="text-danger">No</span> | |
|
|
||||||
| **T631** | **IN predicate with one list element** | <span class="text-success">Yes</span> | |
|
|
||||||
@ -86,7 +86,7 @@ The table below describes how different interval kinds of `Interval` data type a
|
|||||||
|
|
||||||
### Aggregate function parameter binary encoding
|
### Aggregate function parameter binary encoding
|
||||||
|
|
||||||
The table below describes how parameters of `AggragateFunction` and `SimpleAggregateFunction` are encoded.
|
The table below describes how parameters of `AggregateFunction` and `SimpleAggregateFunction` are encoded.
|
||||||
The encoding of a parameter consists of 1 byte indicating the type of the parameter and the value itself.
|
The encoding of a parameter consists of 1 byte indicating the type of the parameter and the value itself.
|
||||||
|
|
||||||
| Parameter type | Binary encoding |
|
| Parameter type | Binary encoding |
|
||||||
@ -106,7 +106,7 @@ The encoding of a parameter consists of 1 byte indicating the type of the parame
|
|||||||
| `String` | `0x0C<var_uint_size><data>` |
|
| `String` | `0x0C<var_uint_size><data>` |
|
||||||
| `Array` | `0x0D<var_uint_size><value_encoding_1>...<value_encoding_N>` |
|
| `Array` | `0x0D<var_uint_size><value_encoding_1>...<value_encoding_N>` |
|
||||||
| `Tuple` | `0x0E<var_uint_size><value_encoding_1>...<value_encoding_N>` |
|
| `Tuple` | `0x0E<var_uint_size><value_encoding_1>...<value_encoding_N>` |
|
||||||
| `Map` | `0x0F<var_uint_size><key_encoding_1><value_encoding_1>...<key_endoding_N><value_encoding_N>` |
|
| `Map` | `0x0F<var_uint_size><key_encoding_1><value_encoding_1>...<key_encoding_N><value_encoding_N>` |
|
||||||
| `IPv4` | `0x10<uint32_little_endian_value>` |
|
| `IPv4` | `0x10<uint32_little_endian_value>` |
|
||||||
| `IPv6` | `0x11<uint128_little_endian_value>` |
|
| `IPv6` | `0x11<uint128_little_endian_value>` |
|
||||||
| `UUID` | `0x12<uuid_value>` |
|
| `UUID` | `0x12<uuid_value>` |
|
||||||
|
|||||||
@ -2010,6 +2010,38 @@ Result:
|
|||||||
|
|
||||||
Converts a date, or date with time, to a UInt8 number containing the ISO Week number.
|
Converts a date, or date with time, to a UInt8 number containing the ISO Week number.
|
||||||
|
|
||||||
|
**Syntax**
|
||||||
|
|
||||||
|
```sql
|
||||||
|
toISOWeek(value)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Arguments**
|
||||||
|
|
||||||
|
- `value` — The value with date or date with time.
|
||||||
|
|
||||||
|
**Returned value**
|
||||||
|
|
||||||
|
- `value` converted to the current ISO week number. [UInt8](../data-types/int-uint.md).
|
||||||
|
|
||||||
|
**Example**
|
||||||
|
|
||||||
|
Query:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT
|
||||||
|
toISOWeek(toDate('2024/10/02')) AS week1,
|
||||||
|
toISOWeek(toDateTime('2024/10/02 01:30:00')) AS week2
|
||||||
|
```
|
||||||
|
|
||||||
|
Response:
|
||||||
|
|
||||||
|
```response
|
||||||
|
┌─week1─┬─week2─┐
|
||||||
|
│ 40 │ 40 │
|
||||||
|
└───────┴───────┘
|
||||||
|
```
|
||||||
|
|
||||||
## toWeek
|
## toWeek
|
||||||
|
|
||||||
This function returns the week number for date or datetime. The two-argument form of `toWeek()` enables you to specify whether the week starts on Sunday or Monday and whether the return value should be in the range from 0 to 53 or from 1 to 53. If the mode argument is omitted, the default mode is 0.
|
This function returns the week number for date or datetime. The two-argument form of `toWeek()` enables you to specify whether the week starts on Sunday or Monday and whether the return value should be in the range from 0 to 53 or from 1 to 53. If the mode argument is omitted, the default mode is 0.
|
||||||
@ -2901,7 +2933,42 @@ The same as ‘today() - 1’.
|
|||||||
|
|
||||||
## timeSlot
|
## timeSlot
|
||||||
|
|
||||||
Rounds the time to the half hour.
|
Round the time to the start of a half-an-hour length interval.
|
||||||
|
|
||||||
|
**Syntax**
|
||||||
|
|
||||||
|
```sql
|
||||||
|
timeSlot(time[, time_zone])
|
||||||
|
```
|
||||||
|
|
||||||
|
**Arguments**
|
||||||
|
|
||||||
|
- `time` — Time to round to the start of a half-an-hour length interval. [DateTime](../data-types/datetime.md)/[Date32](../data-types/date32.md)/[DateTime64](../data-types/datetime64.md).
|
||||||
|
- `time_zone` — A String type const value or an expression representing the time zone. [String](../data-types/string.md).
|
||||||
|
|
||||||
|
:::note
|
||||||
|
Though this function can take values of the extended types `Date32` and `DateTime64` as an argument, passing it a time outside the normal range (year 1970 to 2149 for `Date` / 2106 for `DateTime`) will produce wrong results.
|
||||||
|
:::
|
||||||
|
|
||||||
|
**Return type**
|
||||||
|
|
||||||
|
- Returns the time rounded to the start of a half-an-hour length interval. [DateTime](../data-types/datetime.md).
|
||||||
|
|
||||||
|
**Example**
|
||||||
|
|
||||||
|
Query:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT timeSlot(toDateTime('2000-01-02 03:04:05', 'UTC'));
|
||||||
|
```
|
||||||
|
|
||||||
|
Result:
|
||||||
|
|
||||||
|
```response
|
||||||
|
┌─timeSlot(toDateTime('2000-01-02 03:04:05', 'UTC'))─┐
|
||||||
|
│ 2000-01-02 03:00:00 │
|
||||||
|
└────────────────────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
## toYYYYMM
|
## toYYYYMM
|
||||||
|
|
||||||
|
|||||||
@ -244,24 +244,42 @@ SELECT IPv6CIDRToRange(toIPv6('2001:0db8:0000:85a3:0000:0000:ac1f:8001'), 32);
|
|||||||
└────────────────────────────────────────────────────────────────────────┘
|
└────────────────────────────────────────────────────────────────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
## toIPv4(string)
|
## toIPv4
|
||||||
|
|
||||||
An alias to `IPv4StringToNum()` that takes a string form of IPv4 address and returns value of [IPv4](../data-types/ipv4.md) type, which is binary equal to value returned by `IPv4StringToNum()`.
|
Like [`IPv4StringToNum`](##IPv4NumToString(num)) but takes a string form of IPv4 address and returns value of [IPv4](../data-types/ipv4.md) type.
|
||||||
|
|
||||||
|
**Syntax**
|
||||||
|
|
||||||
|
```sql
|
||||||
|
toIPv4(string)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Arguments**
|
||||||
|
|
||||||
|
- `string` — IPv4 address. [String](../data-types/string.md).
|
||||||
|
|
||||||
|
**Returned value**
|
||||||
|
|
||||||
|
- `string` converted to the IPv4 address. [IPv4](../data-types/ipv4.md).
|
||||||
|
|
||||||
|
**Examples**
|
||||||
|
|
||||||
|
Query:
|
||||||
|
|
||||||
``` sql
|
``` sql
|
||||||
WITH
|
SELECT toIPv4('171.225.130.45');
|
||||||
'171.225.130.45' as IPv4_string
|
|
||||||
SELECT
|
|
||||||
toTypeName(IPv4StringToNum(IPv4_string)),
|
|
||||||
toTypeName(toIPv4(IPv4_string))
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Result:
|
||||||
|
|
||||||
``` text
|
``` text
|
||||||
┌─toTypeName(IPv4StringToNum(IPv4_string))─┬─toTypeName(toIPv4(IPv4_string))─┐
|
┌─toIPv4('171.225.130.45')─┐
|
||||||
│ UInt32 │ IPv4 │
|
│ 171.225.130.45 │
|
||||||
└──────────────────────────────────────────┴─────────────────────────────────┘
|
└──────────────────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Query:
|
||||||
|
|
||||||
``` sql
|
``` sql
|
||||||
WITH
|
WITH
|
||||||
'171.225.130.45' as IPv4_string
|
'171.225.130.45' as IPv4_string
|
||||||
@ -270,91 +288,124 @@ SELECT
|
|||||||
hex(toIPv4(IPv4_string))
|
hex(toIPv4(IPv4_string))
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Result:
|
||||||
|
|
||||||
``` text
|
``` text
|
||||||
┌─hex(IPv4StringToNum(IPv4_string))─┬─hex(toIPv4(IPv4_string))─┐
|
┌─hex(IPv4StringToNum(IPv4_string))─┬─hex(toIPv4(IPv4_string))─┐
|
||||||
│ ABE1822D │ ABE1822D │
|
│ ABE1822D │ ABE1822D │
|
||||||
└───────────────────────────────────┴──────────────────────────┘
|
└───────────────────────────────────┴──────────────────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
## toIPv4OrDefault(string)
|
## toIPv4OrDefault
|
||||||
|
|
||||||
Same as `toIPv4`, but if the IPv4 address has an invalid format, it returns `0.0.0.0` (0 IPv4).
|
Same as `toIPv4`, but if the IPv4 address has an invalid format, it returns `0.0.0.0` (0 IPv4), or the provided IPv4 default.
|
||||||
|
|
||||||
**Syntax**
|
**Syntax**
|
||||||
|
|
||||||
```sql
|
```sql
|
||||||
toIPv4OrDefault(value)
|
toIPv4OrDefault(string[, default])
|
||||||
```
|
```
|
||||||
|
|
||||||
**Arguments**
|
**Arguments**
|
||||||
|
|
||||||
- `value` — A string-encoded IPv4 address. [String](../data-types/string.md)
|
- `value` — IP address. [String](../data-types/string.md).
|
||||||
|
- `default` (optional) — The value to return if `string` has an invalid format. [IPv4](../data-types/ipv4.md).
|
||||||
|
|
||||||
**Returned value**
|
**Returned value**
|
||||||
|
|
||||||
- `value` converted to an IPv4 address. [IPv4](../data-types/ipv4.md).
|
- `string` converted to the current IPv4 address. [String](../data-types/string.md).
|
||||||
|
|
||||||
**Example**
|
**Example**
|
||||||
|
|
||||||
Query:
|
Query:
|
||||||
|
|
||||||
```sql
|
```sql
|
||||||
|
WITH
|
||||||
|
'::ffff:127.0.0.1' AS valid_IPv6_string,
|
||||||
|
'fe80:2030:31:24' AS invalid_IPv6_string
|
||||||
SELECT
|
SELECT
|
||||||
toIPv4OrDefault('192.168.0.1') AS s1,
|
toIPv4OrDefault(valid_IPv6_string) AS valid,
|
||||||
toIPv4OrDefault('192.168.0') AS s2
|
toIPv4OrDefault(invalid_IPv6_string) AS default,
|
||||||
|
toIPv4OrDefault(invalid_IPv6_string, toIPv4('1.1.1.1')) AS provided_default;
|
||||||
```
|
```
|
||||||
|
|
||||||
Result:
|
Result:
|
||||||
|
|
||||||
```response
|
```response
|
||||||
┌─s1──────────┬─s2──────┐
|
┌─valid───┬─default─┬─provided_default─┐
|
||||||
│ 192.168.0.1 │ 0.0.0.0 │
|
│ 0.0.0.0 │ 0.0.0.0 │ 1.1.1.1 │
|
||||||
└─────────────┴─────────┘
|
└─────────┴─────────┴──────────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
## toIPv4OrNull(string)
|
## toIPv4OrNull
|
||||||
|
|
||||||
Same as `toIPv4`, but if the IPv4 address has an invalid format, it returns null.
|
Same as [`toIPv4`](#toipv4), but if the IPv4 address has an invalid format, it returns null.
|
||||||
|
|
||||||
**Syntax**
|
**Syntax**
|
||||||
|
|
||||||
```sql
|
```sql
|
||||||
toIPv4OrNull(value)
|
toIPv4OrNull(string)
|
||||||
```
|
```
|
||||||
|
|
||||||
**Arguments**
|
**Arguments**
|
||||||
|
|
||||||
- `value` — A string-encoded IPv4 address. [String](../data-types/string.md)
|
- `string` — IP address. [String](../data-types/string.md).
|
||||||
|
|
||||||
**Returned value**
|
**Returned value**
|
||||||
|
|
||||||
- `value` converted to an IPv4 address. [IPv4](../data-types/ipv4.md).
|
- `string` converted to the current IPv4 address, or null if `string` is an invalid address. [String](../data-types/string.md).
|
||||||
|
|
||||||
**Example**
|
**Example**
|
||||||
|
|
||||||
Query:
|
Query:
|
||||||
|
|
||||||
```sql
|
``` sql
|
||||||
SELECT
|
WITH 'fe80:2030:31:24' AS invalid_IPv6_string
|
||||||
toIPv4OrNull('192.168.0.1') AS s1,
|
SELECT toIPv4OrNull(invalid_IPv6_string);
|
||||||
toIPv4OrNull('192.168.0') AS s2
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Result:
|
Result:
|
||||||
|
|
||||||
```response
|
``` text
|
||||||
┌─s1──────────┬─s2───┐
|
┌─toIPv4OrNull(invalid_IPv6_string)─┐
|
||||||
│ 192.168.0.1 │ ᴺᵁᴸᴸ │
|
│ ᴺᵁᴸᴸ │
|
||||||
└─────────────┴──────┘
|
└───────────────────────────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
## toIPv6OrDefault(string)
|
## toIPv4OrZero
|
||||||
|
|
||||||
Same as `toIPv6`, but if the IPv6 address has an invalid format, it returns `::` (0 IPv6).
|
Same as [`toIPv4`](#toipv4), but if the IPv4 address has an invalid format, it returns `0.0.0.0`.
|
||||||
|
|
||||||
## toIPv6OrNull(string)
|
**Syntax**
|
||||||
|
|
||||||
Same as `toIPv6`, but if the IPv6 address has an invalid format, it returns null.
|
```sql
|
||||||
|
toIPv4OrZero(string)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Arguments**
|
||||||
|
|
||||||
|
- `string` — IP address. [String](../data-types/string.md).
|
||||||
|
|
||||||
|
**Returned value**
|
||||||
|
|
||||||
|
- `string` converted to the current IPv4 address, or `0.0.0.0` if `string` is an invalid address. [String](../data-types/string.md).
|
||||||
|
|
||||||
|
**Example**
|
||||||
|
|
||||||
|
Query:
|
||||||
|
|
||||||
|
``` sql
|
||||||
|
WITH 'Not an IP address' AS invalid_IPv6_string
|
||||||
|
SELECT toIPv4OrZero(invalid_IPv6_string);
|
||||||
|
```
|
||||||
|
|
||||||
|
Result:
|
||||||
|
|
||||||
|
``` text
|
||||||
|
┌─toIPv4OrZero(invalid_IPv6_string)─┐
|
||||||
|
│ 0.0.0.0 │
|
||||||
|
└───────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
## toIPv6
|
## toIPv6
|
||||||
|
|
||||||
@ -371,7 +422,7 @@ toIPv6(string)
|
|||||||
|
|
||||||
**Argument**
|
**Argument**
|
||||||
|
|
||||||
- `string` — IP address. [String](../data-types/string.md)
|
- `string` — IP address. [String](../data-types/string.md).
|
||||||
|
|
||||||
**Returned value**
|
**Returned value**
|
||||||
|
|
||||||
@ -410,6 +461,117 @@ Result:
|
|||||||
└─────────────────────┘
|
└─────────────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## toIPv6OrDefault
|
||||||
|
|
||||||
|
Same as [`toIPv6`](#toipv6), but if the IPv6 address has an invalid format, it returns `::` (0 IPv6) or the provided IPv6 default.
|
||||||
|
|
||||||
|
**Syntax**
|
||||||
|
|
||||||
|
```sql
|
||||||
|
toIPv6OrDefault(string[, default])
|
||||||
|
```
|
||||||
|
|
||||||
|
**Argument**
|
||||||
|
|
||||||
|
- `string` — IP address. [String](../data-types/string.md).
|
||||||
|
- `default` (optional) — The value to return if `string` has an invalid format. [IPv6](../data-types/ipv6.md).
|
||||||
|
|
||||||
|
**Returned value**
|
||||||
|
|
||||||
|
- IPv6 address [IPv6](../data-types/ipv6.md), otherwise `::` or the provided optional default if `string` has an invalid format.
|
||||||
|
|
||||||
|
**Example**
|
||||||
|
|
||||||
|
Query:
|
||||||
|
|
||||||
|
``` sql
|
||||||
|
WITH
|
||||||
|
'127.0.0.1' AS valid_IPv4_string,
|
||||||
|
'127.0.0.1.6' AS invalid_IPv4_string
|
||||||
|
SELECT
|
||||||
|
toIPv6OrDefault(valid_IPv4_string) AS valid,
|
||||||
|
toIPv6OrDefault(invalid_IPv4_string) AS default,
|
||||||
|
toIPv6OrDefault(invalid_IPv4_string, toIPv6('1.1.1.1')) AS provided_default
|
||||||
|
```
|
||||||
|
|
||||||
|
Result:
|
||||||
|
|
||||||
|
``` text
|
||||||
|
┌─valid────────────┬─default─┬─provided_default─┐
|
||||||
|
│ ::ffff:127.0.0.1 │ :: │ ::ffff:1.1.1.1 │
|
||||||
|
└──────────────────┴─────────┴──────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
## toIPv6OrNull
|
||||||
|
|
||||||
|
Same as [`toIPv6`](#toipv6), but if the IPv6 address has an invalid format, it returns null.
|
||||||
|
|
||||||
|
**Syntax**
|
||||||
|
|
||||||
|
```sql
|
||||||
|
toIPv6OrNull(string)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Argument**
|
||||||
|
|
||||||
|
- `string` — IP address. [String](../data-types/string.md).
|
||||||
|
|
||||||
|
**Returned value**
|
||||||
|
|
||||||
|
- IP address. [IPv6](../data-types/ipv6.md), or null if `string` is not a valid format.
|
||||||
|
|
||||||
|
**Example**
|
||||||
|
|
||||||
|
Query:
|
||||||
|
|
||||||
|
``` sql
|
||||||
|
WITH '127.0.0.1.6' AS invalid_IPv4_string
|
||||||
|
SELECT toIPv6OrNull(invalid_IPv4_string);
|
||||||
|
```
|
||||||
|
|
||||||
|
Result:
|
||||||
|
|
||||||
|
``` text
|
||||||
|
┌─toIPv6OrNull(invalid_IPv4_string)─┐
|
||||||
|
│ ᴺᵁᴸᴸ │
|
||||||
|
└───────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
## toIPv6OrZero
|
||||||
|
|
||||||
|
Same as [`toIPv6`](#toipv6), but if the IPv6 address has an invalid format, it returns `::`.
|
||||||
|
|
||||||
|
**Syntax**
|
||||||
|
|
||||||
|
```sql
|
||||||
|
toIPv6OrZero(string)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Argument**
|
||||||
|
|
||||||
|
- `string` — IP address. [String](../data-types/string.md).
|
||||||
|
|
||||||
|
**Returned value**
|
||||||
|
|
||||||
|
- IP address. [IPv6](../data-types/ipv6.md), or `::` if `string` is not a valid format.
|
||||||
|
|
||||||
|
**Example**
|
||||||
|
|
||||||
|
Query:
|
||||||
|
|
||||||
|
``` sql
|
||||||
|
WITH '127.0.0.1.6' AS invalid_IPv4_string
|
||||||
|
SELECT toIPv6OrZero(invalid_IPv4_string);
|
||||||
|
```
|
||||||
|
|
||||||
|
Result:
|
||||||
|
|
||||||
|
``` text
|
||||||
|
┌─toIPv6OrZero(invalid_IPv4_string)─┐
|
||||||
|
│ :: │
|
||||||
|
└───────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
## IPv6StringToNumOrDefault(s)
|
## IPv6StringToNumOrDefault(s)
|
||||||
|
|
||||||
Same as `toIPv6`, but if the IPv6 address has an invalid format, it returns 0.
|
Same as `toIPv6`, but if the IPv6 address has an invalid format, it returns 0.
|
||||||
|
|||||||
@ -4390,3 +4390,37 @@ Result:
|
|||||||
1. │ ['{ArraySizes}','{ArrayElements, TupleElement(keys), Regular}','{ArrayElements, TupleElement(values), Regular}'] │
|
1. │ ['{ArraySizes}','{ArrayElements, TupleElement(keys), Regular}','{ArrayElements, TupleElement(values), Regular}'] │
|
||||||
└──────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
|
└──────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## globalVariable
|
||||||
|
|
||||||
|
Takes constant string argument and returns the value of global variable with that name. It is intended for compatibility with MySQL.
|
||||||
|
|
||||||
|
**Syntax**
|
||||||
|
|
||||||
|
```sql
|
||||||
|
globalVariable(name)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Arguments**
|
||||||
|
|
||||||
|
- `name` — Global variable name. [String](../data-types/string.md).
|
||||||
|
|
||||||
|
**Returned value**
|
||||||
|
|
||||||
|
- Returns the value of variable `name`.
|
||||||
|
|
||||||
|
**Example**
|
||||||
|
|
||||||
|
Query:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT globalVariable('max_allowed_packet');
|
||||||
|
```
|
||||||
|
|
||||||
|
Result:
|
||||||
|
|
||||||
|
```response
|
||||||
|
┌─globalVariable('max_allowed_packet')─┐
|
||||||
|
│ 67108864 │
|
||||||
|
└──────────────────────────────────────┘
|
||||||
|
```
|
||||||
@ -5230,15 +5230,52 @@ Result:
|
|||||||
|
|
||||||
Also see the `toUnixTimestamp` function.
|
Also see the `toUnixTimestamp` function.
|
||||||
|
|
||||||
## toFixedString(s, N)
|
## toFixedString
|
||||||
|
|
||||||
Converts a [String](../data-types/string.md) type argument to a [FixedString(N)](../data-types/fixedstring.md) type (a string of fixed length N).
|
Converts a [String](../data-types/string.md) type argument to a [FixedString(N)](../data-types/fixedstring.md) type (a string of fixed length N).
|
||||||
If the string has fewer bytes than N, it is padded with null bytes to the right. If the string has more bytes than N, an exception is thrown.
|
If the string has fewer bytes than N, it is padded with null bytes to the right. If the string has more bytes than N, an exception is thrown.
|
||||||
|
|
||||||
## toStringCutToZero(s)
|
**Syntax**
|
||||||
|
|
||||||
|
```sql
|
||||||
|
toFixedString(s, N)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Arguments**
|
||||||
|
|
||||||
|
- `s` — A String to convert to a fixed string. [String](../data-types/string.md).
|
||||||
|
- `N` — Length N. [UInt8](../data-types/int-uint.md)
|
||||||
|
|
||||||
|
**Returned value**
|
||||||
|
|
||||||
|
- An N length fixed string of `s`. [FixedString](../data-types/fixedstring.md).
|
||||||
|
|
||||||
|
**Example**
|
||||||
|
|
||||||
|
Query:
|
||||||
|
|
||||||
|
``` sql
|
||||||
|
SELECT toFixedString('foo', 8) AS s;
|
||||||
|
```
|
||||||
|
|
||||||
|
Result:
|
||||||
|
|
||||||
|
```response
|
||||||
|
┌─s─────────────┐
|
||||||
|
│ foo\0\0\0\0\0 │
|
||||||
|
└───────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
## toStringCutToZero
|
||||||
|
|
||||||
Accepts a String or FixedString argument. Returns the String with the content truncated at the first zero byte found.
|
Accepts a String or FixedString argument. Returns the String with the content truncated at the first zero byte found.
|
||||||
|
|
||||||
|
**Syntax**
|
||||||
|
|
||||||
|
```sql
|
||||||
|
toStringCutToZero(s)
|
||||||
|
```
|
||||||
|
|
||||||
**Example**
|
**Example**
|
||||||
|
|
||||||
Query:
|
Query:
|
||||||
|
|||||||
@ -41,7 +41,7 @@ ORDER BY ts, event_type;
|
|||||||
│ 2020-01-02 00:00:00 │ imp │ 2 │
|
│ 2020-01-02 00:00:00 │ imp │ 2 │
|
||||||
└─────────────────────┴────────────┴─────────────────┘
|
└─────────────────────┴────────────┴─────────────────┘
|
||||||
|
|
||||||
-- Let's add the new measurment `cost`
|
-- Let's add the new measurement `cost`
|
||||||
-- and the new dimension `browser`.
|
-- and the new dimension `browser`.
|
||||||
|
|
||||||
ALTER TABLE events
|
ALTER TABLE events
|
||||||
|
|||||||
@ -46,7 +46,7 @@ The `CHECK TABLE` query supports the following table engines:
|
|||||||
- [StripeLog](../../engines/table-engines/log-family/stripelog.md)
|
- [StripeLog](../../engines/table-engines/log-family/stripelog.md)
|
||||||
- [MergeTree family](../../engines/table-engines/mergetree-family/mergetree.md)
|
- [MergeTree family](../../engines/table-engines/mergetree-family/mergetree.md)
|
||||||
|
|
||||||
Performed over the tables with another table engines causes an `NOT_IMPLEMETED` exception.
|
Performed over the tables with another table engines causes an `NOT_IMPLEMENTED` exception.
|
||||||
|
|
||||||
Engines from the `*Log` family do not provide automatic data recovery on failure. Use the `CHECK TABLE` query to track data loss in a timely manner.
|
Engines from the `*Log` family do not provide automatic data recovery on failure. Use the `CHECK TABLE` query to track data loss in a timely manner.
|
||||||
|
|
||||||
|
|||||||
@ -442,7 +442,7 @@ DEFLATE_QPL is not available in ClickHouse Cloud.
|
|||||||
|
|
||||||
### Specialized Codecs
|
### Specialized Codecs
|
||||||
|
|
||||||
These codecs are designed to make compression more effective by exploiting specific features of the data. Some of these codecs do not compress data themself, they instead preprocess the data such that a second compression stage using a general-purpose codec can achieve a higher data compression rate.
|
These codecs are designed to make compression more effective by exploiting specific features of the data. Some of these codecs do not compress data themselves, they instead preprocess the data such that a second compression stage using a general-purpose codec can achieve a higher data compression rate.
|
||||||
|
|
||||||
#### Delta
|
#### Delta
|
||||||
|
|
||||||
|
|||||||
@ -194,7 +194,7 @@ REFRESH EVERY 1 MONTH OFFSET 5 DAY 2 HOUR -- on 6th day of every month, at 2:00
|
|||||||
REFRESH EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE -- every other Saturday, at 3:10 pm
|
REFRESH EVERY 2 WEEK OFFSET 5 DAY 15 HOUR 10 MINUTE -- every other Saturday, at 3:10 pm
|
||||||
REFRESH EVERY 30 MINUTE -- at 00:00, 00:30, 01:00, 01:30, etc
|
REFRESH EVERY 30 MINUTE -- at 00:00, 00:30, 01:00, 01:30, etc
|
||||||
REFRESH AFTER 30 MINUTE -- 30 minutes after the previous refresh completes, no alignment with time of day
|
REFRESH AFTER 30 MINUTE -- 30 minutes after the previous refresh completes, no alignment with time of day
|
||||||
-- REFRESH AFTER 1 HOUR OFFSET 1 MINUTE -- syntax errror, OFFSET is not allowed with AFTER
|
-- REFRESH AFTER 1 HOUR OFFSET 1 MINUTE -- syntax error, OFFSET is not allowed with AFTER
|
||||||
REFRESH EVERY 1 WEEK 2 DAYS -- every 9 days, not on any particular day of the week or month;
|
REFRESH EVERY 1 WEEK 2 DAYS -- every 9 days, not on any particular day of the week or month;
|
||||||
-- specifically, when day number (since 1969-12-29) is divisible by 9
|
-- specifically, when day number (since 1969-12-29) is divisible by 9
|
||||||
REFRESH EVERY 5 MONTHS -- every 5 months, different months each year (as 12 is not divisible by 5);
|
REFRESH EVERY 5 MONTHS -- every 5 months, different months each year (as 12 is not divisible by 5);
|
||||||
|
|||||||
@ -6,7 +6,7 @@ sidebar_label: iceberg
|
|||||||
|
|
||||||
# iceberg Table Function
|
# iceberg Table Function
|
||||||
|
|
||||||
Provides a read-only table-like interface to Apache [Iceberg](https://iceberg.apache.org/) tables in Amazon S3, Azure or locally stored.
|
Provides a read-only table-like interface to Apache [Iceberg](https://iceberg.apache.org/) tables in Amazon S3, Azure, HDFS or locally stored.
|
||||||
|
|
||||||
## Syntax
|
## Syntax
|
||||||
|
|
||||||
@ -17,13 +17,16 @@ icebergS3(named_collection[, option=value [,..]])
|
|||||||
icebergAzure(connection_string|storage_account_url, container_name, blobpath, [,account_name], [,account_key] [,format] [,compression_method])
|
icebergAzure(connection_string|storage_account_url, container_name, blobpath, [,account_name], [,account_key] [,format] [,compression_method])
|
||||||
icebergAzure(named_collection[, option=value [,..]])
|
icebergAzure(named_collection[, option=value [,..]])
|
||||||
|
|
||||||
|
icebergHDFS(path_to_table, [,format] [,compression_method])
|
||||||
|
icebergHDFS(named_collection[, option=value [,..]])
|
||||||
|
|
||||||
icebergLocal(path_to_table, [,format] [,compression_method])
|
icebergLocal(path_to_table, [,format] [,compression_method])
|
||||||
icebergLocal(named_collection[, option=value [,..]])
|
icebergLocal(named_collection[, option=value [,..]])
|
||||||
```
|
```
|
||||||
|
|
||||||
## Arguments
|
## Arguments
|
||||||
|
|
||||||
Description of the arguments coincides with description of arguments in table functions `s3`, `azureBlobStorage` and `file` correspondingly.
|
Description of the arguments coincides with description of arguments in table functions `s3`, `azureBlobStorage`, `HDFS` and `file` correspondingly.
|
||||||
`format` stands for the format of data files in the Iceberg table.
|
`format` stands for the format of data files in the Iceberg table.
|
||||||
|
|
||||||
**Returned value**
|
**Returned value**
|
||||||
@ -36,7 +39,7 @@ SELECT * FROM icebergS3('http://test.s3.amazonaws.com/clickhouse-bucket/test_tab
|
|||||||
```
|
```
|
||||||
|
|
||||||
:::important
|
:::important
|
||||||
ClickHouse currently supports reading v1 and v2 of the Iceberg format via the `icebergS3`, `icebergAzure` and `icebergLocal` table functions and `IcebergS3`, `icebergAzure` ans `icebergLocal` table engines.
|
ClickHouse currently supports reading v1 and v2 of the Iceberg format via the `icebergS3`, `icebergAzure`, `icebergHDFS` and `icebergLocal` table functions and `IcebergS3`, `icebergAzure`, `IcebergHDFS` and `IcebergLocal` table engines.
|
||||||
:::
|
:::
|
||||||
|
|
||||||
## Defining a named collection
|
## Defining a named collection
|
||||||
|
|||||||
@ -1,2 +0,0 @@
|
|||||||
# Just an empty yaml file. Keep it alone.
|
|
||||||
{}
|
|
||||||
@ -7,190 +7,3 @@ sidebar_label: "[experimental] MaterializedMySQL"
|
|||||||
# [экспериментальный] MaterializedMySQL {#materialized-mysql}
|
# [экспериментальный] MaterializedMySQL {#materialized-mysql}
|
||||||
|
|
||||||
**Это экспериментальный движок, который не следует использовать в продакшене.**
|
**Это экспериментальный движок, который не следует использовать в продакшене.**
|
||||||
|
|
||||||
Создает базу данных ClickHouse со всеми таблицами, существующими в MySQL, и всеми данными в этих таблицах.
|
|
||||||
|
|
||||||
Сервер ClickHouse работает как реплика MySQL. Он читает файл binlog и выполняет DDL and DML-запросы.
|
|
||||||
|
|
||||||
## Создание базы данных {#creating-a-database}
|
|
||||||
|
|
||||||
``` sql
|
|
||||||
CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster]
|
|
||||||
ENGINE = MaterializedMySQL('host:port', ['database' | database], 'user', 'password') [SETTINGS ...]
|
|
||||||
```
|
|
||||||
|
|
||||||
**Параметры движка**
|
|
||||||
|
|
||||||
- `host:port` — адрес сервера MySQL.
|
|
||||||
- `database` — имя базы данных на удалённом сервере.
|
|
||||||
- `user` — пользователь MySQL.
|
|
||||||
- `password` — пароль пользователя.
|
|
||||||
|
|
||||||
**Настройки движка**
|
|
||||||
|
|
||||||
- `max_rows_in_buffer` — максимальное количество строк, содержимое которых может кешироваться в памяти (для одной таблицы и данных кеша, которые невозможно запросить). При превышении количества строк, данные будут материализованы. Значение по умолчанию: `65 505`.
|
|
||||||
- `max_bytes_in_buffer` — максимальное количество байтов, которое разрешено кешировать в памяти (для одной таблицы и данных кеша, которые невозможно запросить). При превышении количества строк, данные будут материализованы. Значение по умолчанию: `1 048 576`.
|
|
||||||
- `max_rows_in_buffers` — максимальное количество строк, содержимое которых может кешироваться в памяти (для базы данных и данных кеша, которые невозможно запросить). При превышении количества строк, данные будут материализованы. Значение по умолчанию: `65 505`.
|
|
||||||
- `max_bytes_in_buffers` — максимальное количество байтов, которое разрешено кешировать данным в памяти (для базы данных и данных кеша, которые невозможно запросить). При превышении количества строк, данные будут материализованы. Значение по умолчанию: `1 048 576`.
|
|
||||||
- `max_flush_data_time` — максимальное время в миллисекундах, в течение которого разрешено кешировать данные в памяти (для базы данных и данных кеша, которые невозможно запросить). При превышении количества указанного периода, данные будут материализованы. Значение по умолчанию: `1000`.
|
|
||||||
- `max_wait_time_when_mysql_unavailable` — интервал между повторными попытками, если MySQL недоступен. Указывается в миллисекундах. Отрицательное значение отключает повторные попытки. Значение по умолчанию: `1000`.
|
|
||||||
- `allows_query_when_mysql_lost` — признак, разрешен ли запрос к материализованной таблице при потере соединения с MySQL. Значение по умолчанию: `0` (`false`).
|
|
||||||
|
|
||||||
```sql
|
|
||||||
CREATE DATABASE mysql ENGINE = MaterializedMySQL('localhost:3306', 'db', 'user', '***')
|
|
||||||
SETTINGS
|
|
||||||
allows_query_when_mysql_lost=true,
|
|
||||||
max_wait_time_when_mysql_unavailable=10000;
|
|
||||||
```
|
|
||||||
|
|
||||||
**Настройки на стороне MySQL-сервера**
|
|
||||||
|
|
||||||
Для правильной работы `MaterializedMySQL` следует обязательно указать на сервере MySQL следующие параметры конфигурации:
|
|
||||||
- `default_authentication_plugin = mysql_native_password` — `MaterializedMySQL` может авторизоваться только с помощью этого метода.
|
|
||||||
- `gtid_mode = on` — ведение журнала на основе GTID является обязательным для обеспечения правильной репликации.
|
|
||||||
|
|
||||||
:::note Внимание
|
|
||||||
При включении `gtid_mode` вы также должны указать `enforce_gtid_consistency = on`.
|
|
||||||
:::
|
|
||||||
## Виртуальные столбцы {#virtual-columns}
|
|
||||||
|
|
||||||
При работе с движком баз данных `MaterializedMySQL` используются таблицы семейства [ReplacingMergeTree](../../engines/table-engines/mergetree-family/replacingmergetree.md) с виртуальными столбцами `_sign` и `_version`.
|
|
||||||
|
|
||||||
- `_version` — счетчик транзакций. Тип [UInt64](../../sql-reference/data-types/int-uint.md).
|
|
||||||
- `_sign` — метка удаления. Тип [Int8](../../sql-reference/data-types/int-uint.md). Возможные значения:
|
|
||||||
- `1` — строка не удалена,
|
|
||||||
- `-1` — строка удалена.
|
|
||||||
|
|
||||||
## Поддержка типов данных {#data_types-support}
|
|
||||||
|
|
||||||
| MySQL | ClickHouse |
|
|
||||||
|-------------------------|--------------------------------------------------------------|
|
|
||||||
| TINY | [Int8](../../sql-reference/data-types/int-uint.md) |
|
|
||||||
| SHORT | [Int16](../../sql-reference/data-types/int-uint.md) |
|
|
||||||
| INT24 | [Int32](../../sql-reference/data-types/int-uint.md) |
|
|
||||||
| LONG | [UInt32](../../sql-reference/data-types/int-uint.md) |
|
|
||||||
| LONGLONG | [UInt64](../../sql-reference/data-types/int-uint.md) |
|
|
||||||
| FLOAT | [Float32](../../sql-reference/data-types/float.md) |
|
|
||||||
| DOUBLE | [Float64](../../sql-reference/data-types/float.md) |
|
|
||||||
| DECIMAL, NEWDECIMAL | [Decimal](../../sql-reference/data-types/decimal.md) |
|
|
||||||
| DATE, NEWDATE | [Date](../../sql-reference/data-types/date.md) |
|
|
||||||
| DATETIME, TIMESTAMP | [DateTime](../../sql-reference/data-types/datetime.md) |
|
|
||||||
| DATETIME2, TIMESTAMP2 | [DateTime64](../../sql-reference/data-types/datetime64.md) |
|
|
||||||
| ENUM | [Enum](../../sql-reference/data-types/enum.md) |
|
|
||||||
| STRING | [String](../../sql-reference/data-types/string.md) |
|
|
||||||
| VARCHAR, VAR_STRING | [String](../../sql-reference/data-types/string.md) |
|
|
||||||
| BLOB | [String](../../sql-reference/data-types/string.md) |
|
|
||||||
| BINARY | [FixedString](../../sql-reference/data-types/fixedstring.md) |
|
|
||||||
|
|
||||||
Тип [Nullable](../../sql-reference/data-types/nullable.md) поддерживается.
|
|
||||||
|
|
||||||
Другие типы не поддерживаются. Если таблица MySQL содержит столбец другого типа, ClickHouse выдаст исключение "Неподдерживаемый тип данных" ("Unhandled data type") и остановит репликацию.
|
|
||||||
|
|
||||||
## Особенности и рекомендации {#specifics-and-recommendations}
|
|
||||||
|
|
||||||
### Ограничения совместимости {#compatibility-restrictions}
|
|
||||||
|
|
||||||
Кроме ограничений на типы данных, существует несколько ограничений по сравнению с базами данных MySQL, которые следует решить до того, как станет возможной репликация:
|
|
||||||
|
|
||||||
- Каждая таблица в MySQL должна содержать `PRIMARY KEY`.
|
|
||||||
- Репликация для таблиц, содержащих строки со значениями полей `ENUM` вне диапазона значений (определяется размерностью `ENUM`), не будет работать.
|
|
||||||
|
|
||||||
### DDL-запросы {#ddl-queries}
|
|
||||||
|
|
||||||
DDL-запросы в MySQL конвертируются в соответствующие DDL-запросы в ClickHouse ([ALTER](../../sql-reference/statements/alter/index.md), [CREATE](../../sql-reference/statements/create/index.md), [DROP](../../sql-reference/statements/drop.md), [RENAME](../../sql-reference/statements/rename.md)). Если ClickHouse не может конвертировать какой-либо DDL-запрос, он его игнорирует.
|
|
||||||
|
|
||||||
### Репликация данных {#data-replication}
|
|
||||||
|
|
||||||
Данные являются неизменяемыми со стороны пользователя ClickHouse, но автоматически обновляются путём репликации следующих запросов из MySQL:
|
|
||||||
|
|
||||||
- Запрос `INSERT` конвертируется в ClickHouse в `INSERT` с `_sign=1`.
|
|
||||||
|
|
||||||
- Запрос `DELETE` конвертируется в ClickHouse в `INSERT` с `_sign=-1`.
|
|
||||||
|
|
||||||
- Запрос `UPDATE` конвертируется в ClickHouse в `INSERT` с `_sign=-1` и `INSERT` с `_sign=1`.
|
|
||||||
|
|
||||||
### Выборка из таблиц движка MaterializedMySQL {#select}
|
|
||||||
|
|
||||||
Запрос `SELECT` из таблиц движка `MaterializedMySQL` имеет некоторую специфику:
|
|
||||||
|
|
||||||
- Если в запросе `SELECT` напрямую не указан столбец `_version`, то используется модификатор [FINAL](../../sql-reference/statements/select/from.md#select-from-final). Таким образом, выбираются только строки с `MAX(_version)`.
|
|
||||||
|
|
||||||
- Если в запросе `SELECT` напрямую не указан столбец `_sign`, то по умолчанию используется `WHERE _sign=1`. Таким образом, удаленные строки не включаются в результирующий набор.
|
|
||||||
|
|
||||||
- Результат включает комментарии к столбцам, если они существуют в таблицах базы данных MySQL.
|
|
||||||
|
|
||||||
### Конвертация индексов {#index-conversion}
|
|
||||||
|
|
||||||
Секции `PRIMARY KEY` и `INDEX` в MySQL конвертируются в кортежи `ORDER BY` в таблицах ClickHouse.
|
|
||||||
|
|
||||||
В таблицах ClickHouse данные физически хранятся в том порядке, который определяется секцией `ORDER BY`. Чтобы физически перегруппировать данные, используйте [материализованные представления](../../sql-reference/statements/create/view.md#materialized).
|
|
||||||
|
|
||||||
**Примечание**
|
|
||||||
|
|
||||||
- Строки с `_sign=-1` физически не удаляются из таблиц.
|
|
||||||
- Каскадные запросы `UPDATE/DELETE` не поддерживаются движком `MaterializedMySQL`.
|
|
||||||
- Репликация может быть легко нарушена.
|
|
||||||
- Прямые операции изменения данных в таблицах и базах данных `MaterializedMySQL` запрещены.
|
|
||||||
- На работу `MaterializedMySQL` влияет настройка [optimize_on_insert](../../operations/settings/settings.md#optimize-on-insert). Когда таблица на MySQL сервере меняется, происходит слияние данных в соответсвующей таблице в базе данных `MaterializedMySQL`.
|
|
||||||
|
|
||||||
## Примеры использования {#examples-of-use}
|
|
||||||
|
|
||||||
Запросы в MySQL:
|
|
||||||
|
|
||||||
``` sql
|
|
||||||
mysql> CREATE DATABASE db;
|
|
||||||
mysql> CREATE TABLE db.test (a INT PRIMARY KEY, b INT);
|
|
||||||
mysql> INSERT INTO db.test VALUES (1, 11), (2, 22);
|
|
||||||
mysql> DELETE FROM db.test WHERE a=1;
|
|
||||||
mysql> ALTER TABLE db.test ADD COLUMN c VARCHAR(16);
|
|
||||||
mysql> UPDATE db.test SET c='Wow!', b=222;
|
|
||||||
mysql> SELECT * FROM test;
|
|
||||||
```
|
|
||||||
|
|
||||||
```text
|
|
||||||
+---+------+------+
|
|
||||||
| a | b | c |
|
|
||||||
+---+------+------+
|
|
||||||
| 2 | 222 | Wow! |
|
|
||||||
+---+------+------+
|
|
||||||
```
|
|
||||||
|
|
||||||
База данных в ClickHouse, обмен данными с сервером MySQL:
|
|
||||||
|
|
||||||
База данных и созданная таблица:
|
|
||||||
|
|
||||||
``` sql
|
|
||||||
CREATE DATABASE mysql ENGINE = MaterializedMySQL('localhost:3306', 'db', 'user', '***');
|
|
||||||
SHOW TABLES FROM mysql;
|
|
||||||
```
|
|
||||||
|
|
||||||
``` text
|
|
||||||
┌─name─┐
|
|
||||||
│ test │
|
|
||||||
└──────┘
|
|
||||||
```
|
|
||||||
|
|
||||||
После вставки данных:
|
|
||||||
|
|
||||||
``` sql
|
|
||||||
SELECT * FROM mysql.test;
|
|
||||||
```
|
|
||||||
|
|
||||||
``` text
|
|
||||||
┌─a─┬──b─┐
|
|
||||||
│ 1 │ 11 │
|
|
||||||
│ 2 │ 22 │
|
|
||||||
└───┴────┘
|
|
||||||
```
|
|
||||||
|
|
||||||
После удаления данных, добавления столбца и обновления:
|
|
||||||
|
|
||||||
``` sql
|
|
||||||
SELECT * FROM mysql.test;
|
|
||||||
```
|
|
||||||
|
|
||||||
``` text
|
|
||||||
┌─a─┬───b─┬─c────┐
|
|
||||||
│ 2 │ 222 │ Wow! │
|
|
||||||
└───┴─────┴──────┘
|
|
||||||
```
|
|
||||||
|
|||||||
@ -33,7 +33,7 @@ sidebar_label: "Отличительные возможности ClickHouse"
|
|||||||
|
|
||||||
## Поддержка SQL {#sql-support}
|
## Поддержка SQL {#sql-support}
|
||||||
|
|
||||||
ClickHouse поддерживает [декларативный язык запросов на основе SQL](../sql-reference/index.md) и во [многих случаях](../sql-reference/ansi.mdx) совпадающий с SQL-стандартом.
|
ClickHouse поддерживает декларативный язык запросов SQL.
|
||||||
|
|
||||||
Поддерживаются [GROUP BY](../sql-reference/statements/select/group-by.md), [ORDER BY](../sql-reference/statements/select/order-by.md), подзапросы в секциях [FROM](../sql-reference/statements/select/from.md), [IN](../sql-reference/operators/in.md), [JOIN](../sql-reference/statements/select/join.md), [функции window](../sql-reference/window-functions/index.mdx), а также скалярные подзапросы.
|
Поддерживаются [GROUP BY](../sql-reference/statements/select/group-by.md), [ORDER BY](../sql-reference/statements/select/order-by.md), подзапросы в секциях [FROM](../sql-reference/statements/select/from.md), [IN](../sql-reference/operators/in.md), [JOIN](../sql-reference/statements/select/join.md), [функции window](../sql-reference/window-functions/index.mdx), а также скалярные подзапросы.
|
||||||
|
|
||||||
|
|||||||
@ -30,7 +30,7 @@ sidebar_label: "Настройки пользователей"
|
|||||||
<profile>profile_name</profile>
|
<profile>profile_name</profile>
|
||||||
|
|
||||||
<quota>default</quota>
|
<quota>default</quota>
|
||||||
<default_database>default<default_database>
|
<default_database>default</default_database>
|
||||||
<databases>
|
<databases>
|
||||||
<database_name>
|
<database_name>
|
||||||
<table_name>
|
<table_name>
|
||||||
|
|||||||
@ -93,7 +93,7 @@ WITH anySimpleState(number) AS c SELECT toTypeName(c), c FROM numbers(1);
|
|||||||
## -Distinct {#agg-functions-combinator-distinct}
|
## -Distinct {#agg-functions-combinator-distinct}
|
||||||
|
|
||||||
При наличии комбинатора Distinct, каждое уникальное значение аргументов, будет учитано в агрегатной функции только один раз.
|
При наличии комбинатора Distinct, каждое уникальное значение аргументов, будет учитано в агрегатной функции только один раз.
|
||||||
Примеры: `sum(DISTINCT x)`, `groupArray(DISTINCT x)`, `corrStableDistinct(DISTINCT x, y)` и т.п.
|
Примеры: `sum(DISTINCT x)` (или `sumDistinct(x)`), `groupArray(DISTINCT x)` (или `groupArrayDistinct(x)`), `corrStable(DISTINCT x, y)` (или `corrStableDistinct(x, y)`) и т.п.
|
||||||
|
|
||||||
## -OrDefault {#agg-functions-combinator-ordefault}
|
## -OrDefault {#agg-functions-combinator-ordefault}
|
||||||
|
|
||||||
|
|||||||
@ -1,10 +0,0 @@
|
|||||||
---
|
|
||||||
slug: /ru/sql-reference/ansi
|
|
||||||
sidebar_position: 40
|
|
||||||
sidebar_label: ANSI Compatibility
|
|
||||||
title: "ANSI Compatibility"
|
|
||||||
---
|
|
||||||
|
|
||||||
import Content from '@site/docs/en/sql-reference/ansi.md';
|
|
||||||
|
|
||||||
<Content />
|
|
||||||
@ -37,7 +37,7 @@ ClickHouse会使用服务器上一切可用的资源，从而以最自然的方
|
|||||||
|
|
||||||
## 支持SQL {#zhi-chi-sql}
|
## 支持SQL {#zhi-chi-sql}
|
||||||
|
|
||||||
ClickHouse支持一种[基于SQL的声明式查询语言](../sql-reference/index.md)，它在许多情况下与[ANSI SQL标准](../sql-reference/ansi.md)相同。
|
ClickHouse支持一种基于SQL的声明式查询语言。
|
||||||
|
|
||||||
支持的查询[GROUP BY](../sql-reference/statements/select/group-by.md), [ORDER BY](../sql-reference/statements/select/order-by.md), [FROM](../sql-reference/statements/select/from.md), [JOIN](../sql-reference/statements/select/join.md), [IN](../sql-reference/operators/in.md)以及非相关子查询。
|
支持的查询[GROUP BY](../sql-reference/statements/select/group-by.md), [ORDER BY](../sql-reference/statements/select/order-by.md), [FROM](../sql-reference/statements/select/from.md), [JOIN](../sql-reference/statements/select/join.md), [IN](../sql-reference/operators/in.md)以及非相关子查询。
|
||||||
|
|
||||||
|
|||||||
@ -1,191 +0,0 @@
|
|||||||
---
|
|
||||||
slug: /zh/sql-reference/ansi
|
|
||||||
sidebar_position: 40
|
|
||||||
sidebar_label: "ANSI\u517C\u5BB9\u6027"
|
|
||||||
---
|
|
||||||
|
|
||||||
# ClickHouse SQL方言 与ANSI SQL的兼容性{#ansi-sql-compatibility-of-clickhouse-sql-dialect}
|
|
||||||
|
|
||||||
:::note
|
|
||||||
本文参考Annex G所著的[ISO/IEC CD 9075-2:2011](https://www.iso.org/obp/ui/#iso:std:iso-iec:9075:-2:ed-4:v1:en:sec:8)标准.
|
|
||||||
:::
|
|
||||||
|
|
||||||
## 行为差异 {#differences-in-behaviour}
|
|
||||||
|
|
||||||
下表列出了ClickHouse能够使用，但与ANSI SQL规定有差异的查询特性。
|
|
||||||
|
|
||||||
| 功能ID | 功能名称 | 差异 |
|
|
||||||
| ------- | --------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
||||||
| E011 | 数值型数据类型 | 带小数点的数字被视为近似值 (`Float64`）而不是精确值 (`Decimal`) |
|
|
||||||
| E051-05 | SELECT 的列可以重命名 | 字段重命名的作用范围不限于进行重命名的SELECT子查询（参考[表达式别名](https://clickhouse.com/docs/zh/sql-reference/syntax/#notes-on-usage)） |
|
|
||||||
| E141-01 | NOT NULL（非空）约束 | ClickHouse表中每一列默认为`NOT NULL` |
|
|
||||||
| E011-04 | 算术运算符 | ClickHouse在运算时会进行溢出，而不是四舍五入。此外会根据自定义规则修改结果数据类型（参考[溢出检查](https://clickhouse.com/docs/zh/sql-reference/data-types/decimal/#yi-chu-jian-cha)） |
|
|
||||||
|
|
||||||
## 功能状态 {#feature-status}
|
|
||||||
|
|
||||||
| 功能ID | 功能名称 | 状态 | 注释 |
|
|
||||||
| -------- | ---------------------------------------------------------------------------------------- | ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
|
||||||
| **E011** | **数值型数据类型** | **部分**{.text-warning} | |
|
|
||||||
| E011-01 | INTEGER （整型）和SMALLINT （小整型）数据类型 | 是 {.text-success} | |
|
|
||||||
| E011-02 | REAL （实数）、DOUBLE PRECISION （双精度浮点数）和FLOAT（单精度浮点数）数据类型数据类型 | 是 {.text-success} | |
|
|
||||||
| E011-03 | DECIMAL （精确数字）和NUMERIC （精确数字）数据类型 | 是 {.text-success} | |
|
|
||||||
| E011-04 | 算术运算符 | 是 {.text-success} | |
|
|
||||||
| E011-05 | 数值比较 | 是 {.text-success} | |
|
|
||||||
| E011-06 | 数值数据类型之间的隐式转换 | 否 {.text-danger} | ANSI SQL允许在数值类型之间进行任意隐式转换，而ClickHouse针对不同数据类型有对应的比较函数和类型转换函数 |
|
|
||||||
| **E021** | **字符串类型** | **部分**{.text-warning} | |
|
|
||||||
| E021-01 | CHARACTER （字符串）数据类型 | 是 {.text-success} | |
|
|
||||||
| E021-02 | CHARACTER VARYING （可变字符串）数据类型 | 是 {.text-success} | |
|
|
||||||
| E021-03 | 字符字面量 | 是 {.text-success} | |
|
|
||||||
| E021-04 | CHARACTER_LENGTH 函数 | 部分 {.text-warning} | 不支持 `using` 从句 |
|
|
||||||
| E021-05 | OCTET_LENGTH 函数 | 否 {.text-danger} | 使用 `LENGTH` 函数代替 |
|
|
||||||
| E021-06 | SUBSTRING | 部分 {.text-warning} | 不支持 `SIMILAR` 和 `ESCAPE` 从句，没有`SUBSTRING_REGEX` 函数 |
|
|
||||||
| E021-07 | 字符串拼接 | 部分 {.text-warning} | 不支持 `COLLATE` 从句 |
|
|
||||||
| E021-08 | 大小写转换 | 是 {.text-success} | |
|
|
||||||
| E021-09 | 裁剪字符串 | 是 {.text-success} | |
|
|
||||||
| E021-10 | 固定长度和可变长度字符串类型之间的隐式转换 | 部分 {.text-warning} | ANSI SQL允许在数据类型之间进行任意隐式转换，而ClickHouse针对不同数据类型有对应的比较函数和类型转换函数 |
|
|
||||||
| E021-11 | POSITION 函数 | 部分 {.text-warning} | 不支持 `IN` 和 `USING` 从句，不支持`POSITION_REGEX`函数 |
|
|
||||||
| E021-12 | 字符串比较 | 是 {.text-success} | |
|
|
||||||
| **E031** | **标识符** | **部分**{.text-warning} | |
|
|
||||||
| E031-01 | 分隔标识符 | 部分 {.text-warning} | Unicode文字支持有限 |
|
|
||||||
| E031-02 | 小写标识符 | 是 {.text-success} | |
|
|
||||||
| E031-03 | 标识符最后加下划线 | 是 {.text-success} | |
|
|
||||||
| **E051** | **基本查询规范** | **部分**{.text-warning} | |
|
|
||||||
| E051-01 | SELECT DISTINCT | 是 {.text-success} | |
|
|
||||||
| E051-02 | GROUP BY 从句 | 是 {.text-success} | |
|
|
||||||
| E051-04 | GROUP BY 从句中的列可以包含不在 `<select list>`中出现的列 | 是 {.text-success} | |
|
|
||||||
| E051-05 | SELECT 的列可以重命名 | 是 {.text-success} | |
|
|
||||||
| E051-06 | HAVING 从句 | 是 {.text-success} | |
|
|
||||||
| E051-07 | SELECT 选择的列中允许出现\* | 是 {.text-success} | |
|
|
||||||
| E051-08 | FROM 从句中的关联名称 | 是 {.text-success} | |
|
|
||||||
| E051-09 | 重命名 FROM 从句中的列 | 否 {.text-danger} | |
|
|
||||||
| **E061** | **基本谓词和搜索条件** | **部分**{.text-warning} | |
|
|
||||||
| E061-01 | 比较谓词 | 是 {.text-success} | |
|
|
||||||
| E061-02 | BETWEEN 谓词 | 部分 {.text-warning} | 不支持 `SYMMETRIC` 和 `ASYMMETRIC` 从句 |
|
|
||||||
| E061-03 | IN 谓词后可接值列表 | 是 {.text-success} | |
|
|
||||||
| E061-04 | LIKE 谓词 | 是 {.text-success} | |
|
|
||||||
| E061-05 | LIKE 谓词后接 ESCAPE 从句 | 否 {.text-danger} | |
|
|
||||||
| E061-06 | NULL 谓词 | 是 {.text-success} | |
|
|
||||||
| E061-07 | 量化比较谓词（ALL、SOME、ANY） | 否 {.text-danger} | |
|
|
||||||
| E061-08 | EXISTS 谓词 | 否 {.text-danger} | |
|
|
||||||
| E061-09 | 比较谓词中的子查询 | 是 {.text-success} | |
|
|
||||||
| E061-11 | IN 谓词中的子查询 | 是 {.text-success} | |
|
|
||||||
| E061-12 | 量化比较谓词（BETWEEN、IN、LIKE）中的子查询 | 否 {.text-danger} | |
|
|
||||||
| E061-13 | 相关子查询 | 否 {.text-danger} | |
|
|
||||||
| E061-14 | 搜索条件 | 是 {.text-success} | |
|
|
||||||
| **E071** | **基本查询表达式** | **部分**{.text-warning} | |
|
|
||||||
| E071-01 | UNION DISTINCT 表运算符 | 是 {.text-success} | |
|
|
||||||
| E071-02 | UNION ALL 表运算符 | 是 {.text-success} | |
|
|
||||||
| E071-03 | EXCEPT DISTINCT 表运算符 | 否 {.text-danger} | |
|
|
||||||
| E071-05 | 通过表运算符组合的列不必具有完全相同的数据类型 | 是 {.text-success} | |
|
|
||||||
| E071-06 | 子查询中的表运算符 | 是 {.text-success} | |
|
|
||||||
| **E081** | **基本权限** | **是**{.text-success} | |
|
|
||||||
| E081-01 | 表级别的SELECT（查询）权限 | 是 {.text-success} | |
|
|
||||||
| E081-02 | DELETE（删除）权限 | 是 {.text-success} | |
|
|
||||||
| E081-03 | 表级别的INSERT（插入）权限 | 是 {.text-success} | |
|
|
||||||
| E081-04 | 表级别的UPDATE（更新）权限 | 是 {.text-success} | |
|
|
||||||
| E081-05 | 列级别的UPDATE（更新）权限 | 是 {.text-success} | |
|
|
||||||
| E081-06 | 表级别的REFERENCES（引用）权限 | 是 {.text-success} | |
|
|
||||||
| E081-07 | 列级别的REFERENCES（引用）权限 | 是 {.text-success} | |
|
|
||||||
| E081-08 | WITH GRANT OPTION | 是 {.text-success} | |
|
|
||||||
| E081-09 | USAGE（使用）权限 | 是 {.text-success} | |
|
|
||||||
| E081-10 | EXECUTE（执行）权限 | 是 {.text-success} | |
|
|
||||||
| **E091** | **集合函数** | **是**{.text-success} | |
|
|
||||||
| E091-01 | AVG | 是 {.text-success} | |
|
|
||||||
| E091-02 | COUNT | 是 {.text-success} | |
|
|
||||||
| E091-03 | MAX | 是 {.text-success} | |
|
|
||||||
| E091-04 | MIN | 是 {.text-success} | |
|
|
||||||
| E091-05 | SUM | 是 {.text-success} | |
|
|
||||||
| E091-06 | ALL修饰词 | 否。 {.text-danger} | |
|
|
||||||
| E091-07 | DISTINCT修饰词 | 是 {.text-success} | 并非所有聚合函数都支持该修饰词 |
|
|
||||||
| **E101** | **基本数据操作** | **部分**{.text-warning} | |
|
|
||||||
| E101-01 | INSERT（插入）语句 | 是 {.text-success} | 注：ClickHouse中的主键并不隐含`UNIQUE` 约束 |
|
|
||||||
| E101-03 | 可指定范围的UPDATE（更新）语句 | 部分 {.text-warning} | `ALTER UPDATE` 语句用来批量更新数据 |
|
|
||||||
| E101-04 | 可指定范围的DELETE（删除）语句 | 部分 {.text-warning} | `ALTER DELETE` 语句用来批量删除数据 |
|
|
||||||
| **E111** | **返回一行的SELECT语句** | **否**{.text-danger} | |
|
|
||||||
| **E121** | **基本游标支持** | **否**{.text-danger} | |
|
|
||||||
| E121-01 | DECLARE CURSOR | 否 {.text-danger} | |
|
|
||||||
| E121-02 | ORDER BY 涉及的列不需要出现在SELECT的列中 | 是 {.text-success} | |
|
|
||||||
| E121-03 | ORDER BY 从句中的表达式 | 是 {.text-success} | |
|
|
||||||
| E121-04 | OPEN 语句 | 否 {.text-danger} | |
|
|
||||||
| E121-06 | 受游标位置控制的 UPDATE 语句 | 否 {.text-danger} | |
|
|
||||||
| E121-07 | 受游标位置控制的 DELETE 语句 | 否 {.text-danger} | |
|
|
||||||
| E121-08 | CLOSE 语句 | 否 {.text-danger} | |
|
|
||||||
| E121-10 | FETCH 语句中包含隐式NEXT | 否 {.text-danger} | |
|
|
||||||
| E121-17 | WITH HOLD 游标 | 否 {.text-danger} | |
|
|
||||||
| **E131** | **空值支持** | **是**{.text-success} | 有部分限制 |
|
|
||||||
| **E141** | **基本完整性约束** | **部分**{.text-warning} | |
|
|
||||||
| E141-01 | NOT NULL（非空）约束 | 是 {.text-success} | 注: 默认情况下ClickHouse表中的列隐含`NOT NULL`约束 |
|
|
||||||
| E141-02 | NOT NULL（非空）列的UNIQUE（唯一）约束 | 否 {.text-danger} | |
|
|
||||||
| E141-03 | 主键约束 | 部分 {.text-warning} | |
|
|
||||||
| E141-04 | 对于引用删除和引用更新操作，基本的FOREIGN KEY（外键）约束默认不进行任何操作（NO ACTION） | 否 {.text-danger} | |
|
|
||||||
| E141-06 | CHECK（检查）约束 | 是 {.text-success} | |
|
|
||||||
| E141-07 | 列默认值 | 是 {.text-success} | |
|
|
||||||
| E141-08 | 在主键上推断非空 | 是 {.text-success} | |
|
|
||||||
| E141-10 | 可以按任何顺序指定外键中的名称 | 否 {.text-danger} | |
|
|
||||||
| **E151** | **事务支持** | **否**{.text-danger} | |
|
|
||||||
| E151-01 | COMMIT（提交）语句 | 否 {.text-danger} | |
|
|
||||||
| E151-02 | ROLLBACK（回滚）语句 | 否 {.text-danger} | |
|
|
||||||
| **E152** | **基本的SET TRANSACTION（设置事务隔离级别）语句** | **否**{.text-danger} | |
|
|
||||||
| E152-01 | SET TRANSACTION语句：ISOLATION LEVEL SERIALIZABLE（隔离级别为串行化）从句 | 否 {.text-danger} | |
|
|
||||||
| E152-02 | SET TRANSACTION语句：READ ONLY（只读）和READ WRITE（读写）从句 | 否 {.text-danger} | |
|
|
||||||
| **E153** | **具有子查询的可更新查询** | **是**{.text-success} | |
|
|
||||||
| **E161** | **使用“--”符号作为SQL注释** | **是**{.text-success} | |
|
|
||||||
| **E171** | **SQLSTATE支持** | **否**{.text-danger} | |
|
|
||||||
| **E182** | **主机语言绑定** | **否**{.text-danger} | |
|
|
||||||
| **F031** | **基本架构操作** | **部分**{.text-warning} | |
|
|
||||||
| F031-01 | 使用 CREATE TABLE 语句创建持久表 | 部分 {.text-warning} | 不支持 `SYSTEM VERSIONING`, `ON COMMIT`, `GLOBAL`, `LOCAL`, `PRESERVE`, `DELETE`, `REF IS`, `WITH OPTIONS`, `UNDER`, `LIKE`, `PERIOD FOR` 从句，不支持用户解析的数据类型 |
|
|
||||||
| F031-02 | CREATE VIEW（创建视图）语句 | 部分 {.text-warning} | 不支持 `RECURSIVE`, `CHECK`, `UNDER`, `WITH OPTIONS` 从句，不支持用户解析的数据类型 |
|
|
||||||
| F031-03 | GRANT（授权）语句 | 是 {.text-success} | |
|
|
||||||
| F031-04 | ALTER TABLE语句：ADD COLUMN从句 | 是 {.text-success} | 不支持 `GENERATED` 从句和以系统时间做参数 |
|
|
||||||
| F031-13 | DROP TABLE语句：RESTRICT从句 | 否 {.text-danger} | |
|
|
||||||
| F031-16 | DROP VIEW语句：RESTRICT子句 | 否 {.text-danger} | |
|
|
||||||
| F031-19 | REVOKE语句：RESTRICT子句 | 否 {.text-danger} | |
|
|
||||||
| **F041** | **基本连接关系** | **部分**{.text-warning} | |
|
|
||||||
| F041-01 | Inner join（但不一定是INNER关键字) | 是 {.text-success} | |
|
|
||||||
| F041-02 | INNER 关键字 | 是 {.text-success} | |
|
|
||||||
| F041-03 | LEFT OUTER JOIN | 是 {.text-success} | |
|
|
||||||
| F041-04 | RIGHT OUTER JOIN | 是 {.text-success} | |
|
|
||||||
| F041-05 | 外连接可嵌套 | 是 {.text-success} | |
|
|
||||||
| F041-07 | 左外部连接或右外连接中的内部表也可用于内部联接 | 是 {.text-success} | |
|
|
||||||
| F041-08 | 支持所有比较运算符（而不仅仅是=) | 否 {.text-danger} | |
|
|
||||||
| **F051** | **基本日期和时间** | **部分**{.text-warning} | |
|
|
||||||
| F051-01 | DATE（日期）数据类型（并支持用于表达日期的字面量) | 是 {.text-success} | |
|
|
||||||
| F051-02 | TIME（时间）数据类型（并支持用于表达时间的字面量），小数秒精度至少为0 | 否 {.text-danger} | |
|
|
||||||
| F051-03 | 时间戳数据类型（并支持用于表达时间戳的字面量），小数秒精度至少为0和6 | 是 {.text-danger} | |
|
|
||||||
| F051-04 | 日期、时间和时间戳数据类型的比较谓词 | 是 {.text-success} | |
|
|
||||||
| F051-05 | DateTime 类型和字符串形式表达的时间之间的显式转换 | 是 {.text-success} | |
|
|
||||||
| F051-06 | CURRENT_DATE | 否 {.text-danger} | 使用`today()`替代 |
|
|
||||||
| F051-07 | LOCALTIME | 否 {.text-danger} | 使用`now()`替代 |
|
|
||||||
| F051-08 | LOCALTIMESTAMP | 否 {.text-danger} | |
|
|
||||||
| **F081** | **视图的UNION和EXCEPT操作** | **部分**{.text-warning} | |
|
|
||||||
| **F131** | **分组操作** | **部分**{.text-warning} | |
|
|
||||||
| F131-01 | 在具有分组视图的查询中支持 WHERE、GROUP BY 和 HAVING 子句 | 是 {.text-success} | |
|
|
||||||
| F131-02 | 在分组视图中支持多张表 | 是 {.text-success} | |
|
|
||||||
| F131-03 | 分组视图的查询中支持集合函数 | 是 {.text-success} | |
|
|
||||||
| F131-04 | 带有 `GROUP BY` 和 `HAVING` 从句，以及分组视图的子查询 | 是 {.text-success} | |
|
|
||||||
| F131-05 | 带有 `GROUP BY` 和 `HAVING` 从句，以及分组视图的仅返回1条记录的SELECT查询 | 否 {.text-danger} | |
|
|
||||||
| **F181** | **多模块支持** | **否**{.text-danger} | |
|
|
||||||
| **F201** | **CAST 函数** | **是**{.text-success} | |
|
|
||||||
| **F221** | **显式默认值** | **否**{.text-danger} | |
|
|
||||||
| **F261** | **CASE 表达式** | **是**{.text-success} | |
|
|
||||||
| F261-01 | 简单 CASE 表达式 | 是 {.text-success} | |
|
|
||||||
| F261-02 | 搜索型 CASE 表达式 | 是 {.text-success} | |
|
|
||||||
| F261-03 | NULLIF | 是 {.text-success} | |
|
|
||||||
| F261-04 | COALESCE | 是 {.text-success} | |
|
|
||||||
| **F311** | **架构定义语句** | **部分**{.text-warning} | |
|
|
||||||
| F311-01 | CREATE SCHEMA | 部分 {.text-warning} | 见`CREATE DATABASE` |
|
|
||||||
| F311-02 | 用于创建持久表的 CREATE TABLE | 是 {.text-success} | |
|
|
||||||
| F311-03 | CREATE VIEW | 是 {.text-success} | |
|
|
||||||
| F311-04 | CREATE VIEW: WITH CHECK OPTION | 否 {.text-danger} | |
|
|
||||||
| F311-05 | GRANT 语句 | 是 {.text-success} | |
|
|
||||||
| **F471** | **标量子查询** | **是**{.text-success} | |
|
|
||||||
| **F481** | **扩展 NULL 谓词** | **是**{.text-success} | |
|
|
||||||
| **F812** | **基本标志位** | **否**{.text-danger} |
|
|
||||||
| **S011** | **用于不重复数据的数据类型** | **否**{.text-danger} |
|
|
||||||
| **T321** | **基本的SQL调用例程** | **否**{.text-danger} | |
|
|
||||||
| T321-01 | 没有重载的用户定义函数 | 否{.text-danger} | |
|
|
||||||
| T321-02 | 没有重载的用户定义存储过程 | 否{.text-danger} | |
|
|
||||||
| T321-03 | 功能调用 | 否 {.text-danger} | |
|
|
||||||
| T321-04 | CALL 语句 | 否 {.text-danger} | |
|
|
||||||
| T321-05 | RETURN 语句 | 否 {.text-danger} | |
|
|
||||||
| **T631** | **IN 谓词后接一个列表** | **是**{.text-success} | |
|
|
||||||
@ -512,6 +512,7 @@ void Client::connect()
|
|||||||
{
|
{
|
||||||
std::cout << "Connected to " << server_name << " server version " << server_version << "." << std::endl << std::endl;
|
std::cout << "Connected to " << server_name << " server version " << server_version << "." << std::endl << std::endl;
|
||||||
|
|
||||||
|
#ifndef CLICKHOUSE_CLOUD
|
||||||
auto client_version_tuple = std::make_tuple(VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH);
|
auto client_version_tuple = std::make_tuple(VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH);
|
||||||
auto server_version_tuple = std::make_tuple(server_version_major, server_version_minor, server_version_patch);
|
auto server_version_tuple = std::make_tuple(server_version_major, server_version_minor, server_version_patch);
|
||||||
|
|
||||||
@ -527,6 +528,7 @@ void Client::connect()
|
|||||||
<< "It may indicate that the server is out of date and can be upgraded." << std::endl
|
<< "It may indicate that the server is out of date and can be upgraded." << std::endl
|
||||||
<< std::endl;
|
<< std::endl;
|
||||||
}
|
}
|
||||||
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!client_context->getSettingsRef()[Setting::use_client_time_zone])
|
if (!client_context->getSettingsRef()[Setting::use_client_time_zone])
|
||||||
|
|||||||
@ -38,18 +38,10 @@ public:
|
|||||||
String path_to = disk_to.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path-to"));
|
String path_to = disk_to.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path-to"));
|
||||||
bool recursive = options.count("recursive");
|
bool recursive = options.count("recursive");
|
||||||
|
|
||||||
if (!disk_from.getDisk()->exists(path_from))
|
if (disk_from.getDisk()->existsFile(path_from))
|
||||||
{
|
|
||||||
throw Exception(
|
|
||||||
ErrorCodes::BAD_ARGUMENTS,
|
|
||||||
"cannot stat '{}' on disk '{}': No such file or directory",
|
|
||||||
path_from,
|
|
||||||
disk_from.getDisk()->getName());
|
|
||||||
}
|
|
||||||
if (disk_from.getDisk()->isFile(path_from))
|
|
||||||
{
|
{
|
||||||
auto target_location = getTargetLocation(path_from, disk_to, path_to);
|
auto target_location = getTargetLocation(path_from, disk_to, path_to);
|
||||||
if (!disk_to.getDisk()->exists(target_location) || disk_to.getDisk()->isFile(target_location))
|
if (!disk_to.getDisk()->existsDirectory(target_location))
|
||||||
{
|
{
|
||||||
disk_from.getDisk()->copyFile(
|
disk_from.getDisk()->copyFile(
|
||||||
path_from,
|
path_from,
|
||||||
@ -65,7 +57,7 @@ public:
|
|||||||
ErrorCodes::BAD_ARGUMENTS, "cannot overwrite directory {} with non-directory {}", target_location, path_from);
|
ErrorCodes::BAD_ARGUMENTS, "cannot overwrite directory {} with non-directory {}", target_location, path_from);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else if (disk_from.getDisk()->isDirectory(path_from))
|
else if (disk_from.getDisk()->existsDirectory(path_from))
|
||||||
{
|
{
|
||||||
if (!recursive)
|
if (!recursive)
|
||||||
{
|
{
|
||||||
@ -73,11 +65,11 @@ public:
|
|||||||
}
|
}
|
||||||
auto target_location = getTargetLocation(path_from, disk_to, path_to);
|
auto target_location = getTargetLocation(path_from, disk_to, path_to);
|
||||||
|
|
||||||
if (disk_to.getDisk()->isFile(target_location))
|
if (disk_to.getDisk()->existsFile(target_location))
|
||||||
{
|
{
|
||||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "cannot overwrite non-directory {} with directory {}", path_to, target_location);
|
throw Exception(ErrorCodes::BAD_ARGUMENTS, "cannot overwrite non-directory {} with directory {}", path_to, target_location);
|
||||||
}
|
}
|
||||||
if (!disk_to.getDisk()->exists(target_location))
|
if (!disk_to.getDisk()->existsDirectory(target_location))
|
||||||
{
|
{
|
||||||
disk_to.getDisk()->createDirectory(target_location);
|
disk_to.getDisk()->createDirectory(target_location);
|
||||||
}
|
}
|
||||||
@ -89,6 +81,14 @@ public:
|
|||||||
/* write_settings= */ {},
|
/* write_settings= */ {},
|
||||||
/* cancellation_hook= */ {});
|
/* cancellation_hook= */ {});
|
||||||
}
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
throw Exception(
|
||||||
|
ErrorCodes::BAD_ARGUMENTS,
|
||||||
|
"cannot stat '{}' on disk '{}': No such file or directory",
|
||||||
|
path_from,
|
||||||
|
disk_from.getDisk()->getName());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@ -30,21 +30,21 @@ public:
|
|||||||
String path_from = disk.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path-from"));
|
String path_from = disk.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path-from"));
|
||||||
String path_to = disk.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path-to"));
|
String path_to = disk.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path-to"));
|
||||||
|
|
||||||
if (disk.getDisk()->isFile(path_from))
|
if (disk.getDisk()->existsFile(path_from))
|
||||||
{
|
{
|
||||||
disk.getDisk()->moveFile(path_from, path_to);
|
disk.getDisk()->moveFile(path_from, path_to);
|
||||||
}
|
}
|
||||||
else if (disk.getDisk()->isDirectory(path_from))
|
else if (disk.getDisk()->existsDirectory(path_from))
|
||||||
{
|
{
|
||||||
auto target_location = getTargetLocation(path_from, disk, path_to);
|
auto target_location = getTargetLocation(path_from, disk, path_to);
|
||||||
if (!disk.getDisk()->exists(target_location))
|
if (!disk.getDisk()->existsDirectory(target_location))
|
||||||
{
|
{
|
||||||
disk.getDisk()->createDirectory(target_location);
|
disk.getDisk()->createDirectory(target_location);
|
||||||
disk.getDisk()->moveDirectory(path_from, target_location);
|
disk.getDisk()->moveDirectory(path_from, target_location);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
if (disk.getDisk()->isFile(target_location))
|
if (disk.getDisk()->existsFile(target_location))
|
||||||
{
|
{
|
||||||
throw Exception(
|
throw Exception(
|
||||||
ErrorCodes::BAD_ARGUMENTS, "cannot overwrite non-directory '{}' with directory '{}'", target_location, path_from);
|
ErrorCodes::BAD_ARGUMENTS, "cannot overwrite non-directory '{}' with directory '{}'", target_location, path_from);
|
||||||
@ -57,7 +57,7 @@ public:
|
|||||||
disk.getDisk()->moveDirectory(path_from, target_location);
|
disk.getDisk()->moveDirectory(path_from, target_location);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else if (!disk.getDisk()->exists(path_from))
|
else
|
||||||
{
|
{
|
||||||
throw Exception(
|
throw Exception(
|
||||||
ErrorCodes::BAD_ARGUMENTS,
|
ErrorCodes::BAD_ARGUMENTS,
|
||||||
|
|||||||
@ -28,11 +28,7 @@ public:
|
|||||||
auto disk = client.getCurrentDiskWithPath();
|
auto disk = client.getCurrentDiskWithPath();
|
||||||
const String & path = disk.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path"));
|
const String & path = disk.getRelativeFromRoot(getValueFromCommandLineOptionsThrow<String>(options, "path"));
|
||||||
bool recursive = options.count("recursive");
|
bool recursive = options.count("recursive");
|
||||||
if (!disk.getDisk()->exists(path))
|
if (disk.getDisk()->existsDirectory(path))
|
||||||
{
|
|
||||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path {} on disk {} doesn't exist", path, disk.getDisk()->getName());
|
|
||||||
}
|
|
||||||
if (disk.getDisk()->isDirectory(path))
|
|
||||||
{
|
{
|
||||||
if (!recursive)
|
if (!recursive)
|
||||||
{
|
{
|
||||||
@ -41,10 +37,12 @@ public:
|
|||||||
|
|
||||||
disk.getDisk()->removeRecursive(path);
|
disk.getDisk()->removeRecursive(path);
|
||||||
}
|
}
|
||||||
else
|
else if (disk.getDisk()->existsFile(path))
|
||||||
{
|
{
|
||||||
disk.getDisk()->removeFileIfExists(path);
|
disk.getDisk()->removeFileIfExists(path);
|
||||||
}
|
}
|
||||||
|
else
|
||||||
|
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path {} on disk {} doesn't exist", path, disk.getDisk()->getName());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@ -1,6 +1,7 @@
|
|||||||
#include <Interpreters/Context.h>
|
#include <Interpreters/Context.h>
|
||||||
#include "ICommand.h"
|
#include "ICommand.h"
|
||||||
|
|
||||||
|
#include <IO/ReadBufferFromEmptyFile.h>
|
||||||
#include <IO/ReadBufferFromFile.h>
|
#include <IO/ReadBufferFromFile.h>
|
||||||
#include <IO/WriteBufferFromFile.h>
|
#include <IO/WriteBufferFromFile.h>
|
||||||
#include <IO/copyData.h>
|
#include <IO/copyData.h>
|
||||||
@ -36,7 +37,11 @@ public:
|
|||||||
return std::make_unique<ReadBufferFromFileDescriptor>(STDIN_FILENO);
|
return std::make_unique<ReadBufferFromFileDescriptor>(STDIN_FILENO);
|
||||||
|
|
||||||
String relative_path_from = disk.getRelativeFromRoot(path_from.value());
|
String relative_path_from = disk.getRelativeFromRoot(path_from.value());
|
||||||
return disk.getDisk()->readFile(relative_path_from, getReadSettings());
|
auto res = disk.getDisk()->readFileIfExists(relative_path_from, getReadSettings());
|
||||||
|
if (res)
|
||||||
|
return res;
|
||||||
|
/// For backward compatibility.
|
||||||
|
return std::make_unique<ReadBufferFromEmptyFile>();
|
||||||
}();
|
}();
|
||||||
|
|
||||||
auto out = disk.getDisk()->writeFile(path_to);
|
auto out = disk.getDisk()->writeFile(path_to);
|
||||||
|
|||||||
@ -29,7 +29,7 @@ DiskWithPath::DiskWithPath(DiskPtr disk_, std::optional<String> path_) : disk(di
|
|||||||
}
|
}
|
||||||
|
|
||||||
String relative_path = normalizePathAndGetAsRelative(path);
|
String relative_path = normalizePathAndGetAsRelative(path);
|
||||||
if (disk->isDirectory(relative_path) || (relative_path.empty() && (disk->isDirectory("/"))))
|
if (disk->existsDirectory(relative_path) || (relative_path.empty() && (disk->existsDirectory("/"))))
|
||||||
{
|
{
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -33,7 +33,7 @@ public:
|
|||||||
|
|
||||||
bool isDirectory(const String & any_path) const
|
bool isDirectory(const String & any_path) const
|
||||||
{
|
{
|
||||||
return disk->isDirectory(getRelativeFromRoot(any_path)) || (getRelativeFromRoot(any_path).empty() && (disk->isDirectory("/")));
|
return disk->existsDirectory(getRelativeFromRoot(any_path)) || (getRelativeFromRoot(any_path).empty() && (disk->existsDirectory("/")));
|
||||||
}
|
}
|
||||||
|
|
||||||
std::vector<String> listAllFilesByPath(const String & any_path) const;
|
std::vector<String> listAllFilesByPath(const String & any_path) const;
|
||||||
|
|||||||
@ -90,7 +90,7 @@ protected:
|
|||||||
|
|
||||||
String getTargetLocation(const String & path_from, DiskWithPath & disk_to, const String & path_to)
|
String getTargetLocation(const String & path_from, DiskWithPath & disk_to, const String & path_to)
|
||||||
{
|
{
|
||||||
if (!disk_to.getDisk()->isDirectory(path_to))
|
if (!disk_to.getDisk()->existsDirectory(path_to))
|
||||||
{
|
{
|
||||||
return path_to;
|
return path_to;
|
||||||
}
|
}
|
||||||
|
|||||||
@ -594,7 +594,10 @@ void CatBoostLibraryBridgeRequestHandler::handleRequest(HTTPServerRequest & requ
|
|||||||
catch (...)
|
catch (...)
|
||||||
{
|
{
|
||||||
tryLogCurrentException(log);
|
tryLogCurrentException(log);
|
||||||
|
out.cancel();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
try
|
try
|
||||||
|
|||||||
@ -77,6 +77,42 @@ namespace Setting
|
|||||||
extern const SettingsLocalFSReadMethod storage_file_read_method;
|
extern const SettingsLocalFSReadMethod storage_file_read_method;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
namespace ServerSetting
|
||||||
|
{
|
||||||
|
extern const ServerSettingsDouble cache_size_to_ram_max_ratio;
|
||||||
|
extern const ServerSettingsUInt64 compiled_expression_cache_elements_size;
|
||||||
|
extern const ServerSettingsUInt64 compiled_expression_cache_size;
|
||||||
|
extern const ServerSettingsUInt64 database_catalog_drop_table_concurrency;
|
||||||
|
extern const ServerSettingsString default_database;
|
||||||
|
extern const ServerSettingsString index_mark_cache_policy;
|
||||||
|
extern const ServerSettingsUInt64 index_mark_cache_size;
|
||||||
|
extern const ServerSettingsDouble index_mark_cache_size_ratio;
|
||||||
|
extern const ServerSettingsString index_uncompressed_cache_policy;
|
||||||
|
extern const ServerSettingsUInt64 index_uncompressed_cache_size;
|
||||||
|
extern const ServerSettingsDouble index_uncompressed_cache_size_ratio;
|
||||||
|
extern const ServerSettingsUInt64 io_thread_pool_queue_size;
|
||||||
|
extern const ServerSettingsString mark_cache_policy;
|
||||||
|
extern const ServerSettingsUInt64 mark_cache_size;
|
||||||
|
extern const ServerSettingsDouble mark_cache_size_ratio;
|
||||||
|
extern const ServerSettingsUInt64 max_active_parts_loading_thread_pool_size;
|
||||||
|
extern const ServerSettingsUInt64 max_io_thread_pool_free_size;
|
||||||
|
extern const ServerSettingsUInt64 max_io_thread_pool_size;
|
||||||
|
extern const ServerSettingsUInt64 max_outdated_parts_loading_thread_pool_size;
|
||||||
|
extern const ServerSettingsUInt64 max_parts_cleaning_thread_pool_size;
|
||||||
|
extern const ServerSettingsUInt64 max_server_memory_usage;
|
||||||
|
extern const ServerSettingsDouble max_server_memory_usage_to_ram_ratio;
|
||||||
|
extern const ServerSettingsUInt64 max_thread_pool_free_size;
|
||||||
|
extern const ServerSettingsUInt64 max_thread_pool_size;
|
||||||
|
extern const ServerSettingsUInt64 max_unexpected_parts_loading_thread_pool_size;
|
||||||
|
extern const ServerSettingsUInt64 mmap_cache_size;
|
||||||
|
extern const ServerSettingsBool show_addresses_in_stack_traces;
|
||||||
|
extern const ServerSettingsUInt64 thread_pool_queue_size;
|
||||||
|
extern const ServerSettingsString uncompressed_cache_policy;
|
||||||
|
extern const ServerSettingsUInt64 uncompressed_cache_size;
|
||||||
|
extern const ServerSettingsDouble uncompressed_cache_size_ratio;
|
||||||
|
extern const ServerSettingsBool use_legacy_mongodb_integration;
|
||||||
|
}
|
||||||
|
|
||||||
namespace ErrorCodes
|
namespace ErrorCodes
|
||||||
{
|
{
|
||||||
extern const int BAD_ARGUMENTS;
|
extern const int BAD_ARGUMENTS;
|
||||||
@ -157,9 +193,9 @@ void LocalServer::initialize(Poco::Util::Application & self)
|
|||||||
server_settings.loadSettingsFromConfig(config());
|
server_settings.loadSettingsFromConfig(config());
|
||||||
|
|
||||||
GlobalThreadPool::initialize(
|
GlobalThreadPool::initialize(
|
||||||
server_settings.max_thread_pool_size,
|
server_settings[ServerSetting::max_thread_pool_size],
|
||||||
server_settings.max_thread_pool_free_size,
|
server_settings[ServerSetting::max_thread_pool_free_size],
|
||||||
server_settings.thread_pool_queue_size);
|
server_settings[ServerSetting::thread_pool_queue_size]);
|
||||||
|
|
||||||
#if USE_AZURE_BLOB_STORAGE
|
#if USE_AZURE_BLOB_STORAGE
|
||||||
/// See the explanation near the same line in Server.cpp
|
/// See the explanation near the same line in Server.cpp
|
||||||
@ -170,17 +206,17 @@ void LocalServer::initialize(Poco::Util::Application & self)
|
|||||||
#endif
|
#endif
|
||||||
|
|
||||||
getIOThreadPool().initialize(
|
getIOThreadPool().initialize(
|
||||||
server_settings.max_io_thread_pool_size,
|
server_settings[ServerSetting::max_io_thread_pool_size],
|
||||||
server_settings.max_io_thread_pool_free_size,
|
server_settings[ServerSetting::max_io_thread_pool_free_size],
|
||||||
server_settings.io_thread_pool_queue_size);
|
server_settings[ServerSetting::io_thread_pool_queue_size]);
|
||||||
|
|
||||||
const size_t active_parts_loading_threads = server_settings.max_active_parts_loading_thread_pool_size;
|
const size_t active_parts_loading_threads = server_settings[ServerSetting::max_active_parts_loading_thread_pool_size];
|
||||||
getActivePartsLoadingThreadPool().initialize(
|
getActivePartsLoadingThreadPool().initialize(
|
||||||
active_parts_loading_threads,
|
active_parts_loading_threads,
|
||||||
0, // We don't need any threads one all the parts will be loaded
|
0, // We don't need any threads one all the parts will be loaded
|
||||||
active_parts_loading_threads);
|
active_parts_loading_threads);
|
||||||
|
|
||||||
const size_t outdated_parts_loading_threads = server_settings.max_outdated_parts_loading_thread_pool_size;
|
const size_t outdated_parts_loading_threads = server_settings[ServerSetting::max_outdated_parts_loading_thread_pool_size];
|
||||||
getOutdatedPartsLoadingThreadPool().initialize(
|
getOutdatedPartsLoadingThreadPool().initialize(
|
||||||
outdated_parts_loading_threads,
|
outdated_parts_loading_threads,
|
||||||
0, // We don't need any threads one all the parts will be loaded
|
0, // We don't need any threads one all the parts will be loaded
|
||||||
@ -188,7 +224,7 @@ void LocalServer::initialize(Poco::Util::Application & self)
|
|||||||
|
|
||||||
getOutdatedPartsLoadingThreadPool().setMaxTurboThreads(active_parts_loading_threads);
|
getOutdatedPartsLoadingThreadPool().setMaxTurboThreads(active_parts_loading_threads);
|
||||||
|
|
||||||
const size_t unexpected_parts_loading_threads = server_settings.max_unexpected_parts_loading_thread_pool_size;
|
const size_t unexpected_parts_loading_threads = server_settings[ServerSetting::max_unexpected_parts_loading_thread_pool_size];
|
||||||
getUnexpectedPartsLoadingThreadPool().initialize(
|
getUnexpectedPartsLoadingThreadPool().initialize(
|
||||||
unexpected_parts_loading_threads,
|
unexpected_parts_loading_threads,
|
||||||
0, // We don't need any threads one all the parts will be loaded
|
0, // We don't need any threads one all the parts will be loaded
|
||||||
@ -196,16 +232,16 @@ void LocalServer::initialize(Poco::Util::Application & self)
|
|||||||
|
|
||||||
getUnexpectedPartsLoadingThreadPool().setMaxTurboThreads(active_parts_loading_threads);
|
getUnexpectedPartsLoadingThreadPool().setMaxTurboThreads(active_parts_loading_threads);
|
||||||
|
|
||||||
const size_t cleanup_threads = server_settings.max_parts_cleaning_thread_pool_size;
|
const size_t cleanup_threads = server_settings[ServerSetting::max_parts_cleaning_thread_pool_size];
|
||||||
getPartsCleaningThreadPool().initialize(
|
getPartsCleaningThreadPool().initialize(
|
||||||
cleanup_threads,
|
cleanup_threads,
|
||||||
0, // We don't need any threads one all the parts will be deleted
|
0, // We don't need any threads one all the parts will be deleted
|
||||||
cleanup_threads);
|
cleanup_threads);
|
||||||
|
|
||||||
getDatabaseCatalogDropTablesThreadPool().initialize(
|
getDatabaseCatalogDropTablesThreadPool().initialize(
|
||||||
server_settings.database_catalog_drop_table_concurrency,
|
server_settings[ServerSetting::database_catalog_drop_table_concurrency],
|
||||||
0, // We don't need any threads if there are no DROP queries.
|
0, // We don't need any threads if there are no DROP queries.
|
||||||
server_settings.database_catalog_drop_table_concurrency);
|
server_settings[ServerSetting::database_catalog_drop_table_concurrency]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -296,9 +332,14 @@ void LocalServer::tryInitPath()
|
|||||||
|
|
||||||
global_context->setUserFilesPath(""); /// user's files are everywhere
|
global_context->setUserFilesPath(""); /// user's files are everywhere
|
||||||
|
|
||||||
std::string user_scripts_path = getClientConfiguration().getString("user_scripts_path", fs::path(path) / "user_scripts/");
|
std::string user_scripts_path = getClientConfiguration().getString("user_scripts_path", fs::path(path) / "user_scripts" / "");
|
||||||
global_context->setUserScriptsPath(user_scripts_path);
|
global_context->setUserScriptsPath(user_scripts_path);
|
||||||
|
|
||||||
|
/// Set path for filesystem caches
|
||||||
|
String filesystem_caches_path(getClientConfiguration().getString("filesystem_caches_path", fs::path(path) / "cache" / ""));
|
||||||
|
if (!filesystem_caches_path.empty())
|
||||||
|
global_context->setFilesystemCachesPath(filesystem_caches_path);
|
||||||
|
|
||||||
/// top_level_domains_lists
|
/// top_level_domains_lists
|
||||||
const std::string & top_level_domains_path = getClientConfiguration().getString("top_level_domains_path", fs::path(path) / "top_level_domains/");
|
const std::string & top_level_domains_path = getClientConfiguration().getString("top_level_domains_path", fs::path(path) / "top_level_domains/");
|
||||||
if (!top_level_domains_path.empty())
|
if (!top_level_domains_path.empty())
|
||||||
@ -470,7 +511,7 @@ try
|
|||||||
UseSSL use_ssl;
|
UseSSL use_ssl;
|
||||||
thread_status.emplace();
|
thread_status.emplace();
|
||||||
|
|
||||||
StackTrace::setShowAddresses(server_settings.show_addresses_in_stack_traces);
|
StackTrace::setShowAddresses(server_settings[ServerSetting::show_addresses_in_stack_traces]);
|
||||||
|
|
||||||
setupSignalHandler();
|
setupSignalHandler();
|
||||||
|
|
||||||
@ -507,10 +548,10 @@ try
|
|||||||
/// Don't initialize DateLUT
|
/// Don't initialize DateLUT
|
||||||
registerFunctions();
|
registerFunctions();
|
||||||
registerAggregateFunctions();
|
registerAggregateFunctions();
|
||||||
registerTableFunctions(server_settings.use_legacy_mongodb_integration);
|
registerTableFunctions(server_settings[ServerSetting::use_legacy_mongodb_integration]);
|
||||||
registerDatabases();
|
registerDatabases();
|
||||||
registerStorages(server_settings.use_legacy_mongodb_integration);
|
registerStorages(server_settings[ServerSetting::use_legacy_mongodb_integration]);
|
||||||
registerDictionaries(server_settings.use_legacy_mongodb_integration);
|
registerDictionaries(server_settings[ServerSetting::use_legacy_mongodb_integration]);
|
||||||
registerDisks(/* global_skip_access_check= */ true);
|
registerDisks(/* global_skip_access_check= */ true);
|
||||||
registerFormats();
|
registerFormats();
|
||||||
|
|
||||||
@ -659,8 +700,8 @@ void LocalServer::processConfig()
|
|||||||
|
|
||||||
const size_t physical_server_memory = getMemoryAmount();
|
const size_t physical_server_memory = getMemoryAmount();
|
||||||
|
|
||||||
size_t max_server_memory_usage = server_settings.max_server_memory_usage;
|
size_t max_server_memory_usage = server_settings[ServerSetting::max_server_memory_usage];
|
||||||
double max_server_memory_usage_to_ram_ratio = server_settings.max_server_memory_usage_to_ram_ratio;
|
double max_server_memory_usage_to_ram_ratio = server_settings[ServerSetting::max_server_memory_usage_to_ram_ratio];
|
||||||
|
|
||||||
size_t default_max_server_memory_usage = static_cast<size_t>(physical_server_memory * max_server_memory_usage_to_ram_ratio);
|
size_t default_max_server_memory_usage = static_cast<size_t>(physical_server_memory * max_server_memory_usage_to_ram_ratio);
|
||||||
|
|
||||||
@ -689,12 +730,12 @@ void LocalServer::processConfig()
|
|||||||
total_memory_tracker.setDescription("(total)");
|
total_memory_tracker.setDescription("(total)");
|
||||||
total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking);
|
total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking);
|
||||||
|
|
||||||
const double cache_size_to_ram_max_ratio = server_settings.cache_size_to_ram_max_ratio;
|
const double cache_size_to_ram_max_ratio = server_settings[ServerSetting::cache_size_to_ram_max_ratio];
|
||||||
const size_t max_cache_size = static_cast<size_t>(physical_server_memory * cache_size_to_ram_max_ratio);
|
const size_t max_cache_size = static_cast<size_t>(physical_server_memory * cache_size_to_ram_max_ratio);
|
||||||
|
|
||||||
String uncompressed_cache_policy = server_settings.uncompressed_cache_policy;
|
String uncompressed_cache_policy = server_settings[ServerSetting::uncompressed_cache_policy];
|
||||||
size_t uncompressed_cache_size = server_settings.uncompressed_cache_size;
|
size_t uncompressed_cache_size = server_settings[ServerSetting::uncompressed_cache_size];
|
||||||
double uncompressed_cache_size_ratio = server_settings.uncompressed_cache_size_ratio;
|
double uncompressed_cache_size_ratio = server_settings[ServerSetting::uncompressed_cache_size_ratio];
|
||||||
if (uncompressed_cache_size > max_cache_size)
|
if (uncompressed_cache_size > max_cache_size)
|
||||||
{
|
{
|
||||||
uncompressed_cache_size = max_cache_size;
|
uncompressed_cache_size = max_cache_size;
|
||||||
@ -702,9 +743,9 @@ void LocalServer::processConfig()
|
|||||||
}
|
}
|
||||||
global_context->setUncompressedCache(uncompressed_cache_policy, uncompressed_cache_size, uncompressed_cache_size_ratio);
|
global_context->setUncompressedCache(uncompressed_cache_policy, uncompressed_cache_size, uncompressed_cache_size_ratio);
|
||||||
|
|
||||||
String mark_cache_policy = server_settings.mark_cache_policy;
|
String mark_cache_policy = server_settings[ServerSetting::mark_cache_policy];
|
||||||
size_t mark_cache_size = server_settings.mark_cache_size;
|
size_t mark_cache_size = server_settings[ServerSetting::mark_cache_size];
|
||||||
double mark_cache_size_ratio = server_settings.mark_cache_size_ratio;
|
double mark_cache_size_ratio = server_settings[ServerSetting::mark_cache_size_ratio];
|
||||||
if (!mark_cache_size)
|
if (!mark_cache_size)
|
||||||
LOG_ERROR(log, "Too low mark cache size will lead to severe performance degradation.");
|
LOG_ERROR(log, "Too low mark cache size will lead to severe performance degradation.");
|
||||||
if (mark_cache_size > max_cache_size)
|
if (mark_cache_size > max_cache_size)
|
||||||
@ -714,9 +755,9 @@ void LocalServer::processConfig()
|
|||||||
}
|
}
|
||||||
global_context->setMarkCache(mark_cache_policy, mark_cache_size, mark_cache_size_ratio);
|
global_context->setMarkCache(mark_cache_policy, mark_cache_size, mark_cache_size_ratio);
|
||||||
|
|
||||||
String index_uncompressed_cache_policy = server_settings.index_uncompressed_cache_policy;
|
String index_uncompressed_cache_policy = server_settings[ServerSetting::index_uncompressed_cache_policy];
|
||||||
size_t index_uncompressed_cache_size = server_settings.index_uncompressed_cache_size;
|
size_t index_uncompressed_cache_size = server_settings[ServerSetting::index_uncompressed_cache_size];
|
||||||
double index_uncompressed_cache_size_ratio = server_settings.index_uncompressed_cache_size_ratio;
|
double index_uncompressed_cache_size_ratio = server_settings[ServerSetting::index_uncompressed_cache_size_ratio];
|
||||||
if (index_uncompressed_cache_size > max_cache_size)
|
if (index_uncompressed_cache_size > max_cache_size)
|
||||||
{
|
{
|
||||||
index_uncompressed_cache_size = max_cache_size;
|
index_uncompressed_cache_size = max_cache_size;
|
||||||
@ -724,9 +765,9 @@ void LocalServer::processConfig()
|
|||||||
}
|
}
|
||||||
global_context->setIndexUncompressedCache(index_uncompressed_cache_policy, index_uncompressed_cache_size, index_uncompressed_cache_size_ratio);
|
global_context->setIndexUncompressedCache(index_uncompressed_cache_policy, index_uncompressed_cache_size, index_uncompressed_cache_size_ratio);
|
||||||
|
|
||||||
String index_mark_cache_policy = server_settings.index_mark_cache_policy;
|
String index_mark_cache_policy = server_settings[ServerSetting::index_mark_cache_policy];
|
||||||
size_t index_mark_cache_size = server_settings.index_mark_cache_size;
|
size_t index_mark_cache_size = server_settings[ServerSetting::index_mark_cache_size];
|
||||||
double index_mark_cache_size_ratio = server_settings.index_mark_cache_size_ratio;
|
double index_mark_cache_size_ratio = server_settings[ServerSetting::index_mark_cache_size_ratio];
|
||||||
if (index_mark_cache_size > max_cache_size)
|
if (index_mark_cache_size > max_cache_size)
|
||||||
{
|
{
|
||||||
index_mark_cache_size = max_cache_size;
|
index_mark_cache_size = max_cache_size;
|
||||||
@ -734,7 +775,7 @@ void LocalServer::processConfig()
|
|||||||
}
|
}
|
||||||
global_context->setIndexMarkCache(index_mark_cache_policy, index_mark_cache_size, index_mark_cache_size_ratio);
|
global_context->setIndexMarkCache(index_mark_cache_policy, index_mark_cache_size, index_mark_cache_size_ratio);
|
||||||
|
|
||||||
size_t mmap_cache_size = server_settings.mmap_cache_size;
|
size_t mmap_cache_size = server_settings[ServerSetting::mmap_cache_size];
|
||||||
if (mmap_cache_size > max_cache_size)
|
if (mmap_cache_size > max_cache_size)
|
||||||
{
|
{
|
||||||
mmap_cache_size = max_cache_size;
|
mmap_cache_size = max_cache_size;
|
||||||
@ -746,8 +787,8 @@ void LocalServer::processConfig()
|
|||||||
global_context->setQueryCache(0, 0, 0, 0);
|
global_context->setQueryCache(0, 0, 0, 0);
|
||||||
|
|
||||||
#if USE_EMBEDDED_COMPILER
|
#if USE_EMBEDDED_COMPILER
|
||||||
size_t compiled_expression_cache_max_size_in_bytes = server_settings.compiled_expression_cache_size;
|
size_t compiled_expression_cache_max_size_in_bytes = server_settings[ServerSetting::compiled_expression_cache_size];
|
||||||
size_t compiled_expression_cache_max_elements = server_settings.compiled_expression_cache_elements_size;
|
size_t compiled_expression_cache_max_elements = server_settings[ServerSetting::compiled_expression_cache_elements_size];
|
||||||
CompiledExpressionCacheFactory::instance().init(compiled_expression_cache_max_size_in_bytes, compiled_expression_cache_max_elements);
|
CompiledExpressionCacheFactory::instance().init(compiled_expression_cache_max_size_in_bytes, compiled_expression_cache_max_elements);
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
@ -767,7 +808,7 @@ void LocalServer::processConfig()
|
|||||||
/// We load temporary database first, because projections need it.
|
/// We load temporary database first, because projections need it.
|
||||||
DatabaseCatalog::instance().initializeAndLoadTemporaryDatabase();
|
DatabaseCatalog::instance().initializeAndLoadTemporaryDatabase();
|
||||||
|
|
||||||
std::string default_database = server_settings.default_database;
|
std::string default_database = server_settings[ServerSetting::default_database];
|
||||||
DatabaseCatalog::instance().attachDatabase(default_database, createClickHouseLocalDatabaseOverlay(default_database, global_context));
|
DatabaseCatalog::instance().attachDatabase(default_database, createClickHouseLocalDatabaseOverlay(default_database, global_context));
|
||||||
global_context->setCurrentDatabase(default_database);
|
global_context->setCurrentDatabase(default_database);
|
||||||
|
|
||||||
@ -853,6 +894,7 @@ void LocalServer::addOptions(OptionsDescription & options_description)
|
|||||||
{
|
{
|
||||||
options_description.main_description->add_options()
|
options_description.main_description->add_options()
|
||||||
("table,N", po::value<std::string>(), "name of the initial table")
|
("table,N", po::value<std::string>(), "name of the initial table")
|
||||||
|
("copy", "shortcut for format conversion, equivalent to: --query 'SELECT * FROM table'")
|
||||||
|
|
||||||
/// If structure argument is omitted then initial query is not generated
|
/// If structure argument is omitted then initial query is not generated
|
||||||
("structure,S", po::value<std::string>(), "structure of the initial table (list of column and type names)")
|
("structure,S", po::value<std::string>(), "structure of the initial table (list of column and type names)")
|
||||||
@ -925,6 +967,12 @@ void LocalServer::processOptions(const OptionsDescription &, const CommandLineOp
|
|||||||
getClientConfiguration().setString("send_logs_level", options["send_logs_level"].as<std::string>());
|
getClientConfiguration().setString("send_logs_level", options["send_logs_level"].as<std::string>());
|
||||||
if (options.count("wait_for_suggestions_to_load"))
|
if (options.count("wait_for_suggestions_to_load"))
|
||||||
getClientConfiguration().setBool("wait_for_suggestions_to_load", true);
|
getClientConfiguration().setBool("wait_for_suggestions_to_load", true);
|
||||||
|
if (options.count("copy"))
|
||||||
|
{
|
||||||
|
if (!queries.empty())
|
||||||
|
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Options '--copy' and '--query' cannot be specified at the same time");
|
||||||
|
queries.emplace_back("SELECT * FROM table");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void LocalServer::readArguments(int argc, char ** argv, Arguments & common_arguments, std::vector<Arguments> &, std::vector<Arguments> &)
|
void LocalServer::readArguments(int argc, char ** argv, Arguments & common_arguments, std::vector<Arguments> &, std::vector<Arguments> &)
|
||||||
|
|||||||
@ -164,6 +164,123 @@ namespace MergeTreeSetting
|
|||||||
extern const MergeTreeSettingsBool allow_remote_fs_zero_copy_replication;
|
extern const MergeTreeSettingsBool allow_remote_fs_zero_copy_replication;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
namespace ServerSetting
|
||||||
|
{
|
||||||
|
extern const ServerSettingsUInt32 asynchronous_heavy_metrics_update_period_s;
|
||||||
|
extern const ServerSettingsUInt32 asynchronous_metrics_update_period_s;
|
||||||
|
extern const ServerSettingsBool async_insert_queue_flush_on_shutdown;
|
||||||
|
extern const ServerSettingsUInt64 async_insert_threads;
|
||||||
|
extern const ServerSettingsBool async_load_databases;
|
||||||
|
extern const ServerSettingsUInt64 background_buffer_flush_schedule_pool_size;
|
||||||
|
extern const ServerSettingsUInt64 background_common_pool_size;
|
||||||
|
extern const ServerSettingsUInt64 background_distributed_schedule_pool_size;
|
||||||
|
extern const ServerSettingsUInt64 background_fetches_pool_size;
|
||||||
|
extern const ServerSettingsFloat background_merges_mutations_concurrency_ratio;
|
||||||
|
extern const ServerSettingsString background_merges_mutations_scheduling_policy;
|
||||||
|
extern const ServerSettingsUInt64 background_message_broker_schedule_pool_size;
|
||||||
|
extern const ServerSettingsUInt64 background_move_pool_size;
|
||||||
|
extern const ServerSettingsUInt64 background_pool_size;
|
||||||
|
extern const ServerSettingsUInt64 background_schedule_pool_size;
|
||||||
|
extern const ServerSettingsUInt64 backups_io_thread_pool_queue_size;
|
||||||
|
extern const ServerSettingsDouble cache_size_to_ram_max_ratio;
|
||||||
|
extern const ServerSettingsDouble cannot_allocate_thread_fault_injection_probability;
|
||||||
|
extern const ServerSettingsUInt64 cgroups_memory_usage_observer_wait_time;
|
||||||
|
extern const ServerSettingsUInt64 compiled_expression_cache_elements_size;
|
||||||
|
extern const ServerSettingsUInt64 compiled_expression_cache_size;
|
||||||
|
extern const ServerSettingsUInt64 concurrent_threads_soft_limit_num;
|
||||||
|
extern const ServerSettingsUInt64 concurrent_threads_soft_limit_ratio_to_cores;
|
||||||
|
extern const ServerSettingsUInt64 config_reload_interval_ms;
|
||||||
|
extern const ServerSettingsUInt64 database_catalog_drop_table_concurrency;
|
||||||
|
extern const ServerSettingsString default_database;
|
||||||
|
extern const ServerSettingsBool disable_internal_dns_cache;
|
||||||
|
extern const ServerSettingsUInt64 disk_connections_soft_limit;
|
||||||
|
extern const ServerSettingsUInt64 disk_connections_store_limit;
|
||||||
|
extern const ServerSettingsUInt64 disk_connections_warn_limit;
|
||||||
|
extern const ServerSettingsBool dns_allow_resolve_names_to_ipv4;
|
||||||
|
extern const ServerSettingsBool dns_allow_resolve_names_to_ipv6;
|
||||||
|
extern const ServerSettingsUInt64 dns_cache_max_entries;
|
||||||
|
extern const ServerSettingsInt32 dns_cache_update_period;
|
||||||
|
extern const ServerSettingsUInt32 dns_max_consecutive_failures;
|
||||||
|
extern const ServerSettingsBool enable_azure_sdk_logging;
|
||||||
|
extern const ServerSettingsBool format_alter_operations_with_parentheses;
|
||||||
|
extern const ServerSettingsUInt64 global_profiler_cpu_time_period_ns;
|
||||||
|
extern const ServerSettingsUInt64 global_profiler_real_time_period_ns;
|
||||||
|
extern const ServerSettingsDouble gwp_asan_force_sample_probability;
|
||||||
|
extern const ServerSettingsUInt64 http_connections_soft_limit;
|
||||||
|
extern const ServerSettingsUInt64 http_connections_store_limit;
|
||||||
|
extern const ServerSettingsUInt64 http_connections_warn_limit;
|
||||||
|
extern const ServerSettingsString index_mark_cache_policy;
|
||||||
|
extern const ServerSettingsUInt64 index_mark_cache_size;
|
||||||
|
extern const ServerSettingsDouble index_mark_cache_size_ratio;
|
||||||
|
extern const ServerSettingsString index_uncompressed_cache_policy;
|
||||||
|
extern const ServerSettingsUInt64 index_uncompressed_cache_size;
|
||||||
|
extern const ServerSettingsDouble index_uncompressed_cache_size_ratio;
|
||||||
|
extern const ServerSettingsUInt64 io_thread_pool_queue_size;
|
||||||
|
extern const ServerSettingsSeconds keep_alive_timeout;
|
||||||
|
extern const ServerSettingsString mark_cache_policy;
|
||||||
|
extern const ServerSettingsUInt64 mark_cache_size;
|
||||||
|
extern const ServerSettingsDouble mark_cache_size_ratio;
|
||||||
|
extern const ServerSettingsUInt64 max_active_parts_loading_thread_pool_size;
|
||||||
|
extern const ServerSettingsUInt64 max_backups_io_thread_pool_free_size;
|
||||||
|
extern const ServerSettingsUInt64 max_backups_io_thread_pool_size;
|
||||||
|
extern const ServerSettingsUInt64 max_concurrent_insert_queries;
|
||||||
|
extern const ServerSettingsUInt64 max_concurrent_queries;
|
||||||
|
extern const ServerSettingsUInt64 max_concurrent_select_queries;
|
||||||
|
extern const ServerSettingsInt32 max_connections;
|
||||||
|
extern const ServerSettingsUInt64 max_database_num_to_warn;
|
||||||
|
extern const ServerSettingsUInt32 max_database_replicated_create_table_thread_pool_size;
|
||||||
|
extern const ServerSettingsUInt64 max_dictionary_num_to_warn;
|
||||||
|
extern const ServerSettingsUInt64 max_io_thread_pool_free_size;
|
||||||
|
extern const ServerSettingsUInt64 max_io_thread_pool_size;
|
||||||
|
extern const ServerSettingsUInt64 max_keep_alive_requests;
|
||||||
|
extern const ServerSettingsUInt64 max_outdated_parts_loading_thread_pool_size;
|
||||||
|
extern const ServerSettingsUInt64 max_partition_size_to_drop;
|
||||||
|
extern const ServerSettingsUInt64 max_part_num_to_warn;
|
||||||
|
extern const ServerSettingsUInt64 max_parts_cleaning_thread_pool_size;
|
||||||
|
extern const ServerSettingsUInt64 max_server_memory_usage;
|
||||||
|
extern const ServerSettingsDouble max_server_memory_usage_to_ram_ratio;
|
||||||
|
extern const ServerSettingsUInt64 max_table_num_to_warn;
|
||||||
|
extern const ServerSettingsUInt64 max_table_size_to_drop;
|
||||||
|
extern const ServerSettingsUInt64 max_temporary_data_on_disk_size;
|
||||||
|
extern const ServerSettingsUInt64 max_thread_pool_free_size;
|
||||||
|
extern const ServerSettingsUInt64 max_thread_pool_size;
|
||||||
|
extern const ServerSettingsUInt64 max_unexpected_parts_loading_thread_pool_size;
|
||||||
|
extern const ServerSettingsUInt64 max_view_num_to_warn;
|
||||||
|
extern const ServerSettingsUInt64 max_waiting_queries;
|
||||||
|
extern const ServerSettingsUInt64 memory_worker_period_ms;
|
||||||
|
extern const ServerSettingsUInt64 merges_mutations_memory_usage_soft_limit;
|
||||||
|
extern const ServerSettingsDouble merges_mutations_memory_usage_to_ram_ratio;
|
||||||
|
extern const ServerSettingsString merge_workload;
|
||||||
|
extern const ServerSettingsUInt64 mmap_cache_size;
|
||||||
|
extern const ServerSettingsString mutation_workload;
|
||||||
|
extern const ServerSettingsUInt64 page_cache_chunk_size;
|
||||||
|
extern const ServerSettingsUInt64 page_cache_mmap_size;
|
||||||
|
extern const ServerSettingsUInt64 page_cache_size;
|
||||||
|
extern const ServerSettingsBool page_cache_use_madv_free;
|
||||||
|
extern const ServerSettingsBool page_cache_use_transparent_huge_pages;
|
||||||
|
extern const ServerSettingsBool prepare_system_log_tables_on_startup;
|
||||||
|
extern const ServerSettingsBool show_addresses_in_stack_traces;
|
||||||
|
extern const ServerSettingsBool shutdown_wait_backups_and_restores;
|
||||||
|
extern const ServerSettingsUInt64 shutdown_wait_unfinished;
|
||||||
|
extern const ServerSettingsBool shutdown_wait_unfinished_queries;
|
||||||
|
extern const ServerSettingsUInt64 storage_connections_soft_limit;
|
||||||
|
extern const ServerSettingsUInt64 storage_connections_store_limit;
|
||||||
|
extern const ServerSettingsUInt64 storage_connections_warn_limit;
|
||||||
|
extern const ServerSettingsUInt64 tables_loader_background_pool_size;
|
||||||
|
extern const ServerSettingsUInt64 tables_loader_foreground_pool_size;
|
||||||
|
extern const ServerSettingsString temporary_data_in_cache;
|
||||||
|
extern const ServerSettingsUInt64 thread_pool_queue_size;
|
||||||
|
extern const ServerSettingsString tmp_policy;
|
||||||
|
extern const ServerSettingsUInt64 total_memory_profiler_sample_max_allocation_size;
|
||||||
|
extern const ServerSettingsUInt64 total_memory_profiler_sample_min_allocation_size;
|
||||||
|
extern const ServerSettingsUInt64 total_memory_profiler_step;
|
||||||
|
extern const ServerSettingsDouble total_memory_tracker_sample_probability;
|
||||||
|
extern const ServerSettingsString uncompressed_cache_policy;
|
||||||
|
extern const ServerSettingsUInt64 uncompressed_cache_size;
|
||||||
|
extern const ServerSettingsDouble uncompressed_cache_size_ratio;
|
||||||
|
extern const ServerSettingsBool use_legacy_mongodb_integration;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
namespace CurrentMetrics
|
namespace CurrentMetrics
|
||||||
@ -672,7 +789,7 @@ static void initializeAzureSDKLogger(
|
|||||||
[[ maybe_unused ]] int server_logs_level)
|
[[ maybe_unused ]] int server_logs_level)
|
||||||
{
|
{
|
||||||
#if USE_AZURE_BLOB_STORAGE
|
#if USE_AZURE_BLOB_STORAGE
|
||||||
if (!server_settings.enable_azure_sdk_logging)
|
if (!server_settings[ServerSetting::enable_azure_sdk_logging])
|
||||||
return;
|
return;
|
||||||
|
|
||||||
using AzureLogsLevel = Azure::Core::Diagnostics::Logger::Level;
|
using AzureLogsLevel = Azure::Core::Diagnostics::Logger::Level;
|
||||||
@ -746,9 +863,9 @@ try
|
|||||||
ServerSettings server_settings;
|
ServerSettings server_settings;
|
||||||
server_settings.loadSettingsFromConfig(config());
|
server_settings.loadSettingsFromConfig(config());
|
||||||
|
|
||||||
ASTAlterCommand::setFormatAlterCommandsWithParentheses(server_settings.format_alter_operations_with_parentheses);
|
ASTAlterCommand::setFormatAlterCommandsWithParentheses(server_settings[ServerSetting::format_alter_operations_with_parentheses]);
|
||||||
|
|
||||||
StackTrace::setShowAddresses(server_settings.show_addresses_in_stack_traces);
|
StackTrace::setShowAddresses(server_settings[ServerSetting::show_addresses_in_stack_traces]);
|
||||||
|
|
||||||
#if USE_HDFS
|
#if USE_HDFS
|
||||||
/// This will point libhdfs3 to the right location for its config.
|
/// This will point libhdfs3 to the right location for its config.
|
||||||
@ -794,10 +911,10 @@ try
|
|||||||
registerInterpreters();
|
registerInterpreters();
|
||||||
registerFunctions();
|
registerFunctions();
|
||||||
registerAggregateFunctions();
|
registerAggregateFunctions();
|
||||||
registerTableFunctions(server_settings.use_legacy_mongodb_integration);
|
registerTableFunctions(server_settings[ServerSetting::use_legacy_mongodb_integration]);
|
||||||
registerDatabases();
|
registerDatabases();
|
||||||
registerStorages(server_settings.use_legacy_mongodb_integration);
|
registerStorages(server_settings[ServerSetting::use_legacy_mongodb_integration]);
|
||||||
registerDictionaries(server_settings.use_legacy_mongodb_integration);
|
registerDictionaries(server_settings[ServerSetting::use_legacy_mongodb_integration]);
|
||||||
registerDisks(/* global_skip_access_check= */ false);
|
registerDisks(/* global_skip_access_check= */ false);
|
||||||
registerFormats();
|
registerFormats();
|
||||||
registerRemoteFileMetadatas();
|
registerRemoteFileMetadatas();
|
||||||
@ -893,37 +1010,37 @@ try
|
|||||||
// nodes (`from_zk`), because ZooKeeper interface uses the pool. We will
|
// nodes (`from_zk`), because ZooKeeper interface uses the pool. We will
|
||||||
// ignore `max_thread_pool_size` in configs we fetch from ZK, but oh well.
|
// ignore `max_thread_pool_size` in configs we fetch from ZK, but oh well.
|
||||||
GlobalThreadPool::initialize(
|
GlobalThreadPool::initialize(
|
||||||
server_settings.max_thread_pool_size,
|
server_settings[ServerSetting::max_thread_pool_size],
|
||||||
server_settings.max_thread_pool_free_size,
|
server_settings[ServerSetting::max_thread_pool_free_size],
|
||||||
server_settings.thread_pool_queue_size,
|
server_settings[ServerSetting::thread_pool_queue_size],
|
||||||
has_trace_collector ? server_settings.global_profiler_real_time_period_ns : 0,
|
has_trace_collector ? server_settings[ServerSetting::global_profiler_real_time_period_ns] : 0,
|
||||||
has_trace_collector ? server_settings.global_profiler_cpu_time_period_ns : 0);
|
has_trace_collector ? server_settings[ServerSetting::global_profiler_cpu_time_period_ns] : 0);
|
||||||
|
|
||||||
if (has_trace_collector)
|
if (has_trace_collector)
|
||||||
{
|
{
|
||||||
global_context->createTraceCollector();
|
global_context->createTraceCollector();
|
||||||
|
|
||||||
/// Set up server-wide memory profiler (for total memory tracker).
|
/// Set up server-wide memory profiler (for total memory tracker).
|
||||||
if (server_settings.total_memory_profiler_step)
|
if (server_settings[ServerSetting::total_memory_profiler_step])
|
||||||
total_memory_tracker.setProfilerStep(server_settings.total_memory_profiler_step);
|
total_memory_tracker.setProfilerStep(server_settings[ServerSetting::total_memory_profiler_step]);
|
||||||
|
|
||||||
if (server_settings.total_memory_tracker_sample_probability > 0.0)
|
if (server_settings[ServerSetting::total_memory_tracker_sample_probability] > 0.0)
|
||||||
total_memory_tracker.setSampleProbability(server_settings.total_memory_tracker_sample_probability);
|
total_memory_tracker.setSampleProbability(server_settings[ServerSetting::total_memory_tracker_sample_probability]);
|
||||||
|
|
||||||
if (server_settings.total_memory_profiler_sample_min_allocation_size)
|
if (server_settings[ServerSetting::total_memory_profiler_sample_min_allocation_size])
|
||||||
total_memory_tracker.setSampleMinAllocationSize(server_settings.total_memory_profiler_sample_min_allocation_size);
|
total_memory_tracker.setSampleMinAllocationSize(server_settings[ServerSetting::total_memory_profiler_sample_min_allocation_size]);
|
||||||
|
|
||||||
if (server_settings.total_memory_profiler_sample_max_allocation_size)
|
if (server_settings[ServerSetting::total_memory_profiler_sample_max_allocation_size])
|
||||||
total_memory_tracker.setSampleMaxAllocationSize(server_settings.total_memory_profiler_sample_max_allocation_size);
|
total_memory_tracker.setSampleMaxAllocationSize(server_settings[ServerSetting::total_memory_profiler_sample_max_allocation_size]);
|
||||||
}
|
}
|
||||||
|
|
||||||
Poco::ThreadPool server_pool(
|
Poco::ThreadPool server_pool(
|
||||||
/* minCapacity */3,
|
/* minCapacity */3,
|
||||||
/* maxCapacity */server_settings.max_connections,
|
/* maxCapacity */server_settings[ServerSetting::max_connections],
|
||||||
/* idleTime */60,
|
/* idleTime */60,
|
||||||
/* stackSize */POCO_THREAD_STACK_SIZE,
|
/* stackSize */POCO_THREAD_STACK_SIZE,
|
||||||
server_settings.global_profiler_real_time_period_ns,
|
server_settings[ServerSetting::global_profiler_real_time_period_ns],
|
||||||
server_settings.global_profiler_cpu_time_period_ns);
|
server_settings[ServerSetting::global_profiler_cpu_time_period_ns]);
|
||||||
|
|
||||||
std::mutex servers_lock;
|
std::mutex servers_lock;
|
||||||
std::vector<ProtocolServerAdapter> servers;
|
std::vector<ProtocolServerAdapter> servers;
|
||||||
@ -937,13 +1054,13 @@ try
|
|||||||
LOG_INFO(log, "Background threads finished in {} ms", watch.elapsedMilliseconds());
|
LOG_INFO(log, "Background threads finished in {} ms", watch.elapsedMilliseconds());
|
||||||
});
|
});
|
||||||
|
|
||||||
MemoryWorker memory_worker(global_context->getServerSettings().memory_worker_period_ms);
|
MemoryWorker memory_worker(global_context->getServerSettings()[ServerSetting::memory_worker_period_ms]);
|
||||||
|
|
||||||
/// This object will periodically calculate some metrics.
|
/// This object will periodically calculate some metrics.
|
||||||
ServerAsynchronousMetrics async_metrics(
|
ServerAsynchronousMetrics async_metrics(
|
||||||
global_context,
|
global_context,
|
||||||
server_settings.asynchronous_metrics_update_period_s,
|
server_settings[ServerSetting::asynchronous_metrics_update_period_s],
|
||||||
server_settings.asynchronous_heavy_metrics_update_period_s,
|
server_settings[ServerSetting::asynchronous_heavy_metrics_update_period_s],
|
||||||
[&]() -> std::vector<ProtocolServerMetrics>
|
[&]() -> std::vector<ProtocolServerMetrics>
|
||||||
{
|
{
|
||||||
std::vector<ProtocolServerMetrics> metrics;
|
std::vector<ProtocolServerMetrics> metrics;
|
||||||
@ -996,7 +1113,7 @@ try
|
|||||||
LOG_INFO(log, "Closed all listening sockets.");
|
LOG_INFO(log, "Closed all listening sockets.");
|
||||||
|
|
||||||
if (current_connections > 0)
|
if (current_connections > 0)
|
||||||
current_connections = waitServersToFinish(servers_to_start_before_tables, servers_lock, server_settings.shutdown_wait_unfinished);
|
current_connections = waitServersToFinish(servers_to_start_before_tables, servers_lock, server_settings[ServerSetting::shutdown_wait_unfinished]);
|
||||||
|
|
||||||
if (current_connections)
|
if (current_connections)
|
||||||
LOG_INFO(log, "Closed connections to servers for tables. But {} remain. Probably some tables of other users cannot finish their connections after context shutdown.", current_connections);
|
LOG_INFO(log, "Closed connections to servers for tables. But {} remain. Probably some tables of other users cannot finish their connections after context shutdown.", current_connections);
|
||||||
@ -1033,47 +1150,47 @@ try
|
|||||||
#endif
|
#endif
|
||||||
|
|
||||||
getIOThreadPool().initialize(
|
getIOThreadPool().initialize(
|
||||||
server_settings.max_io_thread_pool_size,
|
server_settings[ServerSetting::max_io_thread_pool_size],
|
||||||
server_settings.max_io_thread_pool_free_size,
|
server_settings[ServerSetting::max_io_thread_pool_free_size],
|
||||||
server_settings.io_thread_pool_queue_size);
|
server_settings[ServerSetting::io_thread_pool_queue_size]);
|
||||||
|
|
||||||
getBackupsIOThreadPool().initialize(
|
getBackupsIOThreadPool().initialize(
|
||||||
server_settings.max_backups_io_thread_pool_size,
|
server_settings[ServerSetting::max_backups_io_thread_pool_size],
|
||||||
server_settings.max_backups_io_thread_pool_free_size,
|
server_settings[ServerSetting::max_backups_io_thread_pool_free_size],
|
||||||
server_settings.backups_io_thread_pool_queue_size);
|
server_settings[ServerSetting::backups_io_thread_pool_queue_size]);
|
||||||
|
|
||||||
getActivePartsLoadingThreadPool().initialize(
|
getActivePartsLoadingThreadPool().initialize(
|
||||||
server_settings.max_active_parts_loading_thread_pool_size,
|
server_settings[ServerSetting::max_active_parts_loading_thread_pool_size],
|
||||||
0, // We don't need any threads once all the parts will be loaded
|
0, // We don't need any threads once all the parts will be loaded
|
||||||
server_settings.max_active_parts_loading_thread_pool_size);
|
server_settings[ServerSetting::max_active_parts_loading_thread_pool_size]);
|
||||||
|
|
||||||
getOutdatedPartsLoadingThreadPool().initialize(
|
getOutdatedPartsLoadingThreadPool().initialize(
|
||||||
server_settings.max_outdated_parts_loading_thread_pool_size,
|
server_settings[ServerSetting::max_outdated_parts_loading_thread_pool_size],
|
||||||
0, // We don't need any threads once all the parts will be loaded
|
0, // We don't need any threads once all the parts will be loaded
|
||||||
server_settings.max_outdated_parts_loading_thread_pool_size);
|
server_settings[ServerSetting::max_outdated_parts_loading_thread_pool_size]);
|
||||||
|
|
||||||
/// It could grow if we need to synchronously wait until all the data parts will be loaded.
|
/// It could grow if we need to synchronously wait until all the data parts will be loaded.
|
||||||
getOutdatedPartsLoadingThreadPool().setMaxTurboThreads(
|
getOutdatedPartsLoadingThreadPool().setMaxTurboThreads(
|
||||||
server_settings.max_active_parts_loading_thread_pool_size
|
server_settings[ServerSetting::max_active_parts_loading_thread_pool_size]
|
||||||
);
|
);
|
||||||
|
|
||||||
getUnexpectedPartsLoadingThreadPool().initialize(
|
getUnexpectedPartsLoadingThreadPool().initialize(
|
||||||
server_settings.max_unexpected_parts_loading_thread_pool_size,
|
server_settings[ServerSetting::max_unexpected_parts_loading_thread_pool_size],
|
||||||
0, // We don't need any threads once all the parts will be loaded
|
0, // We don't need any threads once all the parts will be loaded
|
||||||
server_settings.max_unexpected_parts_loading_thread_pool_size);
|
server_settings[ServerSetting::max_unexpected_parts_loading_thread_pool_size]);
|
||||||
|
|
||||||
/// It could grow if we need to synchronously wait until all the data parts will be loaded.
|
/// It could grow if we need to synchronously wait until all the data parts will be loaded.
|
||||||
getUnexpectedPartsLoadingThreadPool().setMaxTurboThreads(
|
getUnexpectedPartsLoadingThreadPool().setMaxTurboThreads(
|
||||||
server_settings.max_active_parts_loading_thread_pool_size
|
server_settings[ServerSetting::max_active_parts_loading_thread_pool_size]
|
||||||
);
|
);
|
||||||
|
|
||||||
getPartsCleaningThreadPool().initialize(
|
getPartsCleaningThreadPool().initialize(
|
||||||
server_settings.max_parts_cleaning_thread_pool_size,
|
server_settings[ServerSetting::max_parts_cleaning_thread_pool_size],
|
||||||
0, // We don't need any threads one all the parts will be deleted
|
0, // We don't need any threads one all the parts will be deleted
|
||||||
server_settings.max_parts_cleaning_thread_pool_size);
|
server_settings[ServerSetting::max_parts_cleaning_thread_pool_size]);
|
||||||
|
|
||||||
auto max_database_replicated_create_table_thread_pool_size = server_settings.max_database_replicated_create_table_thread_pool_size
|
auto max_database_replicated_create_table_thread_pool_size = server_settings[ServerSetting::max_database_replicated_create_table_thread_pool_size]
|
||||||
? server_settings.max_database_replicated_create_table_thread_pool_size
|
? server_settings[ServerSetting::max_database_replicated_create_table_thread_pool_size]
|
||||||
: getNumberOfCPUCoresToUse();
|
: getNumberOfCPUCoresToUse();
|
||||||
getDatabaseReplicatedCreateTablesThreadPool().initialize(
|
getDatabaseReplicatedCreateTablesThreadPool().initialize(
|
||||||
max_database_replicated_create_table_thread_pool_size,
|
max_database_replicated_create_table_thread_pool_size,
|
||||||
@ -1081,9 +1198,9 @@ try
|
|||||||
max_database_replicated_create_table_thread_pool_size);
|
max_database_replicated_create_table_thread_pool_size);
|
||||||
|
|
||||||
getDatabaseCatalogDropTablesThreadPool().initialize(
|
getDatabaseCatalogDropTablesThreadPool().initialize(
|
||||||
server_settings.database_catalog_drop_table_concurrency,
|
server_settings[ServerSetting::database_catalog_drop_table_concurrency],
|
||||||
0, // We don't need any threads if there are no DROP queries.
|
0, // We don't need any threads if there are no DROP queries.
|
||||||
server_settings.database_catalog_drop_table_concurrency);
|
server_settings[ServerSetting::database_catalog_drop_table_concurrency]);
|
||||||
|
|
||||||
/// Initialize global local cache for remote filesystem.
|
/// Initialize global local cache for remote filesystem.
|
||||||
if (config().has("local_cache_for_remote_fs"))
|
if (config().has("local_cache_for_remote_fs"))
|
||||||
@ -1320,18 +1437,18 @@ try
|
|||||||
LOG_TRACE(log, "Initialized DateLUT with time zone '{}'.", DateLUT::serverTimezoneInstance().getTimeZone());
|
LOG_TRACE(log, "Initialized DateLUT with time zone '{}'.", DateLUT::serverTimezoneInstance().getTimeZone());
|
||||||
|
|
||||||
/// Storage with temporary data for processing of heavy queries.
|
/// Storage with temporary data for processing of heavy queries.
|
||||||
if (!server_settings.tmp_policy.value.empty())
|
if (!server_settings[ServerSetting::tmp_policy].value.empty())
|
||||||
{
|
{
|
||||||
global_context->setTemporaryStoragePolicy(server_settings.tmp_policy, server_settings.max_temporary_data_on_disk_size);
|
global_context->setTemporaryStoragePolicy(server_settings[ServerSetting::tmp_policy], server_settings[ServerSetting::max_temporary_data_on_disk_size]);
|
||||||
}
|
}
|
||||||
else if (!server_settings.temporary_data_in_cache.value.empty())
|
else if (!server_settings[ServerSetting::temporary_data_in_cache].value.empty())
|
||||||
{
|
{
|
||||||
global_context->setTemporaryStorageInCache(server_settings.temporary_data_in_cache, server_settings.max_temporary_data_on_disk_size);
|
global_context->setTemporaryStorageInCache(server_settings[ServerSetting::temporary_data_in_cache], server_settings[ServerSetting::max_temporary_data_on_disk_size]);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
std::string temporary_path = config().getString("tmp_path", path / "tmp/");
|
std::string temporary_path = config().getString("tmp_path", path / "tmp/");
|
||||||
global_context->setTemporaryStoragePath(temporary_path, server_settings.max_temporary_data_on_disk_size);
|
global_context->setTemporaryStoragePath(temporary_path, server_settings[ServerSetting::max_temporary_data_on_disk_size]);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Directory with 'flags': files indicating temporary settings for the server set by system administrator.
|
/** Directory with 'flags': files indicating temporary settings for the server set by system administrator.
|
||||||
@ -1420,11 +1537,11 @@ try
|
|||||||
|
|
||||||
/// Set up caches.
|
/// Set up caches.
|
||||||
|
|
||||||
const size_t max_cache_size = static_cast<size_t>(physical_server_memory * server_settings.cache_size_to_ram_max_ratio);
|
const size_t max_cache_size = static_cast<size_t>(physical_server_memory * server_settings[ServerSetting::cache_size_to_ram_max_ratio]);
|
||||||
|
|
||||||
String uncompressed_cache_policy = server_settings.uncompressed_cache_policy;
|
String uncompressed_cache_policy = server_settings[ServerSetting::uncompressed_cache_policy];
|
||||||
size_t uncompressed_cache_size = server_settings.uncompressed_cache_size;
|
size_t uncompressed_cache_size = server_settings[ServerSetting::uncompressed_cache_size];
|
||||||
double uncompressed_cache_size_ratio = server_settings.uncompressed_cache_size_ratio;
|
double uncompressed_cache_size_ratio = server_settings[ServerSetting::uncompressed_cache_size_ratio];
|
||||||
if (uncompressed_cache_size > max_cache_size)
|
if (uncompressed_cache_size > max_cache_size)
|
||||||
{
|
{
|
||||||
uncompressed_cache_size = max_cache_size;
|
uncompressed_cache_size = max_cache_size;
|
||||||
@ -1432,9 +1549,9 @@ try
|
|||||||
}
|
}
|
||||||
global_context->setUncompressedCache(uncompressed_cache_policy, uncompressed_cache_size, uncompressed_cache_size_ratio);
|
global_context->setUncompressedCache(uncompressed_cache_policy, uncompressed_cache_size, uncompressed_cache_size_ratio);
|
||||||
|
|
||||||
String mark_cache_policy = server_settings.mark_cache_policy;
|
String mark_cache_policy = server_settings[ServerSetting::mark_cache_policy];
|
||||||
size_t mark_cache_size = server_settings.mark_cache_size;
|
size_t mark_cache_size = server_settings[ServerSetting::mark_cache_size];
|
||||||
double mark_cache_size_ratio = server_settings.mark_cache_size_ratio;
|
double mark_cache_size_ratio = server_settings[ServerSetting::mark_cache_size_ratio];
|
||||||
if (mark_cache_size > max_cache_size)
|
if (mark_cache_size > max_cache_size)
|
||||||
{
|
{
|
||||||
mark_cache_size = max_cache_size;
|
mark_cache_size = max_cache_size;
|
||||||
@ -1442,16 +1559,16 @@ try
|
|||||||
}
|
}
|
||||||
global_context->setMarkCache(mark_cache_policy, mark_cache_size, mark_cache_size_ratio);
|
global_context->setMarkCache(mark_cache_policy, mark_cache_size, mark_cache_size_ratio);
|
||||||
|
|
||||||
size_t page_cache_size = server_settings.page_cache_size;
|
size_t page_cache_size = server_settings[ServerSetting::page_cache_size];
|
||||||
if (page_cache_size != 0)
|
if (page_cache_size != 0)
|
||||||
global_context->setPageCache(
|
global_context->setPageCache(
|
||||||
server_settings.page_cache_chunk_size, server_settings.page_cache_mmap_size,
|
server_settings[ServerSetting::page_cache_chunk_size], server_settings[ServerSetting::page_cache_mmap_size],
|
||||||
page_cache_size, server_settings.page_cache_use_madv_free,
|
page_cache_size, server_settings[ServerSetting::page_cache_use_madv_free],
|
||||||
server_settings.page_cache_use_transparent_huge_pages);
|
server_settings[ServerSetting::page_cache_use_transparent_huge_pages]);
|
||||||
|
|
||||||
String index_uncompressed_cache_policy = server_settings.index_uncompressed_cache_policy;
|
String index_uncompressed_cache_policy = server_settings[ServerSetting::index_uncompressed_cache_policy];
|
||||||
size_t index_uncompressed_cache_size = server_settings.index_uncompressed_cache_size;
|
size_t index_uncompressed_cache_size = server_settings[ServerSetting::index_uncompressed_cache_size];
|
||||||
double index_uncompressed_cache_size_ratio = server_settings.index_uncompressed_cache_size_ratio;
|
double index_uncompressed_cache_size_ratio = server_settings[ServerSetting::index_uncompressed_cache_size_ratio];
|
||||||
if (index_uncompressed_cache_size > max_cache_size)
|
if (index_uncompressed_cache_size > max_cache_size)
|
||||||
{
|
{
|
||||||
index_uncompressed_cache_size = max_cache_size;
|
index_uncompressed_cache_size = max_cache_size;
|
||||||
@ -1459,9 +1576,9 @@ try
|
|||||||
}
|
}
|
||||||
global_context->setIndexUncompressedCache(index_uncompressed_cache_policy, index_uncompressed_cache_size, index_uncompressed_cache_size_ratio);
|
global_context->setIndexUncompressedCache(index_uncompressed_cache_policy, index_uncompressed_cache_size, index_uncompressed_cache_size_ratio);
|
||||||
|
|
||||||
String index_mark_cache_policy = server_settings.index_mark_cache_policy;
|
String index_mark_cache_policy = server_settings[ServerSetting::index_mark_cache_policy];
|
||||||
size_t index_mark_cache_size = server_settings.index_mark_cache_size;
|
size_t index_mark_cache_size = server_settings[ServerSetting::index_mark_cache_size];
|
||||||
double index_mark_cache_size_ratio = server_settings.index_mark_cache_size_ratio;
|
double index_mark_cache_size_ratio = server_settings[ServerSetting::index_mark_cache_size_ratio];
|
||||||
if (index_mark_cache_size > max_cache_size)
|
if (index_mark_cache_size > max_cache_size)
|
||||||
{
|
{
|
||||||
index_mark_cache_size = max_cache_size;
|
index_mark_cache_size = max_cache_size;
|
||||||
@ -1469,7 +1586,7 @@ try
|
|||||||
}
|
}
|
||||||
global_context->setIndexMarkCache(index_mark_cache_policy, index_mark_cache_size, index_mark_cache_size_ratio);
|
global_context->setIndexMarkCache(index_mark_cache_policy, index_mark_cache_size, index_mark_cache_size_ratio);
|
||||||
|
|
||||||
size_t mmap_cache_size = server_settings.mmap_cache_size;
|
size_t mmap_cache_size = server_settings[ServerSetting::mmap_cache_size];
|
||||||
if (mmap_cache_size > max_cache_size)
|
if (mmap_cache_size > max_cache_size)
|
||||||
{
|
{
|
||||||
mmap_cache_size = max_cache_size;
|
mmap_cache_size = max_cache_size;
|
||||||
@ -1489,13 +1606,15 @@ try
|
|||||||
global_context->setQueryCache(query_cache_max_size_in_bytes, query_cache_max_entries, query_cache_query_cache_max_entry_size_in_bytes, query_cache_max_entry_size_in_rows);
|
global_context->setQueryCache(query_cache_max_size_in_bytes, query_cache_max_entries, query_cache_query_cache_max_entry_size_in_bytes, query_cache_max_entry_size_in_rows);
|
||||||
|
|
||||||
#if USE_EMBEDDED_COMPILER
|
#if USE_EMBEDDED_COMPILER
|
||||||
size_t compiled_expression_cache_max_size_in_bytes = server_settings.compiled_expression_cache_size;
|
size_t compiled_expression_cache_max_size_in_bytes = server_settings[ServerSetting::compiled_expression_cache_size];
|
||||||
size_t compiled_expression_cache_max_elements = server_settings.compiled_expression_cache_elements_size;
|
size_t compiled_expression_cache_max_elements = server_settings[ServerSetting::compiled_expression_cache_elements_size];
|
||||||
CompiledExpressionCacheFactory::instance().init(compiled_expression_cache_max_size_in_bytes, compiled_expression_cache_max_elements);
|
CompiledExpressionCacheFactory::instance().init(compiled_expression_cache_max_size_in_bytes, compiled_expression_cache_max_elements);
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
NamedCollectionFactory::instance().loadIfNot();
|
NamedCollectionFactory::instance().loadIfNot();
|
||||||
|
|
||||||
|
FileCacheFactory::instance().loadDefaultCaches(config());
|
||||||
|
|
||||||
/// Initialize main config reloader.
|
/// Initialize main config reloader.
|
||||||
std::string include_from_path = config().getString("include_from", "/etc/metrika.xml");
|
std::string include_from_path = config().getString("include_from", "/etc/metrika.xml");
|
||||||
|
|
||||||
@ -1507,7 +1626,7 @@ try
|
|||||||
std::optional<CgroupsMemoryUsageObserver> cgroups_memory_usage_observer;
|
std::optional<CgroupsMemoryUsageObserver> cgroups_memory_usage_observer;
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
auto wait_time = server_settings.cgroups_memory_usage_observer_wait_time;
|
auto wait_time = server_settings[ServerSetting::cgroups_memory_usage_observer_wait_time];
|
||||||
if (wait_time != 0)
|
if (wait_time != 0)
|
||||||
cgroups_memory_usage_observer.emplace(std::chrono::seconds(wait_time));
|
cgroups_memory_usage_observer.emplace(std::chrono::seconds(wait_time));
|
||||||
}
|
}
|
||||||
@ -1550,8 +1669,8 @@ try
|
|||||||
ServerSettings new_server_settings;
|
ServerSettings new_server_settings;
|
||||||
new_server_settings.loadSettingsFromConfig(*config);
|
new_server_settings.loadSettingsFromConfig(*config);
|
||||||
|
|
||||||
size_t max_server_memory_usage = new_server_settings.max_server_memory_usage;
|
size_t max_server_memory_usage = new_server_settings[ServerSetting::max_server_memory_usage];
|
||||||
double max_server_memory_usage_to_ram_ratio = new_server_settings.max_server_memory_usage_to_ram_ratio;
|
double max_server_memory_usage_to_ram_ratio = new_server_settings[ServerSetting::max_server_memory_usage_to_ram_ratio];
|
||||||
|
|
||||||
size_t current_physical_server_memory = getMemoryAmount(); /// With cgroups, the amount of memory available to the server can be changed dynamically.
|
size_t current_physical_server_memory = getMemoryAmount(); /// With cgroups, the amount of memory available to the server can be changed dynamically.
|
||||||
size_t default_max_server_memory_usage = static_cast<size_t>(current_physical_server_memory * max_server_memory_usage_to_ram_ratio);
|
size_t default_max_server_memory_usage = static_cast<size_t>(current_physical_server_memory * max_server_memory_usage_to_ram_ratio);
|
||||||
@ -1581,9 +1700,9 @@ try
|
|||||||
total_memory_tracker.setDescription("(total)");
|
total_memory_tracker.setDescription("(total)");
|
||||||
total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking);
|
total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking);
|
||||||
|
|
||||||
size_t merges_mutations_memory_usage_soft_limit = new_server_settings.merges_mutations_memory_usage_soft_limit;
|
size_t merges_mutations_memory_usage_soft_limit = new_server_settings[ServerSetting::merges_mutations_memory_usage_soft_limit];
|
||||||
|
|
||||||
size_t default_merges_mutations_server_memory_usage = static_cast<size_t>(current_physical_server_memory * new_server_settings.merges_mutations_memory_usage_to_ram_ratio);
|
size_t default_merges_mutations_server_memory_usage = static_cast<size_t>(current_physical_server_memory * new_server_settings[ServerSetting::merges_mutations_memory_usage_to_ram_ratio]);
|
||||||
if (merges_mutations_memory_usage_soft_limit == 0)
|
if (merges_mutations_memory_usage_soft_limit == 0)
|
||||||
{
|
{
|
||||||
merges_mutations_memory_usage_soft_limit = default_merges_mutations_server_memory_usage;
|
merges_mutations_memory_usage_soft_limit = default_merges_mutations_server_memory_usage;
|
||||||
@ -1591,7 +1710,7 @@ try
|
|||||||
" ({} available * {:.2f} merges_mutations_memory_usage_to_ram_ratio)",
|
" ({} available * {:.2f} merges_mutations_memory_usage_to_ram_ratio)",
|
||||||
formatReadableSizeWithBinarySuffix(merges_mutations_memory_usage_soft_limit),
|
formatReadableSizeWithBinarySuffix(merges_mutations_memory_usage_soft_limit),
|
||||||
formatReadableSizeWithBinarySuffix(current_physical_server_memory),
|
formatReadableSizeWithBinarySuffix(current_physical_server_memory),
|
||||||
new_server_settings.merges_mutations_memory_usage_to_ram_ratio);
|
new_server_settings[ServerSetting::merges_mutations_memory_usage_to_ram_ratio]);
|
||||||
}
|
}
|
||||||
else if (merges_mutations_memory_usage_soft_limit > default_merges_mutations_server_memory_usage)
|
else if (merges_mutations_memory_usage_soft_limit > default_merges_mutations_server_memory_usage)
|
||||||
{
|
{
|
||||||
@ -1600,7 +1719,7 @@ try
|
|||||||
" ({} available * {:.2f} merges_mutations_memory_usage_to_ram_ratio)",
|
" ({} available * {:.2f} merges_mutations_memory_usage_to_ram_ratio)",
|
||||||
formatReadableSizeWithBinarySuffix(merges_mutations_memory_usage_soft_limit),
|
formatReadableSizeWithBinarySuffix(merges_mutations_memory_usage_soft_limit),
|
||||||
formatReadableSizeWithBinarySuffix(current_physical_server_memory),
|
formatReadableSizeWithBinarySuffix(current_physical_server_memory),
|
||||||
new_server_settings.merges_mutations_memory_usage_to_ram_ratio);
|
new_server_settings[ServerSetting::merges_mutations_memory_usage_to_ram_ratio]);
|
||||||
}
|
}
|
||||||
|
|
||||||
LOG_INFO(log, "Merges and mutations memory limit is set to {}",
|
LOG_INFO(log, "Merges and mutations memory limit is set to {}",
|
||||||
@ -1633,31 +1752,31 @@ try
|
|||||||
global_context->setRemoteHostFilter(*config);
|
global_context->setRemoteHostFilter(*config);
|
||||||
global_context->setHTTPHeaderFilter(*config);
|
global_context->setHTTPHeaderFilter(*config);
|
||||||
|
|
||||||
global_context->setMaxTableSizeToDrop(new_server_settings.max_table_size_to_drop);
|
global_context->setMaxTableSizeToDrop(new_server_settings[ServerSetting::max_table_size_to_drop]);
|
||||||
global_context->setMaxPartitionSizeToDrop(new_server_settings.max_partition_size_to_drop);
|
global_context->setMaxPartitionSizeToDrop(new_server_settings[ServerSetting::max_partition_size_to_drop]);
|
||||||
global_context->setMaxTableNumToWarn(new_server_settings.max_table_num_to_warn);
|
global_context->setMaxTableNumToWarn(new_server_settings[ServerSetting::max_table_num_to_warn]);
|
||||||
global_context->setMaxViewNumToWarn(new_server_settings.max_view_num_to_warn);
|
global_context->setMaxViewNumToWarn(new_server_settings[ServerSetting::max_view_num_to_warn]);
|
||||||
global_context->setMaxDictionaryNumToWarn(new_server_settings.max_dictionary_num_to_warn);
|
global_context->setMaxDictionaryNumToWarn(new_server_settings[ServerSetting::max_dictionary_num_to_warn]);
|
||||||
global_context->setMaxDatabaseNumToWarn(new_server_settings.max_database_num_to_warn);
|
global_context->setMaxDatabaseNumToWarn(new_server_settings[ServerSetting::max_database_num_to_warn]);
|
||||||
global_context->setMaxPartNumToWarn(new_server_settings.max_part_num_to_warn);
|
global_context->setMaxPartNumToWarn(new_server_settings[ServerSetting::max_part_num_to_warn]);
|
||||||
/// Only for system.server_settings
|
/// Only for system.server_settings
|
||||||
global_context->setConfigReloaderInterval(new_server_settings.config_reload_interval_ms);
|
global_context->setConfigReloaderInterval(new_server_settings[ServerSetting::config_reload_interval_ms]);
|
||||||
|
|
||||||
SlotCount concurrent_threads_soft_limit = UnlimitedSlots;
|
SlotCount concurrent_threads_soft_limit = UnlimitedSlots;
|
||||||
if (new_server_settings.concurrent_threads_soft_limit_num > 0 && new_server_settings.concurrent_threads_soft_limit_num < concurrent_threads_soft_limit)
|
if (new_server_settings[ServerSetting::concurrent_threads_soft_limit_num] > 0 && new_server_settings[ServerSetting::concurrent_threads_soft_limit_num] < concurrent_threads_soft_limit)
|
||||||
concurrent_threads_soft_limit = new_server_settings.concurrent_threads_soft_limit_num;
|
concurrent_threads_soft_limit = new_server_settings[ServerSetting::concurrent_threads_soft_limit_num];
|
||||||
if (new_server_settings.concurrent_threads_soft_limit_ratio_to_cores > 0)
|
if (new_server_settings[ServerSetting::concurrent_threads_soft_limit_ratio_to_cores] > 0)
|
||||||
{
|
{
|
||||||
auto value = new_server_settings.concurrent_threads_soft_limit_ratio_to_cores * getNumberOfCPUCoresToUse();
|
auto value = new_server_settings[ServerSetting::concurrent_threads_soft_limit_ratio_to_cores] * getNumberOfCPUCoresToUse();
|
||||||
if (value > 0 && value < concurrent_threads_soft_limit)
|
if (value > 0 && value < concurrent_threads_soft_limit)
|
||||||
concurrent_threads_soft_limit = value;
|
concurrent_threads_soft_limit = value;
|
||||||
}
|
}
|
||||||
ConcurrencyControl::instance().setMaxConcurrency(concurrent_threads_soft_limit);
|
ConcurrencyControl::instance().setMaxConcurrency(concurrent_threads_soft_limit);
|
||||||
|
|
||||||
global_context->getProcessList().setMaxSize(new_server_settings.max_concurrent_queries);
|
global_context->getProcessList().setMaxSize(new_server_settings[ServerSetting::max_concurrent_queries]);
|
||||||
global_context->getProcessList().setMaxInsertQueriesAmount(new_server_settings.max_concurrent_insert_queries);
|
global_context->getProcessList().setMaxInsertQueriesAmount(new_server_settings[ServerSetting::max_concurrent_insert_queries]);
|
||||||
global_context->getProcessList().setMaxSelectQueriesAmount(new_server_settings.max_concurrent_select_queries);
|
global_context->getProcessList().setMaxSelectQueriesAmount(new_server_settings[ServerSetting::max_concurrent_select_queries]);
|
||||||
global_context->getProcessList().setMaxWaitingQueriesAmount(new_server_settings.max_waiting_queries);
|
global_context->getProcessList().setMaxWaitingQueriesAmount(new_server_settings[ServerSetting::max_waiting_queries]);
|
||||||
|
|
||||||
if (config->has("keeper_server"))
|
if (config->has("keeper_server"))
|
||||||
global_context->updateKeeperConfiguration(*config);
|
global_context->updateKeeperConfiguration(*config);
|
||||||
@ -1668,72 +1787,72 @@ try
|
|||||||
/// This is done for backward compatibility.
|
/// This is done for backward compatibility.
|
||||||
if (global_context->areBackgroundExecutorsInitialized())
|
if (global_context->areBackgroundExecutorsInitialized())
|
||||||
{
|
{
|
||||||
auto new_pool_size = new_server_settings.background_pool_size;
|
auto new_pool_size = new_server_settings[ServerSetting::background_pool_size];
|
||||||
auto new_ratio = new_server_settings.background_merges_mutations_concurrency_ratio;
|
auto new_ratio = new_server_settings[ServerSetting::background_merges_mutations_concurrency_ratio];
|
||||||
global_context->getMergeMutateExecutor()->increaseThreadsAndMaxTasksCount(new_pool_size, static_cast<size_t>(new_pool_size * new_ratio));
|
global_context->getMergeMutateExecutor()->increaseThreadsAndMaxTasksCount(new_pool_size, static_cast<size_t>(new_pool_size * new_ratio));
|
||||||
global_context->getMergeMutateExecutor()->updateSchedulingPolicy(new_server_settings.background_merges_mutations_scheduling_policy.toString());
|
global_context->getMergeMutateExecutor()->updateSchedulingPolicy(new_server_settings[ServerSetting::background_merges_mutations_scheduling_policy].toString());
|
||||||
}
|
}
|
||||||
|
|
||||||
if (global_context->areBackgroundExecutorsInitialized())
|
if (global_context->areBackgroundExecutorsInitialized())
|
||||||
{
|
{
|
||||||
auto new_pool_size = new_server_settings.background_move_pool_size;
|
auto new_pool_size = new_server_settings[ServerSetting::background_move_pool_size];
|
||||||
global_context->getMovesExecutor()->increaseThreadsAndMaxTasksCount(new_pool_size, new_pool_size);
|
global_context->getMovesExecutor()->increaseThreadsAndMaxTasksCount(new_pool_size, new_pool_size);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (global_context->areBackgroundExecutorsInitialized())
|
if (global_context->areBackgroundExecutorsInitialized())
|
||||||
{
|
{
|
||||||
auto new_pool_size = new_server_settings.background_fetches_pool_size;
|
auto new_pool_size = new_server_settings[ServerSetting::background_fetches_pool_size];
|
||||||
global_context->getFetchesExecutor()->increaseThreadsAndMaxTasksCount(new_pool_size, new_pool_size);
|
global_context->getFetchesExecutor()->increaseThreadsAndMaxTasksCount(new_pool_size, new_pool_size);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (global_context->areBackgroundExecutorsInitialized())
|
if (global_context->areBackgroundExecutorsInitialized())
|
||||||
{
|
{
|
||||||
auto new_pool_size = new_server_settings.background_common_pool_size;
|
auto new_pool_size = new_server_settings[ServerSetting::background_common_pool_size];
|
||||||
global_context->getCommonExecutor()->increaseThreadsAndMaxTasksCount(new_pool_size, new_pool_size);
|
global_context->getCommonExecutor()->increaseThreadsAndMaxTasksCount(new_pool_size, new_pool_size);
|
||||||
}
|
}
|
||||||
|
|
||||||
global_context->getBufferFlushSchedulePool().increaseThreadsCount(new_server_settings.background_buffer_flush_schedule_pool_size);
|
global_context->getBufferFlushSchedulePool().increaseThreadsCount(new_server_settings[ServerSetting::background_buffer_flush_schedule_pool_size]);
|
||||||
global_context->getSchedulePool().increaseThreadsCount(new_server_settings.background_schedule_pool_size);
|
global_context->getSchedulePool().increaseThreadsCount(new_server_settings[ServerSetting::background_schedule_pool_size]);
|
||||||
global_context->getMessageBrokerSchedulePool().increaseThreadsCount(new_server_settings.background_message_broker_schedule_pool_size);
|
global_context->getMessageBrokerSchedulePool().increaseThreadsCount(new_server_settings[ServerSetting::background_message_broker_schedule_pool_size]);
|
||||||
global_context->getDistributedSchedulePool().increaseThreadsCount(new_server_settings.background_distributed_schedule_pool_size);
|
global_context->getDistributedSchedulePool().increaseThreadsCount(new_server_settings[ServerSetting::background_distributed_schedule_pool_size]);
|
||||||
|
|
||||||
global_context->getAsyncLoader().setMaxThreads(TablesLoaderForegroundPoolId, new_server_settings.tables_loader_foreground_pool_size);
|
global_context->getAsyncLoader().setMaxThreads(TablesLoaderForegroundPoolId, new_server_settings[ServerSetting::tables_loader_foreground_pool_size]);
|
||||||
global_context->getAsyncLoader().setMaxThreads(TablesLoaderBackgroundLoadPoolId, new_server_settings.tables_loader_background_pool_size);
|
global_context->getAsyncLoader().setMaxThreads(TablesLoaderBackgroundLoadPoolId, new_server_settings[ServerSetting::tables_loader_background_pool_size]);
|
||||||
global_context->getAsyncLoader().setMaxThreads(TablesLoaderBackgroundStartupPoolId, new_server_settings.tables_loader_background_pool_size);
|
global_context->getAsyncLoader().setMaxThreads(TablesLoaderBackgroundStartupPoolId, new_server_settings[ServerSetting::tables_loader_background_pool_size]);
|
||||||
|
|
||||||
getIOThreadPool().reloadConfiguration(
|
getIOThreadPool().reloadConfiguration(
|
||||||
new_server_settings.max_io_thread_pool_size,
|
new_server_settings[ServerSetting::max_io_thread_pool_size],
|
||||||
new_server_settings.max_io_thread_pool_free_size,
|
new_server_settings[ServerSetting::max_io_thread_pool_free_size],
|
||||||
new_server_settings.io_thread_pool_queue_size);
|
new_server_settings[ServerSetting::io_thread_pool_queue_size]);
|
||||||
|
|
||||||
getBackupsIOThreadPool().reloadConfiguration(
|
getBackupsIOThreadPool().reloadConfiguration(
|
||||||
new_server_settings.max_backups_io_thread_pool_size,
|
new_server_settings[ServerSetting::max_backups_io_thread_pool_size],
|
||||||
new_server_settings.max_backups_io_thread_pool_free_size,
|
new_server_settings[ServerSetting::max_backups_io_thread_pool_free_size],
|
||||||
new_server_settings.backups_io_thread_pool_queue_size);
|
new_server_settings[ServerSetting::backups_io_thread_pool_queue_size]);
|
||||||
|
|
||||||
getActivePartsLoadingThreadPool().reloadConfiguration(
|
getActivePartsLoadingThreadPool().reloadConfiguration(
|
||||||
new_server_settings.max_active_parts_loading_thread_pool_size,
|
new_server_settings[ServerSetting::max_active_parts_loading_thread_pool_size],
|
||||||
0, // We don't need any threads once all the parts will be loaded
|
0, // We don't need any threads once all the parts will be loaded
|
||||||
new_server_settings.max_active_parts_loading_thread_pool_size);
|
new_server_settings[ServerSetting::max_active_parts_loading_thread_pool_size]);
|
||||||
|
|
||||||
getOutdatedPartsLoadingThreadPool().reloadConfiguration(
|
getOutdatedPartsLoadingThreadPool().reloadConfiguration(
|
||||||
new_server_settings.max_outdated_parts_loading_thread_pool_size,
|
new_server_settings[ServerSetting::max_outdated_parts_loading_thread_pool_size],
|
||||||
0, // We don't need any threads once all the parts will be loaded
|
0, // We don't need any threads once all the parts will be loaded
|
||||||
new_server_settings.max_outdated_parts_loading_thread_pool_size);
|
new_server_settings[ServerSetting::max_outdated_parts_loading_thread_pool_size]);
|
||||||
|
|
||||||
/// It could grow if we need to synchronously wait until all the data parts will be loaded.
|
/// It could grow if we need to synchronously wait until all the data parts will be loaded.
|
||||||
getOutdatedPartsLoadingThreadPool().setMaxTurboThreads(
|
getOutdatedPartsLoadingThreadPool().setMaxTurboThreads(
|
||||||
new_server_settings.max_active_parts_loading_thread_pool_size
|
new_server_settings[ServerSetting::max_active_parts_loading_thread_pool_size]
|
||||||
);
|
);
|
||||||
|
|
||||||
getPartsCleaningThreadPool().reloadConfiguration(
|
getPartsCleaningThreadPool().reloadConfiguration(
|
||||||
new_server_settings.max_parts_cleaning_thread_pool_size,
|
new_server_settings[ServerSetting::max_parts_cleaning_thread_pool_size],
|
||||||
0, // We don't need any threads one all the parts will be deleted
|
0, // We don't need any threads one all the parts will be deleted
|
||||||
new_server_settings.max_parts_cleaning_thread_pool_size);
|
new_server_settings[ServerSetting::max_parts_cleaning_thread_pool_size]);
|
||||||
|
|
||||||
|
|
||||||
global_context->setMergeWorkload(new_server_settings.merge_workload);
|
global_context->setMergeWorkload(new_server_settings[ServerSetting::merge_workload]);
|
||||||
global_context->setMutationWorkload(new_server_settings.mutation_workload);
|
global_context->setMutationWorkload(new_server_settings[ServerSetting::mutation_workload]);
|
||||||
|
|
||||||
if (config->has("resources"))
|
if (config->has("resources"))
|
||||||
{
|
{
|
||||||
@ -1778,28 +1897,28 @@ try
|
|||||||
|
|
||||||
HTTPConnectionPools::instance().setLimits(
|
HTTPConnectionPools::instance().setLimits(
|
||||||
HTTPConnectionPools::Limits{
|
HTTPConnectionPools::Limits{
|
||||||
new_server_settings.disk_connections_soft_limit,
|
new_server_settings[ServerSetting::disk_connections_soft_limit],
|
||||||
new_server_settings.disk_connections_warn_limit,
|
new_server_settings[ServerSetting::disk_connections_warn_limit],
|
||||||
new_server_settings.disk_connections_store_limit,
|
new_server_settings[ServerSetting::disk_connections_store_limit],
|
||||||
},
|
},
|
||||||
HTTPConnectionPools::Limits{
|
HTTPConnectionPools::Limits{
|
||||||
new_server_settings.storage_connections_soft_limit,
|
new_server_settings[ServerSetting::storage_connections_soft_limit],
|
||||||
new_server_settings.storage_connections_warn_limit,
|
new_server_settings[ServerSetting::storage_connections_warn_limit],
|
||||||
new_server_settings.storage_connections_store_limit,
|
new_server_settings[ServerSetting::storage_connections_store_limit],
|
||||||
},
|
},
|
||||||
HTTPConnectionPools::Limits{
|
HTTPConnectionPools::Limits{
|
||||||
new_server_settings.http_connections_soft_limit,
|
new_server_settings[ServerSetting::http_connections_soft_limit],
|
||||||
new_server_settings.http_connections_warn_limit,
|
new_server_settings[ServerSetting::http_connections_warn_limit],
|
||||||
new_server_settings.http_connections_store_limit,
|
new_server_settings[ServerSetting::http_connections_store_limit],
|
||||||
});
|
});
|
||||||
|
|
||||||
DNSResolver::instance().setFilterSettings(new_server_settings.dns_allow_resolve_names_to_ipv4, new_server_settings.dns_allow_resolve_names_to_ipv6);
|
DNSResolver::instance().setFilterSettings(new_server_settings[ServerSetting::dns_allow_resolve_names_to_ipv4], new_server_settings[ServerSetting::dns_allow_resolve_names_to_ipv6]);
|
||||||
|
|
||||||
if (global_context->isServerCompletelyStarted())
|
if (global_context->isServerCompletelyStarted())
|
||||||
CannotAllocateThreadFaultInjector::setFaultProbability(new_server_settings.cannot_allocate_thread_fault_injection_probability);
|
CannotAllocateThreadFaultInjector::setFaultProbability(new_server_settings[ServerSetting::cannot_allocate_thread_fault_injection_probability]);
|
||||||
|
|
||||||
#if USE_GWP_ASAN
|
#if USE_GWP_ASAN
|
||||||
GWPAsan::setForceSampleProbability(new_server_settings.gwp_asan_force_sample_probability);
|
GWPAsan::setForceSampleProbability(new_server_settings[ServerSetting::gwp_asan_force_sample_probability]);
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
ProfileEvents::increment(ProfileEvents::MainConfigLoads);
|
ProfileEvents::increment(ProfileEvents::MainConfigLoads);
|
||||||
@ -1995,7 +2114,7 @@ try
|
|||||||
});
|
});
|
||||||
|
|
||||||
/// Limit on total number of concurrently executed queries.
|
/// Limit on total number of concurrently executed queries.
|
||||||
global_context->getProcessList().setMaxSize(server_settings.max_concurrent_queries);
|
global_context->getProcessList().setMaxSize(server_settings[ServerSetting::max_concurrent_queries]);
|
||||||
|
|
||||||
/// Load global settings from default_profile and system_profile.
|
/// Load global settings from default_profile and system_profile.
|
||||||
global_context->setDefaultProfiles(config());
|
global_context->setDefaultProfiles(config());
|
||||||
@ -2004,12 +2123,12 @@ try
|
|||||||
/// This is needed to load proper values of background_pool_size etc.
|
/// This is needed to load proper values of background_pool_size etc.
|
||||||
global_context->initializeBackgroundExecutorsIfNeeded();
|
global_context->initializeBackgroundExecutorsIfNeeded();
|
||||||
|
|
||||||
if (server_settings.async_insert_threads)
|
if (server_settings[ServerSetting::async_insert_threads])
|
||||||
{
|
{
|
||||||
global_context->setAsynchronousInsertQueue(std::make_shared<AsynchronousInsertQueue>(
|
global_context->setAsynchronousInsertQueue(std::make_shared<AsynchronousInsertQueue>(
|
||||||
global_context,
|
global_context,
|
||||||
server_settings.async_insert_threads,
|
server_settings[ServerSetting::async_insert_threads],
|
||||||
server_settings.async_insert_queue_flush_on_shutdown));
|
server_settings[ServerSetting::async_insert_queue_flush_on_shutdown]));
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Set path for format schema files
|
/// Set path for format schema files
|
||||||
@ -2045,7 +2164,7 @@ try
|
|||||||
/// context is destroyed.
|
/// context is destroyed.
|
||||||
/// In addition this object has to be created before the loading of the tables.
|
/// In addition this object has to be created before the loading of the tables.
|
||||||
std::unique_ptr<DNSCacheUpdater> dns_cache_updater;
|
std::unique_ptr<DNSCacheUpdater> dns_cache_updater;
|
||||||
if (server_settings.disable_internal_dns_cache)
|
if (server_settings[ServerSetting::disable_internal_dns_cache])
|
||||||
{
|
{
|
||||||
/// Disable DNS caching at all
|
/// Disable DNS caching at all
|
||||||
DNSResolver::instance().setDisableCacheFlag();
|
DNSResolver::instance().setDisableCacheFlag();
|
||||||
@ -2053,11 +2172,11 @@ try
|
|||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
DNSResolver::instance().setCacheMaxEntries(server_settings.dns_cache_max_entries);
|
DNSResolver::instance().setCacheMaxEntries(server_settings[ServerSetting::dns_cache_max_entries]);
|
||||||
|
|
||||||
/// Initialize a watcher periodically updating DNS cache
|
/// Initialize a watcher periodically updating DNS cache
|
||||||
dns_cache_updater = std::make_unique<DNSCacheUpdater>(
|
dns_cache_updater = std::make_unique<DNSCacheUpdater>(
|
||||||
global_context, server_settings.dns_cache_update_period, server_settings.dns_max_consecutive_failures);
|
global_context, server_settings[ServerSetting::dns_cache_update_period], server_settings[ServerSetting::dns_max_consecutive_failures]);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (dns_cache_updater)
|
if (dns_cache_updater)
|
||||||
@ -2065,7 +2184,7 @@ try
|
|||||||
|
|
||||||
/// Set current database name before loading tables and databases because
|
/// Set current database name before loading tables and databases because
|
||||||
/// system logs may copy global context.
|
/// system logs may copy global context.
|
||||||
std::string default_database = server_settings.default_database.toString();
|
std::string default_database = server_settings[ServerSetting::default_database].toString();
|
||||||
global_context->setCurrentDatabaseNameInGlobalContext(default_database);
|
global_context->setCurrentDatabaseNameInGlobalContext(default_database);
|
||||||
|
|
||||||
LOG_INFO(log, "Loading metadata from {}", path_str);
|
LOG_INFO(log, "Loading metadata from {}", path_str);
|
||||||
@ -2101,7 +2220,7 @@ try
|
|||||||
waitLoad(TablesLoaderForegroundPoolId, system_startup_tasks);
|
waitLoad(TablesLoaderForegroundPoolId, system_startup_tasks);
|
||||||
|
|
||||||
/// Startup scripts can depend on the system log tables.
|
/// Startup scripts can depend on the system log tables.
|
||||||
if (config().has("startup_scripts") && !server_settings.prepare_system_log_tables_on_startup.changed)
|
if (config().has("startup_scripts") && !server_settings[ServerSetting::prepare_system_log_tables_on_startup].changed)
|
||||||
global_context->setServerSetting("prepare_system_log_tables_on_startup", true);
|
global_context->setServerSetting("prepare_system_log_tables_on_startup", true);
|
||||||
|
|
||||||
/// After attaching system databases we can initialize system log.
|
/// After attaching system databases we can initialize system log.
|
||||||
@ -2121,7 +2240,7 @@ try
|
|||||||
database_catalog.loadMarkedAsDroppedTables();
|
database_catalog.loadMarkedAsDroppedTables();
|
||||||
database_catalog.createBackgroundTasks();
|
database_catalog.createBackgroundTasks();
|
||||||
/// Then, load remaining databases (some of them maybe be loaded asynchronously)
|
/// Then, load remaining databases (some of them maybe be loaded asynchronously)
|
||||||
load_metadata_tasks = loadMetadata(global_context, default_database, server_settings.async_load_databases);
|
load_metadata_tasks = loadMetadata(global_context, default_database, server_settings[ServerSetting::async_load_databases]);
|
||||||
/// If we need to convert database engines, disable async tables loading
|
/// If we need to convert database engines, disable async tables loading
|
||||||
convertDatabasesEnginesIfNeed(load_metadata_tasks, global_context);
|
convertDatabasesEnginesIfNeed(load_metadata_tasks, global_context);
|
||||||
database_catalog.startupBackgroundTasks();
|
database_catalog.startupBackgroundTasks();
|
||||||
@ -2272,11 +2391,11 @@ try
|
|||||||
startup_watch.stop();
|
startup_watch.stop();
|
||||||
ProfileEvents::increment(ProfileEvents::ServerStartupMilliseconds, startup_watch.elapsedMilliseconds());
|
ProfileEvents::increment(ProfileEvents::ServerStartupMilliseconds, startup_watch.elapsedMilliseconds());
|
||||||
|
|
||||||
CannotAllocateThreadFaultInjector::setFaultProbability(server_settings.cannot_allocate_thread_fault_injection_probability);
|
CannotAllocateThreadFaultInjector::setFaultProbability(server_settings[ServerSetting::cannot_allocate_thread_fault_injection_probability]);
|
||||||
|
|
||||||
#if USE_GWP_ASAN
|
#if USE_GWP_ASAN
|
||||||
GWPAsan::initFinished();
|
GWPAsan::initFinished();
|
||||||
GWPAsan::setForceSampleProbability(server_settings.gwp_asan_force_sample_probability);
|
GWPAsan::setForceSampleProbability(server_settings[ServerSetting::gwp_asan_force_sample_probability]);
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
try
|
try
|
||||||
@ -2326,15 +2445,15 @@ try
|
|||||||
/// Wait for unfinished backups and restores.
|
/// Wait for unfinished backups and restores.
|
||||||
/// This must be done after closing listening sockets (no more backups/restores) but before ProcessList::killAllQueries
|
/// This must be done after closing listening sockets (no more backups/restores) but before ProcessList::killAllQueries
|
||||||
/// (because killAllQueries() will cancel all running backups/restores).
|
/// (because killAllQueries() will cancel all running backups/restores).
|
||||||
if (server_settings.shutdown_wait_backups_and_restores)
|
if (server_settings[ServerSetting::shutdown_wait_backups_and_restores])
|
||||||
global_context->waitAllBackupsAndRestores();
|
global_context->waitAllBackupsAndRestores();
|
||||||
|
|
||||||
/// Killing remaining queries.
|
/// Killing remaining queries.
|
||||||
if (!server_settings.shutdown_wait_unfinished_queries)
|
if (!server_settings[ServerSetting::shutdown_wait_unfinished_queries])
|
||||||
global_context->getProcessList().killAllQueries();
|
global_context->getProcessList().killAllQueries();
|
||||||
|
|
||||||
if (current_connections)
|
if (current_connections)
|
||||||
current_connections = waitServersToFinish(servers, servers_lock, server_settings.shutdown_wait_unfinished);
|
current_connections = waitServersToFinish(servers, servers_lock, server_settings[ServerSetting::shutdown_wait_unfinished]);
|
||||||
|
|
||||||
if (current_connections)
|
if (current_connections)
|
||||||
LOG_WARNING(log, "Closed connections. But {} remain."
|
LOG_WARNING(log, "Closed connections. But {} remain."
|
||||||
@ -2473,8 +2592,8 @@ void Server::createServers(
|
|||||||
|
|
||||||
Poco::Net::HTTPServerParams::Ptr http_params = new Poco::Net::HTTPServerParams;
|
Poco::Net::HTTPServerParams::Ptr http_params = new Poco::Net::HTTPServerParams;
|
||||||
http_params->setTimeout(settings[Setting::http_receive_timeout]);
|
http_params->setTimeout(settings[Setting::http_receive_timeout]);
|
||||||
http_params->setKeepAliveTimeout(global_context->getServerSettings().keep_alive_timeout);
|
http_params->setKeepAliveTimeout(global_context->getServerSettings()[ServerSetting::keep_alive_timeout]);
|
||||||
http_params->setMaxKeepAliveRequests(static_cast<int>(global_context->getServerSettings().max_keep_alive_requests));
|
http_params->setMaxKeepAliveRequests(static_cast<int>(global_context->getServerSettings()[ServerSetting::max_keep_alive_requests]));
|
||||||
|
|
||||||
Poco::Util::AbstractConfiguration::Keys protocols;
|
Poco::Util::AbstractConfiguration::Keys protocols;
|
||||||
config.keys("protocols", protocols);
|
config.keys("protocols", protocols);
|
||||||
@ -2730,7 +2849,7 @@ void Server::createInterserverServers(
|
|||||||
|
|
||||||
Poco::Net::HTTPServerParams::Ptr http_params = new Poco::Net::HTTPServerParams;
|
Poco::Net::HTTPServerParams::Ptr http_params = new Poco::Net::HTTPServerParams;
|
||||||
http_params->setTimeout(settings[Setting::http_receive_timeout]);
|
http_params->setTimeout(settings[Setting::http_receive_timeout]);
|
||||||
http_params->setKeepAliveTimeout(global_context->getServerSettings().keep_alive_timeout);
|
http_params->setKeepAliveTimeout(global_context->getServerSettings()[ServerSetting::keep_alive_timeout]);
|
||||||
|
|
||||||
/// Now iterate over interserver_listen_hosts
|
/// Now iterate over interserver_listen_hosts
|
||||||
for (const auto & interserver_listen_host : interserver_listen_hosts)
|
for (const auto & interserver_listen_host : interserver_listen_hosts)
|
||||||
|
|||||||
@ -117,20 +117,20 @@ bool operator ==(const AuthenticationData & lhs, const AuthenticationData & rhs)
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void AuthenticationData::setPassword(const String & password_)
|
void AuthenticationData::setPassword(const String & password_, bool validate)
|
||||||
{
|
{
|
||||||
switch (type)
|
switch (type)
|
||||||
{
|
{
|
||||||
case AuthenticationType::PLAINTEXT_PASSWORD:
|
case AuthenticationType::PLAINTEXT_PASSWORD:
|
||||||
setPasswordHashBinary(Util::stringToDigest(password_));
|
setPasswordHashBinary(Util::stringToDigest(password_), validate);
|
||||||
return;
|
return;
|
||||||
|
|
||||||
case AuthenticationType::SHA256_PASSWORD:
|
case AuthenticationType::SHA256_PASSWORD:
|
||||||
setPasswordHashBinary(Util::encodeSHA256(password_));
|
setPasswordHashBinary(Util::encodeSHA256(password_), validate);
|
||||||
return;
|
return;
|
||||||
|
|
||||||
case AuthenticationType::DOUBLE_SHA1_PASSWORD:
|
case AuthenticationType::DOUBLE_SHA1_PASSWORD:
|
||||||
setPasswordHashBinary(Util::encodeDoubleSHA1(password_));
|
setPasswordHashBinary(Util::encodeDoubleSHA1(password_), validate);
|
||||||
return;
|
return;
|
||||||
|
|
||||||
case AuthenticationType::BCRYPT_PASSWORD:
|
case AuthenticationType::BCRYPT_PASSWORD:
|
||||||
@ -149,12 +149,12 @@ void AuthenticationData::setPassword(const String & password_)
|
|||||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "setPassword(): authentication type {} not supported", toString(type));
|
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "setPassword(): authentication type {} not supported", toString(type));
|
||||||
}
|
}
|
||||||
|
|
||||||
void AuthenticationData::setPasswordBcrypt(const String & password_, int workfactor_)
|
void AuthenticationData::setPasswordBcrypt(const String & password_, int workfactor_, bool validate)
|
||||||
{
|
{
|
||||||
if (type != AuthenticationType::BCRYPT_PASSWORD)
|
if (type != AuthenticationType::BCRYPT_PASSWORD)
|
||||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot specify bcrypt password for authentication type {}", toString(type));
|
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot specify bcrypt password for authentication type {}", toString(type));
|
||||||
|
|
||||||
setPasswordHashBinary(Util::encodeBcrypt(password_, workfactor_));
|
setPasswordHashBinary(Util::encodeBcrypt(password_, workfactor_), validate);
|
||||||
}
|
}
|
||||||
|
|
||||||
String AuthenticationData::getPassword() const
|
String AuthenticationData::getPassword() const
|
||||||
@ -165,7 +165,7 @@ String AuthenticationData::getPassword() const
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void AuthenticationData::setPasswordHashHex(const String & hash)
|
void AuthenticationData::setPasswordHashHex(const String & hash, bool validate)
|
||||||
{
|
{
|
||||||
Digest digest;
|
Digest digest;
|
||||||
digest.resize(hash.size() / 2);
|
digest.resize(hash.size() / 2);
|
||||||
@ -179,7 +179,7 @@ void AuthenticationData::setPasswordHashHex(const String & hash)
|
|||||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot read password hash in hex, check for valid characters [0-9a-fA-F] and length");
|
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot read password hash in hex, check for valid characters [0-9a-fA-F] and length");
|
||||||
}
|
}
|
||||||
|
|
||||||
setPasswordHashBinary(digest);
|
setPasswordHashBinary(digest, validate);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -195,7 +195,7 @@ String AuthenticationData::getPasswordHashHex() const
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void AuthenticationData::setPasswordHashBinary(const Digest & hash)
|
void AuthenticationData::setPasswordHashBinary(const Digest & hash, bool validate)
|
||||||
{
|
{
|
||||||
switch (type)
|
switch (type)
|
||||||
{
|
{
|
||||||
@ -217,7 +217,7 @@ void AuthenticationData::setPasswordHashBinary(const Digest & hash)
|
|||||||
|
|
||||||
case AuthenticationType::DOUBLE_SHA1_PASSWORD:
|
case AuthenticationType::DOUBLE_SHA1_PASSWORD:
|
||||||
{
|
{
|
||||||
if (hash.size() != 20)
|
if (validate && hash.size() != 20)
|
||||||
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
||||||
"Password hash for the 'DOUBLE_SHA1_PASSWORD' authentication type has length {} "
|
"Password hash for the 'DOUBLE_SHA1_PASSWORD' authentication type has length {} "
|
||||||
"but must be exactly 20 bytes.", hash.size());
|
"but must be exactly 20 bytes.", hash.size());
|
||||||
@ -231,7 +231,7 @@ void AuthenticationData::setPasswordHashBinary(const Digest & hash)
|
|||||||
/// However the library we use to encode it requires hash string to be 64 characters long,
|
/// However the library we use to encode it requires hash string to be 64 characters long,
|
||||||
/// so we also allow the hash of this length.
|
/// so we also allow the hash of this length.
|
||||||
|
|
||||||
if (hash.size() != 59 && hash.size() != 60 && hash.size() != 64)
|
if (validate && hash.size() != 59 && hash.size() != 60 && hash.size() != 64)
|
||||||
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
||||||
"Password hash for the 'BCRYPT_PASSWORD' authentication type has length {} "
|
"Password hash for the 'BCRYPT_PASSWORD' authentication type has length {} "
|
||||||
"but must be 59 or 60 bytes.", hash.size());
|
"but must be 59 or 60 bytes.", hash.size());
|
||||||
@ -240,10 +240,13 @@ void AuthenticationData::setPasswordHashBinary(const Digest & hash)
|
|||||||
resized.resize(64);
|
resized.resize(64);
|
||||||
|
|
||||||
#if USE_BCRYPT
|
#if USE_BCRYPT
|
||||||
|
if (validate)
|
||||||
|
{
|
||||||
/// Verify that it is a valid hash
|
/// Verify that it is a valid hash
|
||||||
int ret = bcrypt_checkpw("", reinterpret_cast<const char *>(resized.data()));
|
int ret = bcrypt_checkpw("", reinterpret_cast<const char *>(resized.data()));
|
||||||
if (ret == -1)
|
if (ret == -1)
|
||||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Could not decode the provided hash with 'bcrypt_hash'");
|
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Could not decode the provided hash with 'bcrypt_hash'");
|
||||||
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
password_hash = hash;
|
password_hash = hash;
|
||||||
@ -385,7 +388,7 @@ std::shared_ptr<ASTAuthenticationData> AuthenticationData::toAST() const
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & query, ContextPtr context, bool check_password_rules)
|
AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & query, ContextPtr context, bool validate)
|
||||||
{
|
{
|
||||||
if (query.type && query.type == AuthenticationType::NO_PASSWORD)
|
if (query.type && query.type == AuthenticationType::NO_PASSWORD)
|
||||||
return AuthenticationData();
|
return AuthenticationData();
|
||||||
@ -431,7 +434,7 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que
|
|||||||
if (!query.type && !context)
|
if (!query.type && !context)
|
||||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot get default password type without context");
|
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot get default password type without context");
|
||||||
|
|
||||||
if (check_password_rules && !context)
|
if (validate && !context)
|
||||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot check password complexity rules without context");
|
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot check password complexity rules without context");
|
||||||
|
|
||||||
if (query.type == AuthenticationType::BCRYPT_PASSWORD && !context)
|
if (query.type == AuthenticationType::BCRYPT_PASSWORD && !context)
|
||||||
@ -448,13 +451,13 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que
|
|||||||
|
|
||||||
AuthenticationData auth_data(current_type);
|
AuthenticationData auth_data(current_type);
|
||||||
|
|
||||||
if (check_password_rules)
|
if (validate)
|
||||||
context->getAccessControl().checkPasswordComplexityRules(value);
|
context->getAccessControl().checkPasswordComplexityRules(value);
|
||||||
|
|
||||||
if (query.type == AuthenticationType::BCRYPT_PASSWORD)
|
if (query.type == AuthenticationType::BCRYPT_PASSWORD)
|
||||||
{
|
{
|
||||||
int workfactor = context->getAccessControl().getBcryptWorkfactor();
|
int workfactor = context->getAccessControl().getBcryptWorkfactor();
|
||||||
auth_data.setPasswordBcrypt(value, workfactor);
|
auth_data.setPasswordBcrypt(value, workfactor, validate);
|
||||||
return auth_data;
|
return auth_data;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -486,7 +489,7 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que
|
|||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
auth_data.setPassword(value);
|
auth_data.setPassword(value, validate);
|
||||||
return auth_data;
|
return auth_data;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -498,11 +501,11 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que
|
|||||||
|
|
||||||
if (query.type == AuthenticationType::BCRYPT_PASSWORD)
|
if (query.type == AuthenticationType::BCRYPT_PASSWORD)
|
||||||
{
|
{
|
||||||
auth_data.setPasswordHashBinary(AuthenticationData::Util::stringToDigest(value));
|
auth_data.setPasswordHashBinary(AuthenticationData::Util::stringToDigest(value), validate);
|
||||||
return auth_data;
|
return auth_data;
|
||||||
}
|
}
|
||||||
|
|
||||||
auth_data.setPasswordHashHex(value);
|
auth_data.setPasswordHashHex(value, validate);
|
||||||
|
|
||||||
|
|
||||||
if (query.type == AuthenticationType::SHA256_PASSWORD && args_size == 2)
|
if (query.type == AuthenticationType::SHA256_PASSWORD && args_size == 2)
|
||||||
|
|||||||
@ -31,17 +31,17 @@ public:
|
|||||||
AuthenticationType getType() const { return type; }
|
AuthenticationType getType() const { return type; }
|
||||||
|
|
||||||
/// Sets the password and encrypt it using the authentication type set in the constructor.
|
/// Sets the password and encrypt it using the authentication type set in the constructor.
|
||||||
void setPassword(const String & password_);
|
void setPassword(const String & password_, bool validate);
|
||||||
|
|
||||||
/// Returns the password. Allowed to use only for Type::PLAINTEXT_PASSWORD.
|
/// Returns the password. Allowed to use only for Type::PLAINTEXT_PASSWORD.
|
||||||
String getPassword() const;
|
String getPassword() const;
|
||||||
|
|
||||||
/// Sets the password as a string of hexadecimal digits.
|
/// Sets the password as a string of hexadecimal digits.
|
||||||
void setPasswordHashHex(const String & hash);
|
void setPasswordHashHex(const String & hash, bool validate);
|
||||||
String getPasswordHashHex() const;
|
String getPasswordHashHex() const;
|
||||||
|
|
||||||
/// Sets the password in binary form.
|
/// Sets the password in binary form.
|
||||||
void setPasswordHashBinary(const Digest & hash);
|
void setPasswordHashBinary(const Digest & hash, bool validate);
|
||||||
const Digest & getPasswordHashBinary() const { return password_hash; }
|
const Digest & getPasswordHashBinary() const { return password_hash; }
|
||||||
|
|
||||||
/// Sets the salt in String form.
|
/// Sets the salt in String form.
|
||||||
@ -49,7 +49,7 @@ public:
|
|||||||
String getSalt() const;
|
String getSalt() const;
|
||||||
|
|
||||||
/// Sets the password using bcrypt hash with specified workfactor
|
/// Sets the password using bcrypt hash with specified workfactor
|
||||||
void setPasswordBcrypt(const String & password_, int workfactor_);
|
void setPasswordBcrypt(const String & password_, int workfactor_, bool validate);
|
||||||
|
|
||||||
/// Sets the server name for authentication type LDAP.
|
/// Sets the server name for authentication type LDAP.
|
||||||
const String & getLDAPServerName() const { return ldap_server_name; }
|
const String & getLDAPServerName() const { return ldap_server_name; }
|
||||||
@ -77,7 +77,7 @@ public:
|
|||||||
friend bool operator ==(const AuthenticationData & lhs, const AuthenticationData & rhs);
|
friend bool operator ==(const AuthenticationData & lhs, const AuthenticationData & rhs);
|
||||||
friend bool operator !=(const AuthenticationData & lhs, const AuthenticationData & rhs) { return !(lhs == rhs); }
|
friend bool operator !=(const AuthenticationData & lhs, const AuthenticationData & rhs) { return !(lhs == rhs); }
|
||||||
|
|
||||||
static AuthenticationData fromAST(const ASTAuthenticationData & query, ContextPtr context, bool check_password_rules);
|
static AuthenticationData fromAST(const ASTAuthenticationData & query, ContextPtr context, bool validate);
|
||||||
std::shared_ptr<ASTAuthenticationData> toAST() const;
|
std::shared_ptr<ASTAuthenticationData> toAST() const;
|
||||||
|
|
||||||
struct Util
|
struct Util
|
||||||
|
|||||||
@ -121,6 +121,7 @@ namespace
|
|||||||
bool allow_no_password,
|
bool allow_no_password,
|
||||||
bool allow_plaintext_password)
|
bool allow_plaintext_password)
|
||||||
{
|
{
|
||||||
|
const bool validate = true;
|
||||||
auto user = std::make_shared<User>();
|
auto user = std::make_shared<User>();
|
||||||
user->setName(user_name);
|
user->setName(user_name);
|
||||||
String user_config = "users." + user_name;
|
String user_config = "users." + user_name;
|
||||||
@ -157,17 +158,17 @@ namespace
|
|||||||
if (has_password_plaintext)
|
if (has_password_plaintext)
|
||||||
{
|
{
|
||||||
user->authentication_methods.emplace_back(AuthenticationType::PLAINTEXT_PASSWORD);
|
user->authentication_methods.emplace_back(AuthenticationType::PLAINTEXT_PASSWORD);
|
||||||
user->authentication_methods.back().setPassword(config.getString(user_config + ".password"));
|
user->authentication_methods.back().setPassword(config.getString(user_config + ".password"), validate);
|
||||||
}
|
}
|
||||||
else if (has_password_sha256_hex)
|
else if (has_password_sha256_hex)
|
||||||
{
|
{
|
||||||
user->authentication_methods.emplace_back(AuthenticationType::SHA256_PASSWORD);
|
user->authentication_methods.emplace_back(AuthenticationType::SHA256_PASSWORD);
|
||||||
user->authentication_methods.back().setPasswordHashHex(config.getString(user_config + ".password_sha256_hex"));
|
user->authentication_methods.back().setPasswordHashHex(config.getString(user_config + ".password_sha256_hex"), validate);
|
||||||
}
|
}
|
||||||
else if (has_password_double_sha1_hex)
|
else if (has_password_double_sha1_hex)
|
||||||
{
|
{
|
||||||
user->authentication_methods.emplace_back(AuthenticationType::DOUBLE_SHA1_PASSWORD);
|
user->authentication_methods.emplace_back(AuthenticationType::DOUBLE_SHA1_PASSWORD);
|
||||||
user->authentication_methods.back().setPasswordHashHex(config.getString(user_config + ".password_double_sha1_hex"));
|
user->authentication_methods.back().setPasswordHashHex(config.getString(user_config + ".password_double_sha1_hex"), validate);
|
||||||
}
|
}
|
||||||
else if (has_ldap)
|
else if (has_ldap)
|
||||||
{
|
{
|
||||||
|
|||||||
@ -33,6 +33,12 @@ namespace DB
|
|||||||
{
|
{
|
||||||
struct Settings;
|
struct Settings;
|
||||||
|
|
||||||
|
namespace ServerSetting
|
||||||
|
{
|
||||||
|
extern const ServerSettingsGroupArrayActionWhenLimitReached aggregate_function_group_array_action_when_limit_is_reached;
|
||||||
|
extern const ServerSettingsUInt64 aggregate_function_group_array_max_element_size;
|
||||||
|
}
|
||||||
|
|
||||||
namespace ErrorCodes
|
namespace ErrorCodes
|
||||||
{
|
{
|
||||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||||
@ -746,7 +752,7 @@ inline AggregateFunctionPtr createAggregateFunctionGroupArrayImpl(const DataType
|
|||||||
size_t getMaxArraySize()
|
size_t getMaxArraySize()
|
||||||
{
|
{
|
||||||
if (auto context = Context::getGlobalContextInstance())
|
if (auto context = Context::getGlobalContextInstance())
|
||||||
return context->getServerSettings().aggregate_function_group_array_max_element_size;
|
return context->getServerSettings()[ServerSetting::aggregate_function_group_array_max_element_size];
|
||||||
|
|
||||||
return 0xFFFFFF;
|
return 0xFFFFFF;
|
||||||
}
|
}
|
||||||
@ -754,7 +760,7 @@ size_t getMaxArraySize()
|
|||||||
bool discardOnLimitReached()
|
bool discardOnLimitReached()
|
||||||
{
|
{
|
||||||
if (auto context = Context::getGlobalContextInstance())
|
if (auto context = Context::getGlobalContextInstance())
|
||||||
return context->getServerSettings().aggregate_function_group_array_action_when_limit_is_reached
|
return context->getServerSettings()[ServerSetting::aggregate_function_group_array_action_when_limit_is_reached]
|
||||||
== GroupArrayActionWhenLimitReached::DISCARD;
|
== GroupArrayActionWhenLimitReached::DISCARD;
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
|
|||||||
@ -28,11 +28,36 @@ namespace ErrorCodes
|
|||||||
namespace
|
namespace
|
||||||
{
|
{
|
||||||
|
|
||||||
constexpr size_t max_events = 32;
|
constexpr size_t MAX_EVENTS = 32;
|
||||||
|
|
||||||
|
|
||||||
|
template <typename T>
|
||||||
|
void mergeEventsList(T & events_list, size_t prefix_size, bool prefix_sorted, bool suffix_sorted)
|
||||||
|
{
|
||||||
|
/// either sort whole container or do so partially merging ranges afterwards
|
||||||
|
if (!prefix_sorted && !suffix_sorted)
|
||||||
|
std::stable_sort(std::begin(events_list), std::end(events_list));
|
||||||
|
else
|
||||||
|
{
|
||||||
|
const auto begin = std::begin(events_list);
|
||||||
|
const auto middle = std::next(begin, prefix_size);
|
||||||
|
const auto end = std::end(events_list);
|
||||||
|
|
||||||
|
if (!prefix_sorted)
|
||||||
|
std::stable_sort(begin, middle);
|
||||||
|
|
||||||
|
if (!suffix_sorted)
|
||||||
|
std::stable_sort(middle, end);
|
||||||
|
|
||||||
|
std::inplace_merge(begin, middle, end);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
template <typename T>
|
template <typename T>
|
||||||
struct AggregateFunctionWindowFunnelData
|
struct AggregateFunctionWindowFunnelData
|
||||||
{
|
{
|
||||||
|
static constexpr bool strict_once_enabled = false;
|
||||||
|
|
||||||
using TimestampEvent = std::pair<T, UInt8>;
|
using TimestampEvent = std::pair<T, UInt8>;
|
||||||
using TimestampEvents = PODArrayWithStackMemory<TimestampEvent, 64>;
|
using TimestampEvents = PODArrayWithStackMemory<TimestampEvent, 64>;
|
||||||
|
|
||||||
@ -66,24 +91,7 @@ struct AggregateFunctionWindowFunnelData
|
|||||||
|
|
||||||
events_list.insert(std::begin(other.events_list), std::end(other.events_list));
|
events_list.insert(std::begin(other.events_list), std::end(other.events_list));
|
||||||
|
|
||||||
/// either sort whole container or do so partially merging ranges afterwards
|
mergeEventsList(events_list, size, sorted, other.sorted);
|
||||||
if (!sorted && !other.sorted)
|
|
||||||
std::stable_sort(std::begin(events_list), std::end(events_list));
|
|
||||||
else
|
|
||||||
{
|
|
||||||
const auto begin = std::begin(events_list);
|
|
||||||
const auto middle = std::next(begin, size);
|
|
||||||
const auto end = std::end(events_list);
|
|
||||||
|
|
||||||
if (!sorted)
|
|
||||||
std::stable_sort(begin, middle);
|
|
||||||
|
|
||||||
if (!other.sorted)
|
|
||||||
std::stable_sort(middle, end);
|
|
||||||
|
|
||||||
std::inplace_merge(begin, middle, end);
|
|
||||||
}
|
|
||||||
|
|
||||||
sorted = true;
|
sorted = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -133,6 +141,131 @@ struct AggregateFunctionWindowFunnelData
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
template <typename T>
|
||||||
|
struct AggregateFunctionWindowFunnelStrictOnceData
|
||||||
|
{
|
||||||
|
static constexpr bool strict_once_enabled = true;
|
||||||
|
struct TimestampEvent
|
||||||
|
{
|
||||||
|
T timestamp;
|
||||||
|
UInt8 event_type;
|
||||||
|
UInt64 unique_id;
|
||||||
|
|
||||||
|
TimestampEvent(T timestamp_, UInt8 event_type_, UInt64 unique_id_)
|
||||||
|
: timestamp(timestamp_), event_type(event_type_), unique_id(unique_id_) {}
|
||||||
|
|
||||||
|
bool operator<(const TimestampEvent & other) const
|
||||||
|
{
|
||||||
|
return std::tie(timestamp, event_type, unique_id) < std::tie(other.timestamp, other.event_type, other.unique_id);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool operator<=(const TimestampEvent & other) const
|
||||||
|
{
|
||||||
|
return std::tie(timestamp, event_type, unique_id) <= std::tie(other.timestamp, other.event_type, other.unique_id);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
|
||||||
|
using TimestampEvents = PODArrayWithStackMemory<TimestampEvent, 64>;
|
||||||
|
TimestampEvents events_list;
|
||||||
|
|
||||||
|
/// Next unique identifier for events
|
||||||
|
/// Used to distinguish events with the same timestamp that matches several conditions.
|
||||||
|
UInt64 next_unique_id = 1;
|
||||||
|
bool sorted = true;
|
||||||
|
|
||||||
|
size_t size() const
|
||||||
|
{
|
||||||
|
return events_list.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
void advanceId()
|
||||||
|
{
|
||||||
|
++next_unique_id;
|
||||||
|
}
|
||||||
|
|
||||||
|
void add(T timestamp, UInt8 event_type)
|
||||||
|
{
|
||||||
|
TimestampEvent new_event(timestamp, event_type, next_unique_id);
|
||||||
|
/// Check if the new event maintains the sorted order
|
||||||
|
if (sorted && !events_list.empty())
|
||||||
|
sorted = events_list.back() <= new_event;
|
||||||
|
events_list.push_back(new_event);
|
||||||
|
}
|
||||||
|
|
||||||
|
void merge(const AggregateFunctionWindowFunnelStrictOnceData & other)
|
||||||
|
{
|
||||||
|
if (other.events_list.empty())
|
||||||
|
return;
|
||||||
|
|
||||||
|
const auto current_size = events_list.size();
|
||||||
|
|
||||||
|
UInt64 new_next_unique_id = next_unique_id;
|
||||||
|
events_list.reserve(current_size + other.events_list.size());
|
||||||
|
for (auto other_event : other.events_list)
|
||||||
|
{
|
||||||
|
/// Assign unique IDs to the new events to prevent conflicts
|
||||||
|
other_event.unique_id += next_unique_id;
|
||||||
|
new_next_unique_id = std::max(new_next_unique_id, other_event.unique_id + 1);
|
||||||
|
events_list.push_back(other_event);
|
||||||
|
}
|
||||||
|
next_unique_id = new_next_unique_id;
|
||||||
|
|
||||||
|
mergeEventsList(events_list, current_size, sorted, other.sorted);
|
||||||
|
|
||||||
|
sorted = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void sort()
|
||||||
|
{
|
||||||
|
if (!sorted)
|
||||||
|
{
|
||||||
|
std::stable_sort(std::begin(events_list), std::end(events_list));
|
||||||
|
sorted = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void serialize(WriteBuffer & buf) const
|
||||||
|
{
|
||||||
|
writeBinary(sorted, buf);
|
||||||
|
writeBinary(events_list.size(), buf);
|
||||||
|
|
||||||
|
for (const auto & event : events_list)
|
||||||
|
{
|
||||||
|
writeBinary(event.timestamp, buf);
|
||||||
|
writeBinary(event.event_type, buf);
|
||||||
|
writeBinary(event.unique_id, buf);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void deserialize(ReadBuffer & buf)
|
||||||
|
{
|
||||||
|
readBinary(sorted, buf);
|
||||||
|
|
||||||
|
size_t events_size;
|
||||||
|
readBinary(events_size, buf);
|
||||||
|
|
||||||
|
if (events_size > 100'000'000) /// Arbitrary limit to prevent excessive memory allocation
|
||||||
|
throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large size of the state of windowFunnel");
|
||||||
|
|
||||||
|
events_list.clear();
|
||||||
|
events_list.reserve(events_size);
|
||||||
|
|
||||||
|
T timestamp;
|
||||||
|
UInt8 event_type;
|
||||||
|
UInt64 unique_id = 0;
|
||||||
|
|
||||||
|
for (size_t i = 0; i < events_size; ++i)
|
||||||
|
{
|
||||||
|
readBinary(timestamp, buf);
|
||||||
|
readBinary(event_type, buf);
|
||||||
|
readBinary(unique_id, buf);
|
||||||
|
next_unique_id = std::max(next_unique_id, unique_id + 1);
|
||||||
|
events_list.emplace_back(timestamp, event_type, unique_id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
/** Calculates the max event level in a sliding window.
|
/** Calculates the max event level in a sliding window.
|
||||||
* The max size of events is 32, that's enough for funnel analytics
|
* The max size of events is 32, that's enough for funnel analytics
|
||||||
*
|
*
|
||||||
@ -160,22 +293,15 @@ private:
|
|||||||
/// The level path must be 1---2---3---...---check_events_size, find the max event level that satisfied the path in the sliding window.
|
/// The level path must be 1---2---3---...---check_events_size, find the max event level that satisfied the path in the sliding window.
|
||||||
/// If found, returns the max event level, else return 0.
|
/// If found, returns the max event level, else return 0.
|
||||||
/// The algorithm works in O(n) time, but the overall function works in O(n * log(n)) due to sorting.
|
/// The algorithm works in O(n) time, but the overall function works in O(n * log(n)) due to sorting.
|
||||||
UInt8 getEventLevel(Data & data) const
|
UInt8 getEventLevelNonStrictOnce(const AggregateFunctionWindowFunnelData<T>::TimestampEvents & events_list) const
|
||||||
{
|
{
|
||||||
if (data.size() == 0)
|
|
||||||
return 0;
|
|
||||||
if (!strict_order && events_size == 1)
|
|
||||||
return 1;
|
|
||||||
|
|
||||||
data.sort();
|
|
||||||
|
|
||||||
/// events_timestamp stores the timestamp of the first and previous i-th level event happen within time window
|
/// events_timestamp stores the timestamp of the first and previous i-th level event happen within time window
|
||||||
std::vector<std::optional<std::pair<UInt64, UInt64>>> events_timestamp(events_size);
|
std::vector<std::optional<std::pair<UInt64, UInt64>>> events_timestamp(events_size);
|
||||||
bool first_event = false;
|
bool first_event = false;
|
||||||
for (size_t i = 0; i < data.events_list.size(); ++i)
|
for (size_t i = 0; i < events_list.size(); ++i)
|
||||||
{
|
{
|
||||||
const T & timestamp = data.events_list[i].first;
|
const T & timestamp = events_list[i].first;
|
||||||
const auto & event_idx = data.events_list[i].second - 1;
|
const auto & event_idx = events_list[i].second - 1;
|
||||||
if (strict_order && event_idx == -1)
|
if (strict_order && event_idx == -1)
|
||||||
{
|
{
|
||||||
if (first_event)
|
if (first_event)
|
||||||
@ -189,7 +315,7 @@ private:
|
|||||||
}
|
}
|
||||||
else if (strict_deduplication && events_timestamp[event_idx].has_value())
|
else if (strict_deduplication && events_timestamp[event_idx].has_value())
|
||||||
{
|
{
|
||||||
return data.events_list[i - 1].second;
|
return events_list[i - 1].second;
|
||||||
}
|
}
|
||||||
else if (strict_order && first_event && !events_timestamp[event_idx - 1].has_value())
|
else if (strict_order && first_event && !events_timestamp[event_idx - 1].has_value())
|
||||||
{
|
{
|
||||||
@ -222,6 +348,126 @@ private:
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
UInt8 getEventLevelStrictOnce(const AggregateFunctionWindowFunnelStrictOnceData<T>::TimestampEvents & events_list) const
|
||||||
|
{
|
||||||
|
/// Stores the timestamp of the first and last i-th level event happen within time window
|
||||||
|
struct EventMatchTimeWindow
|
||||||
|
{
|
||||||
|
UInt64 first_timestamp;
|
||||||
|
UInt64 last_timestamp;
|
||||||
|
std::array<UInt64, MAX_EVENTS> event_path;
|
||||||
|
|
||||||
|
EventMatchTimeWindow() = default;
|
||||||
|
EventMatchTimeWindow(UInt64 first_ts, UInt64 last_ts)
|
||||||
|
: first_timestamp(first_ts), last_timestamp(last_ts) {}
|
||||||
|
};
|
||||||
|
|
||||||
|
/// We track all possible event sequences up to the current event.
|
||||||
|
/// It's required because one event can meet several conditions.
|
||||||
|
/// For example: for events 'start', 'a', 'b', 'a', 'end'.
|
||||||
|
/// The second occurrence of 'a' should be counted only once in one sequence.
|
||||||
|
/// However, we do not know in advance if the next event will be 'b' or 'end', so we try to keep both paths.
|
||||||
|
std::vector<std::list<EventMatchTimeWindow>> event_sequences(events_size);
|
||||||
|
|
||||||
|
bool has_first_event = false;
|
||||||
|
for (size_t i = 0; i < events_list.size(); ++i)
|
||||||
|
{
|
||||||
|
const auto & current_event = events_list[i];
|
||||||
|
auto timestamp = current_event.timestamp;
|
||||||
|
Int64 event_idx = current_event.event_type - 1;
|
||||||
|
UInt64 unique_id = current_event.unique_id;
|
||||||
|
|
||||||
|
if (strict_order && event_idx == -1)
|
||||||
|
{
|
||||||
|
if (has_first_event)
|
||||||
|
break;
|
||||||
|
else
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
else if (event_idx == 0)
|
||||||
|
{
|
||||||
|
auto & event_seq = event_sequences[0].emplace_back(timestamp, timestamp);
|
||||||
|
event_seq.event_path[0] = unique_id;
|
||||||
|
has_first_event = true;
|
||||||
|
}
|
||||||
|
else if (strict_deduplication && !event_sequences[event_idx].empty())
|
||||||
|
{
|
||||||
|
return events_list[i - 1].event_type;
|
||||||
|
}
|
||||||
|
else if (strict_order && has_first_event && event_sequences[event_idx - 1].empty())
|
||||||
|
{
|
||||||
|
for (size_t event = 0; event < event_sequences.size(); ++event)
|
||||||
|
{
|
||||||
|
if (event_sequences[event].empty())
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (!event_sequences[event_idx - 1].empty())
|
||||||
|
{
|
||||||
|
auto & prev_level = event_sequences[event_idx - 1];
|
||||||
|
for (auto it = prev_level.begin(); it != prev_level.end();)
|
||||||
|
{
|
||||||
|
auto first_ts = it->first_timestamp;
|
||||||
|
bool time_matched = timestamp <= first_ts + window;
|
||||||
|
if (!time_matched && prev_level.size() > 1)
|
||||||
|
{
|
||||||
|
// Remove old events that are out of the window, but keep at least one
|
||||||
|
it = prev_level.erase(it);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto prev_path = it->event_path;
|
||||||
|
chassert(event_idx > 0);
|
||||||
|
|
||||||
|
/// Ensure the unique_id hasn't been used in the path already
|
||||||
|
for (size_t j = 0; j < static_cast<size_t>(event_idx); ++j)
|
||||||
|
{
|
||||||
|
if (!time_matched)
|
||||||
|
break;
|
||||||
|
time_matched = prev_path[j] != unique_id;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (time_matched && strict_increase)
|
||||||
|
time_matched = it->last_timestamp < timestamp;
|
||||||
|
|
||||||
|
if (time_matched)
|
||||||
|
{
|
||||||
|
prev_path[event_idx] = unique_id;
|
||||||
|
|
||||||
|
auto & new_seq = event_sequences[event_idx].emplace_back(first_ts, timestamp);
|
||||||
|
new_seq.event_path = std::move(prev_path);
|
||||||
|
if (event_idx + 1 == events_size)
|
||||||
|
return events_size;
|
||||||
|
}
|
||||||
|
++it;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (size_t event = event_sequences.size(); event > 0; --event)
|
||||||
|
{
|
||||||
|
if (!event_sequences[event - 1].empty())
|
||||||
|
return event;
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
UInt8 getEventLevel(Data & data) const
|
||||||
|
{
|
||||||
|
if (data.size() == 0)
|
||||||
|
return 0;
|
||||||
|
if (!strict_order && events_size == 1)
|
||||||
|
return 1;
|
||||||
|
|
||||||
|
data.sort();
|
||||||
|
|
||||||
|
if constexpr (Data::strict_once_enabled)
|
||||||
|
return getEventLevelStrictOnce(data.events_list);
|
||||||
|
else
|
||||||
|
return getEventLevelNonStrictOnce(data.events_list);
|
||||||
|
}
|
||||||
|
|
||||||
public:
|
public:
|
||||||
String getName() const override
|
String getName() const override
|
||||||
{
|
{
|
||||||
@ -246,6 +492,9 @@ public:
|
|||||||
strict_order = true;
|
strict_order = true;
|
||||||
else if (option == "strict_increase")
|
else if (option == "strict_increase")
|
||||||
strict_increase = true;
|
strict_increase = true;
|
||||||
|
else if (option == "strict_once")
|
||||||
|
/// Checked in factory
|
||||||
|
chassert(Data::strict_once_enabled);
|
||||||
else if (option == "strict")
|
else if (option == "strict")
|
||||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "strict is replaced with strict_deduplication in Aggregate function {}", getName());
|
throw Exception(ErrorCodes::BAD_ARGUMENTS, "strict is replaced with strict_deduplication in Aggregate function {}", getName());
|
||||||
else
|
else
|
||||||
@ -272,6 +521,9 @@ public:
|
|||||||
|
|
||||||
if (strict_order && !has_event)
|
if (strict_order && !has_event)
|
||||||
this->data(place).add(timestamp, 0);
|
this->data(place).add(timestamp, 0);
|
||||||
|
|
||||||
|
if constexpr (Data::strict_once_enabled)
|
||||||
|
this->data(place).advanceId();
|
||||||
}
|
}
|
||||||
|
|
||||||
void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override
|
void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override
|
||||||
@ -296,7 +548,6 @@ public:
|
|||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
template <template <typename> class Data>
|
|
||||||
AggregateFunctionPtr
|
AggregateFunctionPtr
|
||||||
createAggregateFunctionWindowFunnel(const std::string & name, const DataTypes & arguments, const Array & params, const Settings *)
|
createAggregateFunctionWindowFunnel(const std::string & name, const DataTypes & arguments, const Array & params, const Settings *)
|
||||||
{
|
{
|
||||||
@ -309,7 +560,7 @@ createAggregateFunctionWindowFunnel(const std::string & name, const DataTypes &
|
|||||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
|
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
|
||||||
"Aggregate function {} requires one timestamp argument and at least one event condition.", name);
|
"Aggregate function {} requires one timestamp argument and at least one event condition.", name);
|
||||||
|
|
||||||
if (arguments.size() > max_events + 1)
|
if (arguments.size() > MAX_EVENTS + 1)
|
||||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Too many event arguments for aggregate function {}", name);
|
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Too many event arguments for aggregate function {}", name);
|
||||||
|
|
||||||
for (const auto i : collections::range(1, arguments.size()))
|
for (const auto i : collections::range(1, arguments.size()))
|
||||||
@ -321,16 +572,29 @@ createAggregateFunctionWindowFunnel(const std::string & name, const DataTypes &
|
|||||||
cond_arg->getName(), toString(i + 1), name);
|
cond_arg->getName(), toString(i + 1), name);
|
||||||
}
|
}
|
||||||
|
|
||||||
AggregateFunctionPtr res(createWithUnsignedIntegerType<AggregateFunctionWindowFunnel, Data>(*arguments[0], arguments, params));
|
bool strict_once = params.size() > 1 && std::any_of(params.begin() + 1, params.end(), [](const auto & f) { return f.template safeGet<String>() == "strict_once"; });
|
||||||
|
if (strict_once)
|
||||||
|
{
|
||||||
|
AggregateFunctionPtr res(createWithUnsignedIntegerType<AggregateFunctionWindowFunnel, AggregateFunctionWindowFunnelStrictOnceData>(*arguments[0], arguments, params));
|
||||||
WhichDataType which(arguments.front().get());
|
WhichDataType which(arguments.front().get());
|
||||||
if (res)
|
if (res)
|
||||||
return res;
|
return res;
|
||||||
if (which.isDate())
|
if (which.isDate())
|
||||||
return std::make_shared<AggregateFunctionWindowFunnel<DataTypeDate::FieldType, Data<DataTypeDate::FieldType>>>(arguments, params);
|
return std::make_shared<AggregateFunctionWindowFunnel<DataTypeDate::FieldType, AggregateFunctionWindowFunnelStrictOnceData<DataTypeDate::FieldType>>>(arguments, params);
|
||||||
if (which.isDateTime())
|
if (which.isDateTime())
|
||||||
return std::make_shared<AggregateFunctionWindowFunnel<DataTypeDateTime::FieldType, Data<DataTypeDateTime::FieldType>>>(
|
return std::make_shared<AggregateFunctionWindowFunnel<DataTypeDateTime::FieldType, AggregateFunctionWindowFunnelStrictOnceData<DataTypeDateTime::FieldType>>>(arguments, params);
|
||||||
arguments, params);
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
AggregateFunctionPtr res(createWithUnsignedIntegerType<AggregateFunctionWindowFunnel, AggregateFunctionWindowFunnelData>(*arguments[0], arguments, params));
|
||||||
|
WhichDataType which(arguments.front().get());
|
||||||
|
if (res)
|
||||||
|
return res;
|
||||||
|
if (which.isDate())
|
||||||
|
return std::make_shared<AggregateFunctionWindowFunnel<DataTypeDate::FieldType, AggregateFunctionWindowFunnelData<DataTypeDate::FieldType>>>(arguments, params);
|
||||||
|
if (which.isDateTime())
|
||||||
|
return std::make_shared<AggregateFunctionWindowFunnel<DataTypeDateTime::FieldType, AggregateFunctionWindowFunnelData<DataTypeDateTime::FieldType>>>(arguments, params);
|
||||||
|
}
|
||||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||||
"Illegal type {} of first argument of aggregate function {}, must "
|
"Illegal type {} of first argument of aggregate function {}, must "
|
||||||
"be Unsigned Number, Date, DateTime", arguments.front().get()->getName(), name);
|
"be Unsigned Number, Date, DateTime", arguments.front().get()->getName(), name);
|
||||||
@ -340,7 +604,7 @@ createAggregateFunctionWindowFunnel(const std::string & name, const DataTypes &
|
|||||||
|
|
||||||
void registerAggregateFunctionWindowFunnel(AggregateFunctionFactory & factory)
|
void registerAggregateFunctionWindowFunnel(AggregateFunctionFactory & factory)
|
||||||
{
|
{
|
||||||
factory.registerFunction("windowFunnel", createAggregateFunctionWindowFunnel<AggregateFunctionWindowFunnelData>);
|
factory.registerFunction("windowFunnel", createAggregateFunctionWindowFunnel);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@ -78,11 +78,6 @@ struct WindowFunction : public IAggregateFunctionHelper<WindowFunction>, public
|
|||||||
}
|
}
|
||||||
|
|
||||||
String getName() const override { return name; }
|
String getName() const override { return name; }
|
||||||
void create(AggregateDataPtr __restrict) const override { }
|
|
||||||
void destroy(AggregateDataPtr __restrict) const noexcept override { }
|
|
||||||
bool hasTrivialDestructor() const override { return true; }
|
|
||||||
size_t sizeOfData() const override { return 0; }
|
|
||||||
size_t alignOfData() const override { return 1; }
|
|
||||||
void add(AggregateDataPtr __restrict, const IColumn **, size_t, Arena *) const override { fail(); }
|
void add(AggregateDataPtr __restrict, const IColumn **, size_t, Arena *) const override { fail(); }
|
||||||
void merge(AggregateDataPtr __restrict, ConstAggregateDataPtr, Arena *) const override { fail(); }
|
void merge(AggregateDataPtr __restrict, ConstAggregateDataPtr, Arena *) const override { fail(); }
|
||||||
void serialize(ConstAggregateDataPtr __restrict, WriteBuffer &, std::optional<size_t>) const override { fail(); }
|
void serialize(ConstAggregateDataPtr __restrict, WriteBuffer &, std::optional<size_t>) const override { fail(); }
|
||||||
@ -90,6 +85,22 @@ struct WindowFunction : public IAggregateFunctionHelper<WindowFunction>, public
|
|||||||
void insertResultInto(AggregateDataPtr __restrict, IColumn &, Arena *) const override { fail(); }
|
void insertResultInto(AggregateDataPtr __restrict, IColumn &, Arena *) const override { fail(); }
|
||||||
};
|
};
|
||||||
|
|
||||||
|
struct StatelessWindowFunction : public WindowFunction
|
||||||
|
{
|
||||||
|
StatelessWindowFunction(
|
||||||
|
const std::string & name_, const DataTypes & argument_types_, const Array & parameters_, const DataTypePtr & result_type_)
|
||||||
|
: WindowFunction(name_, argument_types_, parameters_, result_type_)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t sizeOfData() const override { return 0; }
|
||||||
|
size_t alignOfData() const override { return 1; }
|
||||||
|
|
||||||
|
void create(AggregateDataPtr __restrict) const override { }
|
||||||
|
void destroy(AggregateDataPtr __restrict) const noexcept override { }
|
||||||
|
bool hasTrivialDestructor() const override { return true; }
|
||||||
|
};
|
||||||
|
|
||||||
template <typename State>
|
template <typename State>
|
||||||
struct StatefulWindowFunction : public WindowFunction
|
struct StatefulWindowFunction : public WindowFunction
|
||||||
{
|
{
|
||||||
@ -100,7 +111,7 @@ struct StatefulWindowFunction : public WindowFunction
|
|||||||
}
|
}
|
||||||
|
|
||||||
size_t sizeOfData() const override { return sizeof(State); }
|
size_t sizeOfData() const override { return sizeof(State); }
|
||||||
size_t alignOfData() const override { return 1; }
|
size_t alignOfData() const override { return alignof(State); }
|
||||||
|
|
||||||
void create(AggregateDataPtr __restrict place) const override { new (place) State(); }
|
void create(AggregateDataPtr __restrict place) const override { new (place) State(); }
|
||||||
|
|
||||||
|
|||||||
@ -34,7 +34,7 @@ namespace ErrorCodes
|
|||||||
namespace
|
namespace
|
||||||
{
|
{
|
||||||
|
|
||||||
void exctractJoinConditions(const QueryTreeNodePtr & node, QueryTreeNodes & equi_conditions, QueryTreeNodes & other)
|
void extractJoinConditions(const QueryTreeNodePtr & node, QueryTreeNodes & equi_conditions, QueryTreeNodes & other)
|
||||||
{
|
{
|
||||||
auto * func = node->as<FunctionNode>();
|
auto * func = node->as<FunctionNode>();
|
||||||
if (!func)
|
if (!func)
|
||||||
@ -52,7 +52,7 @@ void exctractJoinConditions(const QueryTreeNodePtr & node, QueryTreeNodes & equi
|
|||||||
else if (func->getFunctionName() == "and")
|
else if (func->getFunctionName() == "and")
|
||||||
{
|
{
|
||||||
for (const auto & arg : args)
|
for (const auto & arg : args)
|
||||||
exctractJoinConditions(arg, equi_conditions, other);
|
extractJoinConditions(arg, equi_conditions, other);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -118,7 +118,7 @@ public:
|
|||||||
|
|
||||||
QueryTreeNodes equi_conditions;
|
QueryTreeNodes equi_conditions;
|
||||||
QueryTreeNodes other_conditions;
|
QueryTreeNodes other_conditions;
|
||||||
exctractJoinConditions(where_condition, equi_conditions, other_conditions);
|
extractJoinConditions(where_condition, equi_conditions, other_conditions);
|
||||||
bool can_convert_cross_to_inner = false;
|
bool can_convert_cross_to_inner = false;
|
||||||
for (auto & condition : equi_conditions)
|
for (auto & condition : equi_conditions)
|
||||||
{
|
{
|
||||||
|
|||||||
@ -1950,7 +1950,7 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveUnqualifiedMatcher(
|
|||||||
{
|
{
|
||||||
bool table_expression_in_resolve_process = nearest_query_scope->table_expressions_in_resolve_process.contains(table_expression.get());
|
bool table_expression_in_resolve_process = nearest_query_scope->table_expressions_in_resolve_process.contains(table_expression.get());
|
||||||
|
|
||||||
if (auto * /*array_join_node*/ _ = table_expression->as<ArrayJoinNode>())
|
if (table_expression->as<ArrayJoinNode>())
|
||||||
{
|
{
|
||||||
if (table_expressions_column_nodes_with_names_stack.empty())
|
if (table_expressions_column_nodes_with_names_stack.empty())
|
||||||
throw Exception(ErrorCodes::LOGICAL_ERROR,
|
throw Exception(ErrorCodes::LOGICAL_ERROR,
|
||||||
|
|||||||
@ -19,6 +19,8 @@ namespace ErrorCodes
|
|||||||
extern const int BAD_ARGUMENTS;
|
extern const int BAD_ARGUMENTS;
|
||||||
extern const int ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER;
|
extern const int ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER;
|
||||||
extern const int ILLEGAL_PREWHERE;
|
extern const int ILLEGAL_PREWHERE;
|
||||||
|
extern const int UNSUPPORTED_METHOD;
|
||||||
|
extern const int UNEXPECTED_EXPRESSION;
|
||||||
}
|
}
|
||||||
|
|
||||||
namespace
|
namespace
|
||||||
@ -26,11 +28,24 @@ namespace
|
|||||||
|
|
||||||
void validateFilter(const QueryTreeNodePtr & filter_node, std::string_view exception_place_message, const QueryTreeNodePtr & query_node)
|
void validateFilter(const QueryTreeNodePtr & filter_node, std::string_view exception_place_message, const QueryTreeNodePtr & query_node)
|
||||||
{
|
{
|
||||||
if (filter_node->getNodeType() == QueryTreeNodeType::LIST)
|
DataTypePtr filter_node_result_type;
|
||||||
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
try
|
||||||
"Unsupported expression '{}' in filter", filter_node->formatASTForErrorMessage());
|
{
|
||||||
|
filter_node_result_type = filter_node->getResultType();
|
||||||
|
}
|
||||||
|
catch (const DB::Exception &e)
|
||||||
|
{
|
||||||
|
if (e.code() != ErrorCodes::UNSUPPORTED_METHOD)
|
||||||
|
e.rethrow();
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!filter_node_result_type)
|
||||||
|
throw Exception(ErrorCodes::UNEXPECTED_EXPRESSION,
|
||||||
|
"Unexpected expression '{}' in filter in {}. In query {}",
|
||||||
|
filter_node->formatASTForErrorMessage(),
|
||||||
|
exception_place_message,
|
||||||
|
query_node->formatASTForErrorMessage());
|
||||||
|
|
||||||
auto filter_node_result_type = filter_node->getResultType();
|
|
||||||
if (!filter_node_result_type->canBeUsedInBooleanContext())
|
if (!filter_node_result_type->canBeUsedInBooleanContext())
|
||||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER,
|
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER,
|
||||||
"Invalid type for filter in {}: {}. In query {}",
|
"Invalid type for filter in {}: {}. In query {}",
|
||||||
|
|||||||
@ -20,7 +20,7 @@ BackupReaderDisk::~BackupReaderDisk() = default;
|
|||||||
|
|
||||||
bool BackupReaderDisk::fileExists(const String & file_name)
|
bool BackupReaderDisk::fileExists(const String & file_name)
|
||||||
{
|
{
|
||||||
return disk->exists(root_path / file_name);
|
return disk->existsFile(root_path / file_name);
|
||||||
}
|
}
|
||||||
|
|
||||||
UInt64 BackupReaderDisk::getFileSize(const String & file_name)
|
UInt64 BackupReaderDisk::getFileSize(const String & file_name)
|
||||||
@ -68,7 +68,7 @@ BackupWriterDisk::~BackupWriterDisk() = default;
|
|||||||
|
|
||||||
bool BackupWriterDisk::fileExists(const String & file_name)
|
bool BackupWriterDisk::fileExists(const String & file_name)
|
||||||
{
|
{
|
||||||
return disk->exists(root_path / file_name);
|
return disk->existsFile(root_path / file_name);
|
||||||
}
|
}
|
||||||
|
|
||||||
UInt64 BackupWriterDisk::getFileSize(const String & file_name)
|
UInt64 BackupWriterDisk::getFileSize(const String & file_name)
|
||||||
@ -91,7 +91,7 @@ std::unique_ptr<WriteBuffer> BackupWriterDisk::writeFile(const String & file_nam
|
|||||||
void BackupWriterDisk::removeFile(const String & file_name)
|
void BackupWriterDisk::removeFile(const String & file_name)
|
||||||
{
|
{
|
||||||
disk->removeFileIfExists(root_path / file_name);
|
disk->removeFileIfExists(root_path / file_name);
|
||||||
if (disk->isDirectory(root_path) && disk->isDirectoryEmpty(root_path))
|
if (disk->existsDirectory(root_path) && disk->isDirectoryEmpty(root_path))
|
||||||
disk->removeDirectory(root_path);
|
disk->removeDirectory(root_path);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -99,7 +99,7 @@ void BackupWriterDisk::removeFiles(const Strings & file_names)
|
|||||||
{
|
{
|
||||||
for (const auto & file_name : file_names)
|
for (const auto & file_name : file_names)
|
||||||
disk->removeFileIfExists(root_path / file_name);
|
disk->removeFileIfExists(root_path / file_name);
|
||||||
if (disk->isDirectory(root_path) && disk->isDirectoryEmpty(root_path))
|
if (disk->existsDirectory(root_path) && disk->isDirectoryEmpty(root_path))
|
||||||
disk->removeDirectory(root_path);
|
disk->removeDirectory(root_path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -11,6 +11,12 @@
|
|||||||
namespace DB
|
namespace DB
|
||||||
{
|
{
|
||||||
|
|
||||||
|
namespace ServerSetting
|
||||||
|
{
|
||||||
|
extern const ServerSettingsString default_replica_name;
|
||||||
|
extern const ServerSettingsString default_replica_path;
|
||||||
|
}
|
||||||
|
|
||||||
namespace
|
namespace
|
||||||
{
|
{
|
||||||
void visitStorageSystemTableEngine(ASTStorage &, const DDLAdjustingForBackupVisitor::Data & data)
|
void visitStorageSystemTableEngine(ASTStorage &, const DDLAdjustingForBackupVisitor::Data & data)
|
||||||
@ -55,8 +61,8 @@ namespace
|
|||||||
zookeeper_path_arg.replace(uuid_pos, table_uuid_str.size(), "{uuid}");
|
zookeeper_path_arg.replace(uuid_pos, table_uuid_str.size(), "{uuid}");
|
||||||
}
|
}
|
||||||
const auto & server_settings = data.global_context->getServerSettings();
|
const auto & server_settings = data.global_context->getServerSettings();
|
||||||
if ((zookeeper_path_arg == server_settings.default_replica_path.value)
|
if ((zookeeper_path_arg == server_settings[ServerSetting::default_replica_path].value)
|
||||||
&& (replica_name_arg == server_settings.default_replica_name.value)
|
&& (replica_name_arg == server_settings[ServerSetting::default_replica_name].value)
|
||||||
&& ((engine_args.size() == 2) || !engine_args[2]->as<ASTLiteral>()))
|
&& ((engine_args.size() == 2) || !engine_args[2]->as<ASTLiteral>()))
|
||||||
{
|
{
|
||||||
engine_args.erase(engine_args.begin(), engine_args.begin() + 2);
|
engine_args.erase(engine_args.begin(), engine_args.begin() + 2);
|
||||||
|
|||||||
@ -10,6 +10,10 @@ namespace Setting
|
|||||||
{
|
{
|
||||||
extern const SettingsSeconds http_receive_timeout;
|
extern const SettingsSeconds http_receive_timeout;
|
||||||
}
|
}
|
||||||
|
namespace ServerSetting
|
||||||
|
{
|
||||||
|
extern const ServerSettingsSeconds keep_alive_timeout;
|
||||||
|
}
|
||||||
|
|
||||||
LibraryBridgeHelper::LibraryBridgeHelper(ContextPtr context_)
|
LibraryBridgeHelper::LibraryBridgeHelper(ContextPtr context_)
|
||||||
: IBridgeHelper(context_)
|
: IBridgeHelper(context_)
|
||||||
@ -18,7 +22,7 @@ LibraryBridgeHelper::LibraryBridgeHelper(ContextPtr context_)
|
|||||||
, http_timeout(context_->getGlobalContext()->getSettingsRef()[Setting::http_receive_timeout].value)
|
, http_timeout(context_->getGlobalContext()->getSettingsRef()[Setting::http_receive_timeout].value)
|
||||||
, bridge_host(config.getString("library_bridge.host", DEFAULT_HOST))
|
, bridge_host(config.getString("library_bridge.host", DEFAULT_HOST))
|
||||||
, bridge_port(config.getUInt("library_bridge.port", DEFAULT_PORT))
|
, bridge_port(config.getUInt("library_bridge.port", DEFAULT_PORT))
|
||||||
, http_timeouts(ConnectionTimeouts::getHTTPTimeouts(context_->getSettingsRef(), context_->getServerSettings().keep_alive_timeout))
|
, http_timeouts(ConnectionTimeouts::getHTTPTimeouts(context_->getSettingsRef(), context_->getServerSettings()))
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -99,7 +99,7 @@ protected:
|
|||||||
{
|
{
|
||||||
auto buf = BuilderRWBufferFromHTTP(getPingURI())
|
auto buf = BuilderRWBufferFromHTTP(getPingURI())
|
||||||
.withConnectionGroup(HTTPConnectionGroupType::STORAGE)
|
.withConnectionGroup(HTTPConnectionGroupType::STORAGE)
|
||||||
.withTimeouts(getHTTPTimeouts())
|
.withTimeouts(ConnectionTimeouts::getHTTPTimeouts(getContext()->getSettingsRef(), getContext()->getServerSettings()))
|
||||||
.withSettings(getContext()->getReadSettings())
|
.withSettings(getContext()->getReadSettings())
|
||||||
.create(credentials);
|
.create(credentials);
|
||||||
|
|
||||||
@ -165,11 +165,6 @@ private:
|
|||||||
|
|
||||||
Poco::Net::HTTPBasicCredentials credentials{};
|
Poco::Net::HTTPBasicCredentials credentials{};
|
||||||
|
|
||||||
ConnectionTimeouts getHTTPTimeouts()
|
|
||||||
{
|
|
||||||
return ConnectionTimeouts::getHTTPTimeouts(getContext()->getSettingsRef(), getContext()->getServerSettings().keep_alive_timeout);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
using URLParams = std::vector<std::pair<std::string, std::string>>;
|
using URLParams = std::vector<std::pair<std::string, std::string>>;
|
||||||
|
|
||||||
@ -206,7 +201,7 @@ protected:
|
|||||||
auto buf = BuilderRWBufferFromHTTP(uri)
|
auto buf = BuilderRWBufferFromHTTP(uri)
|
||||||
.withConnectionGroup(HTTPConnectionGroupType::STORAGE)
|
.withConnectionGroup(HTTPConnectionGroupType::STORAGE)
|
||||||
.withMethod(Poco::Net::HTTPRequest::HTTP_POST)
|
.withMethod(Poco::Net::HTTPRequest::HTTP_POST)
|
||||||
.withTimeouts(getHTTPTimeouts())
|
.withTimeouts(ConnectionTimeouts::getHTTPTimeouts(getContext()->getSettingsRef(), getContext()->getServerSettings()))
|
||||||
.withSettings(getContext()->getReadSettings())
|
.withSettings(getContext()->getReadSettings())
|
||||||
.create(credentials);
|
.create(credentials);
|
||||||
|
|
||||||
@ -233,7 +228,7 @@ protected:
|
|||||||
auto buf = BuilderRWBufferFromHTTP(uri)
|
auto buf = BuilderRWBufferFromHTTP(uri)
|
||||||
.withConnectionGroup(HTTPConnectionGroupType::STORAGE)
|
.withConnectionGroup(HTTPConnectionGroupType::STORAGE)
|
||||||
.withMethod(Poco::Net::HTTPRequest::HTTP_POST)
|
.withMethod(Poco::Net::HTTPRequest::HTTP_POST)
|
||||||
.withTimeouts(getHTTPTimeouts())
|
.withTimeouts(ConnectionTimeouts::getHTTPTimeouts(getContext()->getSettingsRef(), getContext()->getServerSettings()))
|
||||||
.withSettings(getContext()->getReadSettings())
|
.withSettings(getContext()->getReadSettings())
|
||||||
.create(credentials);
|
.create(credentials);
|
||||||
|
|
||||||
|
|||||||
@ -244,6 +244,7 @@ add_object_library(clickhouse_storages Storages)
|
|||||||
add_object_library(clickhouse_storages_mysql Storages/MySQL)
|
add_object_library(clickhouse_storages_mysql Storages/MySQL)
|
||||||
add_object_library(clickhouse_storages_distributed Storages/Distributed)
|
add_object_library(clickhouse_storages_distributed Storages/Distributed)
|
||||||
add_object_library(clickhouse_storages_mergetree Storages/MergeTree)
|
add_object_library(clickhouse_storages_mergetree Storages/MergeTree)
|
||||||
|
add_object_library(clickhouse_storages_mergetree_merge_selectors Storages/MergeTree/MergeSelectors)
|
||||||
add_object_library(clickhouse_storages_statistics Storages/Statistics)
|
add_object_library(clickhouse_storages_statistics Storages/Statistics)
|
||||||
add_object_library(clickhouse_storages_liveview Storages/LiveView)
|
add_object_library(clickhouse_storages_liveview Storages/LiveView)
|
||||||
add_object_library(clickhouse_storages_windowview Storages/WindowView)
|
add_object_library(clickhouse_storages_windowview Storages/WindowView)
|
||||||
|
|||||||
@ -107,6 +107,7 @@ namespace Setting
|
|||||||
extern const SettingsUInt64 output_format_pretty_max_value_width;
|
extern const SettingsUInt64 output_format_pretty_max_value_width;
|
||||||
extern const SettingsBool partial_result_on_first_cancel;
|
extern const SettingsBool partial_result_on_first_cancel;
|
||||||
extern const SettingsBool throw_if_no_data_to_insert;
|
extern const SettingsBool throw_if_no_data_to_insert;
|
||||||
|
extern const SettingsBool implicit_select;
|
||||||
}
|
}
|
||||||
|
|
||||||
namespace ErrorCodes
|
namespace ErrorCodes
|
||||||
@ -320,7 +321,7 @@ ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, const Setting
|
|||||||
else if (dialect == Dialect::prql)
|
else if (dialect == Dialect::prql)
|
||||||
parser = std::make_unique<ParserPRQLQuery>(max_length, settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]);
|
parser = std::make_unique<ParserPRQLQuery>(max_length, settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]);
|
||||||
else
|
else
|
||||||
parser = std::make_unique<ParserQuery>(end, settings[Setting::allow_settings_after_format_in_insert]);
|
parser = std::make_unique<ParserQuery>(end, settings[Setting::allow_settings_after_format_in_insert], settings[Setting::implicit_select]);
|
||||||
|
|
||||||
if (is_interactive || ignore_error)
|
if (is_interactive || ignore_error)
|
||||||
{
|
{
|
||||||
|
|||||||
@ -15,7 +15,7 @@ namespace DB
|
|||||||
{
|
{
|
||||||
struct ConnectionParameters
|
struct ConnectionParameters
|
||||||
{
|
{
|
||||||
std::string host;
|
String host;
|
||||||
UInt16 port{};
|
UInt16 port{};
|
||||||
std::string default_database;
|
std::string default_database;
|
||||||
std::string user;
|
std::string user;
|
||||||
@ -30,8 +30,8 @@ struct ConnectionParameters
|
|||||||
ConnectionTimeouts timeouts;
|
ConnectionTimeouts timeouts;
|
||||||
|
|
||||||
ConnectionParameters() = default;
|
ConnectionParameters() = default;
|
||||||
ConnectionParameters(const Poco::Util::AbstractConfiguration & config, std::string host);
|
ConnectionParameters(const Poco::Util::AbstractConfiguration & config, String host);
|
||||||
ConnectionParameters(const Poco::Util::AbstractConfiguration & config, std::string host, std::optional<UInt16> port);
|
ConnectionParameters(const Poco::Util::AbstractConfiguration & config, String host, std::optional<UInt16> port);
|
||||||
|
|
||||||
static UInt16 getPortFromConfig(const Poco::Util::AbstractConfiguration & config, const std::string & connection_host);
|
static UInt16 getPortFromConfig(const Poco::Util::AbstractConfiguration & config, const std::string & connection_host);
|
||||||
|
|
||||||
|
|||||||
@ -33,6 +33,7 @@ namespace Setting
|
|||||||
extern const SettingsUInt64 max_parser_backtracks;
|
extern const SettingsUInt64 max_parser_backtracks;
|
||||||
extern const SettingsUInt64 max_parser_depth;
|
extern const SettingsUInt64 max_parser_depth;
|
||||||
extern const SettingsUInt64 max_query_size;
|
extern const SettingsUInt64 max_query_size;
|
||||||
|
extern const SettingsBool implicit_select;
|
||||||
}
|
}
|
||||||
|
|
||||||
namespace ErrorCodes
|
namespace ErrorCodes
|
||||||
@ -178,7 +179,7 @@ void LocalConnection::sendQuery(
|
|||||||
parser
|
parser
|
||||||
= std::make_unique<ParserPRQLQuery>(settings[Setting::max_query_size], settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]);
|
= std::make_unique<ParserPRQLQuery>(settings[Setting::max_query_size], settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]);
|
||||||
else
|
else
|
||||||
parser = std::make_unique<ParserQuery>(end, settings[Setting::allow_settings_after_format_in_insert]);
|
parser = std::make_unique<ParserQuery>(end, settings[Setting::allow_settings_after_format_in_insert], settings[Setting::implicit_select]);
|
||||||
|
|
||||||
ASTPtr parsed_query;
|
ASTPtr parsed_query;
|
||||||
if (dialect == Dialect::kusto)
|
if (dialect == Dialect::kusto)
|
||||||
|
|||||||
@ -216,6 +216,9 @@
|
|||||||
M(ParquetDecoderThreads, "Number of threads in the ParquetBlockInputFormat thread pool.") \
|
M(ParquetDecoderThreads, "Number of threads in the ParquetBlockInputFormat thread pool.") \
|
||||||
M(ParquetDecoderThreadsActive, "Number of threads in the ParquetBlockInputFormat thread pool running a task.") \
|
M(ParquetDecoderThreadsActive, "Number of threads in the ParquetBlockInputFormat thread pool running a task.") \
|
||||||
M(ParquetDecoderThreadsScheduled, "Number of queued or active jobs in the ParquetBlockInputFormat thread pool.") \
|
M(ParquetDecoderThreadsScheduled, "Number of queued or active jobs in the ParquetBlockInputFormat thread pool.") \
|
||||||
|
M(ParquetDecoderIOThreads, "Number of threads in the ParquetBlockInputFormat io thread pool.") \
|
||||||
|
M(ParquetDecoderIOThreadsActive, "Number of threads in the ParquetBlockInputFormat io thread pool running a task.") \
|
||||||
|
M(ParquetDecoderIOThreadsScheduled, "Number of queued or active jobs in the ParquetBlockInputFormat io thread pool.") \
|
||||||
M(ParquetEncoderThreads, "Number of threads in ParquetBlockOutputFormat thread pool.") \
|
M(ParquetEncoderThreads, "Number of threads in ParquetBlockOutputFormat thread pool.") \
|
||||||
M(ParquetEncoderThreadsActive, "Number of threads in ParquetBlockOutputFormat thread pool running a task.") \
|
M(ParquetEncoderThreadsActive, "Number of threads in ParquetBlockOutputFormat thread pool running a task.") \
|
||||||
M(ParquetEncoderThreadsScheduled, "Number of queued or active jobs in ParquetBlockOutputFormat thread pool.") \
|
M(ParquetEncoderThreadsScheduled, "Number of queued or active jobs in ParquetBlockOutputFormat thread pool.") \
|
||||||
@ -291,9 +294,14 @@
|
|||||||
M(CacheWarmerBytesInProgress, "Total size of remote file segments waiting to be asynchronously loaded into filesystem cache.") \
|
M(CacheWarmerBytesInProgress, "Total size of remote file segments waiting to be asynchronously loaded into filesystem cache.") \
|
||||||
M(DistrCacheOpenedConnections, "Number of open connections to Distributed Cache") \
|
M(DistrCacheOpenedConnections, "Number of open connections to Distributed Cache") \
|
||||||
M(DistrCacheUsedConnections, "Number of currently used connections to Distributed Cache") \
|
M(DistrCacheUsedConnections, "Number of currently used connections to Distributed Cache") \
|
||||||
|
M(DistrCacheAllocatedConnections, "Number of currently allocated connections to Distributed Cache connection pool") \
|
||||||
|
M(DistrCacheBorrowedConnections, "Number of currently borrowed connections to Distributed Cache connection pool") \
|
||||||
M(DistrCacheReadRequests, "Number of executed Read requests to Distributed Cache") \
|
M(DistrCacheReadRequests, "Number of executed Read requests to Distributed Cache") \
|
||||||
M(DistrCacheWriteRequests, "Number of executed Write requests to Distributed Cache") \
|
M(DistrCacheWriteRequests, "Number of executed Write requests to Distributed Cache") \
|
||||||
M(DistrCacheServerConnections, "Number of open connections to ClickHouse server from Distributed Cache") \
|
M(DistrCacheServerConnections, "Number of open connections to ClickHouse server from Distributed Cache") \
|
||||||
|
M(DistrCacheRegisteredServers, "Number of distributed cache registered servers") \
|
||||||
|
M(DistrCacheRegisteredServersCurrentAZ, "Number of distributed cache registered servers in current az") \
|
||||||
|
M(DistrCacheServerS3CachedClients, "Number of distributed cache S3 cached clients") \
|
||||||
\
|
\
|
||||||
M(SchedulerIOReadScheduled, "Number of IO reads are being scheduled currently") \
|
M(SchedulerIOReadScheduled, "Number of IO reads are being scheduled currently") \
|
||||||
M(SchedulerIOWriteScheduled, "Number of IO writes are being scheduled currently") \
|
M(SchedulerIOWriteScheduled, "Number of IO writes are being scheduled currently") \
|
||||||
@ -314,6 +322,20 @@
|
|||||||
M(FilteringMarksWithSecondaryKeys, "Number of threads currently doing filtering of mark ranges by secondary keys") \
|
M(FilteringMarksWithSecondaryKeys, "Number of threads currently doing filtering of mark ranges by secondary keys") \
|
||||||
\
|
\
|
||||||
M(DiskS3NoSuchKeyErrors, "The number of `NoSuchKey` errors that occur when reading data from S3 cloud storage through ClickHouse disks.") \
|
M(DiskS3NoSuchKeyErrors, "The number of `NoSuchKey` errors that occur when reading data from S3 cloud storage through ClickHouse disks.") \
|
||||||
|
\
|
||||||
|
M(SharedCatalogStateApplicationThreads, "Number of threads in the threadpool for state application in Shared Catalog.") \
|
||||||
|
M(SharedCatalogStateApplicationThreadsActive, "Number of active threads in the threadpool for state application in Shared Catalog.") \
|
||||||
|
M(SharedCatalogStateApplicationThreadsScheduled, "Number of queued or active jobs in the threadpool for state application in Shared Catalog.") \
|
||||||
|
\
|
||||||
|
M(SharedCatalogDropLocalThreads, "Number of threads in the threadpool for drop of local tables in Shared Catalog.") \
|
||||||
|
M(SharedCatalogDropLocalThreadsActive, "Number of active threads in the threadpool for drop of local tables in Shared Catalog.") \
|
||||||
|
M(SharedCatalogDropLocalThreadsScheduled, "Number of queued or active jobs in the threadpool for drop of local tables in Shared Catalog.") \
|
||||||
|
\
|
||||||
|
M(SharedCatalogDropZooKeeperThreads, "Number of threads in the threadpool for drop of object in ZooKeeper in Shared Catalog.") \
|
||||||
|
M(SharedCatalogDropZooKeeperThreadsActive, "Number of active threads in the threadpool for drop of object in ZooKeeper in Shared Catalog.") \
|
||||||
|
M(SharedCatalogDropZooKeeperThreadsScheduled, "Number of queued or active jobs in the threadpool for drop of object in ZooKeeper in Shared Catalog.") \
|
||||||
|
\
|
||||||
|
M(SharedDatabaseCatalogTablesInLocalDropDetachQueue, "Number of tables in the queue for local drop or detach in Shared Catalog.") \
|
||||||
|
|
||||||
#ifdef APPLY_FOR_EXTERNAL_METRICS
|
#ifdef APPLY_FOR_EXTERNAL_METRICS
|
||||||
#define APPLY_FOR_METRICS(M) APPLY_FOR_BUILTIN_METRICS(M) APPLY_FOR_EXTERNAL_METRICS(M)
|
#define APPLY_FOR_METRICS(M) APPLY_FOR_BUILTIN_METRICS(M) APPLY_FOR_EXTERNAL_METRICS(M)
|
||||||
|
|||||||
@ -614,6 +614,7 @@
|
|||||||
\
|
\
|
||||||
M(900, DISTRIBUTED_CACHE_ERROR) \
|
M(900, DISTRIBUTED_CACHE_ERROR) \
|
||||||
M(901, CANNOT_USE_DISTRIBUTED_CACHE) \
|
M(901, CANNOT_USE_DISTRIBUTED_CACHE) \
|
||||||
|
M(902, PROTOCOL_VERSION_MISMATCH) \
|
||||||
\
|
\
|
||||||
M(999, KEEPER_EXCEPTION) \
|
M(999, KEEPER_EXCEPTION) \
|
||||||
M(1000, POCO_EXCEPTION) \
|
M(1000, POCO_EXCEPTION) \
|
||||||
|
|||||||
@ -49,11 +49,21 @@ static struct InitFiu
|
|||||||
ONCE(smt_commit_write_zk_fail_before_op) \
|
ONCE(smt_commit_write_zk_fail_before_op) \
|
||||||
ONCE(smt_commit_merge_change_version_before_op) \
|
ONCE(smt_commit_merge_change_version_before_op) \
|
||||||
ONCE(smt_merge_mutate_intention_freeze_in_destructor) \
|
ONCE(smt_merge_mutate_intention_freeze_in_destructor) \
|
||||||
|
ONCE(smt_add_part_sleep_after_add_before_commit) \
|
||||||
|
ONCE(smt_sleep_in_constructor) \
|
||||||
ONCE(meta_in_keeper_create_metadata_failure) \
|
ONCE(meta_in_keeper_create_metadata_failure) \
|
||||||
|
ONCE(smt_insert_retry_timeout) \
|
||||||
|
ONCE(smt_insert_fake_hardware_error) \
|
||||||
|
ONCE(smt_sleep_after_hardware_in_insert) \
|
||||||
|
ONCE(smt_throw_keeper_exception_after_successful_insert) \
|
||||||
|
REGULAR(smt_dont_merge_first_part) \
|
||||||
|
REGULAR(smt_sleep_in_schedule_data_processing_job) \
|
||||||
REGULAR(cache_warmer_stall) \
|
REGULAR(cache_warmer_stall) \
|
||||||
REGULAR(check_table_query_delay_for_part) \
|
REGULAR(check_table_query_delay_for_part) \
|
||||||
REGULAR(dummy_failpoint) \
|
REGULAR(dummy_failpoint) \
|
||||||
REGULAR(prefetched_reader_pool_failpoint) \
|
REGULAR(prefetched_reader_pool_failpoint) \
|
||||||
|
REGULAR(shared_set_sleep_during_update) \
|
||||||
|
REGULAR(smt_outdated_parts_exception_response) \
|
||||||
PAUSEABLE_ONCE(replicated_merge_tree_insert_retry_pause) \
|
PAUSEABLE_ONCE(replicated_merge_tree_insert_retry_pause) \
|
||||||
PAUSEABLE_ONCE(finish_set_quorum_failed_parts) \
|
PAUSEABLE_ONCE(finish_set_quorum_failed_parts) \
|
||||||
PAUSEABLE_ONCE(finish_clean_quorum_failed_parts) \
|
PAUSEABLE_ONCE(finish_clean_quorum_failed_parts) \
|
||||||
|
|||||||
@ -196,7 +196,7 @@ void FileChecker::load()
|
|||||||
|
|
||||||
bool FileChecker::fileReallyExists(const String & path_) const
|
bool FileChecker::fileReallyExists(const String & path_) const
|
||||||
{
|
{
|
||||||
return disk ? disk->exists(path_) : fs::exists(path_);
|
return disk ? disk->existsFile(path_) : fs::exists(path_);
|
||||||
}
|
}
|
||||||
|
|
||||||
size_t FileChecker::getRealFileSize(const String & path_) const
|
size_t FileChecker::getRealFileSize(const String & path_) const
|
||||||
|
|||||||
@ -9,11 +9,11 @@
|
|||||||
throw std::runtime_error(error);
|
throw std::runtime_error(error);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::unordered_map<UInt64, std::pair<time_t, size_t>> LogFrequencyLimiterIml::logged_messages;
|
std::unordered_map<UInt64, std::pair<time_t, size_t>> LogFrequencyLimiterImpl::logged_messages;
|
||||||
time_t LogFrequencyLimiterIml::last_cleanup = 0;
|
time_t LogFrequencyLimiterImpl::last_cleanup = 0;
|
||||||
std::mutex LogFrequencyLimiterIml::mutex;
|
std::mutex LogFrequencyLimiterImpl::mutex;
|
||||||
|
|
||||||
void LogFrequencyLimiterIml::log(Poco::Message & message)
|
void LogFrequencyLimiterImpl::log(Poco::Message & message)
|
||||||
{
|
{
|
||||||
std::string_view pattern = message.getFormatString();
|
std::string_view pattern = message.getFormatString();
|
||||||
if (pattern.empty())
|
if (pattern.empty())
|
||||||
@ -68,7 +68,7 @@ void LogFrequencyLimiterIml::log(Poco::Message & message)
|
|||||||
channel->log(message);
|
channel->log(message);
|
||||||
}
|
}
|
||||||
|
|
||||||
void LogFrequencyLimiterIml::cleanup(time_t too_old_threshold_s)
|
void LogFrequencyLimiterImpl::cleanup(time_t too_old_threshold_s)
|
||||||
{
|
{
|
||||||
time_t now = time(nullptr);
|
time_t now = time(nullptr);
|
||||||
time_t old = now - too_old_threshold_s;
|
time_t old = now - too_old_threshold_s;
|
||||||
|
|||||||
@ -230,7 +230,7 @@ template<> struct FormatStringTypeInfo<PreformattedMessage> { static constexpr b
|
|||||||
/// This wrapper helps to avoid too frequent and noisy log messages.
|
/// This wrapper helps to avoid too frequent and noisy log messages.
|
||||||
/// For each pair (logger_name, format_string) it remembers when such a message was logged the last time.
|
/// For each pair (logger_name, format_string) it remembers when such a message was logged the last time.
|
||||||
/// The message will not be logged again if less than min_interval_s seconds passed since the previously logged message.
|
/// The message will not be logged again if less than min_interval_s seconds passed since the previously logged message.
|
||||||
class LogFrequencyLimiterIml
|
class LogFrequencyLimiterImpl
|
||||||
{
|
{
|
||||||
/// Hash(logger_name, format_string) -> (last_logged_time_s, skipped_messages_count)
|
/// Hash(logger_name, format_string) -> (last_logged_time_s, skipped_messages_count)
|
||||||
static std::unordered_map<UInt64, std::pair<time_t, size_t>> logged_messages;
|
static std::unordered_map<UInt64, std::pair<time_t, size_t>> logged_messages;
|
||||||
@ -240,11 +240,11 @@ class LogFrequencyLimiterIml
|
|||||||
LoggerPtr logger;
|
LoggerPtr logger;
|
||||||
time_t min_interval_s;
|
time_t min_interval_s;
|
||||||
public:
|
public:
|
||||||
LogFrequencyLimiterIml(LoggerPtr logger_, time_t min_interval_s_) : logger(std::move(logger_)), min_interval_s(min_interval_s_) {}
|
LogFrequencyLimiterImpl(LoggerPtr logger_, time_t min_interval_s_) : logger(std::move(logger_)), min_interval_s(min_interval_s_) {}
|
||||||
|
|
||||||
LogFrequencyLimiterIml & operator -> () { return *this; }
|
LogFrequencyLimiterImpl & operator -> () { return *this; }
|
||||||
bool is(Poco::Message::Priority priority) { return logger->is(priority); }
|
bool is(Poco::Message::Priority priority) { return logger->is(priority); }
|
||||||
LogFrequencyLimiterIml * getChannel() {return this; }
|
LogFrequencyLimiterImpl * getChannel() {return this; }
|
||||||
const String & name() const { return logger->name(); }
|
const String & name() const { return logger->name(); }
|
||||||
|
|
||||||
void log(Poco::Message & message);
|
void log(Poco::Message & message);
|
||||||
@ -257,9 +257,9 @@ public:
|
|||||||
|
|
||||||
/// This wrapper helps to avoid too noisy log messages from similar objects.
|
/// This wrapper helps to avoid too noisy log messages from similar objects.
|
||||||
/// Once an instance of LogSeriesLimiter type is created the decision is done
|
/// Once an instance of LogSeriesLimiter type is created the decision is done
|
||||||
/// All followed message which use this instance is either printed or muted all together.
|
/// All followed messages which use this instance are either printed or muted altogether.
|
||||||
/// LogSeriesLimiter differs from LogFrequencyLimiterIml in a way that
|
/// LogSeriesLimiter differs from LogFrequencyLimiterImpl in a way that
|
||||||
/// LogSeriesLimiter is useful for accept or mute series of logs when LogFrequencyLimiterIml works for each line independently.
|
/// LogSeriesLimiter is useful for accept or mute series of logs when LogFrequencyLimiterImpl works for each line independently.
|
||||||
class LogSeriesLimiter
|
class LogSeriesLimiter
|
||||||
{
|
{
|
||||||
static std::mutex mutex;
|
static std::mutex mutex;
|
||||||
@ -295,11 +295,11 @@ class LogToStrImpl
|
|||||||
{
|
{
|
||||||
String & out_str;
|
String & out_str;
|
||||||
LoggerPtr logger;
|
LoggerPtr logger;
|
||||||
std::unique_ptr<LogFrequencyLimiterIml> maybe_nested;
|
std::unique_ptr<LogFrequencyLimiterImpl> maybe_nested;
|
||||||
bool propagate_to_actual_log = true;
|
bool propagate_to_actual_log = true;
|
||||||
public:
|
public:
|
||||||
LogToStrImpl(String & out_str_, LoggerPtr logger_) : out_str(out_str_), logger(std::move(logger_)) {}
|
LogToStrImpl(String & out_str_, LoggerPtr logger_) : out_str(out_str_), logger(std::move(logger_)) {}
|
||||||
LogToStrImpl(String & out_str_, std::unique_ptr<LogFrequencyLimiterIml> && maybe_nested_)
|
LogToStrImpl(String & out_str_, std::unique_ptr<LogFrequencyLimiterImpl> && maybe_nested_)
|
||||||
: out_str(out_str_), logger(maybe_nested_->getLogger()), maybe_nested(std::move(maybe_nested_)) {}
|
: out_str(out_str_), logger(maybe_nested_->getLogger()), maybe_nested(std::move(maybe_nested_)) {}
|
||||||
LogToStrImpl & operator -> () { return *this; }
|
LogToStrImpl & operator -> () { return *this; }
|
||||||
bool is(Poco::Message::Priority priority) { propagate_to_actual_log &= logger->is(priority); return true; }
|
bool is(Poco::Message::Priority priority) { propagate_to_actual_log &= logger->is(priority); return true; }
|
||||||
|
|||||||
@ -222,6 +222,8 @@
|
|||||||
M(SelectedBytes, "Number of bytes (uncompressed; for columns as they stored in memory) SELECTed from all tables.", ValueType::Bytes) \
|
M(SelectedBytes, "Number of bytes (uncompressed; for columns as they stored in memory) SELECTed from all tables.", ValueType::Bytes) \
|
||||||
M(RowsReadByMainReader, "Number of rows read from MergeTree tables by the main reader (after PREWHERE step).", ValueType::Number) \
|
M(RowsReadByMainReader, "Number of rows read from MergeTree tables by the main reader (after PREWHERE step).", ValueType::Number) \
|
||||||
M(RowsReadByPrewhereReaders, "Number of rows read from MergeTree tables (in total) by prewhere readers.", ValueType::Number) \
|
M(RowsReadByPrewhereReaders, "Number of rows read from MergeTree tables (in total) by prewhere readers.", ValueType::Number) \
|
||||||
|
M(LoadedDataParts, "Number of data parts loaded by MergeTree tables during initialization.", ValueType::Number) \
|
||||||
|
M(LoadedDataPartsMicroseconds, "Microseconds spent by MergeTree tables for loading data parts during initialization.", ValueType::Microseconds) \
|
||||||
\
|
\
|
||||||
M(WaitMarksLoadMicroseconds, "Time spent loading marks", ValueType::Microseconds) \
|
M(WaitMarksLoadMicroseconds, "Time spent loading marks", ValueType::Microseconds) \
|
||||||
M(BackgroundLoadingMarksTasks, "Number of background tasks for loading marks", ValueType::Number) \
|
M(BackgroundLoadingMarksTasks, "Number of background tasks for loading marks", ValueType::Number) \
|
||||||
@ -241,6 +243,8 @@
|
|||||||
M(MergeVerticalStageExecuteMilliseconds, "Total busy time spent for execution of vertical stage of background merges", ValueType::Milliseconds) \
|
M(MergeVerticalStageExecuteMilliseconds, "Total busy time spent for execution of vertical stage of background merges", ValueType::Milliseconds) \
|
||||||
M(MergeProjectionStageTotalMilliseconds, "Total time spent for projection stage of background merges", ValueType::Milliseconds) \
|
M(MergeProjectionStageTotalMilliseconds, "Total time spent for projection stage of background merges", ValueType::Milliseconds) \
|
||||||
M(MergeProjectionStageExecuteMilliseconds, "Total busy time spent for execution of projection stage of background merges", ValueType::Milliseconds) \
|
M(MergeProjectionStageExecuteMilliseconds, "Total busy time spent for execution of projection stage of background merges", ValueType::Milliseconds) \
|
||||||
|
M(MergePrewarmStageTotalMilliseconds, "Total time spent for prewarm stage of background merges", ValueType::Milliseconds) \
|
||||||
|
M(MergePrewarmStageExecuteMilliseconds, "Total busy time spent for execution of prewarm stage of background merges", ValueType::Milliseconds) \
|
||||||
\
|
\
|
||||||
M(MergingSortedMilliseconds, "Total time spent while merging sorted columns", ValueType::Milliseconds) \
|
M(MergingSortedMilliseconds, "Total time spent while merging sorted columns", ValueType::Milliseconds) \
|
||||||
M(AggregatingSortedMilliseconds, "Total time spent while aggregating sorted columns", ValueType::Milliseconds) \
|
M(AggregatingSortedMilliseconds, "Total time spent while aggregating sorted columns", ValueType::Milliseconds) \
|
||||||
@ -639,6 +643,8 @@ The server successfully detected this situation and will download merged part fr
|
|||||||
M(MetadataFromKeeperBackgroundCleanupTransactions, "Number of times old transaction idempotency token was cleaned up by background task", ValueType::Number) \
|
M(MetadataFromKeeperBackgroundCleanupTransactions, "Number of times old transaction idempotency token was cleaned up by background task", ValueType::Number) \
|
||||||
M(MetadataFromKeeperBackgroundCleanupErrors, "Number of times an error was encountered in background cleanup task", ValueType::Number) \
|
M(MetadataFromKeeperBackgroundCleanupErrors, "Number of times an error was encountered in background cleanup task", ValueType::Number) \
|
||||||
\
|
\
|
||||||
|
M(SharedMergeTreeMetadataCacheHintLoadedFromCache, "Number of times metadata cache hint was found without going to Keeper", ValueType::Number) \
|
||||||
|
\
|
||||||
M(KafkaRebalanceRevocations, "Number of partition revocations (the first stage of consumer group rebalance)", ValueType::Number) \
|
M(KafkaRebalanceRevocations, "Number of partition revocations (the first stage of consumer group rebalance)", ValueType::Number) \
|
||||||
M(KafkaRebalanceAssignments, "Number of partition assignments (the final stage of consumer group rebalance)", ValueType::Number) \
|
M(KafkaRebalanceAssignments, "Number of partition assignments (the final stage of consumer group rebalance)", ValueType::Number) \
|
||||||
M(KafkaRebalanceErrors, "Number of failed consumer group rebalances", ValueType::Number) \
|
M(KafkaRebalanceErrors, "Number of failed consumer group rebalances", ValueType::Number) \
|
||||||
@ -742,29 +748,51 @@ The server successfully detected this situation and will download merged part fr
|
|||||||
M(ConnectionPoolIsFullMicroseconds, "Total time spent waiting for a slot in connection pool.", ValueType::Microseconds) \
|
M(ConnectionPoolIsFullMicroseconds, "Total time spent waiting for a slot in connection pool.", ValueType::Microseconds) \
|
||||||
M(AsyncLoaderWaitMicroseconds, "Total time a query was waiting for async loader jobs.", ValueType::Microseconds) \
|
M(AsyncLoaderWaitMicroseconds, "Total time a query was waiting for async loader jobs.", ValueType::Microseconds) \
|
||||||
\
|
\
|
||||||
M(DistrCacheServerSwitches, "Number of server switches between distributed cache servers in read/write-through cache", ValueType::Number) \
|
M(DistrCacheServerSwitches, "Distributed Cache read buffer event. Number of server switches between distributed cache servers in read/write-through cache", ValueType::Number) \
|
||||||
M(DistrCacheReadMicroseconds, "Time spent reading from distributed cache", ValueType::Microseconds) \
|
M(DistrCacheReadMicroseconds, "Distributed Cache read buffer event. Time spent reading from distributed cache", ValueType::Microseconds) \
|
||||||
M(DistrCacheFallbackReadMicroseconds, "Time spend reading from fallback buffer instead of distribted cache", ValueType::Microseconds) \
|
M(DistrCacheFallbackReadMicroseconds, "Distributed Cache read buffer event. Time spend reading from fallback buffer instead of distributed cache", ValueType::Microseconds) \
|
||||||
M(DistrCachePrecomputeRangesMicroseconds, "Time spent to precompute read ranges", ValueType::Microseconds) \
|
M(DistrCachePrecomputeRangesMicroseconds, "Distributed Cache read buffer event. Time spent to precompute read ranges", ValueType::Microseconds) \
|
||||||
M(DistrCacheNextImplMicroseconds, "Time spend in ReadBufferFromDistributedCache::nextImpl", ValueType::Microseconds) \
|
M(DistrCacheNextImplMicroseconds, "Distributed Cache read buffer event. Time spend in ReadBufferFromDistributedCache::nextImpl", ValueType::Microseconds) \
|
||||||
M(DistrCacheOpenedConnections, "The number of open connections to distributed cache", ValueType::Number) \
|
M(DistrCacheStartRangeMicroseconds, "Distributed Cache read buffer event. Time spent to start a new read range with distributed cache", ValueType::Microseconds) \
|
||||||
M(DistrCacheReusedConnections, "The number of reused connections to distributed cache", ValueType::Number) \
|
M(DistrCacheIgnoredBytesWhileWaitingProfileEvents, "Distributed Cache read buffer event. Ignored bytes while waiting for profile events in distributed cache", ValueType::Number) \
|
||||||
M(DistrCacheHoldConnections, "The number of used connections to distributed cache", ValueType::Number) \
|
M(DistrCacheRangeChange, "Distributed Cache read buffer event. Number of times we changed read range because of seek/last_position change", ValueType::Number) \
|
||||||
\
|
\
|
||||||
M(DistrCacheGetResponseMicroseconds, "Time spend to wait for response from distributed cache", ValueType::Microseconds) \
|
M(DistrCacheGetResponseMicroseconds, "Distributed Cache client event. Time spend to wait for response from distributed cache", ValueType::Microseconds) \
|
||||||
M(DistrCacheStartRangeMicroseconds, "Time spent to start a new read range with distributed cache", ValueType::Microseconds) \
|
M(DistrCacheReadErrors, "Distributed Cache client event. Number of distributed cache errors during read", ValueType::Number) \
|
||||||
M(DistrCacheLockRegistryMicroseconds, "Time spent to take DistributedCacheRegistry lock", ValueType::Microseconds) \
|
M(DistrCacheMakeRequestErrors, "Distributed Cache client event. Number of distributed cache errors when making a request", ValueType::Number) \
|
||||||
M(DistrCacheUnusedPackets, "Number of skipped unused packets from distributed cache", ValueType::Number) \
|
M(DistrCacheReceiveResponseErrors, "Distributed Cache client event. Number of distributed cache errors when receiving response a request", ValueType::Number) \
|
||||||
M(DistrCachePackets, "Total number of packets received from distributed cache", ValueType::Number) \
|
|
||||||
M(DistrCacheUnusedPacketsBytes, "The number of bytes in Data packets which were ignored", ValueType::Bytes) \
|
|
||||||
M(DistrCacheRegistryUpdateMicroseconds, "Time spent updating distributed cache registry", ValueType::Microseconds) \
|
|
||||||
M(DistrCacheRegistryUpdates, "Number of distributed cache registry updates", ValueType::Number) \
|
|
||||||
\
|
\
|
||||||
M(DistrCacheConnectMicroseconds, "The time spent to connect to distributed cache", ValueType::Microseconds) \
|
M(DistrCachePackets, "Distributed Cache client event. Total number of packets received from distributed cache", ValueType::Number) \
|
||||||
M(DistrCacheConnectAttempts, "The number of connection attempts to distributed cache", ValueType::Number) \
|
M(DistrCachePacketsBytes, "Distributed Cache client event. The number of bytes in Data packets which were not ignored", ValueType::Bytes) \
|
||||||
M(DistrCacheGetClient, "Number of client access times", ValueType::Number) \
|
M(DistrCacheUnusedPackets, "Distributed Cache client event. Number of skipped unused packets from distributed cache", ValueType::Number) \
|
||||||
|
M(DistrCacheUnusedPacketsBytes, "Distributed Cache client event. The number of bytes in Data packets which were ignored", ValueType::Bytes) \
|
||||||
|
M(DistrCacheUnusedPacketsBufferAllocations, "Distributed Cache client event. The number of extra buffer allocations in case we could not reuse existing buffer", ValueType::Number) \
|
||||||
\
|
\
|
||||||
M(DistrCacheServerProcessRequestMicroseconds, "Time spent processing request on DistributedCache server side", ValueType::Microseconds) \
|
M(DistrCacheLockRegistryMicroseconds, "Distributed Cache registry event. Time spent to take DistributedCacheRegistry lock", ValueType::Microseconds) \
|
||||||
|
M(DistrCacheRegistryUpdateMicroseconds, "Distributed Cache registry event. Time spent updating distributed cache registry", ValueType::Microseconds) \
|
||||||
|
M(DistrCacheRegistryUpdates, "Distributed Cache registry event. Number of distributed cache registry updates", ValueType::Number) \
|
||||||
|
M(DistrCacheHashRingRebuilds, "Distributed Cache registry event. Number of distributed cache hash ring rebuilds", ValueType::Number) \
|
||||||
|
\
|
||||||
|
M(DistrCacheReadBytesFromCache, "Distributed Cache read buffer event. Bytes read from distributed cache", ValueType::Bytes) \
|
||||||
|
M(DistrCacheReadBytesFromFallbackBuffer, "Distributed Cache read buffer event. Bytes read from fallback buffer", ValueType::Number) \
|
||||||
|
\
|
||||||
|
M(DistrCacheRangeResetBackward, "Distributed Cache read buffer event. Number of times we reset read range because of seek/last_position change", ValueType::Number) \
|
||||||
|
M(DistrCacheRangeResetForward, "Distributed Cache read buffer event. Number of times we reset read range because of seek/last_position change", ValueType::Number) \
|
||||||
|
\
|
||||||
|
M(DistrCacheOpenedConnections, "Distributed Cache connection event. The number of open connections to distributed cache", ValueType::Number) \
|
||||||
|
M(DistrCacheReusedConnections, "Distributed Cache connection event. The number of reused connections to distributed cache", ValueType::Number) \
|
||||||
|
M(DistrCacheOpenedConnectionsBypassingPool, "Distributed Cache connection event. The number of open connections to distributed cache bypassing pool", ValueType::Number) \
|
||||||
|
M(DistrCacheConnectMicroseconds, "Distributed Cache connection event. The time spent to connect to distributed cache", ValueType::Microseconds) \
|
||||||
|
M(DistrCacheConnectAttempts, "Distributed Cache connection event. The number of connection attempts to distributed cache", ValueType::Number) \
|
||||||
|
M(DistrCacheGetClientMicroseconds, "Distributed Cache connection event. Time spent getting client for distributed cache", ValueType::Microseconds) \
|
||||||
|
\
|
||||||
|
M(DistrCacheServerProcessRequestMicroseconds, "Distributed Cache server event. Time spent processing request on DistributedCache server side", ValueType::Microseconds) \
|
||||||
|
M(DistrCacheServerStartRequestPackets, "Distributed Cache server event. Number of StartRequest packets in DistributedCacheServer", ValueType::Number) \
|
||||||
|
M(DistrCacheServerContinueRequestPackets, "Distributed Cache server event. Number of ContinueRequest packets in DistributedCacheServer", ValueType::Number) \
|
||||||
|
M(DistrCacheServerEndRequestPackets, "Distributed Cache server event. Number of EndRequest packets in DistributedCacheServer", ValueType::Number) \
|
||||||
|
M(DistrCacheServerAckRequestPackets, "Distributed Cache server event. Number of AckRequest packets in DistributedCacheServer", ValueType::Number) \
|
||||||
|
M(DistrCacheServerNewS3CachedClients, "Distributed Cache server event. The number of new cached s3 clients", ValueType::Number) \
|
||||||
|
M(DistrCacheServerReusedS3CachedClients, "Distributed Cache server event. The number of reused cached s3 clients", ValueType::Number) \
|
||||||
\
|
\
|
||||||
M(LogTest, "Number of log messages with level Test", ValueType::Number) \
|
M(LogTest, "Number of log messages with level Test", ValueType::Number) \
|
||||||
M(LogTrace, "Number of log messages with level Trace", ValueType::Number) \
|
M(LogTrace, "Number of log messages with level Trace", ValueType::Number) \
|
||||||
@ -788,15 +816,38 @@ The server successfully detected this situation and will download merged part fr
|
|||||||
M(InterfacePostgreSQLReceiveBytes, "Number of bytes received through PostgreSQL interfaces", ValueType::Bytes) \
|
M(InterfacePostgreSQLReceiveBytes, "Number of bytes received through PostgreSQL interfaces", ValueType::Bytes) \
|
||||||
\
|
\
|
||||||
M(ParallelReplicasUsedCount, "Number of replicas used to execute a query with task-based parallel replicas", ValueType::Number) \
|
M(ParallelReplicasUsedCount, "Number of replicas used to execute a query with task-based parallel replicas", ValueType::Number) \
|
||||||
|
M(ParallelReplicasAvailableCount, "Number of replicas available to execute a query with task-based parallel replicas", ValueType::Number) \
|
||||||
|
M(ParallelReplicasUnavailableCount, "Number of replicas which was chosen, but found to be unavailable during query execution with task-based parallel replicas", ValueType::Number) \
|
||||||
\
|
\
|
||||||
|
M(SharedMergeTreeVirtualPartsUpdates, "Virtual parts update count", ValueType::Number) \
|
||||||
|
M(SharedMergeTreeVirtualPartsUpdatesByLeader, "Virtual parts updates by leader", ValueType::Number) \
|
||||||
|
M(SharedMergeTreeVirtualPartsUpdateMicroseconds, "Virtual parts update microseconds", ValueType::Microseconds) \
|
||||||
|
M(SharedMergeTreeVirtualPartsUpdatesFromZooKeeper, "Virtual parts updates count from ZooKeeper", ValueType::Number) \
|
||||||
|
M(SharedMergeTreeVirtualPartsUpdatesFromZooKeeperMicroseconds, "Virtual parts updates from ZooKeeper microseconds", ValueType::Microseconds) \
|
||||||
|
M(SharedMergeTreeVirtualPartsUpdatesPeerNotFound, "Virtual updates from peer failed because no one found", ValueType::Number) \
|
||||||
|
M(SharedMergeTreeVirtualPartsUpdatesFromPeer, "Virtual parts updates count from peer", ValueType::Number) \
|
||||||
|
M(SharedMergeTreeVirtualPartsUpdatesFromPeerMicroseconds, "Virtual parts updates from peer microseconds", ValueType::Microseconds) \
|
||||||
|
M(SharedMergeTreeVirtualPartsUpdatesForMergesOrStatus, "Virtual parts updates from non-default background job", ValueType::Number) \
|
||||||
|
M(SharedMergeTreeVirtualPartsUpdatesLeaderFailedElection, "Virtual parts updates leader election failed", ValueType::Number) \
|
||||||
|
M(SharedMergeTreeVirtualPartsUpdatesLeaderSuccessfulElection, "Virtual parts updates leader election successful", ValueType::Number) \
|
||||||
|
M(SharedMergeTreeMergeMutationAssignmentAttempt, "How many times we tried to assign merge or mutation", ValueType::Number) \
|
||||||
|
M(SharedMergeTreeMergeMutationAssignmentFailedWithNothingToDo, "How many times we tried to assign merge or mutation and failed because nothing to merge", ValueType::Number) \
|
||||||
|
M(SharedMergeTreeMergeMutationAssignmentFailedWithConflict, "How many times we tried to assign merge or mutation and failed because of conflict in Keeper", ValueType::Number) \
|
||||||
|
M(SharedMergeTreeMergeMutationAssignmentSuccessful, "How many times we tried to assign merge or mutation", ValueType::Number) \
|
||||||
|
M(SharedMergeTreeMergePartsMovedToOudated, "How many parts moved to oudated directory", ValueType::Number) \
|
||||||
|
M(SharedMergeTreeMergePartsMovedToCondemned, "How many parts moved to condemned directory", ValueType::Number) \
|
||||||
|
M(SharedMergeTreeOutdatedPartsConfirmationRequest, "How many ZooKeeper requests were used to config outdated parts", ValueType::Number) \
|
||||||
|
M(SharedMergeTreeOutdatedPartsConfirmationInvocations, "How many invocations were made to confirm outdated parts", ValueType::Number) \
|
||||||
|
M(SharedMergeTreeOutdatedPartsHTTPRequest, "How many HTTP requests were send to confirm outdated parts", ValueType::Number) \
|
||||||
|
M(SharedMergeTreeOutdatedPartsHTTPResponse, "How many HTTP responses were send to confirm outdated parts", ValueType::Number) \
|
||||||
|
M(SharedMergeTreeCondemnedPartsKillRequest, "How many ZooKeeper requests were used to remove condemned parts", ValueType::Number) \
|
||||||
|
M(SharedMergeTreeCondemnedPartsLockConfict, "How many times we failed to acquite lock because of conflict", ValueType::Number) \
|
||||||
|
M(SharedMergeTreeCondemnedPartsRemoved, "How many condemned parts were removed", ValueType::Number) \
|
||||||
M(KeeperLogsEntryReadFromLatestCache, "Number of log entries in Keeper being read from latest logs cache", ValueType::Number) \
|
M(KeeperLogsEntryReadFromLatestCache, "Number of log entries in Keeper being read from latest logs cache", ValueType::Number) \
|
||||||
M(KeeperLogsEntryReadFromCommitCache, "Number of log entries in Keeper being read from commit logs cache", ValueType::Number) \
|
M(KeeperLogsEntryReadFromCommitCache, "Number of log entries in Keeper being read from commit logs cache", ValueType::Number) \
|
||||||
M(KeeperLogsEntryReadFromFile, "Number of log entries in Keeper being read directly from the changelog file", ValueType::Number) \
|
M(KeeperLogsEntryReadFromFile, "Number of log entries in Keeper being read directly from the changelog file", ValueType::Number) \
|
||||||
M(KeeperLogsPrefetchedEntries, "Number of log entries in Keeper being prefetched from the changelog file", ValueType::Number) \
|
M(KeeperLogsPrefetchedEntries, "Number of log entries in Keeper being prefetched from the changelog file", ValueType::Number) \
|
||||||
\
|
\
|
||||||
M(ParallelReplicasAvailableCount, "Number of replicas available to execute a query with task-based parallel replicas", ValueType::Number) \
|
|
||||||
M(ParallelReplicasUnavailableCount, "Number of replicas which was chosen, but found to be unavailable during query execution with task-based parallel replicas", ValueType::Number) \
|
|
||||||
\
|
|
||||||
M(StorageConnectionsCreated, "Number of created connections for storages", ValueType::Number) \
|
M(StorageConnectionsCreated, "Number of created connections for storages", ValueType::Number) \
|
||||||
M(StorageConnectionsReused, "Number of reused connections for storages", ValueType::Number) \
|
M(StorageConnectionsReused, "Number of reused connections for storages", ValueType::Number) \
|
||||||
M(StorageConnectionsReset, "Number of reset connections for storages", ValueType::Number) \
|
M(StorageConnectionsReset, "Number of reset connections for storages", ValueType::Number) \
|
||||||
@ -828,12 +879,17 @@ The server successfully detected this situation and will download merged part fr
|
|||||||
M(ReadWriteBufferFromHTTPRequestsSent, "Number of HTTP requests sent by ReadWriteBufferFromHTTP", ValueType::Number) \
|
M(ReadWriteBufferFromHTTPRequestsSent, "Number of HTTP requests sent by ReadWriteBufferFromHTTP", ValueType::Number) \
|
||||||
M(ReadWriteBufferFromHTTPBytes, "Total size of payload bytes received and sent by ReadWriteBufferFromHTTP. Doesn't include HTTP headers.", ValueType::Bytes) \
|
M(ReadWriteBufferFromHTTPBytes, "Total size of payload bytes received and sent by ReadWriteBufferFromHTTP. Doesn't include HTTP headers.", ValueType::Bytes) \
|
||||||
\
|
\
|
||||||
|
M(SharedDatabaseCatalogFailedToApplyState, "Number of failures to apply new state in SharedDatabaseCatalog", ValueType::Number) \
|
||||||
|
M(SharedDatabaseCatalogStateApplicationMicroseconds, "Total time spend on application of new state in SharedDatabaseCatalog", ValueType::Microseconds) \
|
||||||
|
\
|
||||||
M(GWPAsanAllocateSuccess, "Number of successful allocations done by GWPAsan", ValueType::Number) \
|
M(GWPAsanAllocateSuccess, "Number of successful allocations done by GWPAsan", ValueType::Number) \
|
||||||
M(GWPAsanAllocateFailed, "Number of failed allocations done by GWPAsan (i.e. filled pool)", ValueType::Number) \
|
M(GWPAsanAllocateFailed, "Number of failed allocations done by GWPAsan (i.e. filled pool)", ValueType::Number) \
|
||||||
M(GWPAsanFree, "Number of free operations done by GWPAsan", ValueType::Number) \
|
M(GWPAsanFree, "Number of free operations done by GWPAsan", ValueType::Number) \
|
||||||
\
|
\
|
||||||
M(MemoryWorkerRun, "Number of runs done by MemoryWorker in background", ValueType::Number) \
|
M(MemoryWorkerRun, "Number of runs done by MemoryWorker in background", ValueType::Number) \
|
||||||
M(MemoryWorkerRunElapsedMicroseconds, "Total time spent by MemoryWorker for background work", ValueType::Microseconds) \
|
M(MemoryWorkerRunElapsedMicroseconds, "Total time spent by MemoryWorker for background work", ValueType::Microseconds) \
|
||||||
|
\
|
||||||
|
M(ParquetFetchWaitTimeMicroseconds, "Time of waiting fetching parquet data", ValueType::Microseconds) \
|
||||||
|
|
||||||
|
|
||||||
#ifdef APPLY_FOR_EXTERNAL_EVENTS
|
#ifdef APPLY_FOR_EXTERNAL_EVENTS
|
||||||
|
|||||||
@ -38,6 +38,9 @@ namespace ProfileEvents
|
|||||||
};
|
};
|
||||||
Timer(Counters & counters_, Event timer_event_, Resolution resolution_);
|
Timer(Counters & counters_, Event timer_event_, Resolution resolution_);
|
||||||
Timer(Counters & counters_, Event timer_event_, Event counter_event, Resolution resolution_);
|
Timer(Counters & counters_, Event timer_event_, Event counter_event, Resolution resolution_);
|
||||||
|
Timer(Timer && other) noexcept
|
||||||
|
: counters(other.counters), timer_event(std::move(other.timer_event)), watch(std::move(other.watch)), resolution(std::move(other.resolution))
|
||||||
|
{}
|
||||||
~Timer() { end(); }
|
~Timer() { end(); }
|
||||||
void cancel() { watch.reset(); }
|
void cancel() { watch.reset(); }
|
||||||
void restart() { watch.restart(); }
|
void restart() { watch.restart(); }
|
||||||
|
|||||||
@ -31,7 +31,7 @@ std::string RemoteProxyHostFetcherImpl::fetch(const Poco::URI & endpoint, const
|
|||||||
endpoint.toString(),
|
endpoint.toString(),
|
||||||
response.getStatus(),
|
response.getStatus(),
|
||||||
response.getReason(),
|
response.getReason(),
|
||||||
"");
|
/* body_length = */ 0);
|
||||||
|
|
||||||
std::string proxy_host;
|
std::string proxy_host;
|
||||||
Poco::StreamCopier::copyToString(response_body_stream, proxy_host);
|
Poco::StreamCopier::copyToString(response_body_stream, proxy_host);
|
||||||
|
|||||||
@ -10,11 +10,11 @@ namespace ProfileEvents
|
|||||||
namespace DB
|
namespace DB
|
||||||
{
|
{
|
||||||
|
|
||||||
TaskTracker::TaskTracker(ThreadPoolCallbackRunnerUnsafe<void> scheduler_, size_t max_tasks_inflight_, LogSeriesLimiterPtr limitedLog_)
|
TaskTracker::TaskTracker(ThreadPoolCallbackRunnerUnsafe<void> scheduler_, size_t max_tasks_inflight_, LogSeriesLimiterPtr limited_log_)
|
||||||
: is_async(bool(scheduler_))
|
: is_async(bool(scheduler_))
|
||||||
, scheduler(scheduler_ ? std::move(scheduler_) : syncRunner())
|
, scheduler(scheduler_ ? std::move(scheduler_) : syncRunner())
|
||||||
, max_tasks_inflight(max_tasks_inflight_)
|
, max_tasks_inflight(max_tasks_inflight_)
|
||||||
, limitedLog(limitedLog_)
|
, limited_log(limited_log_)
|
||||||
{}
|
{}
|
||||||
|
|
||||||
TaskTracker::~TaskTracker()
|
TaskTracker::~TaskTracker()
|
||||||
@ -142,7 +142,7 @@ void TaskTracker::waitTilInflightShrink()
|
|||||||
return;
|
return;
|
||||||
|
|
||||||
if (futures.size() >= max_tasks_inflight)
|
if (futures.size() >= max_tasks_inflight)
|
||||||
LOG_TEST(limitedLog, "have to wait some tasks finish, in queue {}, limit {}", futures.size(), max_tasks_inflight);
|
LOG_TEST(limited_log, "have to wait some tasks finish, in queue {}, limit {}", futures.size(), max_tasks_inflight);
|
||||||
|
|
||||||
Stopwatch watch;
|
Stopwatch watch;
|
||||||
|
|
||||||
|
|||||||
@ -23,7 +23,7 @@ class TaskTracker
|
|||||||
public:
|
public:
|
||||||
using Callback = std::function<void()>;
|
using Callback = std::function<void()>;
|
||||||
|
|
||||||
TaskTracker(ThreadPoolCallbackRunnerUnsafe<void> scheduler_, size_t max_tasks_inflight_, LogSeriesLimiterPtr limitedLog_);
|
TaskTracker(ThreadPoolCallbackRunnerUnsafe<void> scheduler_, size_t max_tasks_inflight_, LogSeriesLimiterPtr limited_log_);
|
||||||
~TaskTracker();
|
~TaskTracker();
|
||||||
|
|
||||||
static ThreadPoolCallbackRunnerUnsafe<void> syncRunner();
|
static ThreadPoolCallbackRunnerUnsafe<void> syncRunner();
|
||||||
@ -55,7 +55,7 @@ private:
|
|||||||
|
|
||||||
using FutureList = std::list<std::future<void>>;
|
using FutureList = std::list<std::future<void>>;
|
||||||
FutureList futures;
|
FutureList futures;
|
||||||
LogSeriesLimiterPtr limitedLog;
|
LogSeriesLimiterPtr limited_log;
|
||||||
|
|
||||||
std::mutex mutex;
|
std::mutex mutex;
|
||||||
std::condition_variable has_finished TSA_GUARDED_BY(mutex);
|
std::condition_variable has_finished TSA_GUARDED_BY(mutex);
|
||||||
|
|||||||
@ -81,7 +81,7 @@ void ZooKeeperLock::unlock()
|
|||||||
zookeeper->remove(lock_path, -1);
|
zookeeper->remove(lock_path, -1);
|
||||||
LOG_TRACE(log, "Lock on path {} for session {} is unlocked", lock_path, zookeeper->getClientID());
|
LOG_TRACE(log, "Lock on path {} for session {} is unlocked", lock_path, zookeeper->getClientID());
|
||||||
}
|
}
|
||||||
else if (result)
|
else if (result && throw_if_lost) /// NOTE: What if session expired exactly here?
|
||||||
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Lock is lost, it has another owner. Path: {}, message: {}, owner: {}, our id: {}",
|
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Lock is lost, it has another owner. Path: {}, message: {}, owner: {}, our id: {}",
|
||||||
lock_path, lock_message, stat.ephemeralOwner, zookeeper->getClientID());
|
lock_path, lock_message, stat.ephemeralOwner, zookeeper->getClientID());
|
||||||
else if (throw_if_lost)
|
else if (throw_if_lost)
|
||||||
|
|||||||
@ -159,6 +159,8 @@ public:
|
|||||||
const std::string & getLastKeeperErrorMessage() const { return keeper_error.message; }
|
const std::string & getLastKeeperErrorMessage() const { return keeper_error.message; }
|
||||||
|
|
||||||
/// action will be called only once and only after latest failed retry
|
/// action will be called only once and only after latest failed retry
|
||||||
|
/// NOTE: this one will be called only in case when retries finishes with Keeper exception
|
||||||
|
/// if it will be some other exception this function will not be called.
|
||||||
void actionAfterLastFailedRetry(std::function<void()> f) { action_after_last_failed_retry = std::move(f); }
|
void actionAfterLastFailedRetry(std::function<void()> f) { action_after_last_failed_retry = std::move(f); }
|
||||||
|
|
||||||
const std::string & getName() const { return name; }
|
const std::string & getName() const { return name; }
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user