thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 23:21:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into add_performance_tests

Browse Source
This commit is contained in:
alesapin 2021-11-25 10:53:10 +03:00 committed by GitHub
commit 58fad90351
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
89 changed files with 3788 additions and 281 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/master.yml vendored
View File

@ -181,7 +181,7 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 9
BUILD_NUMBER: 8
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH

2
.gitmodules vendored
View File

@ -140,7 +140,7 @@
url = https://github.com/ClickHouse-Extras/libc-headers.git
[submodule "contrib/replxx"]
path = contrib/replxx
url = https://github.com/AmokHuginnsson/replxx.git
url = https://github.com/ClickHouse-Extras/replxx.git
[submodule "contrib/avro"]
path = contrib/avro
url = https://github.com/ClickHouse-Extras/avro.git

24
PreLoad.cmake
View File

@ -18,3 +18,27 @@ if (NOT DEFINED ENV{CLION_IDE} AND NOT DEFINED ENV{XCODE_IDE})
set(CMAKE_GENERATOR "Ninja" CACHE INTERNAL "" FORCE)
endif ()
endif()
# Default toolchain - this is needed to avoid dependency on OS files.
execute_process(COMMAND uname -s OUTPUT_VARIABLE OS)
execute_process(COMMAND uname -m OUTPUT_VARIABLE ARCH)
if (OS MATCHES "Linux"
AND NOT DEFINED CMAKE_TOOLCHAIN_FILE
AND NOT UNBUNDLED
AND NOT DISABLE_HERMETIC_BUILD
AND ($ENV{CC} MATCHES ".*clang.*" OR CMAKE_C_COMPILER MATCHES ".*clang.*")
AND (USE_STATIC_LIBRARIES OR NOT DEFINED USE_STATIC_LIBRARIES))
if (ARCH MATCHES "amd64|x86_64")
set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-x86_64.cmake" CACHE INTERNAL "" FORCE)
elseif (ARCH MATCHES "^(aarch64.*|AARCH64.*|arm64.*|ARM64.*)")
set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-aarch64.cmake" CACHE INTERNAL "" FORCE)
elseif (ARCH MATCHES "^(ppc64le.*|PPC64LE.*)")
set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-ppc64le.cmake" CACHE INTERNAL "" FORCE)
else ()
message (FATAL_ERROR "Unsupported architecture: ${ARCH}")
endif ()
endif()

4
SECURITY.md
View File

@ -19,9 +19,11 @@ The following versions of ClickHouse server are currently being supported with s
| 21.4 | :x: |
| 21.5 | :x: |
| 21.6 | :x: |
| 21.7 | ✅ |
| 21.7 | :x: |
| 21.8 | ✅ |
| 21.9 | ✅ |
| 21.10 | ✅ |
| 21.11 | ✅ |
## Reporting a Vulnerability

6
cmake/add_warning.cmake
View File

@ -12,13 +12,13 @@ macro (add_warning flag)
if (SUPPORTS_CXXFLAG_${underscored_flag})
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W${flag}")
else ()
message (WARNING "Flag -W${flag} is unsupported")
message (STATUS "Flag -W${flag} is unsupported")
endif ()
if (SUPPORTS_CFLAG_${underscored_flag})
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -W${flag}")
else ()
message (WARNING "Flag -W${flag} is unsupported")
message (STATUS "Flag -W${flag} is unsupported")
endif ()
endmacro ()
@ -39,7 +39,7 @@ macro (target_add_warning target flag)
if (SUPPORTS_CXXFLAG_${underscored_flag})
target_compile_options (${target} PRIVATE "-W${flag}")
else ()
message (WARNING "Flag -W${flag} is unsupported")
message (STATUS "Flag -W${flag} is unsupported")
endif ()
endmacro ()

2
cmake/freebsd/toolchain-aarch64.cmake
View File

@ -3,7 +3,7 @@ set (CMAKE_SYSTEM_PROCESSOR "aarch64")
set (CMAKE_C_COMPILER_TARGET "aarch64-unknown-freebsd12")
set (CMAKE_CXX_COMPILER_TARGET "aarch64-unknown-freebsd12")
set (CMAKE_ASM_COMPILER_TARGET "aarch64-unknown-freebsd12")
set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/freebsd-aarch64")
set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/freebsd-aarch64")
set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake

2
cmake/freebsd/toolchain-x86_64.cmake
View File

@ -3,7 +3,7 @@ set (CMAKE_SYSTEM_PROCESSOR "x86_64")
set (CMAKE_C_COMPILER_TARGET "x86_64-pc-freebsd11")
set (CMAKE_CXX_COMPILER_TARGET "x86_64-pc-freebsd11")
set (CMAKE_ASM_COMPILER_TARGET "x86_64-pc-freebsd11")
set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/freebsd-x86_64")
set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/freebsd-x86_64")
set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake

8
cmake/linux/default_libs.cmake
View File

@ -5,8 +5,12 @@ set (DEFAULT_LIBS "-nodefaultlibs")
# We need builtins from Clang's RT even without libcxx - for ubsan+int128.
# See https://bugs.llvm.org/show_bug.cgi?id=16404
if (COMPILER_CLANG AND NOT CMAKE_CROSSCOMPILING)
execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-libgcc-file-name --rtlib=compiler-rt OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
if (COMPILER_CLANG)
execute_process (COMMAND ${CMAKE_CXX_COMPILER} --target=${CMAKE_CXX_COMPILER_TARGET} --print-libgcc-file-name --rtlib=compiler-rt OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
if (NOT EXISTS "${BUILTINS_LIBRARY}")
set (BUILTINS_LIBRARY "-lgcc")
endif ()
else ()
set (BUILTINS_LIBRARY "-lgcc")
endif ()

2
contrib/base64 vendored

@ -1 +1 @@
Subproject commit af9b331f2b4f30b41c70f3a571ff904a8251c1d3
Subproject commit 9499e0c4945589973b9ea1bc927377cfbc84aa46

4
contrib/krb5-cmake/CMakeLists.txt
View File

@ -500,7 +500,6 @@ function(preprocess_et out_var)
COMMAND perl "${KRB5_SOURCE_DIR}/util/et/compile_et" -d "${KRB5_SOURCE_DIR}/util/et" ${in_f}
DEPENDS ${in_f} "${KRB5_SOURCE_DIR}/util/et/compile_et"
WORKING_DIRECTORY ${ET_PATH}
COMMENT "Creating preprocessed file ${F_C}"
VERBATIM
)
list(APPEND result ${F_C})
@ -526,7 +525,6 @@ add_custom_command(
add_custom_target(
ERROR_MAP_H
DEPENDS "${KRB5_SOURCE_DIR}/lib/gssapi/krb5/error_map.h"
COMMENT "generating error_map.h"
VERBATIM
)
@ -539,14 +537,12 @@ add_custom_command(
add_custom_target(
ERRMAP_H
DEPENDS "${KRB5_SOURCE_DIR}/lib/gssapi/generic/errmap.h"
COMMENT "generating errmap.h"
VERBATIM
)
add_custom_target(
KRB_5_H
DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/include/krb5/krb5.h"
COMMENT "generating krb5.h"
VERBATIM
)

2
contrib/libhdfs3 vendored

@ -1 +1 @@
Subproject commit a8c37ee001af1ae88e5dfa637ae5b31b087c96d3
Subproject commit 9194af44588633c1b2dae44bf945804401ff883e

4
contrib/protobuf-cmake/CMakeLists.txt
View File

@ -211,7 +211,9 @@ add_library(protobuf::libprotoc ALIAS libprotoc)
set(protoc_files ${protobuf_source_dir}/src/google/protobuf/compiler/main.cc)
if (NOT CMAKE_CROSSCOMPILING)
if (CMAKE_HOST_SYSTEM_NAME STREQUAL CMAKE_SYSTEM_NAME
AND CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL CMAKE_SYSTEM_PROCESSOR)
add_executable(protoc ${protoc_files})
target_link_libraries(protoc libprotoc libprotobuf pthread)
add_executable(protobuf::protoc ALIAS protoc)

2
contrib/replxx vendored

@ -1 +1 @@
Subproject commit 68410ac01dfb4f09ea76120ac5a2cecda3943aaf
Subproject commit f019cba7ea1bcd1b4feb7826f28ed57fb581b04c

2
contrib/sysroot vendored

@ -1 +1 @@
Subproject commit 1a64956aa7c280448be6526251bb2b8e6d380ab1
Subproject commit 4ef348b7f30f2ad5b02b266268b3c948e51ad457

3
docker/packager/binary/Dockerfile
View File

@ -93,9 +93,6 @@ RUN git clone https://github.com/tpoechtrager/cctools-port.git \
# Download toolchain and SDK for Darwin
RUN wget -nv https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.0.sdk.tar.xz
# Download toolchain for FreeBSD 11.3
RUN wget -nv https://clickhouse-datasets.s3.yandex.net/toolchains/toolchains/freebsd-11.3-toolchain.tar.xz
# NOTE: Seems like gcc-11 is too new for ubuntu20 repository
RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
&& apt-get update \

3
docker/packager/binary/build.sh
View File

@ -6,9 +6,6 @@ mkdir -p build/cmake/toolchain/darwin-x86_64
tar xJf MacOSX11.0.sdk.tar.xz -C build/cmake/toolchain/darwin-x86_64 --strip-components=1
ln -sf darwin-x86_64 build/cmake/toolchain/darwin-aarch64
mkdir -p build/cmake/toolchain/freebsd-x86_64
tar xJf freebsd-11.3-toolchain.tar.xz -C build/cmake/toolchain/freebsd-x86_64 --strip-components=1
# Uncomment to debug ccache. Don't put ccache log in /output right away, or it
# will be confusingly packed into the "performance" package.
# export CCACHE_LOGFILE=/build/ccache.log

1
docker/test/fasttest/run.sh
View File

@ -159,6 +159,7 @@ function clone_submodules
cd "$FASTTEST_SOURCE"
SUBMODULES_TO_UPDATE=(
contrib/sysroot
contrib/magic_enum
contrib/abseil-cpp
contrib/boost

2
docker/test/pvs/Dockerfile
View File

@ -40,7 +40,7 @@ RUN set -x \
ENV CCACHE_DIR=/test_output/ccache
CMD echo "Running PVS version $PKG_VERSION" && mkdir -p $CCACHE_DIR && cd /repo_folder && pvs-studio-analyzer credentials $LICENCE_NAME $LICENCE_KEY -o ./licence.lic \
&& cmake . -D"ENABLE_EMBEDDED_COMPILER"=OFF -D"USE_INTERNAL_PROTOBUF_LIBRARY"=OFF -D"USE_INTERNAL_GRPC_LIBRARY"=OFF -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang\+\+-13 \
&& cmake . -D"ENABLE_EMBEDDED_COMPILER"=OFF -D"DISABLE_HERMETIC_BUILD"=ON -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang\+\+-13 \
&& ninja re2_st clickhouse_grpc_protos \
&& pvs-studio-analyzer analyze -o pvs-studio.log -e contrib -j 4 -l ./licence.lic; \
cp /repo_folder/pvs-studio.log /test_output; \

2
docs/en/getting-started/example-datasets/ontime.md
View File

@ -15,7 +15,7 @@ This dataset can be obtained in two ways:
Downloading data:
``` bash
echo https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_{1987..2021}_{1..12}.zip | xargs -P10 wget --no-check-certificate --continue
wget --no-check-certificate --continue https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_{1987..2021}_{1..12}.zip
```
Creating a table:

21
docs/en/introduction/adopters.md
View File

@ -11,6 +11,7 @@ toc_title: Adopters
| Company | Industry | Usecase | Cluster Size | (Un)Compressed Data Size<abbr title="of single replica"><sup>\*</sup></abbr> | Reference |
|---------|----------|---------|--------------|------------------------------------------------------------------------------|-----------|
| <a href="https://2gis.ru" class="favicon">2gis</a> | Maps | Monitoring | — | — | [Talk in Russian, July 2019](https://youtu.be/58sPkXfq6nw) |
| <a href="https://adapty.io/" class="favicon">Adapty</a> | Subscription Analytics | Main product | — | — | [Tweet, November 2021](https://twitter.com/iwitaly/status/1462698148061659139) |
| <a href="https://getadmiral.com/" class="favicon">Admiral</a> | Martech | Engagement Management | — | — | [Webinar Slides, June 2020](https://altinity.com/presentations/2020/06/16/big-data-in-real-time-how-clickhouse-powers-admirals-visitor-relationships-for-publishers) |
| <a href="http://www.adscribe.tv/" class="favicon">AdScribe</a> | Ads | TV Analytics | — | — | [A quote from CTO](https://altinity.com/24x7-support/) |
| <a href="https://ahrefs.com/" class="favicon">Ahrefs</a> | SEO | Analytics | — | — | [Job listing](https://ahrefs.com/jobs/data-scientist-search) |
@ -19,7 +20,7 @@ toc_title: Adopters
| <a href="https://alohabrowser.com/" class="favicon">Aloha Browser</a> | Mobile App | Browser backend | — | — | [Slides in Russian, May 2019](https://presentations.clickhouse.com/meetup22/aloha.pdf) |
| <a href="https://altinity.com/" class="favicon">Altinity</a> | Cloud, SaaS | Main product | — | — | [Official Website](https://altinity.com/) |
| <a href="https://amadeus.com/" class="favicon">Amadeus</a> | Travel | Analytics | — | — | [Press Release, April 2018](https://www.altinity.com/blog/2018/4/5/amadeus-technologies-launches-investment-and-insights-tool-based-on-machine-learning-and-strategy-algorithms) |
| <a href="https://apiroad.net/" class="favicon">ApiRoad</a> | API marketplace | Analytics | — | — | [Blog post, Nov 2018, Mar 2020](https://pixeljets.com/blog/clickhouse-vs-elasticsearch/) |
| <a href="https://apiroad.net/" class="favicon">ApiRoad</a> | API marketplace | Analytics | — | — | [Blog post, November 2018, March 2020](https://pixeljets.com/blog/clickhouse-vs-elasticsearch/) |
| <a href="https://www.appsflyer.com" class="favicon">Appsflyer</a> | Mobile analytics | Main product | — | — | [Talk in Russian, July 2019](https://www.youtube.com/watch?v=M3wbRlcpBbY) |
| <a href="https://arenadata.tech/" class="favicon">ArenaData</a> | Data Platform | Main product | — | — | [Slides in Russian, December 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup38/indexes.pdf) |
| <a href="https://www.argedor.com/en/clickhouse/" class="favicon">Argedor</a> | ClickHouse support | — | — | — | [Official website](https://www.argedor.com/en/clickhouse/) |
@ -50,6 +51,7 @@ toc_title: Adopters
| <a href="https://cryptology.com/" class="favicon">Cryptology</a> | Digital Assets Trading Platform | — | — | — | [Job advertisement, March 2021](https://career.habr.com/companies/cryptology/vacancies) |
| <a href="https://www.chinatelecomglobal.com/" class="favicon">Dataliance for China Telecom</a> | Telecom | Analytics | — | — | [Slides in Chinese, January 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/telecom.pdf) |
| <a href="https://db.com" class="favicon">Deutsche Bank</a> | Finance | BI Analytics | — | — | [Slides in English, October 2019](https://bigdatadays.ru/wp-content/uploads/2019/10/D2-H3-3_Yakunin-Goihburg.pdf) |
| <a href="https://www.deepl.com/" class="favicon">Deepl</a> | Machine Learning | — | — | — | [Video, October 2021](https://www.youtube.com/watch?v=WIYJiPwxXdM&t=1182s) |
| <a href="https://deeplay.io/eng/" class="favicon">Deeplay</a> | Gaming Analytics | — | — | — | [Job advertisement, 2020](https://career.habr.com/vacancies/1000062568) |
| <a href="https://www.diva-e.com" class="favicon">Diva-e</a> | Digital consulting | Main Product | — | — | [Slides in English, September 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup29/ClickHouse-MeetUp-Unusual-Applications-sd-2019-09-17.pdf) |
| <a href="https://ecommpay.com/" class="favicon">Ecommpay</a> | Payment Processing | Logs | — | — | [Video, Nov 2019](https://www.youtube.com/watch?v=d3GdZTOWGLk) |
@ -65,6 +67,7 @@ toc_title: Adopters
| <a href="https://gigapipe.com/" class="favicon">Gigapipe</a> | Managed ClickHouse | Main product | — | — | [Official website](https://gigapipe.com/) |
| <a href="https://glaber.io/" class="favicon">Glaber</a> | Monitoring | Main product | — | — | [Website](https://glaber.io/) |
| <a href="https://graphcdn.io/" class="favicon">GraphCDN</a> | CDN | Traffic Analytics | — | — | [Blog Post in English, August 2021](https://altinity.com/blog/delivering-insight-on-graphql-apis-with-clickhouse-at-graphcdn/) |
| <a href="https://www.grouparoo.com" class="favicon">Grouparoo</a> | Data Warehouse Integrations | Main product | — | — | [Official Website, November 2021](https://www.grouparoo.com/integrations) |
| <a href="https://www.huya.com/" class="favicon">HUYA</a> | Video Streaming | Analytics | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) |
| <a href="https://www.hydrolix.io/" class="favicon">Hydrolix</a> | Cloud data platform | Main product | — | — | [Documentation](https://docs.hydrolix.io/guide/query) |
| <a href="https://www.the-ica.com/" class="favicon">ICA</a> | FinTech | Risk Management | — | — | [Blog Post in English, Sep 2020](https://altinity.com/blog/clickhouse-vs-redshift-performance-for-fintech-risk-management?utm_campaign=ClickHouse%20vs%20RedShift&utm_content=143520807&utm_medium=social&utm_source=twitter&hss_channel=tw-3894792263) |
@ -89,7 +92,7 @@ toc_title: Adopters
| <a href="https://mcs.mail.ru/" class="favicon">Mail.ru Cloud Solutions</a> | Cloud services | Main product | — | — | [Article in Russian](https://mcs.mail.ru/help/db-create/clickhouse#) |
| <a href="https://maxilect.com/" class="favicon">MAXILECT</a> | Ad Tech, Blockchain, ML, AI | — | — | — | [Job advertisement, 2021](https://www.linkedin.com/feed/update/urn:li:activity:6780842017229430784/) |
| <a href="https://tech.mymarilyn.ru" class="favicon">Marilyn</a> | Advertising | Statistics | — | — | [Talk in Russian, June 2017](https://www.youtube.com/watch?v=iXlIgx2khwc) |
| <a href="https://mellodesign.ru/" class="favicon">Mello</a> | Marketing | Analytics | 1 server | — | [Article, Oct 2020](https://vc.ru/marketing/166180-razrabotka-tipovogo-otcheta-skvoznoy-analitiki) |
| <a href="https://mellodesign.ru/" class="favicon">Mello</a> | Marketing | Analytics | 1 server | — | [Article, October 2020](https://vc.ru/marketing/166180-razrabotka-tipovogo-otcheta-skvoznoy-analitiki) |
| <a href="https://www.messagebird.com" class="favicon">MessageBird</a> | Telecommunications | Statistics | — | — | [Slides in English, November 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup20/messagebird.pdf) |
| <a href="https://clarity.microsoft.com/" class="favicon">Microsoft</a> | Web Analytics | Clarity (Main Product) | — | — | [A question on GitHub](https://github.com/ClickHouse/ClickHouse/issues/21556) |
| <a href="https://www.mindsdb.com/" class="favicon">MindsDB</a> | Machine Learning | Main Product | — | — | [Official Website](https://www.mindsdb.com/blog/machine-learning-models-as-tables-in-ch) |
@ -100,17 +103,16 @@ toc_title: Adopters
| <a href="https://getnoc.com/" class="favicon">NOC Project</a> | Network Monitoring | Analytics | Main Product | — | [Official Website](https://getnoc.com/features/big-data/) |
| <a href="https://www.noction.com" class="favicon">Noction</a> | Network Technology | Main Product | — | — | [Official Website](https://www.noction.com/news/irp-3-11-remote-triggered-blackholing-capability)
| <a href="https://www.nuna.com/" class="favicon">Nuna Inc.</a> | Health Data Analytics | — | — | — | [Talk in English, July 2020](https://youtu.be/GMiXCMFDMow?t=170) |
| <a href="https://ok.ru" class="favicon">Ok.ru</a> | Social Network | — | 72 servers | 810 TB compressed, 50bn rows/day, 1.5 TB/day | [SmartData conference, Oct 2021](https://assets.ctfassets.net/oxjq45e8ilak/4JPHkbJenLgZhBGGyyonFP/57472ec6987003ec4078d0941740703b/____________________ClickHouse_______________________.pdf) |
| <a href="https://omnicomm.ru/" class="favicon">Omnicomm</a> | Transportation Monitoring | — | — | — | [Facebook post, Oct 2021](https://www.facebook.com/OmnicommTeam/posts/2824479777774500) |
| <a href="https://ok.ru" class="favicon">Ok.ru</a> | Social Network | — | 72 servers | 810 TB compressed, 50bn rows/day, 1.5 TB/day | [SmartData conference, October 2021](https://assets.ctfassets.net/oxjq45e8ilak/4JPHkbJenLgZhBGGyyonFP/57472ec6987003ec4078d0941740703b/____________________ClickHouse_______________________.pdf) |
| <a href="https://omnicomm.ru/" class="favicon">Omnicomm</a> | Transportation Monitoring | — | — | — | [Facebook post, October 2021](https://www.facebook.com/OmnicommTeam/posts/2824479777774500) |
| <a href="https://www.oneapm.com/" class="favicon">OneAPM</a> | Monitoring and Data Analysis | Main product | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/8.%20clickhouse在OneAPM的应用%20杜龙.pdf) |
| <a href="https://www.opentargets.org/" class="favicon">Open Targets</a> | Genome Research | Genome Search | — | — | [Tweet, Oct 2021](https://twitter.com/OpenTargets/status/1452570865342758913?s=20), [Blog](https://blog.opentargets.org/graphql/) |
| <a href="https://www.opentargets.org/" class="favicon">Open Targets</a> | Genome Research | Genome Search | — | — | [Tweet, October 2021](https://twitter.com/OpenTargets/status/1452570865342758913?s=20), [Blog](https://blog.opentargets.org/graphql/) |
| <a href="https://corp.ozon.com/" class="favicon">OZON</a> | E-commerce | — | — | — | [Official website](https://job.ozon.ru/vacancy/razrabotchik-clickhouse-ekspluatatsiya-40991870/) |
| <a href="https://panelbear.com/" class="favicon">Panelbear | Analytics | Monitoring and Analytics | — | — | [Tech Stack, November 2020](https://panelbear.com/blog/tech-stack/) |
| <a href="https://www.percent.cn/" class="favicon">Percent 百分点</a> | Analytics | Main Product | — | — | [Slides in Chinese, June 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf) |
| <a href="https://www.percona.com/" class="favicon">Percona</a> | Performance analysis | Percona Monitoring and Management | — | — | [Official website, Mar 2020](https://www.percona.com/blog/2020/03/30/advanced-query-analysis-in-percona-monitoring-and-management-with-direct-clickhouse-access/) |
| <a href="https://piwik.pro/" class="favicon">Piwik PRO</a> | Web Analytics | Main Product | — | — | [Official website, Dec 2018](https://piwik.pro/blog/piwik-pro-clickhouse-faster-efficient-reports/) |
| <a href="https://plausible.io/" class="favicon">Plausible</a> | Analytics | Main Product | — | — | [Blog post, June 2020](https://twitter.com/PlausibleHQ/status/1273889629087969280) |
| <a href="https://posthog.com/" class="favicon">PostHog</a> | Product Analytics | Main Product | — | — | [Release Notes, Oct 2020](https://posthog.com/blog/the-posthog-array-1-15-0) |
| <a href="https://posthog.com/" class="favicon">PostHog</a> | Product Analytics | Main Product | — | — | [Release Notes, October 2020](https://posthog.com/blog/the-posthog-array-1-15-0), [Blog, November 2021](https://posthog.com/blog/how-we-turned-clickhouse-into-our-eventmansion) |
| <a href="https://postmates.com/" class="favicon">Postmates</a> | Delivery | — | — | — | [Talk in English, July 2020](https://youtu.be/GMiXCMFDMow?t=188) |
| <a href="http://www.pragma-innovation.fr/" class="favicon">Pragma Innovation</a> | Telemetry and Big Data Analysis | Main product | — | — | [Slides in English, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/4_pragma_innovation.pdf) |
| <a href="https://prana-system.com/en/" class="favicon">PRANA</a> | Industrial predictive analytics | Main product | — | — | [News (russian), Feb 2021](https://habr.com/en/news/t/541392/) |
@ -152,6 +154,7 @@ toc_title: Adopters
| <a href="https://trafficstars.com/" class="favicon">Traffic Stars</a> | AD network | — | 300 servers in Europe/US | 1.8 PiB, 700 000 insert rps (as of 2021) | [Slides in Russian, May 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/ninja.pdf) |
| <a href="https://www.uber.com" class="favicon">Uber</a> | Taxi | Logging | — | — | [Slides, February 2020](https://presentations.clickhouse.com/meetup40/uber.pdf) |
| <a href="https://hello.utmstat.com/" class="favicon">UTMSTAT</a> | Analytics | Main product | — | — | [Blog post, June 2020](https://vc.ru/tribuna/133956-striming-dannyh-iz-servisa-skvoznoy-analitiki-v-clickhouse) |
| <a href="https://vercel.com/" class="favicon">Vercel</a> | Traffic and Performance Analytics | — | — | — | Direct reference, October 2021 |
| <a href="https://vk.com" class="favicon">VKontakte</a> | Social Network | Statistics, Logging | — | — | [Slides in Russian, August 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/3_vk.pdf) |
| <a href="https://www.vmware.com/" class="favicon">VMware</a> | Cloud | VeloCloud, SDN | — | — | [Product documentation](https://docs.vmware.com/en/vRealize-Operations-Manager/8.3/com.vmware.vcom.metrics.doc/GUID-A9AD72E1-C948-4CA2-971B-919385AB3CA8.html) |
| <a href="https://www.walmartlabs.com/" class="favicon">Walmart Labs</a> | Internet, Retail | — | — | — | [Talk in English, July 2020](https://youtu.be/GMiXCMFDMow?t=144) |
@ -167,6 +170,7 @@ toc_title: Adopters
| <a href="https://market.yandex.ru/" class="favicon">Yandex Market</a> | e-Commerce | Metrics, Logging | — | — | [Talk in Russian, January 2019](https://youtu.be/_l1qP0DyBcA?t=478) |
| <a href="https://metrica.yandex.com" class="favicon">Yandex Metrica</a> | Web analytics | Main product | 630 servers in one cluster, 360 servers in another cluster, 1862 servers in one department | 133 PiB / 8.31 PiB / 120 trillion records | [Slides, February 2020](https://presentations.clickhouse.com/meetup40/introduction/#13) |
| <a href="https://www.yotascale.com/" class="favicon">Yotascale</a> | Cloud | Data pipeline | — | 2 bn records/day | [LinkedIn (Accomplishments)](https://www.linkedin.com/in/adilsaleem/) |
| <a href="https://www.your-analytics.org/" class="favicon">Your Analytics</a> | Product Analytics | Main Product | — | - | [Tweet, November 2021](https://twitter.com/mikenikles/status/1459737241165565953) |
| <a href="https://zagravagames.com/en/" class="favicon">Zagrava Trading</a> | — | — | — | — | [Job offer, May 2021](https://twitter.com/datastackjobs/status/1394707267082063874) |
| <a href="https://htc-cs.ru/" class="favicon">ЦВТ</a> | Software Development | Metrics, Logging | — | — | [Blog Post, March 2019, in Russian](https://vc.ru/dev/62715-kak-my-stroili-monitoring-na-prometheus-clickhouse-i-elk) |
| <a href="https://mkb.ru/" class="favicon">МКБ</a> | Bank | Web-system monitoring | — | — | [Slides in Russian, September 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/mkb.pdf) |
@ -174,8 +178,5 @@ toc_title: Adopters
| <a href="https://promo.croc.ru/digitalworker" class="favicon">Цифровой Рабочий</a> | Industrial IoT, Analytics | — | — | — | [Blog post in Russian, March 2021](https://habr.com/en/company/croc/blog/548018/) |
| <a href="https://shop.okraina.ru/" class="favicon">ООО «МПЗ Богородский»</a> | Agriculture | — | — | — | [Article in Russian, November 2020](https://cloud.yandex.ru/cases/okraina) |
| <a href="https://domclick.ru/" class="favicon">ДомКлик</a> | Real Estate | — | — | — | [Article in Russian, October 2021](https://habr.com/ru/company/domclick/blog/585936/) |
| <a href="https://www.deepl.com/" class="favicon">Deepl</a> | Machine Learning | — | — | — | [Video, October 2021](https://www.youtube.com/watch?v=WIYJiPwxXdM&t=1182s) |
| <a href="https://vercel.com/" class="favicon">Vercel</a> | Traffic and Performance Analytics | — | — | — | Direct reference, October 2021 |
| <a href="https://www.your-analytics.org/" class="favicon">YourAnalytics</a> | Web Analytics | — | — | — | [Tweet, Nov 2021](https://twitter.com/mikenikles/status/1460860140249235461) |
[Original article](https://clickhouse.com/docs/en/introduction/adopters/) <!--hide-->

191
docs/en/operations/clickhouse-keeper.md
View File

@ -49,6 +49,7 @@ Internal coordination settings are located in `<keeper_server>.<coordination_set
- `auto_forwarding` — Allow to forward write requests from followers to the leader (default: true).
- `shutdown_timeout` — Wait to finish internal connections and shutdown (ms) (default: 5000).
- `startup_timeout` — If the server doesn't connect to other quorum participants in the specified timeout it will terminate (ms) (default: 30000).
- `four_letter_word_white_list` — White list of 4lw commands (default: "conf,cons,crst,envi,ruok,srst,srvr,stat,wchc,wchs,dirs,mntr,isro").
Quorum configuration is located in `<keeper_server>.<raft_configuration>` section and contain servers description.
@ -104,6 +105,196 @@ ClickHouse Keeper is bundled into the ClickHouse server package, just add config
clickhouse-keeper --config /etc/your_path_to_config/config.xml --daemon
```
## Four Latter Word Commands
ClickHouse Keeper also provides 4lw commands which are almost the same with Zookeeper. Each command is composed of four letters such as `mntr`, `stat` etc. There are some more interesting commands: `stat` gives some general information about the server and connected clients, while `srvr` and `cons` give extended details on server and connections respectively.
The 4lw commands has a white list configuration `four_letter_word_white_list` which has default value "conf,cons,crst,envi,ruok,srst,srvr,stat,wchc,wchs,dirs,mntr,isro".
You can issue the commands to ClickHouse Keeper via telnet or nc, at the client port.
```
echo mntr | nc localhost 9181
```
Bellow is the detailed 4lw commands:
- ruok : Tests if server is running in a non-error state. The server will respond with imok if it is running. Otherwise it will not respond at all. A response of "imok" does not necessarily indicate that the server has joined the quorum, just that the server process is active and bound to the specified client port. Use "stat" for details on state wrt quorum and client connection information.
```
imok
```
- mntr : Outputs a list of variables that could be used for monitoring the health of the cluster.
```
zk_version v21.11.1.1-prestable-7a4a0b0edef0ad6e0aa662cd3b90c3f4acf796e7
zk_avg_latency 0
zk_max_latency 0
zk_min_latency 0
zk_packets_received 68
zk_packets_sent 68
zk_num_alive_connections 1
zk_outstanding_requests 0
zk_server_state leader
zk_znode_count 4
zk_watch_count 1
zk_ephemerals_count 0
zk_approximate_data_size 723
zk_open_file_descriptor_count 310
zk_max_file_descriptor_count 10240
zk_followers 0
zk_synced_followers 0
```
- srvr : Lists full details for the server.
```
ClickHouse Keeper version: v21.11.1.1-prestable-7a4a0b0edef0ad6e0aa662cd3b90c3f4acf796e7
Latency min/avg/max: 0/0/0
Received: 2
Sent : 2
Connections: 1
Outstanding: 0
Zxid: 34
Mode: leader
Node count: 4
```
- stat : Lists brief details for the server and connected clients.
```
ClickHouse Keeper version: v21.11.1.1-prestable-7a4a0b0edef0ad6e0aa662cd3b90c3f4acf796e7
Clients:
192.168.1.1:52852(recved=0,sent=0)
192.168.1.1:52042(recved=24,sent=48)
Latency min/avg/max: 0/0/0
Received: 4
Sent : 4
Connections: 1
Outstanding: 0
Zxid: 36
Mode: leader
Node count: 4
```
- srst : Reset server statistics. The command will affect the result of `srvr`, `mntr` and `stat`.
```
Server stats reset.
```
- conf : Print details about serving configuration.
```
server_id=1
tcp_port=2181
four_letter_word_white_list=*
log_storage_path=./coordination/logs
snapshot_storage_path=./coordination/snapshots
max_requests_batch_size=100
session_timeout_ms=30000
operation_timeout_ms=10000
dead_session_check_period_ms=500
heart_beat_interval_ms=500
election_timeout_lower_bound_ms=1000
election_timeout_upper_bound_ms=2000
reserved_log_items=1000000000000000
snapshot_distance=10000
auto_forwarding=true
shutdown_timeout=5000
startup_timeout=240000
raft_logs_level=information
snapshots_to_keep=3
rotate_log_storage_interval=100000
stale_log_gap=10000
fresh_log_gap=200
max_requests_batch_size=100
quorum_reads=false
force_sync=false
compress_logs=true
compress_snapshots_with_zstd_format=true
configuration_change_tries_count=20
```
- cons : List full connection/session details for all clients connected to this server. Includes information on numbers of packets received/sent, session id, operation latencies, last operation performed, etc...
```
192.168.1.1:52163(recved=0,sent=0,sid=0xffffffffffffffff,lop=NA,est=1636454787393,to=30000,lzxid=0xffffffffffffffff,lresp=0,llat=0,minlat=0,avglat=0,maxlat=0)
192.168.1.1:52042(recved=9,sent=18,sid=0x0000000000000001,lop=List,est=1636454739887,to=30000,lcxid=0x0000000000000005,lzxid=0x0000000000000005,lresp=1636454739892,llat=0,minlat=0,avglat=0,maxlat=0)
```
- crst : Reset connection/session statistics for all connections.
```
Connection stats reset.
```
- envi : Print details about serving environment
```
Environment:
clickhouse.keeper.version=v21.11.1.1-prestable-7a4a0b0edef0ad6e0aa662cd3b90c3f4acf796e7
host.name=ZBMAC-C02D4054M.local
os.name=Darwin
os.arch=x86_64
os.version=19.6.0
cpu.count=12
user.name=root
user.home=/Users/JackyWoo/
user.dir=/Users/JackyWoo/project/jd/clickhouse/cmake-build-debug/programs/
user.tmp=/var/folders/b4/smbq5mfj7578f2jzwn602tt40000gn/T/
```
- dirs : Shows the total size of snapshot and log files in bytes
```
snapshot_dir_size: 0
log_dir_size: 3875
```
- isro: Tests if server is running in read-only mode. The server will respond with "ro" if in read-only mode or "rw" if not in read-only mode.
```
rw
```
- wchs : Lists brief information on watches for the server.
```
1 connections watching 1 paths
Total watches:1
```
- wchc : Lists detailed information on watches for the server, by session. This outputs a list of sessions(connections) with associated watches (paths). Note, depending on the number of watches this operation may be expensive (ie impact server performance), use it carefully.
```
0x0000000000000001
/clickhouse/task_queue/ddl
```
- wchp : Lists detailed information on watches for the server, by path. This outputs a list of paths (znodes) with associated sessions. Note, depending on the number of watches this operation may be expensive (ie impact server performance), use it carefully.
```
/clickhouse/task_queue/ddl
0x0000000000000001
```
- dump : Lists the outstanding sessions and ephemeral nodes. This only works on the leader.
```
Sessions dump (2):
0x0000000000000001
0x0000000000000002
Sessions with Ephemerals (1):
0x0000000000000001
/clickhouse/task_queue/ddl
```
## [experimental] Migration from ZooKeeper
Seamlessly migration from ZooKeeper to ClickHouse Keeper is impossible you have to stop your ZooKeeper cluster, convert data and start ClickHouse Keeper. `clickhouse-keeper-converter` tool allows converting ZooKeeper logs and snapshots to ClickHouse Keeper snapshot. It works only with ZooKeeper > 3.4. Steps for migration:

2
docs/en/operations/server-configuration-parameters/settings.md
View File

@ -588,7 +588,7 @@ For more information, see the section [Creating replicated tables](../../engines
Approximate size (in bytes) of the cache of marks used by table engines of the [MergeTree](../../engines/table-engines/mergetree-family/mergetree.md) family.
The cache is shared for the server and memory is allocated as needed. The cache size must be at least 5368709120.
The cache is shared for the server and memory is allocated as needed.
**Example**

3
programs/keeper/Keeper.cpp
View File

@ -20,6 +20,7 @@
#include <Poco/Environment.h>
#include <sys/stat.h>
#include <pwd.h>
#include <Coordination/FourLetterCommand.h>
#if !defined(ARCADIA_BUILD)
# include "config_core.h"
@ -367,6 +368,8 @@ int Keeper::main(const std::vector<std::string> & /*args*/)
/// Initialize keeper RAFT. Do nothing if no keeper_server in config.
global_context->initializeKeeperDispatcher(/* start_async = */false);
FourLetterCommandFactory::registerCommands(*global_context->getKeeperDispatcher());
for (const auto & listen_host : listen_hosts)
{
/// TCP Keeper

7
programs/server/Server.cpp
View File

@ -85,9 +85,6 @@
#if !defined(ARCADIA_BUILD)
# include "config_core.h"
# include "Common/config_version.h"
# if USE_OPENCL
# include "Common/BitonicSort.h"
# endif
#endif
#if defined(OS_LINUX)
@ -111,7 +108,8 @@
#endif
#if USE_NURAFT
# include <Server/KeeperTCPHandlerFactory.h>
# include <Coordination/FourLetterCommand.h>
# include <Server/KeeperTCPHandlerFactory.h>
#endif
#if USE_BASE64
@ -1025,6 +1023,7 @@ if (ThreadFuzzer::instance().isEffective())
}
/// Initialize keeper RAFT.
global_context->initializeKeeperDispatcher(can_initialize_keeper_async);
FourLetterCommandFactory::registerCommands(*global_context->getKeeperDispatcher());
for (const auto & listen_host : listen_hosts)
{

2
src/Columns/ColumnMap.cpp
View File

@ -263,7 +263,7 @@ void ColumnMap::getExtremes(Field & min, Field & max) const
void ColumnMap::forEachSubcolumn(ColumnCallback callback)
{
nested->forEachSubcolumn(callback);
callback(nested);
}
bool ColumnMap::structureEquals(const IColumn & rhs) const

46
src/Common/getCurrentProcessFDCount.cpp Normal file
View File

@ -0,0 +1,46 @@
#include <Common/getCurrentProcessFDCount.h>
#include <Common/ShellCommand.h>
#include <IO/WriteBufferFromString.h>
#include <unistd.h>
#include <fmt/format.h>
#include <IO/ReadHelpers.h>
#include <filesystem>
int getCurrentProcessFDCount()
{
namespace fs = std::filesystem;
int result = -1;
#if defined(__linux__) || defined(__APPLE__)
using namespace DB;
Int32 pid = getpid();
auto proc_fs_path = fmt::format("/proc/{}/fd", pid);
if (fs::exists(proc_fs_path))
{
result = std::distance(fs::directory_iterator(proc_fs_path), fs::directory_iterator{});
}
else if (fs::exists("/dev/fd"))
{
result = std::distance(fs::directory_iterator("/dev/fd"), fs::directory_iterator{});
}
else
{
/// Then try lsof command
String by_lsof = fmt::format("lsof -p {} | wc -l", pid);
auto command = ShellCommand::execute(by_lsof);
try
{
readIntText(result, command->out);
command->wait();
}
catch (...)
{
}
}
#endif
return result;
}

5
src/Common/getCurrentProcessFDCount.h Normal file
View File

@ -0,0 +1,5 @@
#pragma once
/// Get current process file descriptor count
/// @return -1 os doesn't support "lsof" command or some error occurs.
int getCurrentProcessFDCount();

36
src/Common/getMaxFileDescriptorCount.cpp Normal file
View File

@ -0,0 +1,36 @@
#include <IO/ReadHelpers.h>
#include <IO/WriteBufferFromString.h>
#include <IO/ReadBufferFromFile.h>
#include <Common/ShellCommand.h>
#include <Common/getMaxFileDescriptorCount.h>
#include <filesystem>
int getMaxFileDescriptorCount()
{
namespace fs = std::filesystem;
int result = -1;
#if defined(__linux__) || defined(__APPLE__)
using namespace DB;
if (fs::exists("/proc/sys/fs/file-max"))
{
ReadBufferFromFile reader("/proc/sys/fs/file-max");
readIntText(result, reader);
}
else
{
auto command = ShellCommand::execute("ulimit -n");
try
{
readIntText(result, command->out);
command->wait();
}
catch (...)
{
}
}
#endif
return result;
}

6
src/Common/getMaxFileDescriptorCount.h Normal file
View File

@ -0,0 +1,6 @@
#pragma once
/// Get process max file descriptor count
/// @return -1 if os does not support ulimit command or some error occurs
int getMaxFileDescriptorCount();

89
src/Common/renameat2.cpp
View File

@ -4,13 +4,6 @@
#include <Poco/Environment.h>
#include <filesystem>
#if defined(linux) || defined(__linux) || defined(__linux__)
#include <unistd.h>
#include <fcntl.h>
#include <sys/syscall.h>
#include <linux/fs.h>
#endif
namespace fs = std::filesystem;
namespace DB
@ -25,25 +18,56 @@ namespace ErrorCodes
extern const int FILE_ALREADY_EXISTS;
}
}
#if defined(__linux__)
#include <unistd.h>
#include <fcntl.h>
#include <sys/syscall.h>
#include <linux/fs.h>
/// For old versions of libc.
#if !defined(RENAME_NOREPLACE)
#define RENAME_NOREPLACE 1
#endif
#if !defined(RENAME_EXCHANGE)
#define RENAME_EXCHANGE 2
#endif
#if !defined(__NR_renameat2)
#if defined(__x86_64__)
#define __NR_renameat2 316
#elif defined(__aarch64__)
#define __NR_renameat2 276
#elif defined(__ppc64__)
#define __NR_renameat2 357
#elif defined(__riscv)
#define __NR_renameat2 276
#else
#error "Unsupported architecture"
#endif
#endif
namespace DB
{
static bool supportsRenameat2Impl()
{
#if defined(__NR_renameat2)
VersionNumber renameat2_minimal_version(3, 15, 0);
VersionNumber linux_version(Poco::Environment::osVersion());
return linux_version >= renameat2_minimal_version;
#else
return false;
#endif
}
#if defined(__NR_renameat2)
static bool renameat2(const std::string & old_path, const std::string & new_path, int flags)
{
if (!supportsRenameat2())
return false;
if (old_path.empty() || new_path.empty())
throw Exception("Cannot rename " + old_path + " to " + new_path + ": path is empty", ErrorCodes::LOGICAL_ERROR);
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot rename {} to {}: path is empty", old_path, new_path);
/// int olddirfd (ignored for absolute oldpath), const char *oldpath,
/// int newdirfd (ignored for absolute newpath), const char *newpath,
@ -63,28 +87,50 @@ static bool renameat2(const std::string & old_path, const std::string & new_path
return false;
if (errno == EEXIST)
throwFromErrno("Cannot rename " + old_path + " to " + new_path + " because the second path already exists", ErrorCodes::ATOMIC_RENAME_FAIL);
throwFromErrno(fmt::format("Cannot rename {} to {} because the second path already exists", old_path, new_path), ErrorCodes::ATOMIC_RENAME_FAIL);
if (errno == ENOENT)
throwFromErrno("Paths cannot be exchanged because " + old_path + " or " + new_path + " does not exist", ErrorCodes::ATOMIC_RENAME_FAIL);
throwFromErrnoWithPath("Cannot rename " + old_path + " to " + new_path, new_path, ErrorCodes::SYSTEM_ERROR);
throwFromErrno(fmt::format("Paths cannot be exchanged because {} or {} does not exist", old_path, new_path), ErrorCodes::ATOMIC_RENAME_FAIL);
throwFromErrnoWithPath(fmt::format("Cannot rename {} to {}", old_path, new_path), new_path, ErrorCodes::SYSTEM_ERROR);
}
bool supportsRenameat2()
{
static bool supports = supportsRenameat2Impl();
return supports;
}
}
#else
#define RENAME_NOREPLACE -1
#define RENAME_EXCHANGE -1
namespace DB
{
static bool renameat2(const std::string &, const std::string &, int)
{
return false;
}
bool supportsRenameat2()
{
return false;
}
}
#endif
namespace DB
{
static void renameNoReplaceFallback(const std::string & old_path, const std::string & new_path)
{
/// NOTE it's unsafe
if (fs::exists(new_path))
throw Exception("File " + new_path + " exists", ErrorCodes::FILE_ALREADY_EXISTS);
throw Exception(ErrorCodes::FILE_ALREADY_EXISTS, "File {} exists", new_path);
fs::rename(old_path, new_path);
}
@ -97,13 +143,6 @@ static void renameExchangeFallback(const std::string &, const std::string &)
}
#pragma GCC diagnostic pop
bool supportsRenameat2()
{
static bool supports = supportsRenameat2Impl();
return supports;
}
void renameNoReplace(const std::string & old_path, const std::string & new_path)
{
if (!renameat2(old_path, new_path, RENAME_NOREPLACE))

2
src/Coordination/Changelog.h
View File

@ -11,7 +11,7 @@
namespace DB
{
using Checksum = UInt64;
using Checksum = uint64_t;
using LogEntryPtr = nuraft::ptr<nuraft::log_entry>;
using LogEntries = std::vector<LogEntryPtr>;

177
src/Coordination/CoordinationSettings.cpp
View File

@ -1,6 +1,10 @@
#include <Coordination/CoordinationSettings.h>
#include <Core/Settings.h>
#include <base/logger_useful.h>
#include <filesystem>
#include <Coordination/Defines.h>
#include <IO/WriteHelpers.h>
#include <IO/WriteIntText.h>
namespace DB
{
@ -32,4 +36,177 @@ void CoordinationSettings::loadFromConfig(const String & config_elem, const Poco
}
}
const String KeeperConfigurationAndSettings::DEFAULT_FOUR_LETTER_WORD_CMD = "conf,cons,crst,envi,ruok,srst,srvr,stat,wchc,wchs,dirs,mntr,isro";
KeeperConfigurationAndSettings::KeeperConfigurationAndSettings()
: server_id(NOT_EXIST)
, tcp_port(NOT_EXIST)
, tcp_port_secure(NOT_EXIST)
, standalone_keeper(false)
, coordination_settings(std::make_shared<CoordinationSettings>())
{
}
void KeeperConfigurationAndSettings::dump(WriteBufferFromOwnString & buf) const
{
auto write_int = [&buf](int64_t value)
{
writeIntText(value, buf);
buf.write('\n');
};
auto write_bool = [&buf](bool value)
{
String str_val = value ? "true" : "false";
writeText(str_val, buf);
buf.write('\n');
};
writeText("server_id=", buf);
write_int(server_id);
if (tcp_port != NOT_EXIST)
{
writeText("tcp_port=", buf);
write_int(tcp_port);
}
if (tcp_port_secure != NOT_EXIST)
{
writeText("tcp_port_secure=", buf);
write_int(tcp_port_secure);
}
writeText("four_letter_word_white_list=", buf);
writeText(four_letter_word_white_list, buf);
buf.write('\n');
writeText("log_storage_path=", buf);
writeText(log_storage_path, buf);
buf.write('\n');
writeText("snapshot_storage_path=", buf);
writeText(snapshot_storage_path, buf);
buf.write('\n');
/// coordination_settings
writeText("max_requests_batch_size=", buf);
write_int(coordination_settings->max_requests_batch_size);
writeText("session_timeout_ms=", buf);
write_int(uint64_t(coordination_settings->session_timeout_ms));
writeText("operation_timeout_ms=", buf);
write_int(uint64_t(coordination_settings->operation_timeout_ms));
writeText("dead_session_check_period_ms=", buf);
write_int(uint64_t(coordination_settings->dead_session_check_period_ms));
writeText("heart_beat_interval_ms=", buf);
write_int(uint64_t(coordination_settings->heart_beat_interval_ms));
writeText("election_timeout_lower_bound_ms=", buf);
write_int(uint64_t(coordination_settings->election_timeout_lower_bound_ms));
writeText("election_timeout_upper_bound_ms=", buf);
write_int(uint64_t(coordination_settings->election_timeout_upper_bound_ms));
writeText("reserved_log_items=", buf);
write_int(coordination_settings->reserved_log_items);
writeText("snapshot_distance=", buf);
write_int(coordination_settings->snapshot_distance);
writeText("auto_forwarding=", buf);
write_bool(coordination_settings->auto_forwarding);
writeText("shutdown_timeout=", buf);
write_int(uint64_t(coordination_settings->shutdown_timeout));
writeText("startup_timeout=", buf);
write_int(uint64_t(coordination_settings->startup_timeout));
writeText("raft_logs_level=", buf);
writeText(coordination_settings->raft_logs_level.toString(), buf);
buf.write('\n');
writeText("snapshots_to_keep=", buf);
write_int(coordination_settings->snapshots_to_keep);
writeText("rotate_log_storage_interval=", buf);
write_int(coordination_settings->rotate_log_storage_interval);
writeText("stale_log_gap=", buf);
write_int(coordination_settings->stale_log_gap);
writeText("fresh_log_gap=", buf);
write_int(coordination_settings->fresh_log_gap);
writeText("max_requests_batch_size=", buf);
write_int(coordination_settings->max_requests_batch_size);
writeText("quorum_reads=", buf);
write_bool(coordination_settings->quorum_reads);
writeText("force_sync=", buf);
write_bool(coordination_settings->force_sync);
writeText("compress_logs=", buf);
write_bool(coordination_settings->compress_logs);
writeText("compress_snapshots_with_zstd_format=", buf);
write_bool(coordination_settings->compress_snapshots_with_zstd_format);
writeText("configuration_change_tries_count=", buf);
write_int(coordination_settings->configuration_change_tries_count);
}
KeeperConfigurationAndSettingsPtr
KeeperConfigurationAndSettings::loadFromConfig(const Poco::Util::AbstractConfiguration & config, bool standalone_keeper_)
{
std::shared_ptr<KeeperConfigurationAndSettings> ret = std::make_shared<KeeperConfigurationAndSettings>();
ret->server_id = config.getInt("keeper_server.server_id");
ret->standalone_keeper = standalone_keeper_;
if (config.has("keeper_server.tcp_port"))
{
ret->tcp_port = config.getInt("keeper_server.tcp_port");
}
if (config.has("keeper_server.tcp_port_secure"))
{
ret->tcp_port_secure = config.getInt("keeper_server.tcp_port_secure");
}
if (config.has("keeper_server.superdigest"))
{
ret->super_digest = config.getString("keeper_server.superdigest");
}
ret->four_letter_word_white_list = config.getString("keeper_server.four_letter_word_white_list", DEFAULT_FOUR_LETTER_WORD_CMD);
ret->log_storage_path = getLogsPathFromConfig(config, standalone_keeper_);
ret->snapshot_storage_path = getSnapshotsPathFromConfig(config, standalone_keeper_);
ret->coordination_settings->loadFromConfig("keeper_server.coordination_settings", config);
return ret;
}
String KeeperConfigurationAndSettings::getLogsPathFromConfig(const Poco::Util::AbstractConfiguration & config, bool standalone_keeper_)
{
/// the most specialized path
if (config.has("keeper_server.log_storage_path"))
return config.getString("keeper_server.log_storage_path");
if (config.has("keeper_server.storage_path"))
return std::filesystem::path{config.getString("keeper_server.storage_path")} / "logs";
if (standalone_keeper_)
return std::filesystem::path{config.getString("path", KEEPER_DEFAULT_PATH)} / "logs";
else
return std::filesystem::path{config.getString("path", DBMS_DEFAULT_PATH)} / "coordination/logs";
}
String KeeperConfigurationAndSettings::getSnapshotsPathFromConfig(const Poco::Util::AbstractConfiguration & config, bool standalone_keeper_)
{
/// the most specialized path
if (config.has("keeper_server.snapshot_storage_path"))
return config.getString("keeper_server.snapshot_storage_path");
if (config.has("keeper_server.storage_path"))
return std::filesystem::path{config.getString("keeper_server.storage_path")} / "snapshots";
if (standalone_keeper_)
return std::filesystem::path{config.getString("path", KEEPER_DEFAULT_PATH)} / "snapshots";
else
return std::filesystem::path{config.getString("path", DBMS_DEFAULT_PATH)} / "coordination/snapshots";
}
}

35
src/Coordination/CoordinationSettings.h
View File

@ -5,6 +5,7 @@
#include <Core/SettingsEnums.h>
#include <Common/ZooKeeper/ZooKeeperConstants.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <IO/WriteBufferFromString.h>
namespace DB
{
@ -51,4 +52,38 @@ struct CoordinationSettings : public BaseSettings<CoordinationSettingsTraits>
using CoordinationSettingsPtr = std::shared_ptr<CoordinationSettings>;
/// Coordination settings + some other parts of keeper configuration
/// which are not stored in settings. Allows to dump configuration
/// with 4lw commands.
struct KeeperConfigurationAndSettings
{
static constexpr int NOT_EXIST = -1;
static const String DEFAULT_FOUR_LETTER_WORD_CMD;
KeeperConfigurationAndSettings();
int server_id;
int tcp_port;
int tcp_port_secure;
String four_letter_word_white_list;
String super_digest;
bool standalone_keeper;
CoordinationSettingsPtr coordination_settings;
String log_storage_path;
String snapshot_storage_path;
void dump(WriteBufferFromOwnString & buf) const;
static std::shared_ptr<KeeperConfigurationAndSettings> loadFromConfig(const Poco::Util::AbstractConfiguration & config, bool standalone_keeper_);
private:
static String getLogsPathFromConfig(const Poco::Util::AbstractConfiguration & config, bool standalone_keeper_);
static String getSnapshotsPathFromConfig(const Poco::Util::AbstractConfiguration & config, bool standalone_keeper_);
};
using KeeperConfigurationAndSettingsPtr = std::shared_ptr<KeeperConfigurationAndSettings>;
}

418
src/Coordination/FourLetterCommand.cpp Normal file
View File

@ -0,0 +1,418 @@
#include <Coordination/FourLetterCommand.h>
#include <Coordination/KeeperDispatcher.h>
#include <Server/KeeperTCPHandler.h>
#include <base/logger_useful.h>
#include <Poco/Environment.h>
#include <Poco/Path.h>
#include <Common/getCurrentProcessFDCount.h>
#include <Common/getMaxFileDescriptorCount.h>
#include <Common/StringUtils/StringUtils.h>
#include <Coordination/Keeper4LWInfo.h>
#include <unistd.h>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
}
IFourLetterCommand::IFourLetterCommand(KeeperDispatcher & keeper_dispatcher_)
: keeper_dispatcher(keeper_dispatcher_)
{
}
int32_t IFourLetterCommand::code()
{
return toCode(name());
}
String IFourLetterCommand::toName(int32_t code)
{
int reverted_code = __builtin_bswap32(code);
return String(reinterpret_cast<char *>(&reverted_code), 4);
}
int32_t IFourLetterCommand::toCode(const String & name)
{
int32_t res = *reinterpret_cast<const int32_t *>(name.data());
/// keep consistent with Coordination::read method by changing big endian to little endian.
return __builtin_bswap32(res);
}
IFourLetterCommand::~IFourLetterCommand() = default;
FourLetterCommandFactory & FourLetterCommandFactory::instance()
{
static FourLetterCommandFactory factory;
return factory;
}
void FourLetterCommandFactory::checkInitialization() const
{
if (!initialized)
throw Exception("Four letter command not initialized", ErrorCodes::LOGICAL_ERROR);
}
bool FourLetterCommandFactory::isKnown(int32_t code)
{
checkInitialization();
return commands.contains(code);
}
FourLetterCommandPtr FourLetterCommandFactory::get(int32_t code)
{
checkInitialization();
return commands.at(code);
}
void FourLetterCommandFactory::registerCommand(FourLetterCommandPtr & command)
{
if (commands.contains(command->code()))
throw Exception("Four letter command " + command->name() + " already registered", ErrorCodes::LOGICAL_ERROR);
commands.emplace(command->code(), std::move(command));
}
void FourLetterCommandFactory::registerCommands(KeeperDispatcher & keeper_dispatcher)
{
FourLetterCommandFactory & factory = FourLetterCommandFactory::instance();
if (!factory.isInitialized())
{
FourLetterCommandPtr ruok_command = std::make_shared<RuokCommand>(keeper_dispatcher);
factory.registerCommand(ruok_command);
FourLetterCommandPtr mntr_command = std::make_shared<MonitorCommand>(keeper_dispatcher);
factory.registerCommand(mntr_command);
FourLetterCommandPtr conf_command = std::make_shared<ConfCommand>(keeper_dispatcher);
factory.registerCommand(conf_command);
FourLetterCommandPtr cons_command = std::make_shared<ConsCommand>(keeper_dispatcher);
factory.registerCommand(cons_command);
FourLetterCommandPtr brief_watch_command = std::make_shared<BriefWatchCommand>(keeper_dispatcher);
factory.registerCommand(brief_watch_command);
FourLetterCommandPtr data_size_command = std::make_shared<DataSizeCommand>(keeper_dispatcher);
factory.registerCommand(data_size_command);
FourLetterCommandPtr dump_command = std::make_shared<DumpCommand>(keeper_dispatcher);
factory.registerCommand(dump_command);
FourLetterCommandPtr envi_command = std::make_shared<EnviCommand>(keeper_dispatcher);
factory.registerCommand(envi_command);
FourLetterCommandPtr is_rad_only_command = std::make_shared<IsReadOnlyCommand>(keeper_dispatcher);
factory.registerCommand(is_rad_only_command);
FourLetterCommandPtr rest_conn_stats_command = std::make_shared<RestConnStatsCommand>(keeper_dispatcher);
factory.registerCommand(rest_conn_stats_command);
FourLetterCommandPtr server_stat_command = std::make_shared<ServerStatCommand>(keeper_dispatcher);
factory.registerCommand(server_stat_command);
FourLetterCommandPtr stat_command = std::make_shared<StatCommand>(keeper_dispatcher);
factory.registerCommand(stat_command);
FourLetterCommandPtr stat_reset_command = std::make_shared<StatResetCommand>(keeper_dispatcher);
factory.registerCommand(stat_reset_command);
FourLetterCommandPtr watch_by_path_command = std::make_shared<WatchByPathCommand>(keeper_dispatcher);
factory.registerCommand(watch_by_path_command);
FourLetterCommandPtr watch_command = std::make_shared<WatchCommand>(keeper_dispatcher);
factory.registerCommand(watch_command);
factory.initializeWhiteList(keeper_dispatcher);
factory.setInitialize(true);
}
}
bool FourLetterCommandFactory::isEnabled(int32_t code)
{
checkInitialization();
if (!white_list.empty() && *white_list.cbegin() == WHITE_LIST_ALL)
return true;
return std::find(white_list.begin(), white_list.end(), code) != white_list.end();
}
void FourLetterCommandFactory::initializeWhiteList(KeeperDispatcher & keeper_dispatcher)
{
const auto & keeper_settings = keeper_dispatcher.getKeeperConfigurationAndSettings();
String list_str = keeper_settings->four_letter_word_white_list;
Strings tokens;
splitInto<','>(tokens, list_str);
for (String token: tokens)
{
trim(token);
if (token == "*")
{
white_list.clear();
white_list.push_back(WHITE_LIST_ALL);
return;
}
else
{
if (commands.contains(IFourLetterCommand::toCode(token)))
{
white_list.push_back(IFourLetterCommand::toCode(token));
}
else
{
auto * log = &Poco::Logger::get("FourLetterCommandFactory");
LOG_WARNING(log, "Find invalid keeper 4lw command {} when initializing, ignore it.", token);
}
}
}
}
String RuokCommand::run()
{
return "imok";
}
namespace
{
void print(IFourLetterCommand::StringBuffer & buf, const String & key, const String & value)
{
writeText("zk_", buf);
writeText(key, buf);
writeText('\t', buf);
writeText(value, buf);
writeText('\n', buf);
}
void print(IFourLetterCommand::StringBuffer & buf, const String & key, uint64_t value)
{
print(buf, key, toString(value));
}
}
String MonitorCommand::run()
{
KeeperConnectionStats stats = keeper_dispatcher.getKeeperConnectionStats();
Keeper4LWInfo keeper_info = keeper_dispatcher.getKeeper4LWInfo();
if (!keeper_info.has_leader)
return "This instance is not currently serving requests";
const auto & state_machine = keeper_dispatcher.getStateMachine();
StringBuffer ret;
print(ret, "version", String(VERSION_DESCRIBE) + "-" + VERSION_GITHASH);
print(ret, "avg_latency", stats.getAvgLatency());
print(ret, "max_latency", stats.getMaxLatency());
print(ret, "min_latency", stats.getMinLatency());
print(ret, "packets_received", stats.getPacketsReceived());
print(ret, "packets_sent", stats.getPacketsSent());
print(ret, "num_alive_connections", keeper_info.alive_connections_count);
print(ret, "outstanding_requests", keeper_info.outstanding_requests_count);
print(ret, "server_state", keeper_info.getRole());
print(ret, "znode_count", state_machine.getNodesCount());
print(ret, "watch_count", state_machine.getTotalWatchesCount());
print(ret, "ephemerals_count", state_machine.getTotalEphemeralNodesCount());
print(ret, "approximate_data_size", state_machine.getApproximateDataSize());
#if defined(__linux__) || defined(__APPLE__)
print(ret, "open_file_descriptor_count", getCurrentProcessFDCount());
print(ret, "max_file_descriptor_count", getMaxFileDescriptorCount());
#endif
if (keeper_info.is_leader)
{
print(ret, "followers", keeper_info.follower_count);
print(ret, "synced_followers", keeper_info.synced_follower_count);
}
return ret.str();
}
String StatResetCommand::run()
{
keeper_dispatcher.resetConnectionStats();
return "Server stats reset.\n";
}
String NopCommand::run()
{
return "";
}
String ConfCommand::run()
{
StringBuffer buf;
keeper_dispatcher.getKeeperConfigurationAndSettings()->dump(buf);
return buf.str();
}
String ConsCommand::run()
{
StringBuffer buf;
KeeperTCPHandler::dumpConnections(buf, false);
return buf.str();
}
String RestConnStatsCommand::run()
{
KeeperTCPHandler::resetConnsStats();
return "Connection stats reset.\n";
}
String ServerStatCommand::run()
{
StringBuffer buf;
auto write = [&buf](const String & key, const String & value)
{
writeText(key, buf);
writeText(": ", buf);
writeText(value, buf);
writeText('\n', buf);
};
KeeperConnectionStats stats = keeper_dispatcher.getKeeperConnectionStats();
Keeper4LWInfo keeper_info = keeper_dispatcher.getKeeper4LWInfo();
write("ClickHouse Keeper version", String(VERSION_DESCRIBE) + "-" + VERSION_GITHASH);
StringBuffer latency;
latency << stats.getMinLatency() << "/" << stats.getAvgLatency() << "/" << stats.getMaxLatency();
write("Latency min/avg/max", latency.str());
write("Received", toString(stats.getPacketsReceived()));
write("Sent ", toString(stats.getPacketsSent()));
write("Connections", toString(keeper_info.alive_connections_count));
write("Outstanding", toString(keeper_info.outstanding_requests_count));
write("Zxid", toString(keeper_info.last_zxid));
write("Mode", keeper_info.getRole());
write("Node count", toString(keeper_info.total_nodes_count));
return buf.str();
}
String StatCommand::run()
{
StringBuffer buf;
auto write = [&buf] (const String & key, const String & value) { buf << key << ": " << value << '\n'; };
KeeperConnectionStats stats = keeper_dispatcher.getKeeperConnectionStats();
Keeper4LWInfo keeper_info = keeper_dispatcher.getKeeper4LWInfo();
write("ClickHouse Keeper version", String(VERSION_DESCRIBE) + "-" + VERSION_GITHASH);
buf << "Clients:\n";
KeeperTCPHandler::dumpConnections(buf, true);
buf << '\n';
StringBuffer latency;
latency << stats.getMinLatency() << "/" << stats.getAvgLatency() << "/" << stats.getMaxLatency();
write("Latency min/avg/max", latency.str());
write("Received", toString(stats.getPacketsReceived()));
write("Sent ", toString(stats.getPacketsSent()));
write("Connections", toString(keeper_info.alive_connections_count));
write("Outstanding", toString(keeper_info.outstanding_requests_count));
write("Zxid", toString(keeper_info.last_zxid));
write("Mode", keeper_info.getRole());
write("Node count", toString(keeper_info.total_nodes_count));
return buf.str();
}
String BriefWatchCommand::run()
{
StringBuffer buf;
const auto & state_machine = keeper_dispatcher.getStateMachine();
buf << state_machine.getSessionsWithWatchesCount() << " connections watching "
<< state_machine.getWatchedPathsCount() << " paths\n";
buf << "Total watches:" << state_machine.getTotalWatchesCount() << "\n";
return buf.str();
}
String WatchCommand::run()
{
StringBuffer buf;
const auto & state_machine = keeper_dispatcher.getStateMachine();
state_machine.dumpWatches(buf);
return buf.str();
}
String WatchByPathCommand::run()
{
StringBuffer buf;
const auto & state_machine = keeper_dispatcher.getStateMachine();
state_machine.dumpWatchesByPath(buf);
return buf.str();
}
String DataSizeCommand::run()
{
StringBuffer buf;
buf << "snapshot_dir_size: " << keeper_dispatcher.getSnapDirSize() << '\n';
buf << "log_dir_size: " << keeper_dispatcher.getLogDirSize() << '\n';
return buf.str();
}
String DumpCommand::run()
{
StringBuffer buf;
const auto & state_machine = keeper_dispatcher.getStateMachine();
state_machine.dumpSessionsAndEphemerals(buf);
return buf.str();
}
String EnviCommand::run()
{
using Poco::Environment;
using Poco::Path;
StringBuffer buf;
buf << "Environment:\n";
buf << "clickhouse.keeper.version=" << (String(VERSION_DESCRIBE) + "-" + VERSION_GITHASH) << '\n';
buf << "host.name=" << Environment::nodeName() << '\n';
buf << "os.name=" << Environment::osDisplayName() << '\n';
buf << "os.arch=" << Environment::osArchitecture() << '\n';
buf << "os.version=" << Environment::osVersion() << '\n';
buf << "cpu.count=" << Environment::processorCount() << '\n';
String os_user;
os_user.resize(256, '\0');
if (0 == getlogin_r(os_user.data(), os_user.size() - 1))
os_user.resize(strlen(os_user.c_str()));
else
os_user.clear(); /// Don't mind if we cannot determine user login.
buf << "user.name=" << os_user << '\n';
buf << "user.home=" << Path::home() << '\n';
buf << "user.dir=" << Path::current() << '\n';
buf << "user.tmp=" << Path::temp() << '\n';
return buf.str();
}
String IsReadOnlyCommand::run()
{
if (keeper_dispatcher.isObserver())
return "ro";
else
return "rw";
}
}

307
src/Coordination/FourLetterCommand.h Normal file
View File

@ -0,0 +1,307 @@
#pragma once
#include <sstream>
#include <string>
#include <unordered_map>
#include <Coordination/KeeperDispatcher.h>
#include <IO/WriteBufferFromString.h>
#include <Common/config_version.h>
namespace DB
{
struct IFourLetterCommand;
using FourLetterCommandPtr = std::shared_ptr<DB::IFourLetterCommand>;
/// Just like zookeeper Four Letter Words commands, CH Keeper responds to a small set of commands.
/// Each command is composed of four letters, these commands are useful to monitor and issue system problems.
/// The feature is based on Zookeeper 3.5.9, details is in https://zookeeper.apache.org/doc/r3.5.9/zookeeperAdmin.html#sc_zkCommands.
struct IFourLetterCommand
{
public:
using StringBuffer = DB::WriteBufferFromOwnString;
explicit IFourLetterCommand(KeeperDispatcher & keeper_dispatcher_);
virtual String name() = 0;
virtual String run() = 0;
virtual ~IFourLetterCommand();
int32_t code();
static String toName(int32_t code);
static inline int32_t toCode(const String & name);
protected:
KeeperDispatcher & keeper_dispatcher;
};
struct FourLetterCommandFactory : private boost::noncopyable
{
public:
using Commands = std::unordered_map<int32_t, FourLetterCommandPtr>;
using WhiteList = std::vector<int32_t>;
///represent '*' which is used in white list
static constexpr int32_t WHITE_LIST_ALL = 0;
bool isKnown(int32_t code);
bool isEnabled(int32_t code);
FourLetterCommandPtr get(int32_t code);
/// There is no need to make it thread safe, because registration is no initialization and get is after startup.
void registerCommand(FourLetterCommandPtr & command);
void initializeWhiteList(KeeperDispatcher & keeper_dispatcher);
void checkInitialization() const;
bool isInitialized() const { return initialized; }
void setInitialize(bool flag) { initialized = flag; }
static FourLetterCommandFactory & instance();
static void registerCommands(KeeperDispatcher & keeper_dispatcher);
private:
std::atomic<bool> initialized = false;
Commands commands;
WhiteList white_list;
};
/**Tests if server is running in a non-error state. The server will respond with imok if it is running.
* Otherwise it will not respond at all.
*
* A response of "imok" does not necessarily indicate that the server has joined the quorum,
* just that the server process is active and bound to the specified client port.
* Use "stat" for details on state wrt quorum and client connection information.
*/
struct RuokCommand : public IFourLetterCommand
{
explicit RuokCommand(KeeperDispatcher & keeper_dispatcher_) : IFourLetterCommand(keeper_dispatcher_) { }
String name() override { return "ruok"; }
String run() override;
~RuokCommand() override = default;
};
/**
* Outputs a list of variables that could be used for monitoring the health of the cluster.
*
* echo mntr | nc localhost 2181
* zk_version 3.5.9
* zk_avg_latency 0
* zk_max_latency 0
* zk_min_latency 0
* zk_packets_received 70
* zk_packets_sent 69
* zk_outstanding_requests 0
* zk_server_state leader
* zk_znode_count 4
* zk_watch_count 0
* zk_ephemerals_count 0
* zk_approximate_data_size 27
* zk_open_file_descriptor_count 23 - only available on Unix platforms
* zk_max_file_descriptor_count 1024 - only available on Unix platforms
* zk_followers 2 - only exposed by the Leader
* zk_synced_followers 2 - only exposed by the Leader
* zk_pending_syncs 0 - only exposed by the Leader
*/
struct MonitorCommand : public IFourLetterCommand
{
explicit MonitorCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "mntr"; }
String run() override;
~MonitorCommand() override = default;
};
struct StatResetCommand : public IFourLetterCommand
{
explicit StatResetCommand(KeeperDispatcher & keeper_dispatcher_) :
IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "srst"; }
String run() override;
~StatResetCommand() override = default;
};
/// A command that does not do anything except reply to client with predefined message.
///It is used to inform clients who execute none white listed four letter word commands.
struct NopCommand : public IFourLetterCommand
{
explicit NopCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "nopc"; }
String run() override;
~NopCommand() override = default;
};
struct ConfCommand : public IFourLetterCommand
{
explicit ConfCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "conf"; }
String run() override;
~ConfCommand() override = default;
};
/// List full connection/session details for all clients connected to this server.
/// Includes information on numbers of packets received/sent, session id, operation latencies, last operation performed, etc...
struct ConsCommand : public IFourLetterCommand
{
explicit ConsCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "cons"; }
String run() override;
~ConsCommand() override = default;
};
/// Reset connection/session statistics for all connections.
struct RestConnStatsCommand : public IFourLetterCommand
{
explicit RestConnStatsCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "crst"; }
String run() override;
~RestConnStatsCommand() override = default;
};
/// Lists full details for the server.
struct ServerStatCommand : public IFourLetterCommand
{
explicit ServerStatCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "srvr"; }
String run() override;
~ServerStatCommand() override = default;
};
/// Lists brief details for the server and connected clients.
struct StatCommand : public IFourLetterCommand
{
explicit StatCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "stat"; }
String run() override;
~StatCommand() override = default;
};
/// Lists brief information on watches for the server.
struct BriefWatchCommand : public IFourLetterCommand
{
explicit BriefWatchCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "wchs"; }
String run() override;
~BriefWatchCommand() override = default;
};
/// Lists detailed information on watches for the server, by session.
/// This outputs a list of sessions(connections) with associated watches (paths).
/// Note, depending on the number of watches this operation may be expensive (ie impact server performance), use it carefully.
struct WatchCommand : public IFourLetterCommand
{
explicit WatchCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "wchc"; }
String run() override;
~WatchCommand() override = default;
};
/// Lists detailed information on watches for the server, by path.
/// This outputs a list of paths (znodes) with associated sessions.
/// Note, depending on the number of watches this operation may be expensive (ie impact server performance), use it carefully.
struct WatchByPathCommand : public IFourLetterCommand
{
explicit WatchByPathCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "wchp"; }
String run() override;
~WatchByPathCommand() override = default;
};
/// Lists the outstanding sessions and ephemeral nodes. This only works on the leader.
struct DumpCommand : public IFourLetterCommand
{
explicit DumpCommand(KeeperDispatcher & keeper_dispatcher_):
IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "dump"; }
String run() override;
~DumpCommand() override = default;
};
/// Print details about serving environment
struct EnviCommand : public IFourLetterCommand
{
explicit EnviCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "envi"; }
String run() override;
~EnviCommand() override = default;
};
/// Shows the total size of snapshot and log files in bytes
struct DataSizeCommand : public IFourLetterCommand
{
explicit DataSizeCommand(KeeperDispatcher & keeper_dispatcher_):
IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "dirs"; }
String run() override;
~DataSizeCommand() override = default;
};
/// Tests if server is running in read-only mode.
/// The server will respond with "ro" if in read-only mode or "rw" if not in read-only mode.
struct IsReadOnlyCommand : public IFourLetterCommand
{
explicit IsReadOnlyCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "isro"; }
String run() override;
~IsReadOnlyCommand() override = default;
};
}

47
src/Coordination/Keeper4LWInfo.h Normal file
View File

@ -0,0 +1,47 @@
#pragma once
#include <string>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
}
/// Keeper server related information for different 4lw commands
struct Keeper4LWInfo
{
bool is_leader;
bool is_observer;
bool is_follower;
bool is_standalone;
bool has_leader;
uint64_t alive_connections_count;
uint64_t outstanding_requests_count;
uint64_t follower_count;
uint64_t synced_follower_count;
uint64_t total_nodes_count;
int64_t last_zxid;
String getRole() const
{
if (is_standalone)
return "standalone";
if (is_leader)
return "leader";
if (is_observer)
return "observer";
if (is_follower)
return "follower";
throw Exception(ErrorCodes::LOGICAL_ERROR, "RAFT server has undefined state state, it's a bug");
}
};
}

85
src/Coordination/KeeperConnectionStats.cpp Normal file
View File

@ -0,0 +1,85 @@
#include <Coordination/KeeperConnectionStats.h>
namespace DB
{
uint64_t KeeperConnectionStats::getMinLatency() const
{
return min_latency;
}
uint64_t KeeperConnectionStats::getMaxLatency() const
{
return max_latency;
}
uint64_t KeeperConnectionStats::getAvgLatency() const
{
if (count != 0)
return total_latency / count;
return 0;
}
uint64_t KeeperConnectionStats::getLastLatency() const
{
return last_latency;
}
uint64_t KeeperConnectionStats::getPacketsReceived() const
{
return packets_received;
}
uint64_t KeeperConnectionStats::getPacketsSent() const
{
return packets_sent;
}
void KeeperConnectionStats::incrementPacketsReceived()
{
packets_received++;
}
void KeeperConnectionStats::incrementPacketsSent()
{
packets_sent++;
}
void KeeperConnectionStats::updateLatency(uint64_t latency_ms)
{
last_latency = latency_ms;
total_latency += (latency_ms);
count++;
if (latency_ms < min_latency)
{
min_latency = latency_ms;
}
if (latency_ms > max_latency)
{
max_latency = latency_ms;
}
}
void KeeperConnectionStats::reset()
{
resetLatency();
resetRequestCounters();
}
void KeeperConnectionStats::resetLatency()
{
total_latency = 0;
count = 0;
max_latency = 0;
min_latency = 0;
}
void KeeperConnectionStats::resetRequestCounters()
{
packets_received = 0;
packets_sent = 0;
}
}

52
src/Coordination/KeeperConnectionStats.h Normal file
View File

@ -0,0 +1,52 @@
#pragma once
#include <base/types.h>
#include <memory>
#include <cstdint>
namespace DB
{
/// Request statistics for connection or dispatcher
class KeeperConnectionStats
{
public:
KeeperConnectionStats() = default;
uint64_t getMinLatency() const;
uint64_t getMaxLatency() const;
uint64_t getAvgLatency() const;
uint64_t getLastLatency() const;
uint64_t getPacketsReceived() const;
uint64_t getPacketsSent() const;
void incrementPacketsReceived();
void incrementPacketsSent();
void updateLatency(uint64_t latency_ms);
void reset();
private:
void resetLatency();
void resetRequestCounters();
/// all response with watch response included
uint64_t packets_sent = 0;
/// All user requests
uint64_t packets_received = 0;
/// For consistent with zookeeper measured by millisecond,
/// otherwise maybe microsecond is better
uint64_t total_latency = 0;
uint64_t max_latency = 0;
uint64_t min_latency = 0;
/// last operation latency
uint64_t last_latency = 0;
uint64_t count = 0;
};
}

93
src/Coordination/KeeperDispatcher.cpp
View File

@ -3,6 +3,12 @@
#include <Common/ZooKeeper/KeeperException.h>
#include <future>
#include <chrono>
#include <Poco/Path.h>
#include <Common/hex.h>
#include <filesystem>
#include <Common/checkStackSize.h>
namespace fs = std::filesystem;
namespace DB
{
@ -14,9 +20,10 @@ namespace ErrorCodes
extern const int SYSTEM_ERROR;
}
KeeperDispatcher::KeeperDispatcher()
: coordination_settings(std::make_shared<CoordinationSettings>())
, responses_queue(std::numeric_limits<size_t>::max())
: responses_queue(std::numeric_limits<size_t>::max())
, configuration_and_settings(std::make_shared<KeeperConfigurationAndSettings>())
, log(&Poco::Logger::get("KeeperDispatcher"))
{
}
@ -36,7 +43,8 @@ void KeeperDispatcher::requestThread()
{
KeeperStorage::RequestForSession request;
UInt64 max_wait = UInt64(coordination_settings->operation_timeout_ms.totalMilliseconds());
auto coordination_settings = configuration_and_settings->coordination_settings;
uint64_t max_wait = coordination_settings->operation_timeout_ms.totalMilliseconds();
uint64_t max_batch_size = coordination_settings->max_requests_batch_size;
/// The code below do a very simple thing: batch all write (quorum) requests into vector until
@ -141,7 +149,7 @@ void KeeperDispatcher::responseThread()
{
KeeperStorage::ResponseForSession response_for_session;
UInt64 max_wait = UInt64(coordination_settings->operation_timeout_ms.totalMilliseconds());
uint64_t max_wait = configuration_and_settings->coordination_settings->operation_timeout_ms.totalMilliseconds();
if (responses_queue.tryPop(response_for_session, max_wait))
{
@ -245,28 +253,25 @@ bool KeeperDispatcher::putRequest(const Coordination::ZooKeeperRequestPtr & requ
if (!requests_queue->push(std::move(request_info)))
throw Exception("Cannot push request to queue", ErrorCodes::SYSTEM_ERROR);
}
else if (!requests_queue->tryPush(std::move(request_info), coordination_settings->operation_timeout_ms.totalMilliseconds()))
else if (!requests_queue->tryPush(std::move(request_info), configuration_and_settings->coordination_settings->operation_timeout_ms.totalMilliseconds()))
{
throw Exception("Cannot push request to queue within operation timeout", ErrorCodes::TIMEOUT_EXCEEDED);
}
return true;
}
void KeeperDispatcher::initialize(const Poco::Util::AbstractConfiguration & config, bool standalone_keeper, bool start_async)
{
LOG_DEBUG(log, "Initializing storage dispatcher");
int myid = config.getInt("keeper_server.server_id");
coordination_settings->loadFromConfig("keeper_server.coordination_settings", config);
requests_queue = std::make_unique<RequestsQueue>(coordination_settings->max_requests_batch_size);
configuration_and_settings = KeeperConfigurationAndSettings::loadFromConfig(config, standalone_keeper);
requests_queue = std::make_unique<RequestsQueue>(configuration_and_settings->coordination_settings->max_requests_batch_size);
request_thread = ThreadFromGlobalPool([this] { requestThread(); });
responses_thread = ThreadFromGlobalPool([this] { responseThread(); });
snapshot_thread = ThreadFromGlobalPool([this] { snapshotThread(); });
server = std::make_unique<KeeperServer>(
myid, coordination_settings, config, responses_queue, snapshots_queue, standalone_keeper);
server = std::make_unique<KeeperServer>(configuration_and_settings, config, responses_queue, snapshots_queue);
try
{
@ -413,7 +418,8 @@ void KeeperDispatcher::sessionCleanerTask()
tryLogCurrentException(__PRETTY_FUNCTION__);
}
std::this_thread::sleep_for(std::chrono::milliseconds(coordination_settings->dead_session_check_period_ms.totalMilliseconds()));
auto time_to_sleep = configuration_and_settings->coordination_settings->dead_session_check_period_ms.totalMilliseconds();
std::this_thread::sleep_for(std::chrono::milliseconds(time_to_sleep));
}
}
@ -580,4 +586,67 @@ void KeeperDispatcher::updateConfiguration(const Poco::Util::AbstractConfigurati
}
}
void KeeperDispatcher::updateKeeperStatLatency(uint64_t process_time_ms)
{
std::lock_guard lock(keeper_stats_mutex);
keeper_stats.updateLatency(process_time_ms);
}
static uint64_t getDirSize(const fs::path & dir)
{
checkStackSize();
if (!fs::exists(dir))
return 0;
fs::directory_iterator it(dir);
fs::directory_iterator end;
uint64_t size{0};
while (it != end)
{
if (it->is_regular_file())
size += fs::file_size(*it);
else
size += getDirSize(it->path());
++it;
}
return size;
}
uint64_t KeeperDispatcher::getLogDirSize() const
{
return getDirSize(configuration_and_settings->log_storage_path);
}
uint64_t KeeperDispatcher::getSnapDirSize() const
{
return getDirSize(configuration_and_settings->snapshot_storage_path);
}
Keeper4LWInfo KeeperDispatcher::getKeeper4LWInfo() const
{
Keeper4LWInfo result;
result.is_follower = server->isFollower();
result.is_standalone = !result.is_follower && server->getFollowerCount() == 0;
result.is_leader = isLeader();
result.is_observer = server->isObserver();
result.has_leader = hasLeader();
{
std::lock_guard lock(push_request_mutex);
result.outstanding_requests_count = requests_queue->size();
}
{
std::lock_guard lock(session_to_response_callback_mutex);
result.alive_connections_count = session_to_response_callback.size();
}
if (result.is_leader)
{
result.follower_count = server->getFollowerCount();
result.synced_follower_count = server->getSyncedFollowerCount();
}
result.total_nodes_count = server->getKeeperStateMachine()->getNodesCount();
result.last_zxid = server->getKeeperStateMachine()->getLastProcessedZxid();
return result;
}
}

76
src/Coordination/KeeperDispatcher.h
View File

@ -13,22 +13,20 @@
#include <functional>
#include <Coordination/KeeperServer.h>
#include <Coordination/CoordinationSettings.h>
#include <Coordination/Keeper4LWInfo.h>
#include <Coordination/KeeperConnectionStats.h>
namespace DB
{
using ZooKeeperResponseCallback = std::function<void(const Coordination::ZooKeeperResponsePtr & response)>;
/// Highlevel wrapper for ClickHouse Keeper.
/// Process user requests via consensus and return responses.
class KeeperDispatcher
{
private:
std::mutex push_request_mutex;
mutable std::mutex push_request_mutex;
CoordinationSettingsPtr coordination_settings;
using RequestsQueue = ConcurrentBoundedQueue<KeeperStorage::RequestForSession>;
using SessionToResponseCallback = std::unordered_map<int64_t, ZooKeeperResponseCallback>;
using UpdateConfigurationQueue = ConcurrentBoundedQueue<ConfigUpdateAction>;
@ -43,7 +41,7 @@ private:
std::atomic<bool> shutdown_called{false};
std::mutex session_to_response_callback_mutex;
mutable std::mutex session_to_response_callback_mutex;
/// These two maps looks similar, but serves different purposes.
/// The first map is subscription map for normal responses like
/// (get, set, list, etc.). Dispatcher determines callback for each response
@ -70,6 +68,11 @@ private:
/// RAFT wrapper.
std::unique_ptr<KeeperServer> server;
mutable std::mutex keeper_stats_mutex;
KeeperConnectionStats keeper_stats;
KeeperConfigurationAndSettingsPtr configuration_and_settings;
Poco::Logger * log;
/// Counter for new session_id requests.
@ -123,6 +126,18 @@ public:
/// Put request to ClickHouse Keeper
bool putRequest(const Coordination::ZooKeeperRequestPtr & request, int64_t session_id);
/// Get new session ID
int64_t getSessionID(int64_t session_timeout_ms);
/// Register session and subscribe for responses with callback
void registerSession(int64_t session_id, ZooKeeperResponseCallback callback);
/// Call if we don't need any responses for this session no more (session was expired)
void finishSession(int64_t session_id);
/// Invoked when a request completes.
void updateKeeperStatLatency(uint64_t process_time_ms);
/// Are we leader
bool isLeader() const
{
@ -134,14 +149,51 @@ public:
return server->isLeaderAlive();
}
/// Get new session ID
int64_t getSessionID(int64_t session_timeout_ms);
bool isObserver() const
{
return server->isObserver();
}
/// Register session and subscribe for responses with callback
void registerSession(int64_t session_id, ZooKeeperResponseCallback callback);
uint64_t getLogDirSize() const;
/// Call if we don't need any responses for this session no more (session was expired)
void finishSession(int64_t session_id);
uint64_t getSnapDirSize() const;
/// Request statistics such as qps, latency etc.
KeeperConnectionStats getKeeperConnectionStats() const
{
std::lock_guard lock(keeper_stats_mutex);
return keeper_stats;
}
Keeper4LWInfo getKeeper4LWInfo() const;
const KeeperStateMachine & getStateMachine() const
{
return *server->getKeeperStateMachine();
}
const KeeperConfigurationAndSettingsPtr & getKeeperConfigurationAndSettings() const
{
return configuration_and_settings;
}
void incrementPacketsSent()
{
std::lock_guard lock(keeper_stats_mutex);
keeper_stats.incrementPacketsSent();
}
void incrementPacketsReceived()
{
std::lock_guard lock(keeper_stats_mutex);
keeper_stats.incrementPacketsReceived();
}
void resetConnectionStats()
{
std::lock_guard lock(keeper_stats_mutex);
keeper_stats.reset();
}
};
}

72
src/Coordination/KeeperServer.cpp
View File

@ -61,27 +61,12 @@ void setSSLParams(nuraft::asio_service::options & asio_opts)
}
#endif
std::string getSnapshotsPathFromConfig(const Poco::Util::AbstractConfiguration & config, bool standalone_keeper)
std::string checkAndGetSuperdigest(const String & user_and_digest)
{
/// the most specialized path
if (config.has("keeper_server.snapshot_storage_path"))
return config.getString("keeper_server.snapshot_storage_path");
if (config.has("keeper_server.storage_path"))
return std::filesystem::path{config.getString("keeper_server.storage_path")} / "snapshots";
if (standalone_keeper)
return std::filesystem::path{config.getString("path", KEEPER_DEFAULT_PATH)} / "snapshots";
else
return std::filesystem::path{config.getString("path", DBMS_DEFAULT_PATH)} / "coordination/snapshots";
}
std::string checkAndGetSuperdigest(const Poco::Util::AbstractConfiguration & config)
{
if (!config.has("keeper_server.superdigest"))
if (user_and_digest.empty())
return "";
auto user_and_digest = config.getString("keeper_server.superdigest");
std::vector<std::string> scheme_and_id;
boost::split(scheme_and_id, user_and_digest, [](char c) { return c == ':'; });
if (scheme_and_id.size() != 2 || scheme_and_id[0] != "super")
@ -93,20 +78,18 @@ std::string checkAndGetSuperdigest(const Poco::Util::AbstractConfiguration & con
}
KeeperServer::KeeperServer(
int server_id_,
const CoordinationSettingsPtr & coordination_settings_,
const KeeperConfigurationAndSettingsPtr & configuration_and_settings_,
const Poco::Util::AbstractConfiguration & config,
ResponsesQueue & responses_queue_,
SnapshotsQueue & snapshots_queue_,
bool standalone_keeper)
: server_id(server_id_)
, coordination_settings(coordination_settings_)
SnapshotsQueue & snapshots_queue_)
: server_id(configuration_and_settings_->server_id)
, coordination_settings(configuration_and_settings_->coordination_settings)
, state_machine(nuraft::cs_new<KeeperStateMachine>(
responses_queue_, snapshots_queue_,
getSnapshotsPathFromConfig(config, standalone_keeper),
configuration_and_settings_->snapshot_storage_path,
coordination_settings,
checkAndGetSuperdigest(config)))
, state_manager(nuraft::cs_new<KeeperStateManager>(server_id, "keeper_server", config, coordination_settings, standalone_keeper))
checkAndGetSuperdigest(configuration_and_settings_->super_digest)))
, state_manager(nuraft::cs_new<KeeperStateManager>(server_id, "keeper_server", configuration_and_settings_->log_storage_path, config, coordination_settings))
, log(&Poco::Logger::get("KeeperServer"))
{
if (coordination_settings->quorum_reads)
@ -302,11 +285,46 @@ bool KeeperServer::isLeader() const
return raft_instance->is_leader();
}
bool KeeperServer::isObserver() const
{
auto srv_config = state_manager->get_srv_config();
return srv_config->is_learner();
}
bool KeeperServer::isFollower() const
{
return !isLeader() && !isObserver();
}
bool KeeperServer::isLeaderAlive() const
{
return raft_instance->is_leader_alive();
}
/// TODO test whether taking failed peer in count
uint64_t KeeperServer::getFollowerCount() const
{
return raft_instance->get_peer_info_all().size();
}
uint64_t KeeperServer::getSyncedFollowerCount() const
{
uint64_t last_log_idx = raft_instance->get_last_log_idx();
const auto followers = raft_instance->get_peer_info_all();
uint64_t stale_followers = 0;
const uint64_t stale_follower_gap = raft_instance->get_current_params().stale_log_gap_;
for (const auto & fl : followers)
{
if (last_log_idx > fl.last_log_idx_ + stale_follower_gap)
stale_followers++;
}
return followers.size() - stale_followers;
}
nuraft::cb_func::ReturnCode KeeperServer::callbackFunc(nuraft::cb_func::Type type, nuraft::cb_func::Param * param)
{
if (initialized_flag)

24
src/Coordination/KeeperServer.h
View File

@ -6,7 +6,6 @@
#include <Coordination/KeeperStateMachine.h>
#include <Coordination/KeeperStorage.h>
#include <Coordination/CoordinationSettings.h>
#include <unordered_map>
#include <base/logger_useful.h>
namespace DB
@ -52,12 +51,10 @@ private:
public:
KeeperServer(
int server_id_,
const CoordinationSettingsPtr & coordination_settings_,
const Poco::Util::AbstractConfiguration & config,
const KeeperConfigurationAndSettingsPtr & settings_,
const Poco::Util::AbstractConfiguration & config_,
ResponsesQueue & responses_queue_,
SnapshotsQueue & snapshots_queue_,
bool standalone_keeper);
SnapshotsQueue & snapshots_queue_);
/// Load state machine from the latest snapshot and load log storage. Start NuRaft with required settings.
void startup();
@ -73,10 +70,25 @@ public:
/// Return set of the non-active sessions
std::vector<int64_t> getDeadSessions();
nuraft::ptr<KeeperStateMachine> getKeeperStateMachine() const
{
return state_machine;
}
bool isLeader() const;
bool isFollower() const;
bool isObserver() const;
bool isLeaderAlive() const;
/// @return follower count if node is not leader return 0
uint64_t getFollowerCount() const;
/// @return synced follower count if node is not leader return 0
uint64_t getSyncedFollowerCount() const;
/// Wait server initialization (see callbackFunc)
void waitInit();

70
src/Coordination/KeeperStateMachine.cpp
View File

@ -326,16 +326,82 @@ void KeeperStateMachine::processReadRequest(const KeeperStorage::RequestForSessi
throw Exception(ErrorCodes::SYSTEM_ERROR, "Could not push response with session id {} into responses queue", response.session_id);
}
void KeeperStateMachine::shutdownStorage()
{
std::lock_guard lock(storage_and_responses_lock);
storage->finalize();
}
std::vector<int64_t> KeeperStateMachine::getDeadSessions()
{
std::lock_guard lock(storage_and_responses_lock);
return storage->getDeadSessions();
}
void KeeperStateMachine::shutdownStorage()
uint64_t KeeperStateMachine::getLastProcessedZxid() const
{
std::lock_guard lock(storage_and_responses_lock);
storage->finalize();
return storage->getZXID();
}
uint64_t KeeperStateMachine::getNodesCount() const
{
std::lock_guard lock(storage_and_responses_lock);
return storage->getNodesCount();
}
uint64_t KeeperStateMachine::getTotalWatchesCount() const
{
std::lock_guard lock(storage_and_responses_lock);
return storage->getTotalWatchesCount();
}
uint64_t KeeperStateMachine::getWatchedPathsCount() const
{
std::lock_guard lock(storage_and_responses_lock);
return storage->getWatchedPathsCount();
}
uint64_t KeeperStateMachine::getSessionsWithWatchesCount() const
{
std::lock_guard lock(storage_and_responses_lock);
return storage->getSessionsWithWatchesCount();
}
uint64_t KeeperStateMachine::getTotalEphemeralNodesCount() const
{
std::lock_guard lock(storage_and_responses_lock);
return storage->getTotalEphemeralNodesCount();
}
uint64_t KeeperStateMachine::getSessionWithEphemeralNodesCount() const
{
std::lock_guard lock(storage_and_responses_lock);
return storage->getSessionWithEphemeralNodesCount();
}
void KeeperStateMachine::dumpWatches(WriteBufferFromOwnString & buf) const
{
std::lock_guard lock(storage_and_responses_lock);
storage->dumpWatches(buf);
}
void KeeperStateMachine::dumpWatchesByPath(WriteBufferFromOwnString & buf) const
{
std::lock_guard lock(storage_and_responses_lock);
storage->dumpWatchesByPath(buf);
}
void KeeperStateMachine::dumpSessionsAndEphemerals(WriteBufferFromOwnString & buf) const
{
std::lock_guard lock(storage_and_responses_lock);
storage->dumpSessionsAndEphemerals(buf);
}
uint64_t KeeperStateMachine::getApproximateDataSize() const
{
std::lock_guard lock(storage_and_responses_lock);
return storage->getApproximateDataSize();
}
ClusterConfigPtr KeeperStateMachine::getClusterConfig() const

23
src/Coordination/KeeperStateMachine.h
View File

@ -67,19 +67,36 @@ public:
nuraft::ptr<nuraft::buffer> & data_out,
bool & is_last_obj) override;
/// just for test
KeeperStorage & getStorage()
{
return *storage;
}
void shutdownStorage();
ClusterConfigPtr getClusterConfig() const;
/// Process local read request
void processReadRequest(const KeeperStorage::RequestForSession & request_for_session);
std::vector<int64_t> getDeadSessions();
void shutdownStorage();
/// Introspection functions for 4lw commands
uint64_t getLastProcessedZxid() const;
ClusterConfigPtr getClusterConfig() const;
uint64_t getNodesCount() const;
uint64_t getTotalWatchesCount() const;
uint64_t getWatchedPathsCount() const;
uint64_t getSessionsWithWatchesCount() const;
void dumpWatches(WriteBufferFromOwnString & buf) const;
void dumpWatchesByPath(WriteBufferFromOwnString & buf) const;
void dumpSessionsAndEphemerals(WriteBufferFromOwnString & buf) const;
uint64_t getSessionWithEphemeralNodesCount() const;
uint64_t getTotalEphemeralNodesCount() const;
uint64_t getApproximateDataSize() const;
private:
@ -110,7 +127,7 @@ private:
/// we can get strange cases when, for example client send read request with
/// watch and after that receive watch response and only receive response
/// for request.
std::mutex storage_and_responses_lock;
mutable std::mutex storage_and_responses_lock;
/// Last committed Raft log number.
std::atomic<uint64_t> last_committed_idx;

44
src/Coordination/KeeperStateManager.cpp
View File

@ -1,4 +1,5 @@
#include <Coordination/KeeperStateManager.h>
#include <Coordination/Defines.h>
#include <Common/Exception.h>
#include <filesystem>
@ -11,28 +12,7 @@ namespace ErrorCodes
extern const int RAFT_ERROR;
}
namespace
{
std::string getLogsPathFromConfig(
const std::string & config_prefix, const Poco::Util::AbstractConfiguration & config, bool standalone_keeper)
{
/// the most specialized path
if (config.has(config_prefix + ".log_storage_path"))
return config.getString(config_prefix + ".log_storage_path");
if (config.has(config_prefix + ".storage_path"))
return std::filesystem::path{config.getString(config_prefix + ".storage_path")} / "logs";
if (standalone_keeper)
return std::filesystem::path{config.getString("path", KEEPER_DEFAULT_PATH)} / "logs";
else
return std::filesystem::path{config.getString("path", DBMS_DEFAULT_PATH)} / "coordination/logs";
}
}
KeeperConfigurationWrapper KeeperStateManager::parseServersConfiguration(const Poco::Util::AbstractConfiguration & config, bool allow_without_us) const
KeeperStateManager::KeeperConfigurationWrapper KeeperStateManager::parseServersConfiguration(const Poco::Util::AbstractConfiguration & config, bool allow_without_us) const
{
KeeperConfigurationWrapper result;
result.cluster_config = std::make_shared<nuraft::cluster_config>();
@ -78,9 +58,9 @@ KeeperConfigurationWrapper KeeperStateManager::parseServersConfiguration(const P
}
KeeperStateManager::KeeperStateManager(int server_id_, const std::string & host, int port, const std::string & logs_path)
: my_server_id(server_id_)
, secure(false)
, log_store(nuraft::cs_new<KeeperLogStore>(logs_path, 5000, false, false))
: my_server_id(server_id_)
, secure(false)
, log_store(nuraft::cs_new<KeeperLogStore>(logs_path, 5000, false, false))
{
auto peer_config = nuraft::cs_new<nuraft::srv_config>(my_server_id, host + ":" + std::to_string(port));
configuration_wrapper.cluster_config = nuraft::cs_new<nuraft::cluster_config>();
@ -90,18 +70,20 @@ KeeperStateManager::KeeperStateManager(int server_id_, const std::string & host,
}
KeeperStateManager::KeeperStateManager(
int server_id_,
int my_server_id_,
const std::string & config_prefix_,
const std::string & log_storage_path,
const Poco::Util::AbstractConfiguration & config,
const CoordinationSettingsPtr & coordination_settings,
bool standalone_keeper)
: my_server_id(server_id_)
const CoordinationSettingsPtr & coordination_settings)
: my_server_id(my_server_id_)
, secure(config.getBool(config_prefix_ + ".raft_configuration.secure", false))
, config_prefix(config_prefix_)
, configuration_wrapper(parseServersConfiguration(config, false))
, log_store(nuraft::cs_new<KeeperLogStore>(
getLogsPathFromConfig(config_prefix_, config, standalone_keeper),
coordination_settings->rotate_log_storage_interval, coordination_settings->force_sync, coordination_settings->compress_logs))
log_storage_path,
coordination_settings->rotate_log_storage_interval,
coordination_settings->force_sync,
coordination_settings->compress_logs))
{
}

32
src/Coordination/KeeperStateManager.h
View File

@ -13,20 +13,6 @@ namespace DB
using KeeperServerConfigPtr = nuraft::ptr<nuraft::srv_config>;
/// Wrapper struct for Keeper cluster config. We parse this
/// info from XML files.
struct KeeperConfigurationWrapper
{
/// Our port
int port;
/// Our config
KeeperServerConfigPtr config;
/// Servers id's to start as followers
std::unordered_set<int> servers_start_as_followers;
/// Cluster config
ClusterConfigPtr cluster_config;
};
/// When our configuration changes the following action types
/// can happen
enum class ConfigUpdateActionType
@ -52,9 +38,9 @@ public:
KeeperStateManager(
int server_id_,
const std::string & config_prefix_,
const std::string & log_storage_path,
const Poco::Util::AbstractConfiguration & config,
const CoordinationSettingsPtr & coordination_settings,
bool standalone_keeper);
const CoordinationSettingsPtr & coordination_settings);
/// Constructor for tests
KeeperStateManager(
@ -121,6 +107,20 @@ public:
ConfigUpdateActions getConfigurationDiff(const Poco::Util::AbstractConfiguration & config) const;
private:
/// Wrapper struct for Keeper cluster config. We parse this
/// info from XML files.
struct KeeperConfigurationWrapper
{
/// Our port
int port;
/// Our config
KeeperServerConfigPtr config;
/// Servers id's to start as followers
std::unordered_set<int> servers_start_as_followers;
/// Cluster config
ClusterConfigPtr cluster_config;
};
int my_server_id;
bool secure;
std::string config_prefix;

108
src/Coordination/KeeperStorage.cpp
View File

@ -10,6 +10,7 @@
#include <Poco/SHA1Engine.h>
#include <Poco/Base64Encoder.h>
#include <boost/algorithm/string.hpp>
#include <Common/hex.h>
namespace DB
{
@ -132,6 +133,21 @@ static bool fixupACL(
return valid_found;
}
uint64_t KeeperStorage::Node::sizeInBytes() const
{
uint64_t total_size{0};
for (const auto & child : children)
total_size += child.size();
total_size += data.size();
total_size += sizeof(acl_id);
total_size += sizeof(is_sequental);
total_size += sizeof(stat);
total_size += sizeof(seq_num);
return total_size;
}
static KeeperStorage::ResponsesForSessions processWatchesImpl(const String & path, KeeperStorage::Watches & watches, KeeperStorage::Watches & list_watches, Coordination::Event event_type)
{
KeeperStorage::ResponsesForSessions result;
@ -1220,4 +1236,96 @@ void KeeperStorage::clearDeadWatches(int64_t session_id)
}
}
void KeeperStorage::dumpWatches(WriteBufferFromOwnString & buf) const
{
for (const auto & [session_id, watches_paths] : sessions_and_watchers)
{
buf << "0x" << getHexUIntLowercase(session_id) << "\n";
for (const String & path : watches_paths)
buf << "\t" << path << "\n";
}
}
void KeeperStorage::dumpWatchesByPath(WriteBufferFromOwnString & buf) const
{
auto write_int_vec = [&buf](const std::vector<int64_t> & session_ids)
{
for (int64_t session_id : session_ids)
{
buf << "\t0x" << getHexUIntLowercase(session_id) << "\n";
}
};
for (const auto & [watch_path, sessions] : watches)
{
buf << watch_path << "\n";
write_int_vec(sessions);
}
for (const auto & [watch_path, sessions] : list_watches)
{
buf << watch_path << "\n";
write_int_vec(sessions);
}
}
void KeeperStorage::dumpSessionsAndEphemerals(WriteBufferFromOwnString & buf) const
{
auto write_str_set = [&buf](const std::unordered_set<String> & ephemeral_paths)
{
for (const String & path : ephemeral_paths)
{
buf << "\t" << path << "\n";
}
};
buf << "Sessions dump (" << session_and_timeout.size() << "):\n";
for (const auto & [session_id, _] : session_and_timeout)
{
buf << "0x" << getHexUIntLowercase(session_id) << "\n";
}
buf << "Sessions with Ephemerals (" << getSessionWithEphemeralNodesCount() << "):\n";
for (const auto & [session_id, ephemeral_paths] : ephemerals)
{
buf << "0x" << getHexUIntLowercase(session_id) << "\n";
write_str_set(ephemeral_paths);
}
}
uint64_t KeeperStorage::getTotalWatchesCount() const
{
uint64_t ret = 0;
for (const auto & [path, subscribed_sessions] : watches)
ret += subscribed_sessions.size();
for (const auto & [path, subscribed_sessions] : list_watches)
ret += subscribed_sessions.size();
return ret;
}
uint64_t KeeperStorage::getSessionsWithWatchesCount() const
{
std::unordered_set<int64_t> counter;
for (const auto & [path, subscribed_sessions] : watches)
counter.insert(subscribed_sessions.begin(), subscribed_sessions.end());
for (const auto & [path, subscribed_sessions] : list_watches)
counter.insert(subscribed_sessions.begin(), subscribed_sessions.end());
return counter.size();
}
uint64_t KeeperStorage::getTotalEphemeralNodesCount() const
{
uint64_t ret = 0;
for (const auto & [session_id, nodes] : ephemerals)
ret += nodes.size();
return ret;
}
}

45
src/Coordination/KeeperStorage.h
View File

@ -1,6 +1,5 @@
#pragma once
#include <Common/ThreadPool.h>
#include <Common/ZooKeeper/IKeeper.h>
#include <Common/ConcurrentBoundedQueue.h>
#include <Common/ZooKeeper/ZooKeeperCommon.h>
@ -14,7 +13,6 @@
namespace DB
{
using namespace DB;
struct KeeperStorageRequestProcessor;
using KeeperStorageRequestProcessorPtr = std::shared_ptr<KeeperStorageRequestProcessor>;
using ResponseCallback = std::function<void(const Coordination::ZooKeeperResponsePtr &)>;
@ -29,8 +27,6 @@ struct KeeperStorageSnapshot;
class KeeperStorage
{
public:
int64_t session_id_counter{1};
struct Node
{
String data;
@ -39,6 +35,9 @@ public:
Coordination::Stat stat{};
int32_t seq_num = 0;
ChildrenSet children{};
/// Object memory size
uint64_t sizeInBytes() const;
};
struct ResponseForSession
@ -46,7 +45,6 @@ public:
int64_t session_id;
Coordination::ZooKeeperResponsePtr response;
};
using ResponsesForSessions = std::vector<ResponseForSession>;
struct RequestForSession
@ -76,10 +74,13 @@ public:
/// Just vector of SHA1 from user:password
using AuthIDs = std::vector<AuthID>;
using SessionAndAuth = std::unordered_map<int64_t, AuthIDs>;
SessionAndAuth session_and_auth;
using Watches = std::map<String /* path, relative of root_path */, SessionIDs>;
public:
int64_t session_id_counter{1};
SessionAndAuth session_and_auth;
/// Main hashtable with nodes. Contain all information about data.
/// All other structures expect session_and_timeout can be restored from
/// container.
@ -176,6 +177,36 @@ public:
{
return session_expiry_queue.getExpiredSessions();
}
/// Introspection functions mostly used in 4-letter commands
uint64_t getNodesCount() const
{
return container.size();
}
uint64_t getApproximateDataSize() const
{
return container.getApproximateDataSize();
}
uint64_t getTotalWatchesCount() const;
uint64_t getWatchedPathsCount() const
{
return watches.size() + list_watches.size();
}
uint64_t getSessionsWithWatchesCount() const;
uint64_t getSessionWithEphemeralNodesCount() const
{
return ephemerals.size();
}
uint64_t getTotalEphemeralNodesCount() const;
void dumpWatches(WriteBufferFromOwnString & buf) const;
void dumpWatchesByPath(WriteBufferFromOwnString & buf) const;
void dumpSessionsAndEphemerals(WriteBufferFromOwnString & buf) const;
};
using KeeperStoragePtr = std::unique_ptr<KeeperStorage>;

105
src/Coordination/SnapshotableHashTable.h
View File

@ -15,6 +15,7 @@ struct ListNode
bool active_in_map;
};
template <class V>
class SnapshotableHashTable
{
@ -28,6 +29,82 @@ private:
IndexMap map;
bool snapshot_mode{false};
uint64_t approximate_data_size{0};
enum OperationType
{
INSERT = 0,
INSERT_OR_REPLACE = 1,
ERASE = 2,
UPDATE_VALUE = 3,
GET_VALUE = 4,
FIND = 5,
CONTAINS = 6,
CLEAR = 7,
CLEAR_OUTDATED_NODES = 8
};
/// Update hash table approximate data size
/// op_type: operation type
/// key_size: key size
/// value_size: size of value to add
/// old_value_size: size of value to minus
/// old_value_size=0 means there is no old value with the same key.
void updateDataSize(OperationType op_type, uint64_t key_size, uint64_t value_size, uint64_t old_value_size)
{
switch (op_type)
{
case INSERT:
approximate_data_size += key_size;
approximate_data_size += value_size;
break;
case INSERT_OR_REPLACE:
/// replace
if (old_value_size != 0)
{
approximate_data_size += key_size;
approximate_data_size += value_size;
if (!snapshot_mode)
{
approximate_data_size += key_size;
approximate_data_size -= old_value_size;
}
}
/// insert
else
{
approximate_data_size += key_size;
approximate_data_size += value_size;
}
break;
case UPDATE_VALUE:
approximate_data_size += key_size;
approximate_data_size += value_size;
if (!snapshot_mode)
{
approximate_data_size -= key_size;
approximate_data_size -= old_value_size;
}
break;
case ERASE:
if (!snapshot_mode)
{
approximate_data_size -= key_size;
approximate_data_size -= old_value_size;
}
break;
case CLEAR:
approximate_data_size = 0;
break;
case CLEAR_OUTDATED_NODES:
approximate_data_size -= key_size;
approximate_data_size -= value_size;
break;
default:
break;
}
}
public:
using iterator = typename List::iterator;
@ -44,6 +121,7 @@ public:
ListElem elem{key, value, true};
auto itr = list.insert(list.end(), elem);
map.emplace(itr->key, itr);
updateDataSize(INSERT, key.size(), value.sizeInBytes(), 0);
return true;
}
@ -54,6 +132,8 @@ public:
void insertOrReplace(const std::string & key, const V & value)
{
auto it = map.find(key);
uint64_t old_value_size = it == map.end() ? 0 : it->second->value.sizeInBytes();
if (it == map.end())
{
ListElem elem{key, value, true};
@ -76,6 +156,7 @@ public:
list_itr->value = value;
}
}
updateDataSize(INSERT_OR_REPLACE, key.size(), value.sizeInBytes(), old_value_size);
}
bool erase(const std::string & key)
@ -85,6 +166,7 @@ public:
return false;
auto list_itr = it->second;
uint64_t old_data_size = list_itr->value.sizeInBytes();
if (snapshot_mode)
{
list_itr->active_in_map = false;
@ -96,6 +178,7 @@ public:
list.erase(list_itr);
}
updateDataSize(ERASE, key.size(), 0, old_data_size);
return true;
}
@ -108,23 +191,29 @@ public:
{
auto it = map.find(key);
assert(it != map.end());
auto list_itr = it->second;
uint64_t old_value_size = list_itr->value.sizeInBytes();
const_iterator ret;
if (snapshot_mode)
{
auto list_itr = it->second;
auto elem_copy = *(list_itr);
list_itr->active_in_map = false;
map.erase(it);
updater(elem_copy.value);
auto itr = list.insert(list.end(), elem_copy);
map.emplace(itr->key, itr);
return itr;
ret = itr;
}
else
{
auto list_itr = it->second;
updater(list_itr->value);
return list_itr;
ret = list_itr;
}
updateDataSize(UPDATE_VALUE, key.size(), ret->value.sizeInBytes(), old_value_size);
return ret;
}
const_iterator find(const std::string & key) const
@ -149,7 +238,10 @@ public:
for (auto itr = start; itr != end;)
{
if (!itr->active_in_map)
{
updateDataSize(CLEAR_OUTDATED_NODES, itr->key.size(), itr->value.sizeInBytes(), 0);
itr = list.erase(itr);
}
else
itr++;
}
@ -159,6 +251,7 @@ public:
{
list.clear();
map.clear();
updateDataSize(CLEAR, 0, 0, 0);
}
void enableSnapshotMode()
@ -181,6 +274,10 @@ public:
return list.size();
}
uint64_t getApproximateDataSize() const
{
return approximate_data_size;
}
iterator begin() { return list.begin(); }
const_iterator begin() const { return list.cbegin(); }

89
src/Coordination/tests/gtest_coordination.cpp
View File

@ -829,15 +829,29 @@ TEST_P(CoordinationTest, ChangelogTestLostFiles)
EXPECT_FALSE(fs::exists("./logs/changelog_21_40.bin" + params.extension));
}
struct IntNode
{
int value;
IntNode(int value_) : value(value_) { } // NOLINT(google-explicit-constructor)
UInt64 sizeInBytes() const { return sizeof value; }
IntNode & operator=(int rhs)
{
this->value = rhs;
return *this;
}
bool operator==(const int & rhs) const { return value == rhs; }
bool operator!=(const int & rhs) const { return rhs != this->value; }
};
TEST_P(CoordinationTest, SnapshotableHashMapSimple)
{
DB::SnapshotableHashTable<int> hello;
DB::SnapshotableHashTable<IntNode> hello;
EXPECT_TRUE(hello.insert("hello", 5));
EXPECT_TRUE(hello.contains("hello"));
EXPECT_EQ(hello.getValue("hello"), 5);
EXPECT_FALSE(hello.insert("hello", 145));
EXPECT_EQ(hello.getValue("hello"), 5);
hello.updateValue("hello", [](int & value) { value = 7; });
hello.updateValue("hello", [](IntNode & value) { value = 7; });
EXPECT_EQ(hello.getValue("hello"), 7);
EXPECT_EQ(hello.size(), 1);
EXPECT_TRUE(hello.erase("hello"));
@ -846,12 +860,12 @@ TEST_P(CoordinationTest, SnapshotableHashMapSimple)
TEST_P(CoordinationTest, SnapshotableHashMapTrySnapshot)
{
DB::SnapshotableHashTable<int> map_snp;
DB::SnapshotableHashTable<IntNode> map_snp;
EXPECT_TRUE(map_snp.insert("/hello", 7));
EXPECT_FALSE(map_snp.insert("/hello", 145));
map_snp.enableSnapshotMode();
EXPECT_FALSE(map_snp.insert("/hello", 145));
map_snp.updateValue("/hello", [](int & value) { value = 554; });
map_snp.updateValue("/hello", [](IntNode & value) { value = 554; });
EXPECT_EQ(map_snp.getValue("/hello"), 554);
EXPECT_EQ(map_snp.snapshotSize(), 2);
EXPECT_EQ(map_snp.size(), 1);
@ -921,6 +935,73 @@ TEST_P(CoordinationTest, SnapshotableHashMapTrySnapshot)
map_snp.disableSnapshotMode();
}
TEST_P(CoordinationTest, SnapshotableHashMapDataSize)
{
/// int
DB::SnapshotableHashTable<IntNode> hello;
hello.disableSnapshotMode();
EXPECT_EQ(hello.getApproximateDataSize(), 0);
hello.insert("hello", 1);
EXPECT_EQ(hello.getApproximateDataSize(), 9);
hello.updateValue("hello", [](IntNode & value) { value = 2; });
EXPECT_EQ(hello.getApproximateDataSize(), 9);
hello.erase("hello");
EXPECT_EQ(hello.getApproximateDataSize(), 0);
hello.clear();
EXPECT_EQ(hello.getApproximateDataSize(), 0);
hello.enableSnapshotMode();
hello.insert("hello", 1);
EXPECT_EQ(hello.getApproximateDataSize(), 9);
hello.updateValue("hello", [](IntNode & value) { value = 2; });
EXPECT_EQ(hello.getApproximateDataSize(), 18);
hello.clearOutdatedNodes();
EXPECT_EQ(hello.getApproximateDataSize(), 9);
hello.erase("hello");
EXPECT_EQ(hello.getApproximateDataSize(), 9);
hello.clearOutdatedNodes();
EXPECT_EQ(hello.getApproximateDataSize(), 0);
/// Node
using Node = DB::KeeperStorage::Node;
DB::SnapshotableHashTable<Node> world;
Node n1;
n1.data = "1234";
Node n2;
n2.data = "123456";
n2.children.insert("");
world.disableSnapshotMode();
world.insert("world", n1);
EXPECT_EQ(world.getApproximateDataSize(), 94);
world.updateValue("world", [&](Node & value) { value = n2; });
EXPECT_EQ(world.getApproximateDataSize(), 96);
world.erase("world");
EXPECT_EQ(world.getApproximateDataSize(), 0);
world.enableSnapshotMode();
world.insert("world", n1);
EXPECT_EQ(world.getApproximateDataSize(), 94);
world.updateValue("world", [&](Node & value) { value = n2; });
EXPECT_EQ(world.getApproximateDataSize(), 190);
world.clearOutdatedNodes();
EXPECT_EQ(world.getApproximateDataSize(), 96);
world.erase("world");
EXPECT_EQ(world.getApproximateDataSize(), 96);
world.clear();
EXPECT_EQ(world.getApproximateDataSize(), 0);
}
void addNode(DB::KeeperStorage & storage, const std::string & path, const std::string & data, int64_t ephemeral_owner=0)
{
using Node = DB::KeeperStorage::Node;

13
src/Databases/DatabaseAtomic.cpp
View File

@ -229,15 +229,8 @@ void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_
StoragePtr table = getTableUnlocked(table_name, db_lock);
if (table->isDictionary() && !dictionary)
{
if (exchange)
throw Exception(ErrorCodes::INCORRECT_QUERY,
"Use EXCHANGE DICTIONARIES for dictionaries and EXCHANGE TABLES for tables.");
else
throw Exception(ErrorCodes::INCORRECT_QUERY,
"Use RENAME DICTIONARY for dictionaries and RENAME TABLE for tables.");
}
if (dictionary && !table->isDictionary())
throw Exception(ErrorCodes::INCORRECT_QUERY, "Use RENAME/EXCHANGE TABLE (instead of RENAME/EXCHANGE DICTIONARY) for tables");
table->checkTableCanBeRenamed();
assert_can_move_mat_view(table);
@ -245,6 +238,8 @@ void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_
if (exchange)
{
other_table = other_db.getTableUnlocked(to_table_name, other_db_lock);
if (dictionary && !other_table->isDictionary())
throw Exception(ErrorCodes::INCORRECT_QUERY, "Use RENAME/EXCHANGE TABLE (instead of RENAME/EXCHANGE DICTIONARY) for tables");
other_table->checkTableCanBeRenamed();
assert_can_move_mat_view(other_table);
}

7
src/Databases/DatabaseOnDisk.cpp
View File

@ -40,6 +40,7 @@ namespace ErrorCodes
extern const int TABLE_ALREADY_EXISTS;
extern const int EMPTY_LIST_OF_COLUMNS_PASSED;
extern const int DATABASE_NOT_EMPTY;
extern const int INCORRECT_QUERY;
}
@ -349,8 +350,6 @@ void DatabaseOnDisk::renameTable(
{
if (exchange)
throw Exception("Tables can be exchanged only in Atomic databases", ErrorCodes::NOT_IMPLEMENTED);
if (dictionary)
throw Exception("Dictionaries can be renamed only in Atomic databases", ErrorCodes::NOT_IMPLEMENTED);
bool from_ordinary_to_atomic = false;
bool from_atomic_to_ordinary = false;
@ -372,7 +371,11 @@ void DatabaseOnDisk::renameTable(
ASTPtr attach_query;
/// DatabaseLazy::detachTable may return nullptr even if table exists, so we need tryGetTable for this case.
StoragePtr table = tryGetTable(table_name, local_context);
if (dictionary && table && !table->isDictionary())
throw Exception("Use RENAME/EXCHANGE TABLE (instead of RENAME/EXCHANGE DICTIONARY) for tables", ErrorCodes::INCORRECT_QUERY);
detachTable(local_context, table_name);
UUID prev_uuid = UUIDHelpers::Nil;
try
{

2
src/IO/ThreadPoolReader.cpp
View File

@ -30,6 +30,8 @@
#define SYS_preadv2 286
#elif defined(__ppc64__)
#define SYS_preadv2 380
#elif defined(__riscv)
#define SYS_preadv2 286
#else
#error "Unsupported architecture"
#endif

3
src/IO/WriteBufferValidUTF8.cpp
View File

@ -117,9 +117,6 @@ void WriteBufferValidUTF8::nextImpl()
memory[i] = p[i];
working_buffer = Buffer(&memory[cnt], memory.data() + memory.size());
/// Propagate next() to the output buffer
output_buffer.next();
}
WriteBufferValidUTF8::~WriteBufferValidUTF8()

2
src/Processors/Transforms/JoiningTransform.h
View File

@ -83,7 +83,7 @@ private:
/// Fills Join with block from right table.
/// Has single input and single output port.
/// Output port has empty header. It is closed when al data is inserted in join.
/// Output port has empty header. It is closed when all data is inserted in join.
class FillingRightJoinSideTransform : public IProcessor
{
public:

234
src/Server/KeeperTCPHandler.cpp
View File

@ -18,6 +18,9 @@
#include <IO/ReadBufferFromFileDescriptor.h>
#include <queue>
#include <mutex>
#include <Coordination/FourLetterCommand.h>
#include <Common/hex.h>
#ifdef POCO_HAVE_FD_EPOLL
#include <sys/epoll.h>
@ -29,6 +32,16 @@
namespace DB
{
struct LastOp
{
public:
String name{"NA"};
int64_t last_cxid{-1};
int64_t last_zxid{-1};
int64_t last_response_time{0};
};
static const LastOp EMPTY_LAST_OP {"NA", -1, -1, 0};
namespace ErrorCodes
{
@ -199,7 +212,9 @@ KeeperTCPHandler::KeeperTCPHandler(IServer & server_, const Poco::Net::StreamSoc
, session_timeout(0, global_context->getConfigRef().getUInt("keeper_server.session_timeout_ms", Coordination::DEFAULT_SESSION_TIMEOUT_MS) * 1000)
, poll_wrapper(std::make_unique<SocketInterruptablePollWrapper>(socket_))
, responses(std::make_unique<ThreadSafeResponseQueue>(std::numeric_limits<size_t>::max()))
, last_op(std::make_unique<LastOp>(EMPTY_LAST_OP))
{
KeeperTCPHandler::registerConnection(this);
}
void KeeperTCPHandler::sendHandshake(bool has_leader)
@ -222,16 +237,15 @@ void KeeperTCPHandler::run()
runImpl();
}
Poco::Timespan KeeperTCPHandler::receiveHandshake()
Poco::Timespan KeeperTCPHandler::receiveHandshake(int32_t handshake_length)
{
int32_t handshake_length;
int32_t protocol_version;
int64_t last_zxid_seen;
int32_t timeout_ms;
int64_t previous_session_id = 0; /// We don't support session restore. So previous session_id is always zero.
std::array<char, Coordination::PASSWORD_LENGTH> passwd {};
Coordination::read(handshake_length, *in);
if (handshake_length != Coordination::CLIENT_HANDSHAKE_LENGTH && handshake_length != Coordination::CLIENT_HANDSHAKE_LENGTH_WITH_READONLY)
if (!isHandShake(handshake_length))
throw Exception("Unexpected handshake length received: " + toString(handshake_length), ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT);
Coordination::read(protocol_version, *in);
@ -274,9 +288,32 @@ void KeeperTCPHandler::runImpl()
return;
}
int32_t header;
try
{
auto client_timeout = receiveHandshake();
Coordination::read(header, *in);
}
catch (const Exception & e)
{
LOG_WARNING(log, "Error while read connection header {}", e.displayText());
return;
}
/// All four letter word command code is larger than 2^24 or lower than 0.
/// Hand shake package length must be lower than 2^24 and larger than 0.
/// So collision never happens.
int32_t four_letter_cmd = header;
if (!isHandShake(four_letter_cmd))
{
tryExecuteFourLetterWordCmd(four_letter_cmd);
return;
}
try
{
int32_t handshake_length = header;
auto client_timeout = receiveHandshake(handshake_length);
if (client_timeout != 0)
session_timeout = std::min(client_timeout, session_timeout);
}
@ -345,6 +382,7 @@ void KeeperTCPHandler::runImpl()
session_stopwatch.start();
bool close_received = false;
try
{
while (true)
@ -356,6 +394,7 @@ void KeeperTCPHandler::runImpl()
if (result.has_requests && !close_received)
{
auto [received_op, received_xid] = receiveRequest();
packageReceived();
log_long_operation("Receiving request");
if (received_op == Coordination::OpNum::Close)
@ -368,6 +407,8 @@ void KeeperTCPHandler::runImpl()
{
LOG_TRACE(log, "Received heartbeat for session #{}", session_id);
}
else
operations[received_xid] = Poco::Timestamp();
/// Each request restarts session stopwatch
session_stopwatch.restart();
@ -390,6 +431,9 @@ void KeeperTCPHandler::runImpl()
return;
}
updateStats(response);
packageSent();
response->write(*out);
log_long_operation("Sending response");
if (response->error == Coordination::Error::ZSESSIONEXPIRED)
@ -422,6 +466,44 @@ void KeeperTCPHandler::runImpl()
}
}
bool KeeperTCPHandler::isHandShake(int32_t handshake_length)
{
return handshake_length == Coordination::CLIENT_HANDSHAKE_LENGTH
|| handshake_length == Coordination::CLIENT_HANDSHAKE_LENGTH_WITH_READONLY;
}
bool KeeperTCPHandler::tryExecuteFourLetterWordCmd(int32_t command)
{
if (!FourLetterCommandFactory::instance().isKnown(command))
{
LOG_WARNING(log, "invalid four letter command {}", IFourLetterCommand::toName(command));
return false;
}
else if (!FourLetterCommandFactory::instance().isEnabled(command))
{
LOG_WARNING(log, "Not enabled four letter command {}", IFourLetterCommand::toName(command));
return false;
}
else
{
auto command_ptr = FourLetterCommandFactory::instance().get(command);
LOG_DEBUG(log, "Receive four letter command {}", command_ptr->name());
try
{
String res = command_ptr->run();
out->write(res.data(), res.size());
out->next();
}
catch (...)
{
tryLogCurrentException(log, "Error when executing four letter command " + command_ptr->name());
}
return true;
}
}
std::pair<Coordination::OpNum, Coordination::XID> KeeperTCPHandler::receiveRequest()
{
int32_t length;
@ -441,6 +523,148 @@ std::pair<Coordination::OpNum, Coordination::XID> KeeperTCPHandler::receiveReque
return std::make_pair(opnum, xid);
}
void KeeperTCPHandler::packageSent()
{
{
std::lock_guard lock(conn_stats_mutex);
conn_stats.incrementPacketsSent();
}
keeper_dispatcher->incrementPacketsSent();
}
void KeeperTCPHandler::packageReceived()
{
{
std::lock_guard lock(conn_stats_mutex);
conn_stats.incrementPacketsReceived();
}
keeper_dispatcher->incrementPacketsReceived();
}
void KeeperTCPHandler::updateStats(Coordination::ZooKeeperResponsePtr & response)
{
/// update statistics ignoring watch response and heartbeat.
if (response->xid != Coordination::WATCH_XID && response->getOpNum() != Coordination::OpNum::Heartbeat)
{
Int64 elapsed = (Poco::Timestamp() - operations[response->xid]) / 1000;
{
std::lock_guard lock(conn_stats_mutex);
conn_stats.updateLatency(elapsed);
}
keeper_dispatcher->updateKeeperStatLatency(elapsed);
last_op.set(std::make_unique<LastOp>(LastOp{
.name = Coordination::toString(response->getOpNum()),
.last_cxid = response->xid,
.last_zxid = response->zxid,
.last_response_time = Poco::Timestamp().epochMicroseconds() / 1000,
}));
}
}
KeeperConnectionStats KeeperTCPHandler::getConnectionStats() const
{
std::lock_guard lock(conn_stats_mutex);
return conn_stats;
}
void KeeperTCPHandler::dumpStats(WriteBufferFromOwnString & buf, bool brief)
{
KeeperConnectionStats stats = getConnectionStats();
writeText(' ', buf);
writeText(socket().peerAddress().toString(), buf);
writeText("(recved=", buf);
writeIntText(stats.getPacketsReceived(), buf);
writeText(",sent=", buf);
writeIntText(stats.getPacketsSent(), buf);
if (!brief)
{
if (session_id != 0)
{
writeText(",sid=0x", buf);
writeText(getHexUIntLowercase(session_id), buf);
writeText(",lop=", buf);
LastOpPtr op = last_op.get();
writeText(op->name, buf);
writeText(",est=", buf);
writeIntText(established.epochMicroseconds() / 1000, buf);
writeText(",to=", buf);
writeIntText(session_timeout.totalMilliseconds(), buf);
int64_t last_cxid = op->last_cxid;
if (last_cxid >= 0)
{
writeText(",lcxid=0x", buf);
writeText(getHexUIntLowercase(last_cxid), buf);
}
writeText(",lzxid=0x", buf);
writeText(getHexUIntLowercase(op->last_zxid), buf);
writeText(",lresp=", buf);
writeIntText(op->last_response_time, buf);
writeText(",llat=", buf);
writeIntText(stats.getLastLatency(), buf);
writeText(",minlat=", buf);
writeIntText(stats.getMinLatency(), buf);
writeText(",avglat=", buf);
writeIntText(stats.getAvgLatency(), buf);
writeText(",maxlat=", buf);
writeIntText(stats.getMaxLatency(), buf);
}
}
writeText(')', buf);
writeText('\n', buf);
}
void KeeperTCPHandler::resetStats()
{
{
std::lock_guard lock(conn_stats_mutex);
conn_stats.reset();
}
last_op.set(std::make_unique<LastOp>(EMPTY_LAST_OP));
}
KeeperTCPHandler::~KeeperTCPHandler()
{
KeeperTCPHandler::unregisterConnection(this);
}
std::mutex KeeperTCPHandler::conns_mutex;
std::unordered_set<KeeperTCPHandler *> KeeperTCPHandler::connections;
void KeeperTCPHandler::registerConnection(KeeperTCPHandler * conn)
{
std::lock_guard lock(conns_mutex);
connections.insert(conn);
}
void KeeperTCPHandler::unregisterConnection(KeeperTCPHandler * conn)
{
std::lock_guard lock(conns_mutex);
connections.erase(conn);
}
void KeeperTCPHandler::dumpConnections(WriteBufferFromOwnString & buf, bool brief)
{
std::lock_guard lock(conns_mutex);
for (auto * conn : connections)
{
conn->dumpStats(buf, brief);
}
}
void KeeperTCPHandler::resetConnsStats()
{
std::lock_guard lock(conns_mutex);
for (auto * conn : connections)
{
conn->resetStats();
}
}
}
#endif

48
src/Server/KeeperTCPHandler.h
View File

@ -6,6 +6,7 @@
#if USE_NURAFT
#include <Poco/Net/TCPServerConnection.h>
#include <Common/MultiVersion.h>
#include "IServer.h"
#include <Common/Stopwatch.h>
#include <Interpreters/Context.h>
@ -16,6 +17,8 @@
#include <IO/WriteBufferFromPocoSocket.h>
#include <IO/ReadBufferFromPocoSocket.h>
#include <unordered_map>
#include <Coordination/KeeperConnectionStats.h>
#include <Poco/Timestamp.h>
namespace DB
{
@ -24,14 +27,36 @@ struct SocketInterruptablePollWrapper;
using SocketInterruptablePollWrapperPtr = std::unique_ptr<SocketInterruptablePollWrapper>;
using ThreadSafeResponseQueue = ConcurrentBoundedQueue<Coordination::ZooKeeperResponsePtr>;
using ThreadSafeResponseQueuePtr = std::unique_ptr<ThreadSafeResponseQueue>;
struct LastOp;
using LastOpMultiVersion = MultiVersion<LastOp>;
using LastOpPtr = LastOpMultiVersion::Version;
class KeeperTCPHandler : public Poco::Net::TCPServerConnection
{
public:
static void registerConnection(KeeperTCPHandler * conn);
static void unregisterConnection(KeeperTCPHandler * conn);
/// dump all connections statistics
static void dumpConnections(WriteBufferFromOwnString & buf, bool brief);
static void resetConnsStats();
private:
static std::mutex conns_mutex;
/// all connections
static std::unordered_set<KeeperTCPHandler *> connections;
public:
KeeperTCPHandler(IServer & server_, const Poco::Net::StreamSocket & socket_);
void run() override;
KeeperConnectionStats getConnectionStats() const;
void dumpStats(WriteBufferFromOwnString & buf, bool brief);
void resetStats();
~KeeperTCPHandler() override;
private:
IServer & server;
Poco::Logger * log;
@ -54,9 +79,28 @@ private:
void runImpl();
void sendHandshake(bool has_leader);
Poco::Timespan receiveHandshake();
Poco::Timespan receiveHandshake(int32_t handshake_length);
static bool isHandShake(int32_t handshake_length);
bool tryExecuteFourLetterWordCmd(int32_t command);
std::pair<Coordination::OpNum, Coordination::XID> receiveRequest();
void packageSent();
void packageReceived();
void updateStats(Coordination::ZooKeeperResponsePtr & response);
Poco::Timestamp established;
using Operations = std::map<Coordination::XID, Poco::Timestamp>;
Operations operations;
LastOpMultiVersion last_op;
mutable std::mutex conn_stats_mutex;
KeeperConnectionStats conn_stats;
};
}

2
src/Server/KeeperTCPHandlerFactory.h
View File

@ -21,6 +21,7 @@ private:
using Poco::Net::TCPServerConnection::TCPServerConnection;
void run() override {}
};
public:
KeeperTCPHandlerFactory(IServer & server_, bool secure)
: server(server_)
@ -41,6 +42,7 @@ public:
return new DummyTCPHandler(socket);
}
}
};
}

2
src/Storages/FileLog/StorageFileLog.cpp
View File

@ -470,7 +470,6 @@ void StorageFileLog::openFilesAndSetPos()
void StorageFileLog::closeFilesAndStoreMeta(size_t start, size_t end)
{
assert(start >= 0);
assert(start < end);
assert(end <= file_infos.file_names.size());
@ -491,7 +490,6 @@ void StorageFileLog::closeFilesAndStoreMeta(size_t start, size_t end)
void StorageFileLog::storeMetas(size_t start, size_t end)
{
assert(start >= 0);
assert(start < end);
assert(end <= file_infos.file_names.size());

36
src/Storages/StorageDictionary.cpp
View File

@ -212,19 +212,39 @@ void StorageDictionary::renameInMemory(const StorageID & new_table_id)
auto old_table_id = getStorageID();
IStorage::renameInMemory(new_table_id);
bool has_configuration = false;
assert((location == Location::SameDatabaseAndNameAsDictionary) == (getConfiguration().get() != nullptr));
if (location != Location::SameDatabaseAndNameAsDictionary)
return;
/// It's DDL dictionary, need to update configuration and reload
bool move_to_atomic = old_table_id.uuid == UUIDHelpers::Nil && new_table_id.uuid != UUIDHelpers::Nil;
bool move_to_ordinary = old_table_id.uuid != UUIDHelpers::Nil && new_table_id.uuid == UUIDHelpers::Nil;
assert(old_table_id.uuid == new_table_id.uuid || move_to_atomic || move_to_ordinary);
{
std::lock_guard<std::mutex> lock(dictionary_config_mutex);
if (configuration)
{
has_configuration = true;
configuration->setString("dictionary.database", new_table_id.database_name);
configuration->setString("dictionary.name", new_table_id.table_name);
}
configuration->setString("dictionary.database", new_table_id.database_name);
configuration->setString("dictionary.name", new_table_id.table_name);
if (move_to_atomic)
configuration->setString("dictionary.uuid", toString(new_table_id.uuid));
else if (move_to_ordinary)
configuration->remove("dictionary.uuid");
}
if (has_configuration)
/// Dictionary is moving between databases of different engines or is renaming inside Ordinary database
bool recreate_dictionary = old_table_id.uuid == UUIDHelpers::Nil || new_table_id.uuid == UUIDHelpers::Nil;
if (recreate_dictionary)
{
/// It's too hard to update both name and uuid, better to reload dictionary with new name
removeDictionaryConfigurationFromRepository();
auto repository = std::make_unique<ExternalLoaderDictionaryStorageConfigRepository>(*this);
remove_repository_callback = getContext()->getExternalDictionariesLoader().addConfigRepository(std::move(repository));
/// Dictionary will be reloaded lazily to avoid exceptions in the middle of renaming
}
else
{
const auto & external_dictionaries_loader = getContext()->getExternalDictionariesLoader();
auto result = external_dictionaries_loader.getLoadResult(old_table_id.getInternalDictionaryName());

18
tests/ci/docker_images_check.py
View File

@ -39,17 +39,13 @@ def get_changed_docker_images(pr_info, repo_path, image_file_path):
if image_description['name'].startswith('clickhouse/'):
dockerhub_repo_name = 'clickhouse'
if 'release' in pr_info.labels:
logging.info("Release PR, will rebuild all images from branch, including %s", dockerfile_dir)
changed_images.append(dockerfile_dir)
else:
for f in files_changed:
if f.startswith(dockerfile_dir):
logging.info(
"Found changed file '%s' which affects docker image '%s' with path '%s'",
f, image_description['name'], dockerfile_dir)
changed_images.append(dockerfile_dir)
break
for f in files_changed:
if f.startswith(dockerfile_dir):
logging.info(
"Found changed file '%s' which affects docker image '%s' with path '%s'",
f, image_description['name'], dockerfile_dir)
changed_images.append(dockerfile_dir)
break
# The order is important: dependents should go later than bases, so that
# they are built with updated base versions.

13
tests/ci/integration_test_check.py
View File

@ -34,12 +34,12 @@ IMAGES = [
"yandex/clickhouse-integration-helper",
]
def get_json_params_dict(check_name, commit_sha, pr_number, docker_images):
def get_json_params_dict(check_name, pr_info, docker_images):
return {
'context_name': check_name,
'commit': commit_sha,
'pull_request': pr_number,
'pr_info': None,
'commit': pr_info.sha,
'pull_request': pr_info.number,
'pr_info': {'changed_files' : list(pr_info.changed_files)},
'docker_images_with_versions': docker_images,
'shuffle_test_groups': False,
'use_tmpfs': False,
@ -111,7 +111,8 @@ if __name__ == "__main__":
with open(os.getenv('GITHUB_EVENT_PATH'), 'r', encoding='utf-8') as event_file:
event = json.load(event_file)
pr_info = PRInfo(event)
is_flaky_check = 'flaky' in check_name
pr_info = PRInfo(event, need_changed_files=is_flaky_check)
gh = Github(get_best_robot_token())
@ -135,7 +136,7 @@ if __name__ == "__main__":
json_path = os.path.join(work_path, 'params.json')
with open(json_path, 'w', encoding='utf-8') as json_params:
json_params.write(json.dumps(get_json_params_dict(check_name, pr_info.sha, pr_info.number, images_with_versions)))
json_params.write(json.dumps(get_json_params_dict(check_name, pr_info, images_with_versions)))
output_path_log = os.path.join(result_path, "main_script_log.txt")

20
tests/ci/pr_info.py
View File

@ -90,15 +90,21 @@ class PRInfo:
self.pr_html_url = pull_request['html_url']
if need_changed_files:
commit_before = github_event['before']
response = requests.get(f"https://api.github.com/repos/{os.getenv('GITHUB_REPOSITORY')}/compare/{commit_before}...{self.sha}")
response.raise_for_status()
diff = response.json()
if self.number == 0:
commit_before = github_event['before']
response = requests.get(f"https://api.github.com/repos/{os.getenv('GITHUB_REPOSITORY')}/compare/{commit_before}...{self.sha}")
response.raise_for_status()
diff = response.json()
if 'files' in diff:
self.changed_files = [f['filename'] for f in diff['files']]
if 'files' in diff:
self.changed_files = [f['filename'] for f in diff['files']]
else:
self.changed_files = set([])
else:
self.changed_files = set([])
diff_url = pull_request['diff_url']
diff = urllib.request.urlopen(diff_url)
diff_object = PatchSet(diff, diff.headers.get_charsets()[0])
self.changed_files = { f.path for f in diff_object }
else:
self.changed_files = set([])
else:

2
tests/ci/run_check.py
View File

@ -37,7 +37,7 @@ TRUSTED_CONTRIBUTORS = {
"codyrobert", # Flickerbox engineer
"damozhaeva", # DOCSUP
"den-crane",
"gyuton", # DOCSUP
"flickerbox-tom", # Flickerbox
"gyuton", # technical writer, Yandex
"hagen1778", # Roman Khavronenko, seasoned contributor
"hczhcz",

1
tests/integration/test_keeper_four_word_command/__init__.py Normal file
View File

@ -0,0 +1 @@
#!/usr/bin/env python3

42
tests/integration/test_keeper_four_word_command/configs/enable_keeper1.xml Normal file
View File

@ -0,0 +1,42 @@
<yandex>
<keeper_server>
<tcp_port>9181</tcp_port>
<server_id>1</server_id>
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
<four_letter_word_white_list>*</four_letter_word_white_list>
<coordination_settings>
<operation_timeout_ms>5000</operation_timeout_ms>
<session_timeout_ms>30000</session_timeout_ms>
<snapshot_distance>75</snapshot_distance>
<raft_logs_level>trace</raft_logs_level>
</coordination_settings>
<raft_configuration>
<server>
<id>1</id>
<hostname>node1</hostname>
<port>44444</port>
<can_become_leader>true</can_become_leader>
<priority>3</priority>
</server>
<server>
<id>2</id>
<hostname>node2</hostname>
<port>44444</port>
<can_become_leader>false</can_become_leader>
<start_as_follower>true</start_as_follower>
<priority>2</priority>
</server>
<server>
<id>3</id>
<hostname>node3</hostname>
<port>44444</port>
<can_become_leader>false</can_become_leader>
<start_as_follower>true</start_as_follower>
<priority>1</priority>
</server>
</raft_configuration>
</keeper_server>
</yandex>

42
tests/integration/test_keeper_four_word_command/configs/enable_keeper2.xml Normal file
View File

@ -0,0 +1,42 @@
<yandex>
<keeper_server>
<tcp_port>9181</tcp_port>
<server_id>2</server_id>
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
<four_letter_word_white_list>*</four_letter_word_white_list>
<coordination_settings>
<operation_timeout_ms>5000</operation_timeout_ms>
<session_timeout_ms>30000</session_timeout_ms>
<snapshot_distance>75</snapshot_distance>
<raft_logs_level>trace</raft_logs_level>
</coordination_settings>
<raft_configuration>
<server>
<id>1</id>
<hostname>node1</hostname>
<port>44444</port>
<can_become_leader>true</can_become_leader>
<priority>3</priority>
</server>
<server>
<id>2</id>
<hostname>node2</hostname>
<port>44444</port>
<can_become_leader>false</can_become_leader>
<start_as_follower>true</start_as_follower>
<priority>2</priority>
</server>
<server>
<id>3</id>
<hostname>node3</hostname>
<port>44444</port>
<can_become_leader>false</can_become_leader>
<start_as_follower>true</start_as_follower>
<priority>1</priority>
</server>
</raft_configuration>
</keeper_server>
</yandex>

42
tests/integration/test_keeper_four_word_command/configs/enable_keeper3.xml Normal file
View File

@ -0,0 +1,42 @@
<yandex>
<keeper_server>
<tcp_port>9181</tcp_port>
<server_id>3</server_id>
<log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
<snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
<four_letter_word_white_list>*</four_letter_word_white_list>
<coordination_settings>
<operation_timeout_ms>5000</operation_timeout_ms>
<session_timeout_ms>30000</session_timeout_ms>
<snapshot_distance>75</snapshot_distance>
<raft_logs_level>trace</raft_logs_level>
</coordination_settings>
<raft_configuration>
<server>
<id>1</id>
<hostname>node1</hostname>
<port>44444</port>
<can_become_leader>true</can_become_leader>
<priority>3</priority>
</server>
<server>
<id>2</id>
<hostname>node2</hostname>
<port>44444</port>
<can_become_leader>false</can_become_leader>
<start_as_follower>true</start_as_follower>
<priority>2</priority>
</server>
<server>
<id>3</id>
<hostname>node3</hostname>
<port>44444</port>
<can_become_leader>false</can_become_leader>
<start_as_follower>true</start_as_follower>
<priority>1</priority>
</server>
</raft_configuration>
</keeper_server>
</yandex>

38
tests/integration/test_keeper_four_word_command/configs/keeper_config_with_white_list.xml Normal file
View File

@ -0,0 +1,38 @@
<yandex>
<keeper_server>
<tcp_port>9181</tcp_port>
<server_id>1</server_id>
<four_letter_word_white_list>ruok, conf</four_letter_word_white_list>
<raft_configuration>
<server>
<id>1</id>
<hostname>node1</hostname>
<port>44444</port>
</server>
<server>
<id>2</id>
<hostname>node2</hostname>
<port>44444</port>
</server>
<server>
<id>3</id>
<hostname>node3</hostname>
<port>44444</port>
</server>
</raft_configuration>
</keeper_server>
<zookeeper>
<node index="1">
<host>node1</host>
<port>9181</port>
</node>
<node index="2">
<host>node2</host>
<port>9181</port>
</node>
<node index="3">
<host>node3</host>
<port>9181</port>
</node>
</zookeeper>
</yandex>

38
tests/integration/test_keeper_four_word_command/configs/keeper_config_with_white_list_all.xml Normal file
View File

@ -0,0 +1,38 @@
<yandex>
<keeper_server>
<tcp_port>9181</tcp_port>
<server_id>3</server_id>
<four_letter_word_white_list>*</four_letter_word_white_list>
<raft_configuration>
<server>
<id>1</id>
<hostname>node1</hostname>
<port>44444</port>
</server>
<server>
<id>2</id>
<hostname>node2</hostname>
<port>44444</port>
</server>
<server>
<id>3</id>
<hostname>node3</hostname>
<port>44444</port>
</server>
</raft_configuration>
</keeper_server>
<zookeeper>
<node index="1">
<host>node1</host>
<port>9181</port>
</node>
<node index="2">
<host>node2</host>
<port>9181</port>
</node>
<node index="3">
<host>node3</host>
<port>9181</port>
</node>
</zookeeper>
</yandex>

37
tests/integration/test_keeper_four_word_command/configs/keeper_config_without_white_list.xml Normal file
View File

@ -0,0 +1,37 @@
<yandex>
<keeper_server>
<tcp_port>9181</tcp_port>
<server_id>2</server_id>
<raft_configuration>
<server>
<id>1</id>
<hostname>node1</hostname>
<port>44444</port>
</server>
<server>
<id>2</id>
<hostname>node2</hostname>
<port>44444</port>
</server>
<server>
<id>3</id>
<hostname>node3</hostname>
<port>44444</port>
</server>
</raft_configuration>
</keeper_server>
<zookeeper>
<node index="1">
<host>node1</host>
<port>9181</port>
</node>
<node index="2">
<host>node2</host>
<port>9181</port>
</node>
<node index="3">
<host>node3</host>
<port>9181</port>
</node>
</zookeeper>
</yandex>

611
tests/integration/test_keeper_four_word_command/test.py Normal file
View File

@ -0,0 +1,611 @@
import socket
import pytest
from helpers.cluster import ClickHouseCluster
import random
import string
import os
import time
from multiprocessing.dummy import Pool
from helpers.network import PartitionManager
from helpers.test_tools import assert_eq_with_retry
from io import StringIO
import csv
import re
cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', main_configs=['configs/enable_keeper1.xml'],
stay_alive=True)
node2 = cluster.add_instance('node2', main_configs=['configs/enable_keeper2.xml'],
stay_alive=True)
node3 = cluster.add_instance('node3', main_configs=['configs/enable_keeper3.xml'],
stay_alive=True)
from kazoo.client import KazooClient, KazooState
@pytest.fixture(scope="module")
def started_cluster():
try:
cluster.start()
yield cluster
finally:
cluster.shutdown()
def destroy_zk_client(zk):
try:
if zk:
zk.stop()
zk.close()
except:
pass
def clear_znodes():
zk = None
try:
zk = get_fake_zk(node3.name, timeout=30.0)
nodes = zk.get_children('/')
for node in [n for n in nodes if 'test_4lw_' in n]:
zk.delete('/' + node)
finally:
destroy_zk_client(zk)
def wait_node(node):
for _ in range(100):
zk = None
try:
zk = get_fake_zk(node.name, timeout=30.0)
# zk.create("/test", sequence=True)
print("node", node.name, "ready")
break
except Exception as ex:
time.sleep(0.2)
print("Waiting until", node.name, "will be ready, exception", ex)
finally:
destroy_zk_client(zk)
else:
raise Exception("Can't wait node", node.name, "to become ready")
def wait_nodes():
for n in [node1, node2, node3]:
wait_node(n)
def get_fake_zk(nodename, timeout=30.0):
_fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout)
_fake_zk_instance.start()
return _fake_zk_instance
def get_keeper_socket(node_name):
hosts = cluster.get_instance_ip(node_name)
client = socket.socket()
client.settimeout(10)
client.connect((hosts, 9181))
return client
def close_keeper_socket(cli):
if cli is not None:
cli.close()
def reset_node_stats(node_name=node1.name):
client = None
try:
client = get_keeper_socket(node_name)
client.send(b'srst')
client.recv(10)
finally:
if client is not None:
client.close()
def send_4lw_cmd(node_name=node1.name, cmd='ruok'):
client = None
try:
client = get_keeper_socket(node_name)
client.send(cmd.encode())
data = client.recv(100_000)
data = data.decode()
return data
finally:
if client is not None:
client.close()
def reset_conn_stats(node_name=node1.name):
client = None
try:
client = get_keeper_socket(node_name)
client.send(b'crst')
client.recv(10_000)
finally:
if client is not None:
client.close()
def test_cmd_ruok(started_cluster):
client = None
try:
wait_nodes()
data = send_4lw_cmd(cmd='ruok')
assert data == 'imok'
finally:
close_keeper_socket(client)
def do_some_action(zk, create_cnt=0, get_cnt=0, set_cnt=0, ephemeral_cnt=0, watch_cnt=0, delete_cnt=0):
assert create_cnt >= get_cnt
assert create_cnt >= set_cnt
assert create_cnt >= watch_cnt
assert create_cnt >= delete_cnt
# ensure not delete watched node
assert create_cnt >= (delete_cnt + watch_cnt)
for i in range(create_cnt):
zk.create("/test_4lw_normal_node_" + str(i), b"")
for i in range(get_cnt):
zk.get("/test_4lw_normal_node_" + str(i))
for i in range(set_cnt):
zk.set("/test_4lw_normal_node_" + str(i), b"new-value")
for i in range(ephemeral_cnt):
zk.create("/test_4lw_ephemeral_node_" + str(i), ephemeral=True)
fake_ephemeral_event = None
def fake_ephemeral_callback(event):
print("Fake watch triggered")
nonlocal fake_ephemeral_event
fake_ephemeral_event = event
for i in range(watch_cnt):
zk.exists("/test_4lw_normal_node_" + str(i), watch=fake_ephemeral_callback)
for i in range(create_cnt - delete_cnt, create_cnt):
zk.delete("/test_4lw_normal_node_" + str(i))
def test_cmd_mntr(started_cluster):
zk = None
try:
wait_nodes()
clear_znodes()
# reset stat first
reset_node_stats(node1.name)
zk = get_fake_zk(node1.name, timeout=30.0)
do_some_action(zk, create_cnt=10, get_cnt=10, set_cnt=5, ephemeral_cnt=2, watch_cnt=2, delete_cnt=2)
data = send_4lw_cmd(cmd='mntr')
# print(data.decode())
reader = csv.reader(data.split('\n'), delimiter='\t')
result = {}
for row in reader:
if len(row) != 0:
result[row[0]] = row[1]
assert len(result["zk_version"]) != 0
assert int(result["zk_avg_latency"]) >= 0
assert int(result["zk_max_latency"]) >= 0
assert int(result["zk_min_latency"]) >= 0
assert int(result["zk_min_latency"]) <= int(result["zk_avg_latency"])
assert int(result["zk_max_latency"]) >= int(result["zk_avg_latency"])
assert int(result["zk_num_alive_connections"]) == 1
assert int(result["zk_outstanding_requests"]) == 0
assert result["zk_server_state"] == "leader"
# contains:
# 10 nodes created by test
# 3 nodes created by clickhouse "/clickhouse/task_queue/ddl"
# 1 root node
assert int(result["zk_znode_count"]) == 11
assert int(result["zk_watch_count"]) == 2
assert int(result["zk_ephemerals_count"]) == 2
assert int(result["zk_approximate_data_size"]) > 0
assert int(result["zk_open_file_descriptor_count"]) > 0
assert int(result["zk_max_file_descriptor_count"]) > 0
assert int(result["zk_followers"]) == 2
assert int(result["zk_synced_followers"]) == 2
# contains 31 user request response and some responses for server startup
assert int(result["zk_packets_sent"]) >= 31
assert int(result["zk_packets_received"]) >= 31
finally:
destroy_zk_client(zk)
def test_cmd_srst(started_cluster):
client = None
try:
wait_nodes()
clear_znodes()
data = send_4lw_cmd(cmd='srst')
assert data.strip() == "Server stats reset."
data = send_4lw_cmd(cmd='mntr')
assert len(data) != 0
# print(data)
reader = csv.reader(data.split('\n'), delimiter='\t')
result = {}
for row in reader:
if len(row) != 0:
result[row[0]] = row[1]
assert int(result["zk_packets_received"]) == 0
assert int(result["zk_packets_sent"]) == 0
finally:
close_keeper_socket(client)
def test_cmd_conf(started_cluster):
client = None
try:
wait_nodes()
clear_znodes()
data = send_4lw_cmd(cmd='conf')
reader = csv.reader(data.split('\n'), delimiter='=')
result = {}
for row in reader:
if len(row) != 0:
print(row)
result[row[0]] = row[1]
assert result["server_id"] == "1"
assert result["tcp_port"] == "9181"
assert "tcp_port_secure" not in result
assert "superdigest" not in result
assert result["four_letter_word_white_list"] == "*"
assert result["log_storage_path"] == "/var/lib/clickhouse/coordination/log"
assert result["snapshot_storage_path"] == "/var/lib/clickhouse/coordination/snapshots"
assert result["session_timeout_ms"] == "30000"
assert result["operation_timeout_ms"] == "5000"
assert result["dead_session_check_period_ms"] == "500"
assert result["heart_beat_interval_ms"] == "500"
assert result["election_timeout_lower_bound_ms"] == "1000"
assert result["election_timeout_upper_bound_ms"] == "2000"
assert result["reserved_log_items"] == "100000"
assert result["snapshot_distance"] == "75"
assert result["auto_forwarding"] == "true"
assert result["shutdown_timeout"] == "5000"
assert result["startup_timeout"] == "180000"
assert result["raft_logs_level"] == "trace"
assert result["rotate_log_storage_interval"] == "100000"
assert result["snapshots_to_keep"] == "3"
assert result["stale_log_gap"] == "10000"
assert result["fresh_log_gap"] == "200"
assert result["max_requests_batch_size"] == "100"
assert result["quorum_reads"] == "false"
assert result["force_sync"] == "true"
assert result["compress_logs"] == "true"
assert result["compress_snapshots_with_zstd_format"] == "true"
assert result["configuration_change_tries_count"] == "20"
finally:
close_keeper_socket(client)
def test_cmd_isro(started_cluster):
wait_nodes()
assert send_4lw_cmd(node1.name, 'isro') == 'rw'
assert send_4lw_cmd(node2.name, 'isro') == 'ro'
def test_cmd_srvr(started_cluster):
zk = None
try:
wait_nodes()
clear_znodes()
reset_node_stats(node1.name)
zk = get_fake_zk(node1.name, timeout=30.0)
do_some_action(zk, create_cnt=10)
data = send_4lw_cmd(cmd='srvr')
print("srvr output -------------------------------------")
print(data)
reader = csv.reader(data.split('\n'), delimiter=':')
result = {}
for row in reader:
if len(row) != 0:
result[row[0].strip()] = row[1].strip()
assert 'ClickHouse Keeper version' in result
assert 'Latency min/avg/max' in result
assert result['Received'] == '10'
assert result['Sent'] == '10'
assert int(result['Connections']) == 1
assert int(result['Zxid']) > 14
assert result['Mode'] == 'leader'
assert result['Node count'] == '11'
finally:
destroy_zk_client(zk)
def test_cmd_stat(started_cluster):
zk = None
try:
wait_nodes()
clear_znodes()
reset_node_stats(node1.name)
reset_conn_stats(node1.name)
zk = get_fake_zk(node1.name, timeout=30.0)
do_some_action(zk, create_cnt=10)
data = send_4lw_cmd(cmd='stat')
print("stat output -------------------------------------")
print(data)
# keeper statistics
stats = [n for n in data.split('\n') if '=' not in n]
reader = csv.reader(stats, delimiter=':')
result = {}
for row in reader:
if len(row) != 0:
result[row[0].strip()] = row[1].strip()
assert 'ClickHouse Keeper version' in result
assert 'Latency min/avg/max' in result
assert result['Received'] == '10'
assert result['Sent'] == '10'
assert int(result['Connections']) == 1
assert int(result['Zxid']) > 14
assert result['Mode'] == 'leader'
assert result['Node count'] == '11'
# filter connection statistics
cons = [n for n in data.split('\n') if '=' in n]
# filter connection created by 'cons'
cons = [n for n in cons if 'recved=0' not in n and len(n) > 0]
assert len(cons) == 1
conn_stat = re.match(r'(.*?)[:].*[(](.*?)[)].*', cons[0].strip(), re.S).group(2)
assert conn_stat is not None
result = {}
for col in conn_stat.split(','):
col = col.strip().split('=')
result[col[0]] = col[1]
assert result['recved'] == '10'
assert result['sent'] == '10'
finally:
destroy_zk_client(zk)
def test_cmd_cons(started_cluster):
zk = None
try:
wait_nodes()
clear_znodes()
reset_conn_stats()
zk = get_fake_zk(node1.name, timeout=30.0)
do_some_action(zk, create_cnt=10)
data = send_4lw_cmd(cmd='cons')
print("cons output -------------------------------------")
print(data)
# filter connection created by 'cons'
cons = [n for n in data.split('\n') if 'recved=0' not in n and len(n) > 0]
assert len(cons) == 1
conn_stat = re.match(r'(.*?)[:].*[(](.*?)[)].*', cons[0].strip(), re.S).group(2)
assert conn_stat is not None
result = {}
for col in conn_stat.split(','):
col = col.strip().split('=')
result[col[0]] = col[1]
assert result['recved'] == '10'
assert result['sent'] == '10'
assert 'sid' in result
assert result['lop'] == 'Create'
assert 'est' in result
assert result['to'] == '30000'
assert result['lcxid'] == '0x000000000000000a'
assert 'lzxid' in result
assert 'lresp' in result
assert int(result['llat']) >= 0
assert int(result['minlat']) >= 0
assert int(result['avglat']) >= 0
assert int(result['maxlat']) >= 0
finally:
destroy_zk_client(zk)
def test_cmd_crst(started_cluster):
zk = None
try:
wait_nodes()
clear_znodes()
reset_conn_stats()
zk = get_fake_zk(node1.name, timeout=30.0)
do_some_action(zk, create_cnt=10)
data = send_4lw_cmd(cmd='crst')
print("crst output -------------------------------------")
print(data)
data = send_4lw_cmd(cmd='cons')
# 2 connections, 1 for 'cons' command, 1 for zk
cons = [n for n in data.split('\n') if len(n) > 0]
assert len(cons) == 2
conn_stat = re.match(r'(.*?)[:].*[(](.*?)[)].*', cons[0].strip(), re.S).group(2)
assert conn_stat is not None
result = {}
for col in conn_stat.split(','):
col = col.strip().split('=')
result[col[0]] = col[1]
assert result['recved'] == '0'
assert result['sent'] == '0'
assert 'sid' in result
assert result['lop'] == 'NA'
assert 'est' in result
assert result['to'] == '30000'
assert 'lcxid' not in result
assert result['lzxid'] == '0xffffffffffffffff'
assert result['lresp'] == '0'
assert int(result['llat']) == 0
assert int(result['minlat']) == 0
assert int(result['avglat']) == 0
assert int(result['maxlat']) == 0
finally:
destroy_zk_client(zk)
def test_cmd_dump(started_cluster):
zk = None
try:
wait_nodes()
clear_znodes()
reset_node_stats()
zk = get_fake_zk(node1.name, timeout=30.0)
do_some_action(zk, ephemeral_cnt=2)
data = send_4lw_cmd(cmd='dump')
print("dump output -------------------------------------")
print(data)
list_data = data.split('\n')
session_count = int(re.match(r'.*[(](.*?)[)].*', list_data[0], re.S).group(1))
assert session_count == 1
assert '\t' + '/test_4lw_ephemeral_node_0' in list_data
assert '\t' + '/test_4lw_ephemeral_node_1' in list_data
finally:
destroy_zk_client(zk)
def test_cmd_wchs(started_cluster):
zk = None
try:
wait_nodes()
clear_znodes()
reset_node_stats()
zk = get_fake_zk(node1.name, timeout=30.0)
do_some_action(zk, create_cnt=2, watch_cnt=2)
data = send_4lw_cmd(cmd='wchs')
print("wchs output -------------------------------------")
print(data)
list_data = [n for n in data.split('\n') if len(n.strip()) > 0]
# 37 connections watching 632141 paths
# Total watches:632141
matcher = re.match(r'([0-9].*) connections watching ([0-9].*) paths', list_data[0], re.S)
conn_count = int(matcher.group(1))
watch_path_count = int(matcher.group(2))
watch_count = int(re.match(r'Total watches:([0-9].*)', list_data[1], re.S).group(1))
assert conn_count == 1
assert watch_path_count == 2
assert watch_count == 2
finally:
destroy_zk_client(zk)
def test_cmd_wchc(started_cluster):
zk = None
try:
wait_nodes()
clear_znodes()
reset_node_stats()
zk = get_fake_zk(node1.name, timeout=30.0)
do_some_action(zk, create_cnt=2, watch_cnt=2)
data = send_4lw_cmd(cmd='wchc')
print("wchc output -------------------------------------")
print(data)
list_data = [n for n in data.split('\n') if len(n.strip()) > 0]
assert len(list_data) == 3
assert '\t' + '/test_4lw_normal_node_0' in list_data
assert '\t' + '/test_4lw_normal_node_1' in list_data
finally:
destroy_zk_client(zk)
def test_cmd_wchp(started_cluster):
zk = None
try:
wait_nodes()
clear_znodes()
reset_node_stats()
zk = get_fake_zk(node1.name, timeout=30.0)
do_some_action(zk, create_cnt=2, watch_cnt=2)
data = send_4lw_cmd(cmd='wchp')
print("wchp output -------------------------------------")
print(data)
list_data = [n for n in data.split('\n') if len(n.strip()) > 0]
assert len(list_data) == 4
assert '/test_4lw_normal_node_0' in list_data
assert '/test_4lw_normal_node_1' in list_data
finally:
destroy_zk_client(zk)

98
tests/integration/test_keeper_four_word_command/test_white_list.py Normal file
View File

@ -0,0 +1,98 @@
import socket
import pytest
from helpers.cluster import ClickHouseCluster
import time
cluster = ClickHouseCluster(__file__)
node1 = cluster.add_instance('node1', main_configs=['configs/keeper_config_with_white_list.xml'], stay_alive=True)
node2 = cluster.add_instance('node2', main_configs=['configs/keeper_config_without_white_list.xml'], stay_alive=True)
node3 = cluster.add_instance('node3', main_configs=['configs/keeper_config_with_white_list_all.xml'], stay_alive=True)
from kazoo.client import KazooClient, KazooState
@pytest.fixture(scope="module")
def started_cluster():
try:
cluster.start()
yield cluster
finally:
cluster.shutdown()
def destroy_zk_client(zk):
try:
if zk:
zk.stop()
zk.close()
except:
pass
def wait_node(node):
for _ in range(100):
zk = None
try:
node.query("SELECT * FROM system.zookeeper WHERE path = '/'")
zk = get_fake_zk(node.name, timeout=30.0)
# zk.create("/test", sequence=True)
print("node", node.name, "ready")
break
except Exception as ex:
time.sleep(0.2)
print("Waiting until", node.name, "will be ready, exception", ex)
finally:
destroy_zk_client(zk)
else:
raise Exception("Can't wait node", node.name, "to become ready")
def wait_nodes():
for n in [node1, node2, node3]:
wait_node(n)
def get_keeper_socket(nodename):
hosts = cluster.get_instance_ip(nodename)
client = socket.socket()
client.settimeout(10)
client.connect((hosts, 9181))
return client
def get_fake_zk(nodename, timeout=30.0):
_fake_zk_instance = KazooClient(hosts=cluster.get_instance_ip(nodename) + ":9181", timeout=timeout)
_fake_zk_instance.start()
return _fake_zk_instance
def close_keeper_socket(cli):
if cli is not None:
print("close socket")
cli.close()
def send_cmd(node_name, command = "ruok"):
client = None
try:
wait_nodes()
client = get_keeper_socket(node_name)
client.send(command.encode())
data = client.recv(4)
return data.decode()
finally:
close_keeper_socket(client)
def test_white_list(started_cluster):
client = None
try:
wait_nodes()
assert send_cmd(node1.name) == 'imok'
assert send_cmd(node1.name, command = 'mntr') == ''
assert send_cmd(node2.name) == 'imok'
assert send_cmd(node3.name) == 'imok'
finally:
close_keeper_socket(client)

4
tests/integration/test_keeper_two_nodes_cluster/test.py
View File

@ -83,7 +83,7 @@ def test_read_write_two_nodes(started_cluster):
finally:
try:
for zk_conn in [node1_zk, node2_zk, node3_zk]:
for zk_conn in [node1_zk, node2_zk]:
zk_conn.stop()
zk_conn.close()
except:
@ -156,7 +156,7 @@ def test_read_write_two_nodes_with_blocade(started_cluster):
finally:
try:
for zk_conn in [node1_zk, node2_zk, node3_zk]:
for zk_conn in [node1_zk, node2_zk]:
zk_conn.stop()
zk_conn.close()
except:

26
tests/performance/writing_valid_utf8.xml Normal file
View File

@ -0,0 +1,26 @@
<test>
<settings>
<output_format_parallel_formatting>0</output_format_parallel_formatting>
</settings>
<substitutions>
<substitution>
<name>format</name>
<values>
<value>JSON</value>
<value>JSONCompact</value>
<value>XML</value>
</values>
</substitution>
</substitutions>
<preconditions>
<table_exists>test.hits</table_exists>
</preconditions>
<create_query>CREATE TABLE IF NOT EXISTS table_{format} ENGINE = File({format}, '/dev/null') AS SELECT SearchPhrase, ClientIP6, URL, Referer, URLDomain FROM test.hits limit 0</create_query>
<query>INSERT INTO table_{format} SELECT SearchPhrase, ClientIP6, URL, Referer, URLDomain FROM test.hits LIMIT 100000</query>
<drop_query>DROP TABLE IF EXISTS table_{format}</drop_query>
</test>

2
tests/queries/0_stateless/00623_replicated_truncate_table_zookeeper_long.sql
View File

@ -16,7 +16,7 @@ SELECT * FROM replicated_truncate1 ORDER BY k;
SELECT * FROM replicated_truncate2 ORDER BY k;
SELECT '======After Truncate And Empty======';
TRUNCATE TABLE replicated_truncate1;
TRUNCATE TABLE replicated_truncate1 SETTINGS replication_alter_partitions_sync=2;
SELECT * FROM replicated_truncate1 ORDER BY k;
SELECT * FROM replicated_truncate2 ORDER BY k;

23
tests/queries/0_stateless/01155_rename_move_materialized_view.reference
View File

@ -4,8 +4,13 @@
1 mv1 before moving tablesmv1
1 mv2 before moving tablesmv2
1 src before moving tables
asdf
asdf
test_01155_ordinary dict1 00000000-0000-0000-0000-000000000000
asdf
ordinary:
.inner.mv1
dict
dist
dst
mv1
@ -14,6 +19,7 @@ src
ordinary after rename:
atomic after rename:
.inner_id.
dict
dist
dst
mv1
@ -33,12 +39,14 @@ src
3 src after moving tables
3 src after renaming database
3 src before moving tables
.inner_id.
dist
dst
mv1
mv2
src
asdf
test_01155_ordinary .inner_id.
test_01155_ordinary dict
test_01155_ordinary dist
test_01155_ordinary dst
test_01155_ordinary mv1
test_01155_ordinary mv2
test_01155_ordinary src
CREATE DATABASE test_01155_atomic\nENGINE = Atomic
4 .inner.mv1 after renaming databasemv1
4 .inner.mv1 after renaming tablesmv1
@ -60,8 +68,11 @@ CREATE DATABASE test_01155_atomic\nENGINE = Atomic
4 src after renaming database
4 src after renaming tables
4 src before moving tables
asdf
test_01155_ordinary dict 00000000-0000-0000-0000-000000000000
test_01155_ordinary:
.inner.mv1
dict
dist
dst
mv1

42
tests/queries/0_stateless/01155_rename_move_materialized_view.sql
View File

@ -7,15 +7,27 @@ CREATE DATABASE test_01155_ordinary ENGINE=Ordinary;
CREATE DATABASE test_01155_atomic ENGINE=Atomic;
USE test_01155_ordinary;
CREATE TABLE src (s String) ENGINE=MergeTree() PARTITION BY tuple() ORDER BY s;
CREATE MATERIALIZED VIEW mv1 (s String) ENGINE=MergeTree() PARTITION BY tuple() ORDER BY s AS SELECT (*,).1 || 'mv1' as s FROM src;
CREATE TABLE dst (s String) ENGINE=MergeTree() PARTITION BY tuple() ORDER BY s;
CREATE MATERIALIZED VIEW mv2 TO dst (s String) AS SELECT (*,).1 || 'mv2' as s FROM src;
CREATE TABLE dist (s String) Engine=Distributed(test_shard_localhost, test_01155_ordinary, src);
INSERT INTO dist VALUES ('before moving tables');
CREATE TABLE src (s String, x String DEFAULT 'a') ENGINE=MergeTree() PARTITION BY tuple() ORDER BY s;
CREATE MATERIALIZED VIEW mv1 (s String, x String DEFAULT 'b') ENGINE=MergeTree() PARTITION BY tuple() ORDER BY s AS SELECT (*,).1 || 'mv1' as s FROM src;
CREATE TABLE dst (s String, x String DEFAULT 'c') ENGINE=MergeTree() PARTITION BY tuple() ORDER BY s;
CREATE MATERIALIZED VIEW mv2 TO dst (s String, x String DEFAULT 'd') AS SELECT (*,).1 || 'mv2' as s FROM src;
CREATE TABLE dist (s String, x String DEFAULT 'asdf') ENGINE=Distributed(test_shard_localhost, test_01155_ordinary, src);
INSERT INTO dist(s) VALUES ('before moving tables');
SYSTEM FLUSH DISTRIBUTED dist;
CREATE DICTIONARY dict (s String, x String DEFAULT 'qwerty') PRIMARY KEY s
SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'dist' DB 'test_01155_ordinary'))
LIFETIME(MIN 0 MAX 2) LAYOUT(COMPLEX_KEY_CACHE(SIZE_IN_CELLS 123));
-- FIXME Cannot convert column `1` because it is non constant in source stream but must be constant in result
SELECT materialize(1), substr(_table, 1, 10), s FROM merge('test_01155_ordinary', '') ORDER BY _table, s;
SELECT dictGet('test_01155_ordinary.dict', 'x', 'before moving tables');
RENAME DICTIONARY test_01155_ordinary.dict TO test_01155_ordinary.dict1;
SELECT dictGet('test_01155_ordinary.dict1', 'x', 'before moving tables');
SELECT database, name, uuid FROM system.dictionaries WHERE database='test_01155_ordinary';
RENAME TABLE test_01155_ordinary.dict1 TO test_01155_ordinary.dict;
SELECT dictGet('test_01155_ordinary.dict', 'x', 'before moving tables');
-- Move tables with materialized views from Ordinary to Atomic
SELECT 'ordinary:';
@ -24,7 +36,10 @@ RENAME TABLE test_01155_ordinary.mv1 TO test_01155_atomic.mv1;
RENAME TABLE test_01155_ordinary.mv2 TO test_01155_atomic.mv2;
RENAME TABLE test_01155_ordinary.dst TO test_01155_atomic.dst;
RENAME TABLE test_01155_ordinary.src TO test_01155_atomic.src;
SET check_table_dependencies=0;
RENAME TABLE test_01155_ordinary.dist TO test_01155_atomic.dist;
SET check_table_dependencies=1;
RENAME DICTIONARY test_01155_ordinary.dict TO test_01155_atomic.dict;
SELECT 'ordinary after rename:';
SELECT substr(name, 1, 10) FROM system.tables WHERE database='test_01155_ordinary';
SELECT 'atomic after rename:';
@ -32,17 +47,19 @@ SELECT substr(name, 1, 10) FROM system.tables WHERE database='test_01155_atomic'
DROP DATABASE test_01155_ordinary;
USE default;
INSERT INTO test_01155_atomic.src VALUES ('after moving tables');
SELECT materialize(2), substr(_table, 1, 10), s FROM merge('test_01155_atomic', '') ORDER BY _table, s; -- { serverError 81 }
INSERT INTO test_01155_atomic.src(s) VALUES ('after moving tables');
--SELECT materialize(2), substr(_table, 1, 10), s FROM merge('test_01155_atomic', '') ORDER BY _table, s; -- { serverError 81 }
--SELECT dictGet('test_01155_ordinary.dict', 'x', 'after moving tables'); -- { serverError 36 }
RENAME DATABASE test_01155_atomic TO test_01155_ordinary;
USE test_01155_ordinary;
INSERT INTO dist VALUES ('after renaming database');
INSERT INTO dist(s) VALUES ('after renaming database');
SYSTEM FLUSH DISTRIBUTED dist;
SELECT materialize(3), substr(_table, 1, 10), s FROM merge('test_01155_ordinary', '') ORDER BY _table, s;
SELECT dictGet('test_01155_ordinary.dict', 'x', 'after renaming database');
SELECT substr(name, 1, 10) FROM system.tables WHERE database='test_01155_ordinary';
SELECT database, substr(name, 1, 10) FROM system.tables WHERE database like 'test_01155_%';
-- Move tables back
RENAME DATABASE test_01155_ordinary TO test_01155_atomic;
@ -55,10 +72,13 @@ RENAME TABLE test_01155_atomic.mv2 TO test_01155_ordinary.mv2;
RENAME TABLE test_01155_atomic.dst TO test_01155_ordinary.dst;
RENAME TABLE test_01155_atomic.src TO test_01155_ordinary.src;
RENAME TABLE test_01155_atomic.dist TO test_01155_ordinary.dist;
RENAME DICTIONARY test_01155_atomic.dict TO test_01155_ordinary.dict;
INSERT INTO dist VALUES ('after renaming tables');
INSERT INTO dist(s) VALUES ('after renaming tables');
SYSTEM FLUSH DISTRIBUTED dist;
SELECT materialize(4), substr(_table, 1, 10), s FROM merge('test_01155_ordinary', '') ORDER BY _table, s;
SELECT dictGet('test_01155_ordinary.dict', 'x', 'after renaming tables');
SELECT database, name, uuid FROM system.dictionaries WHERE database='test_01155_ordinary';
SELECT 'test_01155_ordinary:';
SHOW TABLES FROM test_01155_ordinary;
SELECT 'test_01155_atomic:';

8
tests/queries/0_stateless/01191_rename_dictionary.reference
View File

@ -1,7 +1,13 @@
dict NOT_LOADED
_ Memory
dict Dictionary
dict1 NOT_LOADED
t Memory
t NOT_LOADED
_ Memory
dict Memory
t Dictionary
test
dict1 LOADED
_ Memory
dict1 Dictionary
test

13
tests/queries/0_stateless/01191_rename_dictionary.sql
View File

@ -4,6 +4,7 @@ DROP DATABASE IF EXISTS test_01191;
CREATE DATABASE test_01191 ENGINE=Atomic;
CREATE TABLE test_01191._ (n UInt64, s String) ENGINE = Memory();
CREATE TABLE test_01191.t (n UInt64, s String) ENGINE = Memory();
CREATE DICTIONARY test_01191.dict (n UInt64, s String)
PRIMARY KEY n
@ -16,9 +17,15 @@ SELECT name, status FROM system.dictionaries WHERE database='test_01191';
SELECT name, engine FROM system.tables WHERE database='test_01191' ORDER BY name;
RENAME DICTIONARY test_01191.table TO test_01191.table1; -- {serverError 60}
EXCHANGE TABLES test_01191.table AND test_01191.dict; -- {serverError 60}
EXCHANGE TABLES test_01191.dict AND test_01191.table; -- {serverError 80}
RENAME TABLE test_01191.dict TO test_01191.dict1; -- {serverError 80}
EXCHANGE DICTIONARIES test_01191._ AND test_01191.dict; -- {serverError 80}
EXCHANGE TABLES test_01191.t AND test_01191.dict;
SELECT name, status FROM system.dictionaries WHERE database='test_01191';
SELECT name, engine FROM system.tables WHERE database='test_01191' ORDER BY name;
SELECT dictGet(test_01191.t, 's', toUInt64(42));
EXCHANGE TABLES test_01191.dict AND test_01191.t;
RENAME DICTIONARY test_01191.t TO test_01191.dict1; -- {serverError 80}
DROP DICTIONARY test_01191.t; -- {serverError 80}
DROP TABLE test_01191.t;
CREATE DATABASE dummy_db ENGINE=Atomic;
RENAME DICTIONARY test_01191.dict TO dummy_db.dict1;

10
tests/queries/0_stateless/02010_lc_native.python
View File

@ -8,7 +8,6 @@ import uuid
CLICKHOUSE_HOST = os.environ.get('CLICKHOUSE_HOST', '127.0.0.1')
CLICKHOUSE_PORT = int(os.environ.get('CLICKHOUSE_PORT_TCP', '900000'))
CLICKHOUSE_DATABASE = os.environ.get('CLICKHOUSE_DATABASE', 'default')
CLICKHOUSE_QUERY_ID = uuid.uuid4().hex
def writeVarUInt(x, ba):
for _ in range(0, 9):
@ -111,9 +110,9 @@ def receiveHello(s):
# print("Version patch: ", server_version_patch)
def serializeClientInfo(ba):
def serializeClientInfo(ba, query_id):
writeStringBinary('default', ba) # initial_user
writeStringBinary(CLICKHOUSE_QUERY_ID, ba) # initial_query_id
writeStringBinary(query_id, ba) # initial_query_id
writeStringBinary('127.0.0.1:9000', ba) # initial_address
ba.extend([0] * 8) # initial_query_start_time_microseconds
ba.append(1) # TCP
@ -131,13 +130,14 @@ def serializeClientInfo(ba):
def sendQuery(s, query):
ba = bytearray()
query_id = uuid.uuid4().hex
writeVarUInt(1, ba) # query
writeStringBinary(CLICKHOUSE_QUERY_ID, ba)
writeStringBinary(query_id, ba)
ba.append(1) # INITIAL_QUERY
# client info
serializeClientInfo(ba)
serializeClientInfo(ba, query_id)
writeStringBinary('', ba) # No settings
writeStringBinary('', ba) # No interserver secret

9
tests/queries/0_stateless/02050_clickhouse_client_local_exception.sh
View File

@ -4,5 +4,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
touch test_exception
$CLICKHOUSE_LOCAL --query="SELECT 1 INTO OUTFILE 'test_exception' FORMAT Native" 2>&1 | grep -q "Code: 76. DB::ErrnoException:" && echo 'OK' || echo 'FAIL' ||:
touch "${CLICKHOUSE_TMP}/test_exception"
function cleanup()
{
rm "${CLICKHOUSE_TMP}/test_exception"
}
trap cleanup EXIT
$CLICKHOUSE_LOCAL --query="SELECT 1 INTO OUTFILE '${CLICKHOUSE_TMP}/test_exception' FORMAT Native" 2>&1 | grep -q "Code: 76. DB::ErrnoException:" && echo 'OK' || echo 'FAIL' ||:

1
tests/queries/0_stateless/02122_4letter_words_stress_zookeeper.reference Normal file
View File

@ -0,0 +1 @@
1

47
tests/queries/0_stateless/02122_4letter_words_stress_zookeeper.sh Executable file
View File

@ -0,0 +1,47 @@
#!/usr/bin/env bash
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
function four_letter_thread()
{
declare -a FOUR_LETTER_COMMANDS=("conf" "cons" "crst" "envi" "ruok" "srst" "srvr" "stat" "wchc" "wchs" "dirs" "mntr" "isro")
while true; do
command=${FOUR_LETTER_COMMANDS[$RANDOM % ${#FOUR_LETTER_COMMANDS[@]} ]}
echo $command | nc ${CLICKHOUSE_HOST} ${CLICKHOUSE_PORT_KEEPER} 1>/dev/null
done
}
function create_drop_thread()
{
while true; do
num=$RANDOM
$CLICKHOUSE_CLIENT --query "CREATE TABLE test_table$num (key UInt64, value1 UInt8, value2 UInt8) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/test_table$num', '0') ORDER BY key"
sleep 0.$RANDOM
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test_table$num"
done
}
export -f four_letter_thread;
export -f create_drop_thread;
TIMEOUT=15
timeout $TIMEOUT bash -c four_letter_thread 2> /dev/null &
timeout $TIMEOUT bash -c four_letter_thread 2> /dev/null &
timeout $TIMEOUT bash -c four_letter_thread 2> /dev/null &
timeout $TIMEOUT bash -c four_letter_thread 2> /dev/null &
timeout $TIMEOUT bash -c create_drop_thread 2> /dev/null &
timeout $TIMEOUT bash -c create_drop_thread 2> /dev/null &
timeout $TIMEOUT bash -c create_drop_thread 2> /dev/null &
timeout $TIMEOUT bash -c create_drop_thread 2> /dev/null &
wait
# still alive
$CLICKHOUSE_CLIENT --query "SELECT 1"

1
tests/queries/0_stateless/02124_buffer_with_type_map_long.reference Normal file
View File

@ -0,0 +1 @@
OK

35
tests/queries/0_stateless/02124_buffer_with_type_map_long.sh Executable file
View File

@ -0,0 +1,35 @@
#!/usr/bin/env bash
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_buffer_map"
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_buffer_map(m1 Map(String, UInt64), m2 Map(String, String)) ENGINE = Buffer('', '', 1, 1, 1, 1000000000000, 1000000000000, 1000000000000, 1000000000000)"
function insert1
{
while true; do
$CLICKHOUSE_CLIENT -q "INSERT INTO t_buffer_map SELECT (range(10), range(10)), (range(10), range(10)) from numbers(100)"
done
}
function select1
{
while true; do
$CLICKHOUSE_CLIENT -q "SELECT * FROM t_buffer_map" 2> /dev/null > /dev/null
done
}
TIMEOUT=10
export -f insert1
export -f select1
timeout $TIMEOUT bash -c insert1 &
timeout $TIMEOUT bash -c select1 &
wait
echo "OK"
$CLICKHOUSE_CLIENT -q "DROP TABLE t_buffer_map"

2
tests/queries/shell_config.sh
View File

@ -73,6 +73,8 @@ export CLICKHOUSE_PORT_MYSQL=${CLICKHOUSE_PORT_MYSQL:=$(${CLICKHOUSE_EXTRACT_CON
export CLICKHOUSE_PORT_MYSQL=${CLICKHOUSE_PORT_MYSQL:="9004"}
export CLICKHOUSE_PORT_POSTGRESQL=${CLICKHOUSE_PORT_POSTGRESQL:=$(${CLICKHOUSE_EXTRACT_CONFIG} --try --key=postgresql_port 2>/dev/null)} 2>/dev/null
export CLICKHOUSE_PORT_POSTGRESQL=${CLICKHOUSE_PORT_POSTGRESQL:="9005"}
export CLICKHOUSE_PORT_KEEPER=${CLICKHOUSE_PORT_KEEPER:=$(${CLICKHOUSE_EXTRACT_CONFIG} --try --key=keeper_server.tcp_port 2>/dev/null)} 2>/dev/null
export CLICKHOUSE_PORT_KEEPER=${CLICKHOUSE_PORT_KEEPER:="9181"}
export CLICKHOUSE_CLIENT_SECURE=${CLICKHOUSE_CLIENT_SECURE:=$(echo "${CLICKHOUSE_CLIENT}" | sed 's/'"--port=${CLICKHOUSE_PORT_TCP}"'//g; s/$/'"--secure --port=${CLICKHOUSE_PORT_TCP_SECURE}"'/g')}

52
website/benchmark/hardware/results/gcp_n2d.json
View File

@ -50,5 +50,57 @@
[0.021, 0.015, 0.015],
[0.006, 0.008, 0.006]
]
},
{
"system": "GCP n2d-standard-8",
"system_full": "GCP n2d-standard-8, AMD EPYC 7B12, 8vCPU, 64 GiB RAM",
"time": "2021-11-23 00:00:00",
"kind": "cloud",
"result":
[
[0.002, 0.002, 0.002],
[0.054, 0.019, 0.023],
[0.109, 0.053, 0.061],
[0.282, 0.102, 0.098],
[0.261, 0.183, 0.185],
[0.812, 0.697, 0.699],
[0.058, 0.034, 0.036],
[0.048, 0.023, 0.020],
[1.164, 1.191, 1.206],
[1.332, 1.223, 1.354],
[0.378, 0.388, 0.314],
[0.448, 0.464, 0.376],
[1.593, 1.719, 1.699],
[2.137, 1.940, 2.271],
[1.957, 2.048, 1.895],
[1.797, 1.711, 1.998],
[6.016, 5.934, 5.659],
[3.573, 3.571, 3.396],
[11.690, 11.552, 11.673],
[0.251, 0.124, 0.125],
[3.106, 2.289, 2.330],
[3.490, 2.253, 2.563],
[6.666, 4.699, 5.062],
[8.078, 2.300, 2.726],
[0.883, 0.639, 0.604],
[0.513, 0.534, 0.507],
[0.879, 0.626, 0.625],
[3.068, 1.940, 1.656],
[3.389, 3.633, 3.075],
[2.576, 2.445, 2.623],
[1.431, 1.464, 1.441],
[2.365, 2.076, 2.067],
[10.904, 11.111, 13.093],
[8.407, 7.915, 8.475],
[8.005, 8.124, 9.201],
[2.701, 3.199, 3.036],
[0.244, 0.239, 0.204],
[0.118, 0.092, 0.089],
[0.103, 0.093, 0.060],
[0.539, 0.457, 0.445],
[0.048, 0.022, 0.022],
[0.038, 0.067, 0.016],
[0.011, 0.004, 0.004]
]
}
]

54
website/benchmark/hardware/results/upcloud_8cpu_32gb.json Normal file
View File

@ -0,0 +1,54 @@
[
{
"system": "UpCloud 8CPU 32GiB",
"system_full": "UpCloud 8CPU 32GiB, AMD EPYC 7542",
"time": "2021-11-23 00:00:00",
"kind": "cloud",
"result":
[
[0.002, 0.001, 0.001],
[0.024, 0.020, 0.019],
[0.123, 0.040, 0.048],
[0.448, 0.061, 0.058],
[0.517, 0.125, 0.119],
[0.849, 0.357, 0.350],
[0.037, 0.025, 0.033],
[0.038, 0.018, 0.018],
[0.804, 0.549, 0.551],
[0.920, 0.602, 0.612],
[0.542, 0.234, 0.231],
[0.573, 0.272, 0.274],
[1.032, 0.764, 0.777],
[1.617, 0.991, 1.001],
[1.219, 1.010, 0.994],
[1.056, 0.999, 0.992],
[2.643, 2.488, 2.498],
[1.659, 1.583, 1.533],
[4.957, 4.929, 4.960],
[0.453, 0.056, 0.055],
[6.116, 0.994, 0.951],
[6.914, 1.035, 1.028],
[12.878, 2.481, 2.450],
[9.791, 1.195, 1.139],
[1.682, 0.318, 0.319],
[0.784, 0.286, 0.273],
[1.706, 0.322, 0.318],
[6.060, 0.968, 0.948],
[5.184, 1.488, 1.450],
[1.151, 1.206, 1.219],
[1.520, 0.715, 0.723],
[3.469, 0.910, 0.911],
[5.128, 4.690, 4.771],
[7.003, 3.448, 3.492],
[7.022, 3.478, 3.430],
[1.266, 1.246, 1.257],
[0.225, 0.200, 0.199],
[0.087, 0.079, 0.070],
[0.076, 0.064, 0.063],
[0.450, 0.458, 0.411],
[0.034, 0.024, 0.026],
[0.024, 0.020, 0.021],
[0.006, 0.005, 0.006]
]
}
]