thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-26 09:32:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'ClickHouse:master' into master

Browse Source
This commit is contained in:
michael1589 2021-11-24 21:35:36 +08:00 committed by GitHub
commit 8b99138701
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
119 changed files with 4024 additions and 900 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/master.yml vendored
View File

@ -181,7 +181,7 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 9
BUILD_NUMBER: 8
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH

2
.gitmodules vendored
View File

@ -140,7 +140,7 @@
url = https://github.com/ClickHouse-Extras/libc-headers.git
[submodule "contrib/replxx"]
path = contrib/replxx
url = https://github.com/AmokHuginnsson/replxx.git
url = https://github.com/ClickHouse-Extras/replxx.git
[submodule "contrib/avro"]
path = contrib/avro
url = https://github.com/ClickHouse-Extras/avro.git

8
CMakeLists.txt
View File

@ -392,6 +392,8 @@ if (COMPILER_CLANG)
option(ENABLE_THINLTO "Clang-specific link time optimization" ON)
endif()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fstrict-vtable-pointers")
# Set new experimental pass manager, it's a performance, build time and binary size win.
# Can be removed after https://reviews.llvm.org/D66490 merged and released to at least two versions of clang.
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fexperimental-new-pass-manager")
@ -402,9 +404,9 @@ if (COMPILER_CLANG)
# completely.
if (ENABLE_THINLTO AND NOT ENABLE_TESTS AND NOT SANITIZE)
# Link time optimization
set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -flto=thin")
set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -flto=thin")
set (CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -flto=thin")
set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -flto=thin -fwhole-program-vtables")
set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -flto=thin -fwhole-program-vtables")
set (CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -flto=thin -fwhole-program-vtables")
elseif (ENABLE_THINLTO)
message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot enable ThinLTO")
endif ()

10
base/daemon/BaseDaemon.cpp
View File

@ -13,30 +13,21 @@
#if defined(__linux__)
#include <sys/prctl.h>
#endif
#include <fcntl.h>
#include <errno.h>
#include <string.h>
#include <signal.h>
#include <cxxabi.h>
#include <unistd.h>
#include <typeinfo>
#include <iostream>
#include <fstream>
#include <sstream>
#include <memory>
#include <base/scope_guard.h>
#include <Poco/Observer.h>
#include <Poco/AutoPtr.h>
#include <Poco/PatternFormatter.h>
#include <Poco/Message.h>
#include <Poco/Util/Application.h>
#include <Poco/Exception.h>
#include <Poco/ErrorHandler.h>
#include <Poco/Condition.h>
#include <Poco/SyslogChannel.h>
#include <Poco/DirectoryIterator.h>
#include <base/logger_useful.h>
#include <base/ErrorHandlers.h>
@ -56,7 +47,6 @@
#include <Common/getMultipleKeysFromConfig.h>
#include <Common/ClickHouseRevision.h>
#include <Common/Config/ConfigProcessor.h>
#include <Common/MemorySanitizer.h>
#include <Common/SymbolIndex.h>
#include <Common/getExecutablePath.h>
#include <Common/getHashOfLoadedBinary.h>

4
base/loggers/OwnPatternFormatter.cpp
View File

@ -1,16 +1,12 @@
#include "OwnPatternFormatter.h"
#include <functional>
#include <optional>
#include <sys/time.h>
#include <IO/WriteBufferFromString.h>
#include <IO/WriteHelpers.h>
#include <Common/HashTable/Hash.h>
#include <Interpreters/InternalTextLogsQueue.h>
#include <Common/CurrentThread.h>
#include <base/getThreadId.h>
#include <base/terminalColors.h>
#include "Loggers.h"
OwnPatternFormatter::OwnPatternFormatter(bool color_)

2
contrib/base64 vendored

@ -1 +1 @@
Subproject commit af9b331f2b4f30b41c70f3a571ff904a8251c1d3
Subproject commit 9499e0c4945589973b9ea1bc927377cfbc84aa46

2
contrib/replxx vendored

@ -1 +1 @@
Subproject commit 68410ac01dfb4f09ea76120ac5a2cecda3943aaf
Subproject commit f019cba7ea1bcd1b4feb7826f28ed57fb581b04c

2
docs/en/getting-started/example-datasets/ontime.md
View File

@ -15,7 +15,7 @@ This dataset can be obtained in two ways:
Downloading data:
``` bash
echo https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_{1987..2021}_{1..12}.zip | xargs -P10 wget --no-check-certificate --continue
wget --no-check-certificate --continue https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_{1987..2021}_{1..12}.zip
```
Creating a table:

21
docs/en/introduction/adopters.md
View File

@ -11,6 +11,7 @@ toc_title: Adopters
| Company | Industry | Usecase | Cluster Size | (Un)Compressed Data Size<abbr title="of single replica"><sup>\*</sup></abbr> | Reference |
|---------|----------|---------|--------------|------------------------------------------------------------------------------|-----------|
| <a href="https://2gis.ru" class="favicon">2gis</a> | Maps | Monitoring | — | — | [Talk in Russian, July 2019](https://youtu.be/58sPkXfq6nw) |
| <a href="https://adapty.io/" class="favicon">Adapty</a> | Subscription Analytics | Main product | — | — | [Tweet, November 2021](https://twitter.com/iwitaly/status/1462698148061659139) |
| <a href="https://getadmiral.com/" class="favicon">Admiral</a> | Martech | Engagement Management | — | — | [Webinar Slides, June 2020](https://altinity.com/presentations/2020/06/16/big-data-in-real-time-how-clickhouse-powers-admirals-visitor-relationships-for-publishers) |
| <a href="http://www.adscribe.tv/" class="favicon">AdScribe</a> | Ads | TV Analytics | — | — | [A quote from CTO](https://altinity.com/24x7-support/) |
| <a href="https://ahrefs.com/" class="favicon">Ahrefs</a> | SEO | Analytics | — | — | [Job listing](https://ahrefs.com/jobs/data-scientist-search) |
@ -19,7 +20,7 @@ toc_title: Adopters
| <a href="https://alohabrowser.com/" class="favicon">Aloha Browser</a> | Mobile App | Browser backend | — | — | [Slides in Russian, May 2019](https://presentations.clickhouse.com/meetup22/aloha.pdf) |
| <a href="https://altinity.com/" class="favicon">Altinity</a> | Cloud, SaaS | Main product | — | — | [Official Website](https://altinity.com/) |
| <a href="https://amadeus.com/" class="favicon">Amadeus</a> | Travel | Analytics | — | — | [Press Release, April 2018](https://www.altinity.com/blog/2018/4/5/amadeus-technologies-launches-investment-and-insights-tool-based-on-machine-learning-and-strategy-algorithms) |
| <a href="https://apiroad.net/" class="favicon">ApiRoad</a> | API marketplace | Analytics | — | — | [Blog post, Nov 2018, Mar 2020](https://pixeljets.com/blog/clickhouse-vs-elasticsearch/) |
| <a href="https://apiroad.net/" class="favicon">ApiRoad</a> | API marketplace | Analytics | — | — | [Blog post, November 2018, March 2020](https://pixeljets.com/blog/clickhouse-vs-elasticsearch/) |
| <a href="https://www.appsflyer.com" class="favicon">Appsflyer</a> | Mobile analytics | Main product | — | — | [Talk in Russian, July 2019](https://www.youtube.com/watch?v=M3wbRlcpBbY) |
| <a href="https://arenadata.tech/" class="favicon">ArenaData</a> | Data Platform | Main product | — | — | [Slides in Russian, December 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup38/indexes.pdf) |
| <a href="https://www.argedor.com/en/clickhouse/" class="favicon">Argedor</a> | ClickHouse support | — | — | — | [Official website](https://www.argedor.com/en/clickhouse/) |
@ -50,6 +51,7 @@ toc_title: Adopters
| <a href="https://cryptology.com/" class="favicon">Cryptology</a> | Digital Assets Trading Platform | — | — | — | [Job advertisement, March 2021](https://career.habr.com/companies/cryptology/vacancies) |
| <a href="https://www.chinatelecomglobal.com/" class="favicon">Dataliance for China Telecom</a> | Telecom | Analytics | — | — | [Slides in Chinese, January 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup12/telecom.pdf) |
| <a href="https://db.com" class="favicon">Deutsche Bank</a> | Finance | BI Analytics | — | — | [Slides in English, October 2019](https://bigdatadays.ru/wp-content/uploads/2019/10/D2-H3-3_Yakunin-Goihburg.pdf) |
| <a href="https://www.deepl.com/" class="favicon">Deepl</a> | Machine Learning | — | — | — | [Video, October 2021](https://www.youtube.com/watch?v=WIYJiPwxXdM&t=1182s) |
| <a href="https://deeplay.io/eng/" class="favicon">Deeplay</a> | Gaming Analytics | — | — | — | [Job advertisement, 2020](https://career.habr.com/vacancies/1000062568) |
| <a href="https://www.diva-e.com" class="favicon">Diva-e</a> | Digital consulting | Main Product | — | — | [Slides in English, September 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup29/ClickHouse-MeetUp-Unusual-Applications-sd-2019-09-17.pdf) |
| <a href="https://ecommpay.com/" class="favicon">Ecommpay</a> | Payment Processing | Logs | — | — | [Video, Nov 2019](https://www.youtube.com/watch?v=d3GdZTOWGLk) |
@ -65,6 +67,7 @@ toc_title: Adopters
| <a href="https://gigapipe.com/" class="favicon">Gigapipe</a> | Managed ClickHouse | Main product | — | — | [Official website](https://gigapipe.com/) |
| <a href="https://glaber.io/" class="favicon">Glaber</a> | Monitoring | Main product | — | — | [Website](https://glaber.io/) |
| <a href="https://graphcdn.io/" class="favicon">GraphCDN</a> | CDN | Traffic Analytics | — | — | [Blog Post in English, August 2021](https://altinity.com/blog/delivering-insight-on-graphql-apis-with-clickhouse-at-graphcdn/) |
| <a href="https://www.grouparoo.com" class="favicon">Grouparoo</a> | Data Warehouse Integrations | Main product | — | — | [Official Website, November 2021](https://www.grouparoo.com/integrations) |
| <a href="https://www.huya.com/" class="favicon">HUYA</a> | Video Streaming | Analytics | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/7.%20ClickHouse万亿数据分析实践%20李本旺(sundy-li)%20虎牙.pdf) |
| <a href="https://www.hydrolix.io/" class="favicon">Hydrolix</a> | Cloud data platform | Main product | — | — | [Documentation](https://docs.hydrolix.io/guide/query) |
| <a href="https://www.the-ica.com/" class="favicon">ICA</a> | FinTech | Risk Management | — | — | [Blog Post in English, Sep 2020](https://altinity.com/blog/clickhouse-vs-redshift-performance-for-fintech-risk-management?utm_campaign=ClickHouse%20vs%20RedShift&utm_content=143520807&utm_medium=social&utm_source=twitter&hss_channel=tw-3894792263) |
@ -89,7 +92,7 @@ toc_title: Adopters
| <a href="https://mcs.mail.ru/" class="favicon">Mail.ru Cloud Solutions</a> | Cloud services | Main product | — | — | [Article in Russian](https://mcs.mail.ru/help/db-create/clickhouse#) |
| <a href="https://maxilect.com/" class="favicon">MAXILECT</a> | Ad Tech, Blockchain, ML, AI | — | — | — | [Job advertisement, 2021](https://www.linkedin.com/feed/update/urn:li:activity:6780842017229430784/) |
| <a href="https://tech.mymarilyn.ru" class="favicon">Marilyn</a> | Advertising | Statistics | — | — | [Talk in Russian, June 2017](https://www.youtube.com/watch?v=iXlIgx2khwc) |
| <a href="https://mellodesign.ru/" class="favicon">Mello</a> | Marketing | Analytics | 1 server | — | [Article, Oct 2020](https://vc.ru/marketing/166180-razrabotka-tipovogo-otcheta-skvoznoy-analitiki) |
| <a href="https://mellodesign.ru/" class="favicon">Mello</a> | Marketing | Analytics | 1 server | — | [Article, October 2020](https://vc.ru/marketing/166180-razrabotka-tipovogo-otcheta-skvoznoy-analitiki) |
| <a href="https://www.messagebird.com" class="favicon">MessageBird</a> | Telecommunications | Statistics | — | — | [Slides in English, November 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup20/messagebird.pdf) |
| <a href="https://clarity.microsoft.com/" class="favicon">Microsoft</a> | Web Analytics | Clarity (Main Product) | — | — | [A question on GitHub](https://github.com/ClickHouse/ClickHouse/issues/21556) |
| <a href="https://www.mindsdb.com/" class="favicon">MindsDB</a> | Machine Learning | Main Product | — | — | [Official Website](https://www.mindsdb.com/blog/machine-learning-models-as-tables-in-ch) |
@ -100,17 +103,16 @@ toc_title: Adopters
| <a href="https://getnoc.com/" class="favicon">NOC Project</a> | Network Monitoring | Analytics | Main Product | — | [Official Website](https://getnoc.com/features/big-data/) |
| <a href="https://www.noction.com" class="favicon">Noction</a> | Network Technology | Main Product | — | — | [Official Website](https://www.noction.com/news/irp-3-11-remote-triggered-blackholing-capability)
| <a href="https://www.nuna.com/" class="favicon">Nuna Inc.</a> | Health Data Analytics | — | — | — | [Talk in English, July 2020](https://youtu.be/GMiXCMFDMow?t=170) |
| <a href="https://ok.ru" class="favicon">Ok.ru</a> | Social Network | — | 72 servers | 810 TB compressed, 50bn rows/day, 1.5 TB/day | [SmartData conference, Oct 2021](https://assets.ctfassets.net/oxjq45e8ilak/4JPHkbJenLgZhBGGyyonFP/57472ec6987003ec4078d0941740703b/____________________ClickHouse_______________________.pdf) |
| <a href="https://omnicomm.ru/" class="favicon">Omnicomm</a> | Transportation Monitoring | — | — | — | [Facebook post, Oct 2021](https://www.facebook.com/OmnicommTeam/posts/2824479777774500) |
| <a href="https://ok.ru" class="favicon">Ok.ru</a> | Social Network | — | 72 servers | 810 TB compressed, 50bn rows/day, 1.5 TB/day | [SmartData conference, October 2021](https://assets.ctfassets.net/oxjq45e8ilak/4JPHkbJenLgZhBGGyyonFP/57472ec6987003ec4078d0941740703b/____________________ClickHouse_______________________.pdf) |
| <a href="https://omnicomm.ru/" class="favicon">Omnicomm</a> | Transportation Monitoring | — | — | — | [Facebook post, October 2021](https://www.facebook.com/OmnicommTeam/posts/2824479777774500) |
| <a href="https://www.oneapm.com/" class="favicon">OneAPM</a> | Monitoring and Data Analysis | Main product | — | — | [Slides in Chinese, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup19/8.%20clickhouse在OneAPM的应用%20杜龙.pdf) |
| <a href="https://www.opentargets.org/" class="favicon">Open Targets</a> | Genome Research | Genome Search | — | — | [Tweet, Oct 2021](https://twitter.com/OpenTargets/status/1452570865342758913?s=20), [Blog](https://blog.opentargets.org/graphql/) |
| <a href="https://www.opentargets.org/" class="favicon">Open Targets</a> | Genome Research | Genome Search | — | — | [Tweet, October 2021](https://twitter.com/OpenTargets/status/1452570865342758913?s=20), [Blog](https://blog.opentargets.org/graphql/) |
| <a href="https://corp.ozon.com/" class="favicon">OZON</a> | E-commerce | — | — | — | [Official website](https://job.ozon.ru/vacancy/razrabotchik-clickhouse-ekspluatatsiya-40991870/) |
| <a href="https://panelbear.com/" class="favicon">Panelbear | Analytics | Monitoring and Analytics | — | — | [Tech Stack, November 2020](https://panelbear.com/blog/tech-stack/) |
| <a href="https://www.percent.cn/" class="favicon">Percent 百分点</a> | Analytics | Main Product | — | — | [Slides in Chinese, June 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup24/4.%20ClickHouse万亿数据双中心的设计与实践%20.pdf) |
| <a href="https://www.percona.com/" class="favicon">Percona</a> | Performance analysis | Percona Monitoring and Management | — | — | [Official website, Mar 2020](https://www.percona.com/blog/2020/03/30/advanced-query-analysis-in-percona-monitoring-and-management-with-direct-clickhouse-access/) |
| <a href="https://piwik.pro/" class="favicon">Piwik PRO</a> | Web Analytics | Main Product | — | — | [Official website, Dec 2018](https://piwik.pro/blog/piwik-pro-clickhouse-faster-efficient-reports/) |
| <a href="https://plausible.io/" class="favicon">Plausible</a> | Analytics | Main Product | — | — | [Blog post, June 2020](https://twitter.com/PlausibleHQ/status/1273889629087969280) |
| <a href="https://posthog.com/" class="favicon">PostHog</a> | Product Analytics | Main Product | — | — | [Release Notes, Oct 2020](https://posthog.com/blog/the-posthog-array-1-15-0) |
| <a href="https://posthog.com/" class="favicon">PostHog</a> | Product Analytics | Main Product | — | — | [Release Notes, October 2020](https://posthog.com/blog/the-posthog-array-1-15-0), [Blog, November 2021](https://posthog.com/blog/how-we-turned-clickhouse-into-our-eventmansion) |
| <a href="https://postmates.com/" class="favicon">Postmates</a> | Delivery | — | — | — | [Talk in English, July 2020](https://youtu.be/GMiXCMFDMow?t=188) |
| <a href="http://www.pragma-innovation.fr/" class="favicon">Pragma Innovation</a> | Telemetry and Big Data Analysis | Main product | — | — | [Slides in English, October 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup18/4_pragma_innovation.pdf) |
| <a href="https://prana-system.com/en/" class="favicon">PRANA</a> | Industrial predictive analytics | Main product | — | — | [News (russian), Feb 2021](https://habr.com/en/news/t/541392/) |
@ -152,6 +154,7 @@ toc_title: Adopters
| <a href="https://trafficstars.com/" class="favicon">Traffic Stars</a> | AD network | — | 300 servers in Europe/US | 1.8 PiB, 700 000 insert rps (as of 2021) | [Slides in Russian, May 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup15/lightning/ninja.pdf) |
| <a href="https://www.uber.com" class="favicon">Uber</a> | Taxi | Logging | — | — | [Slides, February 2020](https://presentations.clickhouse.com/meetup40/uber.pdf) |
| <a href="https://hello.utmstat.com/" class="favicon">UTMSTAT</a> | Analytics | Main product | — | — | [Blog post, June 2020](https://vc.ru/tribuna/133956-striming-dannyh-iz-servisa-skvoznoy-analitiki-v-clickhouse) |
| <a href="https://vercel.com/" class="favicon">Vercel</a> | Traffic and Performance Analytics | — | — | — | Direct reference, October 2021 |
| <a href="https://vk.com" class="favicon">VKontakte</a> | Social Network | Statistics, Logging | — | — | [Slides in Russian, August 2018](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup17/3_vk.pdf) |
| <a href="https://www.vmware.com/" class="favicon">VMware</a> | Cloud | VeloCloud, SDN | — | — | [Product documentation](https://docs.vmware.com/en/vRealize-Operations-Manager/8.3/com.vmware.vcom.metrics.doc/GUID-A9AD72E1-C948-4CA2-971B-919385AB3CA8.html) |
| <a href="https://www.walmartlabs.com/" class="favicon">Walmart Labs</a> | Internet, Retail | — | — | — | [Talk in English, July 2020](https://youtu.be/GMiXCMFDMow?t=144) |
@ -167,6 +170,7 @@ toc_title: Adopters
| <a href="https://market.yandex.ru/" class="favicon">Yandex Market</a> | e-Commerce | Metrics, Logging | — | — | [Talk in Russian, January 2019](https://youtu.be/_l1qP0DyBcA?t=478) |
| <a href="https://metrica.yandex.com" class="favicon">Yandex Metrica</a> | Web analytics | Main product | 630 servers in one cluster, 360 servers in another cluster, 1862 servers in one department | 133 PiB / 8.31 PiB / 120 trillion records | [Slides, February 2020](https://presentations.clickhouse.com/meetup40/introduction/#13) |
| <a href="https://www.yotascale.com/" class="favicon">Yotascale</a> | Cloud | Data pipeline | — | 2 bn records/day | [LinkedIn (Accomplishments)](https://www.linkedin.com/in/adilsaleem/) |
| <a href="https://www.your-analytics.org/" class="favicon">Your Analytics</a> | Product Analytics | Main Product | — | - | [Tweet, November 2021](https://twitter.com/mikenikles/status/1459737241165565953) |
| <a href="https://zagravagames.com/en/" class="favicon">Zagrava Trading</a> | — | — | — | — | [Job offer, May 2021](https://twitter.com/datastackjobs/status/1394707267082063874) |
| <a href="https://htc-cs.ru/" class="favicon">ЦВТ</a> | Software Development | Metrics, Logging | — | — | [Blog Post, March 2019, in Russian](https://vc.ru/dev/62715-kak-my-stroili-monitoring-na-prometheus-clickhouse-i-elk) |
| <a href="https://mkb.ru/" class="favicon">МКБ</a> | Bank | Web-system monitoring | — | — | [Slides in Russian, September 2019](https://github.com/ClickHouse/clickhouse-presentations/blob/master/meetup28/mkb.pdf) |
@ -174,8 +178,5 @@ toc_title: Adopters
| <a href="https://promo.croc.ru/digitalworker" class="favicon">Цифровой Рабочий</a> | Industrial IoT, Analytics | — | — | — | [Blog post in Russian, March 2021](https://habr.com/en/company/croc/blog/548018/) |
| <a href="https://shop.okraina.ru/" class="favicon">ООО «МПЗ Богородский»</a> | Agriculture | — | — | — | [Article in Russian, November 2020](https://cloud.yandex.ru/cases/okraina) |
| <a href="https://domclick.ru/" class="favicon">ДомКлик</a> | Real Estate | — | — | — | [Article in Russian, October 2021](https://habr.com/ru/company/domclick/blog/585936/) |
| <a href="https://www.deepl.com/" class="favicon">Deepl</a> | Machine Learning | — | — | — | [Video, October 2021](https://www.youtube.com/watch?v=WIYJiPwxXdM&t=1182s) |
| <a href="https://vercel.com/" class="favicon">Vercel</a> | Traffic and Performance Analytics | — | — | — | Direct reference, October 2021 |
| <a href="https://www.your-analytics.org/" class="favicon">YourAnalytics</a> | Web Analytics | — | — | — | [Tweet, Nov 2021](https://twitter.com/mikenikles/status/1460860140249235461) |
[Original article](https://clickhouse.com/docs/en/introduction/adopters/) <!--hide-->

191
docs/en/operations/clickhouse-keeper.md
View File

@ -49,6 +49,7 @@ Internal coordination settings are located in `<keeper_server>.<coordination_set
- `auto_forwarding` — Allow to forward write requests from followers to the leader (default: true).
- `shutdown_timeout` — Wait to finish internal connections and shutdown (ms) (default: 5000).
- `startup_timeout` — If the server doesn't connect to other quorum participants in the specified timeout it will terminate (ms) (default: 30000).
- `four_letter_word_white_list` — White list of 4lw commands (default: "conf,cons,crst,envi,ruok,srst,srvr,stat,wchc,wchs,dirs,mntr,isro").
Quorum configuration is located in `<keeper_server>.<raft_configuration>` section and contain servers description.
@ -104,6 +105,196 @@ ClickHouse Keeper is bundled into the ClickHouse server package, just add config
clickhouse-keeper --config /etc/your_path_to_config/config.xml --daemon
```
## Four Latter Word Commands
ClickHouse Keeper also provides 4lw commands which are almost the same with Zookeeper. Each command is composed of four letters such as `mntr`, `stat` etc. There are some more interesting commands: `stat` gives some general information about the server and connected clients, while `srvr` and `cons` give extended details on server and connections respectively.
The 4lw commands has a white list configuration `four_letter_word_white_list` which has default value "conf,cons,crst,envi,ruok,srst,srvr,stat,wchc,wchs,dirs,mntr,isro".
You can issue the commands to ClickHouse Keeper via telnet or nc, at the client port.
```
echo mntr | nc localhost 9181
```
Bellow is the detailed 4lw commands:
- ruok : Tests if server is running in a non-error state. The server will respond with imok if it is running. Otherwise it will not respond at all. A response of "imok" does not necessarily indicate that the server has joined the quorum, just that the server process is active and bound to the specified client port. Use "stat" for details on state wrt quorum and client connection information.
```
imok
```
- mntr : Outputs a list of variables that could be used for monitoring the health of the cluster.
```
zk_version v21.11.1.1-prestable-7a4a0b0edef0ad6e0aa662cd3b90c3f4acf796e7
zk_avg_latency 0
zk_max_latency 0
zk_min_latency 0
zk_packets_received 68
zk_packets_sent 68
zk_num_alive_connections 1
zk_outstanding_requests 0
zk_server_state leader
zk_znode_count 4
zk_watch_count 1
zk_ephemerals_count 0
zk_approximate_data_size 723
zk_open_file_descriptor_count 310
zk_max_file_descriptor_count 10240
zk_followers 0
zk_synced_followers 0
```
- srvr : Lists full details for the server.
```
ClickHouse Keeper version: v21.11.1.1-prestable-7a4a0b0edef0ad6e0aa662cd3b90c3f4acf796e7
Latency min/avg/max: 0/0/0
Received: 2
Sent : 2
Connections: 1
Outstanding: 0
Zxid: 34
Mode: leader
Node count: 4
```
- stat : Lists brief details for the server and connected clients.
```
ClickHouse Keeper version: v21.11.1.1-prestable-7a4a0b0edef0ad6e0aa662cd3b90c3f4acf796e7
Clients:
192.168.1.1:52852(recved=0,sent=0)
192.168.1.1:52042(recved=24,sent=48)
Latency min/avg/max: 0/0/0
Received: 4
Sent : 4
Connections: 1
Outstanding: 0
Zxid: 36
Mode: leader
Node count: 4
```
- srst : Reset server statistics. The command will affect the result of `srvr`, `mntr` and `stat`.
```
Server stats reset.
```
- conf : Print details about serving configuration.
```
server_id=1
tcp_port=2181
four_letter_word_white_list=*
log_storage_path=./coordination/logs
snapshot_storage_path=./coordination/snapshots
max_requests_batch_size=100
session_timeout_ms=30000
operation_timeout_ms=10000
dead_session_check_period_ms=500
heart_beat_interval_ms=500
election_timeout_lower_bound_ms=1000
election_timeout_upper_bound_ms=2000
reserved_log_items=1000000000000000
snapshot_distance=10000
auto_forwarding=true
shutdown_timeout=5000
startup_timeout=240000
raft_logs_level=information
snapshots_to_keep=3
rotate_log_storage_interval=100000
stale_log_gap=10000
fresh_log_gap=200
max_requests_batch_size=100
quorum_reads=false
force_sync=false
compress_logs=true
compress_snapshots_with_zstd_format=true
configuration_change_tries_count=20
```
- cons : List full connection/session details for all clients connected to this server. Includes information on numbers of packets received/sent, session id, operation latencies, last operation performed, etc...
```
192.168.1.1:52163(recved=0,sent=0,sid=0xffffffffffffffff,lop=NA,est=1636454787393,to=30000,lzxid=0xffffffffffffffff,lresp=0,llat=0,minlat=0,avglat=0,maxlat=0)
192.168.1.1:52042(recved=9,sent=18,sid=0x0000000000000001,lop=List,est=1636454739887,to=30000,lcxid=0x0000000000000005,lzxid=0x0000000000000005,lresp=1636454739892,llat=0,minlat=0,avglat=0,maxlat=0)
```
- crst : Reset connection/session statistics for all connections.
```
Connection stats reset.
```
- envi : Print details about serving environment
```
Environment:
clickhouse.keeper.version=v21.11.1.1-prestable-7a4a0b0edef0ad6e0aa662cd3b90c3f4acf796e7
host.name=ZBMAC-C02D4054M.local
os.name=Darwin
os.arch=x86_64
os.version=19.6.0
cpu.count=12
user.name=root
user.home=/Users/JackyWoo/
user.dir=/Users/JackyWoo/project/jd/clickhouse/cmake-build-debug/programs/
user.tmp=/var/folders/b4/smbq5mfj7578f2jzwn602tt40000gn/T/
```
- dirs : Shows the total size of snapshot and log files in bytes
```
snapshot_dir_size: 0
log_dir_size: 3875
```
- isro: Tests if server is running in read-only mode. The server will respond with "ro" if in read-only mode or "rw" if not in read-only mode.
```
rw
```
- wchs : Lists brief information on watches for the server.
```
1 connections watching 1 paths
Total watches:1
```
- wchc : Lists detailed information on watches for the server, by session. This outputs a list of sessions(connections) with associated watches (paths). Note, depending on the number of watches this operation may be expensive (ie impact server performance), use it carefully.
```
0x0000000000000001
/clickhouse/task_queue/ddl
```
- wchp : Lists detailed information on watches for the server, by path. This outputs a list of paths (znodes) with associated sessions. Note, depending on the number of watches this operation may be expensive (ie impact server performance), use it carefully.
```
/clickhouse/task_queue/ddl
0x0000000000000001
```
- dump : Lists the outstanding sessions and ephemeral nodes. This only works on the leader.
```
Sessions dump (2):
0x0000000000000001
0x0000000000000002
Sessions with Ephemerals (1):
0x0000000000000001
/clickhouse/task_queue/ddl
```
## [experimental] Migration from ZooKeeper
Seamlessly migration from ZooKeeper to ClickHouse Keeper is impossible you have to stop your ZooKeeper cluster, convert data and start ClickHouse Keeper. `clickhouse-keeper-converter` tool allows converting ZooKeeper logs and snapshots to ClickHouse Keeper snapshot. It works only with ZooKeeper > 3.4. Steps for migration:

2
docs/en/sql-reference/functions/string-search-functions.md
View File

@ -373,7 +373,7 @@ The same as `multiMatchAny`, but returns the array of all indicies that match th
## multiFuzzyMatchAny(haystack, distance, \[pattern<sub>1</sub>, pattern<sub>2</sub>, …, pattern<sub>n</sub>\]) {#multifuzzymatchanyhaystack-distance-pattern1-pattern2-patternn}
The same as `multiMatchAny`, but returns 1 if any pattern matches the haystack within a constant [edit distance](https://en.wikipedia.org/wiki/Edit_distance). This function is also in an experimental mode and can be extremely slow. For more information see [hyperscan documentation](https://intel.github.io/hyperscan/dev-reference/compilation.html#approximate-matching).
The same as `multiMatchAny`, but returns 1 if any pattern matches the haystack within a constant [edit distance](https://en.wikipedia.org/wiki/Edit_distance). This function relies on the experimental feature of [hyperscan](https://intel.github.io/hyperscan/dev-reference/compilation.html#approximate-matching) library, and can be slow for some corner cases. The performance depends on the edit distance value and patterns used, but it's always more expensive compared to a non-fuzzy variants.
## multiFuzzyMatchAnyIndex(haystack, distance, \[pattern<sub>1</sub>, pattern<sub>2</sub>, …, pattern<sub>n</sub>\]) {#multifuzzymatchanyindexhaystack-distance-pattern1-pattern2-patternn}

3
programs/keeper/Keeper.cpp
View File

@ -20,6 +20,7 @@
#include <Poco/Environment.h>
#include <sys/stat.h>
#include <pwd.h>
#include <Coordination/FourLetterCommand.h>
#if !defined(ARCADIA_BUILD)
# include "config_core.h"
@ -367,6 +368,8 @@ int Keeper::main(const std::vector<std::string> & /*args*/)
/// Initialize keeper RAFT. Do nothing if no keeper_server in config.
global_context->initializeKeeperDispatcher(/* start_async = */false);
FourLetterCommandFactory::registerCommands(*global_context->getKeeperDispatcher());
for (const auto & listen_host : listen_hosts)
{
/// TCP Keeper

4
programs/server/Server.cpp
View File

@ -111,7 +111,8 @@
#endif
#if USE_NURAFT
# include <Server/KeeperTCPHandlerFactory.h>
# include <Coordination/FourLetterCommand.h>
# include <Server/KeeperTCPHandlerFactory.h>
#endif
#if USE_BASE64
@ -1025,6 +1026,7 @@ if (ThreadFuzzer::instance().isEffective())
}
/// Initialize keeper RAFT.
global_context->initializeKeeperDispatcher(can_initialize_keeper_async);
FourLetterCommandFactory::registerCommands(*global_context->getKeeperDispatcher());
for (const auto & listen_host : listen_hosts)
{

2
src/Columns/ColumnAggregateFunction.cpp
View File

@ -518,7 +518,7 @@ StringRef ColumnAggregateFunction::serializeValueIntoArena(size_t n, Arena & are
{
WriteBufferFromArena out(arena, begin);
func->serialize(data[n], out);
return out.finish();
return out.complete();
}
const char * ColumnAggregateFunction::deserializeAndInsertFromArena(const char * src_arena)

46
src/Common/getCurrentProcessFDCount.cpp Normal file
View File

@ -0,0 +1,46 @@
#include <Common/getCurrentProcessFDCount.h>
#include <Common/ShellCommand.h>
#include <IO/WriteBufferFromString.h>
#include <unistd.h>
#include <fmt/format.h>
#include <IO/ReadHelpers.h>
#include <filesystem>
int getCurrentProcessFDCount()
{
namespace fs = std::filesystem;
int result = -1;
#if defined(__linux__) || defined(__APPLE__)
using namespace DB;
Int32 pid = getpid();
auto proc_fs_path = fmt::format("/proc/{}/fd", pid);
if (fs::exists(proc_fs_path))
{
result = std::distance(fs::directory_iterator(proc_fs_path), fs::directory_iterator{});
}
else if (fs::exists("/dev/fd"))
{
result = std::distance(fs::directory_iterator("/dev/fd"), fs::directory_iterator{});
}
else
{
/// Then try lsof command
String by_lsof = fmt::format("lsof -p {} | wc -l", pid);
auto command = ShellCommand::execute(by_lsof);
try
{
readIntText(result, command->out);
command->wait();
}
catch (...)
{
}
}
#endif
return result;
}

5
src/Common/getCurrentProcessFDCount.h Normal file
View File

@ -0,0 +1,5 @@
#pragma once
/// Get current process file descriptor count
/// @return -1 os doesn't support "lsof" command or some error occurs.
int getCurrentProcessFDCount();

36
src/Common/getMaxFileDescriptorCount.cpp Normal file
View File

@ -0,0 +1,36 @@
#include <IO/ReadHelpers.h>
#include <IO/WriteBufferFromString.h>
#include <IO/ReadBufferFromFile.h>
#include <Common/ShellCommand.h>
#include <Common/getMaxFileDescriptorCount.h>
#include <filesystem>
int getMaxFileDescriptorCount()
{
namespace fs = std::filesystem;
int result = -1;
#if defined(__linux__) || defined(__APPLE__)
using namespace DB;
if (fs::exists("/proc/sys/fs/file-max"))
{
ReadBufferFromFile reader("/proc/sys/fs/file-max");
readIntText(result, reader);
}
else
{
auto command = ShellCommand::execute("ulimit -n");
try
{
readIntText(result, command->out);
command->wait();
}
catch (...)
{
}
}
#endif
return result;
}

6
src/Common/getMaxFileDescriptorCount.h Normal file
View File

@ -0,0 +1,6 @@
#pragma once
/// Get process max file descriptor count
/// @return -1 if os does not support ulimit command or some error occurs
int getMaxFileDescriptorCount();

17
src/Compression/CompressedWriteBuffer.cpp
View File

@ -7,9 +7,6 @@
#include "CompressedWriteBuffer.h"
#include <Compression/CompressionFactory.h>
#include <Common/MemorySanitizer.h>
#include <Common/MemoryTracker.h>
namespace DB
{
@ -35,13 +32,11 @@ void CompressedWriteBuffer::nextImpl()
out.write(compressed_buffer.data(), compressed_size);
}
void CompressedWriteBuffer::finalize()
CompressedWriteBuffer::~CompressedWriteBuffer()
{
next();
finalize();
}
CompressedWriteBuffer::CompressedWriteBuffer(
WriteBuffer & out_,
CompressionCodecPtr codec_,
@ -50,12 +45,4 @@ CompressedWriteBuffer::CompressedWriteBuffer(
{
}
CompressedWriteBuffer::~CompressedWriteBuffer()
{
/// FIXME move final flush into the caller
MemoryTracker::LockExceptionInThread lock;
next();
}
}

18
src/Compression/CompressedWriteBuffer.h
View File

@ -15,21 +15,13 @@ namespace DB
class CompressedWriteBuffer : public BufferWithOwnMemory<WriteBuffer>
{
private:
WriteBuffer & out;
CompressionCodecPtr codec;
PODArray<char> compressed_buffer;
void nextImpl() override;
public:
CompressedWriteBuffer(
WriteBuffer & out_,
CompressionCodecPtr codec_ = CompressionCodecFactory::instance().getDefaultCodec(),
size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE);
void finalize() override;
~CompressedWriteBuffer() override;
/// The amount of compressed data
size_t getCompressedBytes()
@ -51,7 +43,13 @@ public:
return offset();
}
~CompressedWriteBuffer() override;
private:
void nextImpl() override;
WriteBuffer & out;
CompressionCodecPtr codec;
PODArray<char> compressed_buffer;
};
}

18
src/Coordination/Changelog.cpp
View File

@ -79,7 +79,7 @@ class ChangelogWriter
public:
ChangelogWriter(const std::string & filepath_, WriteMode mode, uint64_t start_index_)
: filepath(filepath_)
, file_buf(filepath, DBMS_DEFAULT_BUFFER_SIZE, mode == WriteMode::Rewrite ? -1 : (O_APPEND | O_CREAT | O_WRONLY))
, file_buf(std::make_unique<WriteBufferFromFile>(filepath, DBMS_DEFAULT_BUFFER_SIZE, mode == WriteMode::Rewrite ? -1 : (O_APPEND | O_CREAT | O_WRONLY)))
, start_index(start_index_)
{
auto compression_method = chooseCompressionMethod(filepath_, "");
@ -89,7 +89,7 @@ public:
}
else if (compression_method == CompressionMethod::Zstd)
{
compressed_buffer = std::make_unique<ZstdDeflatingAppendableWriteBuffer>(file_buf, /* compression level = */ 3, /* append_to_existing_stream = */ mode == WriteMode::Append);
compressed_buffer = std::make_unique<ZstdDeflatingAppendableWriteBuffer>(std::move(file_buf), /* compression level = */ 3, /* append_to_existing_stream = */ mode == WriteMode::Append);
}
else
{
@ -120,12 +120,14 @@ public:
compressed_buffer->next();
}
/// Flush working buffer to file system
file_buf.next();
WriteBuffer * working_buf = compressed_buffer ? compressed_buffer->getNestedBuffer() : file_buf.get();
/// Flush working buffer to file system
working_buf->next();
/// Fsync file system if needed
if (force_fsync)
file_buf.sync();
working_buf->sync();
}
uint64_t getStartIndex() const
@ -138,12 +140,12 @@ private:
{
if (compressed_buffer)
return *compressed_buffer;
return file_buf;
return *file_buf;
}
std::string filepath;
WriteBufferFromFile file_buf;
std::unique_ptr<WriteBuffer> compressed_buffer;
std::unique_ptr<WriteBufferFromFile> file_buf;
std::unique_ptr<WriteBufferWithOwnMemoryDecorator> compressed_buffer;
uint64_t start_index;
};

2
src/Coordination/Changelog.h
View File

@ -11,7 +11,7 @@
namespace DB
{
using Checksum = UInt64;
using Checksum = uint64_t;
using LogEntryPtr = nuraft::ptr<nuraft::log_entry>;
using LogEntries = std::vector<LogEntryPtr>;

177
src/Coordination/CoordinationSettings.cpp
View File

@ -1,6 +1,10 @@
#include <Coordination/CoordinationSettings.h>
#include <Core/Settings.h>
#include <base/logger_useful.h>
#include <filesystem>
#include <Coordination/Defines.h>
#include <IO/WriteHelpers.h>
#include <IO/WriteIntText.h>
namespace DB
{
@ -32,4 +36,177 @@ void CoordinationSettings::loadFromConfig(const String & config_elem, const Poco
}
}
const String KeeperConfigurationAndSettings::DEFAULT_FOUR_LETTER_WORD_CMD = "conf,cons,crst,envi,ruok,srst,srvr,stat,wchc,wchs,dirs,mntr,isro";
KeeperConfigurationAndSettings::KeeperConfigurationAndSettings()
: server_id(NOT_EXIST)
, tcp_port(NOT_EXIST)
, tcp_port_secure(NOT_EXIST)
, standalone_keeper(false)
, coordination_settings(std::make_shared<CoordinationSettings>())
{
}
void KeeperConfigurationAndSettings::dump(WriteBufferFromOwnString & buf) const
{
auto write_int = [&buf](int64_t value)
{
writeIntText(value, buf);
buf.write('\n');
};
auto write_bool = [&buf](bool value)
{
String str_val = value ? "true" : "false";
writeText(str_val, buf);
buf.write('\n');
};
writeText("server_id=", buf);
write_int(server_id);
if (tcp_port != NOT_EXIST)
{
writeText("tcp_port=", buf);
write_int(tcp_port);
}
if (tcp_port_secure != NOT_EXIST)
{
writeText("tcp_port_secure=", buf);
write_int(tcp_port_secure);
}
writeText("four_letter_word_white_list=", buf);
writeText(four_letter_word_white_list, buf);
buf.write('\n');
writeText("log_storage_path=", buf);
writeText(log_storage_path, buf);
buf.write('\n');
writeText("snapshot_storage_path=", buf);
writeText(snapshot_storage_path, buf);
buf.write('\n');
/// coordination_settings
writeText("max_requests_batch_size=", buf);
write_int(coordination_settings->max_requests_batch_size);
writeText("session_timeout_ms=", buf);
write_int(uint64_t(coordination_settings->session_timeout_ms));
writeText("operation_timeout_ms=", buf);
write_int(uint64_t(coordination_settings->operation_timeout_ms));
writeText("dead_session_check_period_ms=", buf);
write_int(uint64_t(coordination_settings->dead_session_check_period_ms));
writeText("heart_beat_interval_ms=", buf);
write_int(uint64_t(coordination_settings->heart_beat_interval_ms));
writeText("election_timeout_lower_bound_ms=", buf);
write_int(uint64_t(coordination_settings->election_timeout_lower_bound_ms));
writeText("election_timeout_upper_bound_ms=", buf);
write_int(uint64_t(coordination_settings->election_timeout_upper_bound_ms));
writeText("reserved_log_items=", buf);
write_int(coordination_settings->reserved_log_items);
writeText("snapshot_distance=", buf);
write_int(coordination_settings->snapshot_distance);
writeText("auto_forwarding=", buf);
write_bool(coordination_settings->auto_forwarding);
writeText("shutdown_timeout=", buf);
write_int(uint64_t(coordination_settings->shutdown_timeout));
writeText("startup_timeout=", buf);
write_int(uint64_t(coordination_settings->startup_timeout));
writeText("raft_logs_level=", buf);
writeText(coordination_settings->raft_logs_level.toString(), buf);
buf.write('\n');
writeText("snapshots_to_keep=", buf);
write_int(coordination_settings->snapshots_to_keep);
writeText("rotate_log_storage_interval=", buf);
write_int(coordination_settings->rotate_log_storage_interval);
writeText("stale_log_gap=", buf);
write_int(coordination_settings->stale_log_gap);
writeText("fresh_log_gap=", buf);
write_int(coordination_settings->fresh_log_gap);
writeText("max_requests_batch_size=", buf);
write_int(coordination_settings->max_requests_batch_size);
writeText("quorum_reads=", buf);
write_bool(coordination_settings->quorum_reads);
writeText("force_sync=", buf);
write_bool(coordination_settings->force_sync);
writeText("compress_logs=", buf);
write_bool(coordination_settings->compress_logs);
writeText("compress_snapshots_with_zstd_format=", buf);
write_bool(coordination_settings->compress_snapshots_with_zstd_format);
writeText("configuration_change_tries_count=", buf);
write_int(coordination_settings->configuration_change_tries_count);
}
KeeperConfigurationAndSettingsPtr
KeeperConfigurationAndSettings::loadFromConfig(const Poco::Util::AbstractConfiguration & config, bool standalone_keeper_)
{
std::shared_ptr<KeeperConfigurationAndSettings> ret = std::make_shared<KeeperConfigurationAndSettings>();
ret->server_id = config.getInt("keeper_server.server_id");
ret->standalone_keeper = standalone_keeper_;
if (config.has("keeper_server.tcp_port"))
{
ret->tcp_port = config.getInt("keeper_server.tcp_port");
}
if (config.has("keeper_server.tcp_port_secure"))
{
ret->tcp_port_secure = config.getInt("keeper_server.tcp_port_secure");
}
if (config.has("keeper_server.superdigest"))
{
ret->super_digest = config.getString("keeper_server.superdigest");
}
ret->four_letter_word_white_list = config.getString("keeper_server.four_letter_word_white_list", DEFAULT_FOUR_LETTER_WORD_CMD);
ret->log_storage_path = getLogsPathFromConfig(config, standalone_keeper_);
ret->snapshot_storage_path = getSnapshotsPathFromConfig(config, standalone_keeper_);
ret->coordination_settings->loadFromConfig("keeper_server.coordination_settings", config);
return ret;
}
String KeeperConfigurationAndSettings::getLogsPathFromConfig(const Poco::Util::AbstractConfiguration & config, bool standalone_keeper_)
{
/// the most specialized path
if (config.has("keeper_server.log_storage_path"))
return config.getString("keeper_server.log_storage_path");
if (config.has("keeper_server.storage_path"))
return std::filesystem::path{config.getString("keeper_server.storage_path")} / "logs";
if (standalone_keeper_)
return std::filesystem::path{config.getString("path", KEEPER_DEFAULT_PATH)} / "logs";
else
return std::filesystem::path{config.getString("path", DBMS_DEFAULT_PATH)} / "coordination/logs";
}
String KeeperConfigurationAndSettings::getSnapshotsPathFromConfig(const Poco::Util::AbstractConfiguration & config, bool standalone_keeper_)
{
/// the most specialized path
if (config.has("keeper_server.snapshot_storage_path"))
return config.getString("keeper_server.snapshot_storage_path");
if (config.has("keeper_server.storage_path"))
return std::filesystem::path{config.getString("keeper_server.storage_path")} / "snapshots";
if (standalone_keeper_)
return std::filesystem::path{config.getString("path", KEEPER_DEFAULT_PATH)} / "snapshots";
else
return std::filesystem::path{config.getString("path", DBMS_DEFAULT_PATH)} / "coordination/snapshots";
}
}

35
src/Coordination/CoordinationSettings.h
View File

@ -5,6 +5,7 @@
#include <Core/SettingsEnums.h>
#include <Common/ZooKeeper/ZooKeeperConstants.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <IO/WriteBufferFromString.h>
namespace DB
{
@ -51,4 +52,38 @@ struct CoordinationSettings : public BaseSettings<CoordinationSettingsTraits>
using CoordinationSettingsPtr = std::shared_ptr<CoordinationSettings>;
/// Coordination settings + some other parts of keeper configuration
/// which are not stored in settings. Allows to dump configuration
/// with 4lw commands.
struct KeeperConfigurationAndSettings
{
static constexpr int NOT_EXIST = -1;
static const String DEFAULT_FOUR_LETTER_WORD_CMD;
KeeperConfigurationAndSettings();
int server_id;
int tcp_port;
int tcp_port_secure;
String four_letter_word_white_list;
String super_digest;
bool standalone_keeper;
CoordinationSettingsPtr coordination_settings;
String log_storage_path;
String snapshot_storage_path;
void dump(WriteBufferFromOwnString & buf) const;
static std::shared_ptr<KeeperConfigurationAndSettings> loadFromConfig(const Poco::Util::AbstractConfiguration & config, bool standalone_keeper_);
private:
static String getLogsPathFromConfig(const Poco::Util::AbstractConfiguration & config, bool standalone_keeper_);
static String getSnapshotsPathFromConfig(const Poco::Util::AbstractConfiguration & config, bool standalone_keeper_);
};
using KeeperConfigurationAndSettingsPtr = std::shared_ptr<KeeperConfigurationAndSettings>;
}

418
src/Coordination/FourLetterCommand.cpp Normal file
View File

@ -0,0 +1,418 @@
#include <Coordination/FourLetterCommand.h>
#include <Coordination/KeeperDispatcher.h>
#include <Server/KeeperTCPHandler.h>
#include <base/logger_useful.h>
#include <Poco/Environment.h>
#include <Poco/Path.h>
#include <Common/getCurrentProcessFDCount.h>
#include <Common/getMaxFileDescriptorCount.h>
#include <Common/StringUtils/StringUtils.h>
#include <Coordination/Keeper4LWInfo.h>
#include <unistd.h>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
}
IFourLetterCommand::IFourLetterCommand(KeeperDispatcher & keeper_dispatcher_)
: keeper_dispatcher(keeper_dispatcher_)
{
}
int32_t IFourLetterCommand::code()
{
return toCode(name());
}
String IFourLetterCommand::toName(int32_t code)
{
int reverted_code = __builtin_bswap32(code);
return String(reinterpret_cast<char *>(&reverted_code), 4);
}
int32_t IFourLetterCommand::toCode(const String & name)
{
int32_t res = *reinterpret_cast<const int32_t *>(name.data());
/// keep consistent with Coordination::read method by changing big endian to little endian.
return __builtin_bswap32(res);
}
IFourLetterCommand::~IFourLetterCommand() = default;
FourLetterCommandFactory & FourLetterCommandFactory::instance()
{
static FourLetterCommandFactory factory;
return factory;
}
void FourLetterCommandFactory::checkInitialization() const
{
if (!initialized)
throw Exception("Four letter command not initialized", ErrorCodes::LOGICAL_ERROR);
}
bool FourLetterCommandFactory::isKnown(int32_t code)
{
checkInitialization();
return commands.contains(code);
}
FourLetterCommandPtr FourLetterCommandFactory::get(int32_t code)
{
checkInitialization();
return commands.at(code);
}
void FourLetterCommandFactory::registerCommand(FourLetterCommandPtr & command)
{
if (commands.contains(command->code()))
throw Exception("Four letter command " + command->name() + " already registered", ErrorCodes::LOGICAL_ERROR);
commands.emplace(command->code(), std::move(command));
}
void FourLetterCommandFactory::registerCommands(KeeperDispatcher & keeper_dispatcher)
{
FourLetterCommandFactory & factory = FourLetterCommandFactory::instance();
if (!factory.isInitialized())
{
FourLetterCommandPtr ruok_command = std::make_shared<RuokCommand>(keeper_dispatcher);
factory.registerCommand(ruok_command);
FourLetterCommandPtr mntr_command = std::make_shared<MonitorCommand>(keeper_dispatcher);
factory.registerCommand(mntr_command);
FourLetterCommandPtr conf_command = std::make_shared<ConfCommand>(keeper_dispatcher);
factory.registerCommand(conf_command);
FourLetterCommandPtr cons_command = std::make_shared<ConsCommand>(keeper_dispatcher);
factory.registerCommand(cons_command);
FourLetterCommandPtr brief_watch_command = std::make_shared<BriefWatchCommand>(keeper_dispatcher);
factory.registerCommand(brief_watch_command);
FourLetterCommandPtr data_size_command = std::make_shared<DataSizeCommand>(keeper_dispatcher);
factory.registerCommand(data_size_command);
FourLetterCommandPtr dump_command = std::make_shared<DumpCommand>(keeper_dispatcher);
factory.registerCommand(dump_command);
FourLetterCommandPtr envi_command = std::make_shared<EnviCommand>(keeper_dispatcher);
factory.registerCommand(envi_command);
FourLetterCommandPtr is_rad_only_command = std::make_shared<IsReadOnlyCommand>(keeper_dispatcher);
factory.registerCommand(is_rad_only_command);
FourLetterCommandPtr rest_conn_stats_command = std::make_shared<RestConnStatsCommand>(keeper_dispatcher);
factory.registerCommand(rest_conn_stats_command);
FourLetterCommandPtr server_stat_command = std::make_shared<ServerStatCommand>(keeper_dispatcher);
factory.registerCommand(server_stat_command);
FourLetterCommandPtr stat_command = std::make_shared<StatCommand>(keeper_dispatcher);
factory.registerCommand(stat_command);
FourLetterCommandPtr stat_reset_command = std::make_shared<StatResetCommand>(keeper_dispatcher);
factory.registerCommand(stat_reset_command);
FourLetterCommandPtr watch_by_path_command = std::make_shared<WatchByPathCommand>(keeper_dispatcher);
factory.registerCommand(watch_by_path_command);
FourLetterCommandPtr watch_command = std::make_shared<WatchCommand>(keeper_dispatcher);
factory.registerCommand(watch_command);
factory.initializeWhiteList(keeper_dispatcher);
factory.setInitialize(true);
}
}
bool FourLetterCommandFactory::isEnabled(int32_t code)
{
checkInitialization();
if (!white_list.empty() && *white_list.cbegin() == WHITE_LIST_ALL)
return true;
return std::find(white_list.begin(), white_list.end(), code) != white_list.end();
}
void FourLetterCommandFactory::initializeWhiteList(KeeperDispatcher & keeper_dispatcher)
{
const auto & keeper_settings = keeper_dispatcher.getKeeperConfigurationAndSettings();
String list_str = keeper_settings->four_letter_word_white_list;
Strings tokens;
splitInto<','>(tokens, list_str);
for (String token: tokens)
{
trim(token);
if (token == "*")
{
white_list.clear();
white_list.push_back(WHITE_LIST_ALL);
return;
}
else
{
if (commands.contains(IFourLetterCommand::toCode(token)))
{
white_list.push_back(IFourLetterCommand::toCode(token));
}
else
{
auto * log = &Poco::Logger::get("FourLetterCommandFactory");
LOG_WARNING(log, "Find invalid keeper 4lw command {} when initializing, ignore it.", token);
}
}
}
}
String RuokCommand::run()
{
return "imok";
}
namespace
{
void print(IFourLetterCommand::StringBuffer & buf, const String & key, const String & value)
{
writeText("zk_", buf);
writeText(key, buf);
writeText('\t', buf);
writeText(value, buf);
writeText('\n', buf);
}
void print(IFourLetterCommand::StringBuffer & buf, const String & key, uint64_t value)
{
print(buf, key, toString(value));
}
}
String MonitorCommand::run()
{
KeeperConnectionStats stats = keeper_dispatcher.getKeeperConnectionStats();
Keeper4LWInfo keeper_info = keeper_dispatcher.getKeeper4LWInfo();
if (!keeper_info.has_leader)
return "This instance is not currently serving requests";
const auto & state_machine = keeper_dispatcher.getStateMachine();
StringBuffer ret;
print(ret, "version", String(VERSION_DESCRIBE) + "-" + VERSION_GITHASH);
print(ret, "avg_latency", stats.getAvgLatency());
print(ret, "max_latency", stats.getMaxLatency());
print(ret, "min_latency", stats.getMinLatency());
print(ret, "packets_received", stats.getPacketsReceived());
print(ret, "packets_sent", stats.getPacketsSent());
print(ret, "num_alive_connections", keeper_info.alive_connections_count);
print(ret, "outstanding_requests", keeper_info.outstanding_requests_count);
print(ret, "server_state", keeper_info.getRole());
print(ret, "znode_count", state_machine.getNodesCount());
print(ret, "watch_count", state_machine.getTotalWatchesCount());
print(ret, "ephemerals_count", state_machine.getTotalEphemeralNodesCount());
print(ret, "approximate_data_size", state_machine.getApproximateDataSize());
#if defined(__linux__) || defined(__APPLE__)
print(ret, "open_file_descriptor_count", getCurrentProcessFDCount());
print(ret, "max_file_descriptor_count", getMaxFileDescriptorCount());
#endif
if (keeper_info.is_leader)
{
print(ret, "followers", keeper_info.follower_count);
print(ret, "synced_followers", keeper_info.synced_follower_count);
}
return ret.str();
}
String StatResetCommand::run()
{
keeper_dispatcher.resetConnectionStats();
return "Server stats reset.\n";
}
String NopCommand::run()
{
return "";
}
String ConfCommand::run()
{
StringBuffer buf;
keeper_dispatcher.getKeeperConfigurationAndSettings()->dump(buf);
return buf.str();
}
String ConsCommand::run()
{
StringBuffer buf;
KeeperTCPHandler::dumpConnections(buf, false);
return buf.str();
}
String RestConnStatsCommand::run()
{
KeeperTCPHandler::resetConnsStats();
return "Connection stats reset.\n";
}
String ServerStatCommand::run()
{
StringBuffer buf;
auto write = [&buf](const String & key, const String & value)
{
writeText(key, buf);
writeText(": ", buf);
writeText(value, buf);
writeText('\n', buf);
};
KeeperConnectionStats stats = keeper_dispatcher.getKeeperConnectionStats();
Keeper4LWInfo keeper_info = keeper_dispatcher.getKeeper4LWInfo();
write("ClickHouse Keeper version", String(VERSION_DESCRIBE) + "-" + VERSION_GITHASH);
StringBuffer latency;
latency << stats.getMinLatency() << "/" << stats.getAvgLatency() << "/" << stats.getMaxLatency();
write("Latency min/avg/max", latency.str());
write("Received", toString(stats.getPacketsReceived()));
write("Sent ", toString(stats.getPacketsSent()));
write("Connections", toString(keeper_info.alive_connections_count));
write("Outstanding", toString(keeper_info.outstanding_requests_count));
write("Zxid", toString(keeper_info.last_zxid));
write("Mode", keeper_info.getRole());
write("Node count", toString(keeper_info.total_nodes_count));
return buf.str();
}
String StatCommand::run()
{
StringBuffer buf;
auto write = [&buf] (const String & key, const String & value) { buf << key << ": " << value << '\n'; };
KeeperConnectionStats stats = keeper_dispatcher.getKeeperConnectionStats();
Keeper4LWInfo keeper_info = keeper_dispatcher.getKeeper4LWInfo();
write("ClickHouse Keeper version", String(VERSION_DESCRIBE) + "-" + VERSION_GITHASH);
buf << "Clients:\n";
KeeperTCPHandler::dumpConnections(buf, true);
buf << '\n';
StringBuffer latency;
latency << stats.getMinLatency() << "/" << stats.getAvgLatency() << "/" << stats.getMaxLatency();
write("Latency min/avg/max", latency.str());
write("Received", toString(stats.getPacketsReceived()));
write("Sent ", toString(stats.getPacketsSent()));
write("Connections", toString(keeper_info.alive_connections_count));
write("Outstanding", toString(keeper_info.outstanding_requests_count));
write("Zxid", toString(keeper_info.last_zxid));
write("Mode", keeper_info.getRole());
write("Node count", toString(keeper_info.total_nodes_count));
return buf.str();
}
String BriefWatchCommand::run()
{
StringBuffer buf;
const auto & state_machine = keeper_dispatcher.getStateMachine();
buf << state_machine.getSessionsWithWatchesCount() << " connections watching "
<< state_machine.getWatchedPathsCount() << " paths\n";
buf << "Total watches:" << state_machine.getTotalWatchesCount() << "\n";
return buf.str();
}
String WatchCommand::run()
{
StringBuffer buf;
const auto & state_machine = keeper_dispatcher.getStateMachine();
state_machine.dumpWatches(buf);
return buf.str();
}
String WatchByPathCommand::run()
{
StringBuffer buf;
const auto & state_machine = keeper_dispatcher.getStateMachine();
state_machine.dumpWatchesByPath(buf);
return buf.str();
}
String DataSizeCommand::run()
{
StringBuffer buf;
buf << "snapshot_dir_size: " << keeper_dispatcher.getSnapDirSize() << '\n';
buf << "log_dir_size: " << keeper_dispatcher.getLogDirSize() << '\n';
return buf.str();
}
String DumpCommand::run()
{
StringBuffer buf;
const auto & state_machine = keeper_dispatcher.getStateMachine();
state_machine.dumpSessionsAndEphemerals(buf);
return buf.str();
}
String EnviCommand::run()
{
using Poco::Environment;
using Poco::Path;
StringBuffer buf;
buf << "Environment:\n";
buf << "clickhouse.keeper.version=" << (String(VERSION_DESCRIBE) + "-" + VERSION_GITHASH) << '\n';
buf << "host.name=" << Environment::nodeName() << '\n';
buf << "os.name=" << Environment::osDisplayName() << '\n';
buf << "os.arch=" << Environment::osArchitecture() << '\n';
buf << "os.version=" << Environment::osVersion() << '\n';
buf << "cpu.count=" << Environment::processorCount() << '\n';
String os_user;
os_user.resize(256, '\0');
if (0 == getlogin_r(os_user.data(), os_user.size() - 1))
os_user.resize(strlen(os_user.c_str()));
else
os_user.clear(); /// Don't mind if we cannot determine user login.
buf << "user.name=" << os_user << '\n';
buf << "user.home=" << Path::home() << '\n';
buf << "user.dir=" << Path::current() << '\n';
buf << "user.tmp=" << Path::temp() << '\n';
return buf.str();
}
String IsReadOnlyCommand::run()
{
if (keeper_dispatcher.isObserver())
return "ro";
else
return "rw";
}
}

307
src/Coordination/FourLetterCommand.h Normal file
View File

@ -0,0 +1,307 @@
#pragma once
#include <sstream>
#include <string>
#include <unordered_map>
#include <Coordination/KeeperDispatcher.h>
#include <IO/WriteBufferFromString.h>
#include <Common/config_version.h>
namespace DB
{
struct IFourLetterCommand;
using FourLetterCommandPtr = std::shared_ptr<DB::IFourLetterCommand>;
/// Just like zookeeper Four Letter Words commands, CH Keeper responds to a small set of commands.
/// Each command is composed of four letters, these commands are useful to monitor and issue system problems.
/// The feature is based on Zookeeper 3.5.9, details is in https://zookeeper.apache.org/doc/r3.5.9/zookeeperAdmin.html#sc_zkCommands.
struct IFourLetterCommand
{
public:
using StringBuffer = DB::WriteBufferFromOwnString;
explicit IFourLetterCommand(KeeperDispatcher & keeper_dispatcher_);
virtual String name() = 0;
virtual String run() = 0;
virtual ~IFourLetterCommand();
int32_t code();
static String toName(int32_t code);
static inline int32_t toCode(const String & name);
protected:
KeeperDispatcher & keeper_dispatcher;
};
struct FourLetterCommandFactory : private boost::noncopyable
{
public:
using Commands = std::unordered_map<int32_t, FourLetterCommandPtr>;
using WhiteList = std::vector<int32_t>;
///represent '*' which is used in white list
static constexpr int32_t WHITE_LIST_ALL = 0;
bool isKnown(int32_t code);
bool isEnabled(int32_t code);
FourLetterCommandPtr get(int32_t code);
/// There is no need to make it thread safe, because registration is no initialization and get is after startup.
void registerCommand(FourLetterCommandPtr & command);
void initializeWhiteList(KeeperDispatcher & keeper_dispatcher);
void checkInitialization() const;
bool isInitialized() const { return initialized; }
void setInitialize(bool flag) { initialized = flag; }
static FourLetterCommandFactory & instance();
static void registerCommands(KeeperDispatcher & keeper_dispatcher);
private:
std::atomic<bool> initialized = false;
Commands commands;
WhiteList white_list;
};
/**Tests if server is running in a non-error state. The server will respond with imok if it is running.
* Otherwise it will not respond at all.
*
* A response of "imok" does not necessarily indicate that the server has joined the quorum,
* just that the server process is active and bound to the specified client port.
* Use "stat" for details on state wrt quorum and client connection information.
*/
struct RuokCommand : public IFourLetterCommand
{
explicit RuokCommand(KeeperDispatcher & keeper_dispatcher_) : IFourLetterCommand(keeper_dispatcher_) { }
String name() override { return "ruok"; }
String run() override;
~RuokCommand() override = default;
};
/**
* Outputs a list of variables that could be used for monitoring the health of the cluster.
*
* echo mntr | nc localhost 2181
* zk_version 3.5.9
* zk_avg_latency 0
* zk_max_latency 0
* zk_min_latency 0
* zk_packets_received 70
* zk_packets_sent 69
* zk_outstanding_requests 0
* zk_server_state leader
* zk_znode_count 4
* zk_watch_count 0
* zk_ephemerals_count 0
* zk_approximate_data_size 27
* zk_open_file_descriptor_count 23 - only available on Unix platforms
* zk_max_file_descriptor_count 1024 - only available on Unix platforms
* zk_followers 2 - only exposed by the Leader
* zk_synced_followers 2 - only exposed by the Leader
* zk_pending_syncs 0 - only exposed by the Leader
*/
struct MonitorCommand : public IFourLetterCommand
{
explicit MonitorCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "mntr"; }
String run() override;
~MonitorCommand() override = default;
};
struct StatResetCommand : public IFourLetterCommand
{
explicit StatResetCommand(KeeperDispatcher & keeper_dispatcher_) :
IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "srst"; }
String run() override;
~StatResetCommand() override = default;
};
/// A command that does not do anything except reply to client with predefined message.
///It is used to inform clients who execute none white listed four letter word commands.
struct NopCommand : public IFourLetterCommand
{
explicit NopCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "nopc"; }
String run() override;
~NopCommand() override = default;
};
struct ConfCommand : public IFourLetterCommand
{
explicit ConfCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "conf"; }
String run() override;
~ConfCommand() override = default;
};
/// List full connection/session details for all clients connected to this server.
/// Includes information on numbers of packets received/sent, session id, operation latencies, last operation performed, etc...
struct ConsCommand : public IFourLetterCommand
{
explicit ConsCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "cons"; }
String run() override;
~ConsCommand() override = default;
};
/// Reset connection/session statistics for all connections.
struct RestConnStatsCommand : public IFourLetterCommand
{
explicit RestConnStatsCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "crst"; }
String run() override;
~RestConnStatsCommand() override = default;
};
/// Lists full details for the server.
struct ServerStatCommand : public IFourLetterCommand
{
explicit ServerStatCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "srvr"; }
String run() override;
~ServerStatCommand() override = default;
};
/// Lists brief details for the server and connected clients.
struct StatCommand : public IFourLetterCommand
{
explicit StatCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "stat"; }
String run() override;
~StatCommand() override = default;
};
/// Lists brief information on watches for the server.
struct BriefWatchCommand : public IFourLetterCommand
{
explicit BriefWatchCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "wchs"; }
String run() override;
~BriefWatchCommand() override = default;
};
/// Lists detailed information on watches for the server, by session.
/// This outputs a list of sessions(connections) with associated watches (paths).
/// Note, depending on the number of watches this operation may be expensive (ie impact server performance), use it carefully.
struct WatchCommand : public IFourLetterCommand
{
explicit WatchCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "wchc"; }
String run() override;
~WatchCommand() override = default;
};
/// Lists detailed information on watches for the server, by path.
/// This outputs a list of paths (znodes) with associated sessions.
/// Note, depending on the number of watches this operation may be expensive (ie impact server performance), use it carefully.
struct WatchByPathCommand : public IFourLetterCommand
{
explicit WatchByPathCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "wchp"; }
String run() override;
~WatchByPathCommand() override = default;
};
/// Lists the outstanding sessions and ephemeral nodes. This only works on the leader.
struct DumpCommand : public IFourLetterCommand
{
explicit DumpCommand(KeeperDispatcher & keeper_dispatcher_):
IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "dump"; }
String run() override;
~DumpCommand() override = default;
};
/// Print details about serving environment
struct EnviCommand : public IFourLetterCommand
{
explicit EnviCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "envi"; }
String run() override;
~EnviCommand() override = default;
};
/// Shows the total size of snapshot and log files in bytes
struct DataSizeCommand : public IFourLetterCommand
{
explicit DataSizeCommand(KeeperDispatcher & keeper_dispatcher_):
IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "dirs"; }
String run() override;
~DataSizeCommand() override = default;
};
/// Tests if server is running in read-only mode.
/// The server will respond with "ro" if in read-only mode or "rw" if not in read-only mode.
struct IsReadOnlyCommand : public IFourLetterCommand
{
explicit IsReadOnlyCommand(KeeperDispatcher & keeper_dispatcher_)
: IFourLetterCommand(keeper_dispatcher_)
{
}
String name() override { return "isro"; }
String run() override;
~IsReadOnlyCommand() override = default;
};
}

47
src/Coordination/Keeper4LWInfo.h Normal file
View File

@ -0,0 +1,47 @@
#pragma once
#include <string>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
}
/// Keeper server related information for different 4lw commands
struct Keeper4LWInfo
{
bool is_leader;
bool is_observer;
bool is_follower;
bool is_standalone;
bool has_leader;
uint64_t alive_connections_count;
uint64_t outstanding_requests_count;
uint64_t follower_count;
uint64_t synced_follower_count;
uint64_t total_nodes_count;
int64_t last_zxid;
String getRole() const
{
if (is_standalone)
return "standalone";
if (is_leader)
return "leader";
if (is_observer)
return "observer";
if (is_follower)
return "follower";
throw Exception(ErrorCodes::LOGICAL_ERROR, "RAFT server has undefined state state, it's a bug");
}
};
}

85
src/Coordination/KeeperConnectionStats.cpp Normal file
View File

@ -0,0 +1,85 @@
#include <Coordination/KeeperConnectionStats.h>
namespace DB
{
uint64_t KeeperConnectionStats::getMinLatency() const
{
return min_latency;
}
uint64_t KeeperConnectionStats::getMaxLatency() const
{
return max_latency;
}
uint64_t KeeperConnectionStats::getAvgLatency() const
{
if (count != 0)
return total_latency / count;
return 0;
}
uint64_t KeeperConnectionStats::getLastLatency() const
{
return last_latency;
}
uint64_t KeeperConnectionStats::getPacketsReceived() const
{
return packets_received;
}
uint64_t KeeperConnectionStats::getPacketsSent() const
{
return packets_sent;
}
void KeeperConnectionStats::incrementPacketsReceived()
{
packets_received++;
}
void KeeperConnectionStats::incrementPacketsSent()
{
packets_sent++;
}
void KeeperConnectionStats::updateLatency(uint64_t latency_ms)
{
last_latency = latency_ms;
total_latency += (latency_ms);
count++;
if (latency_ms < min_latency)
{
min_latency = latency_ms;
}
if (latency_ms > max_latency)
{
max_latency = latency_ms;
}
}
void KeeperConnectionStats::reset()
{
resetLatency();
resetRequestCounters();
}
void KeeperConnectionStats::resetLatency()
{
total_latency = 0;
count = 0;
max_latency = 0;
min_latency = 0;
}
void KeeperConnectionStats::resetRequestCounters()
{
packets_received = 0;
packets_sent = 0;
}
}

52
src/Coordination/KeeperConnectionStats.h Normal file
View File

@ -0,0 +1,52 @@
#pragma once
#include <base/types.h>
#include <memory>
#include <cstdint>
namespace DB
{
/// Request statistics for connection or dispatcher
class KeeperConnectionStats
{
public:
KeeperConnectionStats() = default;
uint64_t getMinLatency() const;
uint64_t getMaxLatency() const;
uint64_t getAvgLatency() const;
uint64_t getLastLatency() const;
uint64_t getPacketsReceived() const;
uint64_t getPacketsSent() const;
void incrementPacketsReceived();
void incrementPacketsSent();
void updateLatency(uint64_t latency_ms);
void reset();
private:
void resetLatency();
void resetRequestCounters();
/// all response with watch response included
uint64_t packets_sent = 0;
/// All user requests
uint64_t packets_received = 0;
/// For consistent with zookeeper measured by millisecond,
/// otherwise maybe microsecond is better
uint64_t total_latency = 0;
uint64_t max_latency = 0;
uint64_t min_latency = 0;
/// last operation latency
uint64_t last_latency = 0;
uint64_t count = 0;
};
}

93
src/Coordination/KeeperDispatcher.cpp
View File

@ -3,6 +3,12 @@
#include <Common/ZooKeeper/KeeperException.h>
#include <future>
#include <chrono>
#include <Poco/Path.h>
#include <Common/hex.h>
#include <filesystem>
#include <Common/checkStackSize.h>
namespace fs = std::filesystem;
namespace DB
{
@ -14,9 +20,10 @@ namespace ErrorCodes
extern const int SYSTEM_ERROR;
}
KeeperDispatcher::KeeperDispatcher()
: coordination_settings(std::make_shared<CoordinationSettings>())
, responses_queue(std::numeric_limits<size_t>::max())
: responses_queue(std::numeric_limits<size_t>::max())
, configuration_and_settings(std::make_shared<KeeperConfigurationAndSettings>())
, log(&Poco::Logger::get("KeeperDispatcher"))
{
}
@ -36,7 +43,8 @@ void KeeperDispatcher::requestThread()
{
KeeperStorage::RequestForSession request;
UInt64 max_wait = UInt64(coordination_settings->operation_timeout_ms.totalMilliseconds());
auto coordination_settings = configuration_and_settings->coordination_settings;
uint64_t max_wait = coordination_settings->operation_timeout_ms.totalMilliseconds();
uint64_t max_batch_size = coordination_settings->max_requests_batch_size;
/// The code below do a very simple thing: batch all write (quorum) requests into vector until
@ -141,7 +149,7 @@ void KeeperDispatcher::responseThread()
{
KeeperStorage::ResponseForSession response_for_session;
UInt64 max_wait = UInt64(coordination_settings->operation_timeout_ms.totalMilliseconds());
uint64_t max_wait = configuration_and_settings->coordination_settings->operation_timeout_ms.totalMilliseconds();
if (responses_queue.tryPop(response_for_session, max_wait))
{
@ -245,28 +253,25 @@ bool KeeperDispatcher::putRequest(const Coordination::ZooKeeperRequestPtr & requ
if (!requests_queue->push(std::move(request_info)))
throw Exception("Cannot push request to queue", ErrorCodes::SYSTEM_ERROR);
}
else if (!requests_queue->tryPush(std::move(request_info), coordination_settings->operation_timeout_ms.totalMilliseconds()))
else if (!requests_queue->tryPush(std::move(request_info), configuration_and_settings->coordination_settings->operation_timeout_ms.totalMilliseconds()))
{
throw Exception("Cannot push request to queue within operation timeout", ErrorCodes::TIMEOUT_EXCEEDED);
}
return true;
}
void KeeperDispatcher::initialize(const Poco::Util::AbstractConfiguration & config, bool standalone_keeper, bool start_async)
{
LOG_DEBUG(log, "Initializing storage dispatcher");
int myid = config.getInt("keeper_server.server_id");
coordination_settings->loadFromConfig("keeper_server.coordination_settings", config);
requests_queue = std::make_unique<RequestsQueue>(coordination_settings->max_requests_batch_size);
configuration_and_settings = KeeperConfigurationAndSettings::loadFromConfig(config, standalone_keeper);
requests_queue = std::make_unique<RequestsQueue>(configuration_and_settings->coordination_settings->max_requests_batch_size);
request_thread = ThreadFromGlobalPool([this] { requestThread(); });
responses_thread = ThreadFromGlobalPool([this] { responseThread(); });
snapshot_thread = ThreadFromGlobalPool([this] { snapshotThread(); });
server = std::make_unique<KeeperServer>(
myid, coordination_settings, config, responses_queue, snapshots_queue, standalone_keeper);
server = std::make_unique<KeeperServer>(configuration_and_settings, config, responses_queue, snapshots_queue);
try
{
@ -413,7 +418,8 @@ void KeeperDispatcher::sessionCleanerTask()
tryLogCurrentException(__PRETTY_FUNCTION__);
}
std::this_thread::sleep_for(std::chrono::milliseconds(coordination_settings->dead_session_check_period_ms.totalMilliseconds()));
auto time_to_sleep = configuration_and_settings->coordination_settings->dead_session_check_period_ms.totalMilliseconds();
std::this_thread::sleep_for(std::chrono::milliseconds(time_to_sleep));
}
}
@ -580,4 +586,67 @@ void KeeperDispatcher::updateConfiguration(const Poco::Util::AbstractConfigurati
}
}
void KeeperDispatcher::updateKeeperStatLatency(uint64_t process_time_ms)
{
std::lock_guard lock(keeper_stats_mutex);
keeper_stats.updateLatency(process_time_ms);
}
static uint64_t getDirSize(const fs::path & dir)
{
checkStackSize();
if (!fs::exists(dir))
return 0;
fs::directory_iterator it(dir);
fs::directory_iterator end;
uint64_t size{0};
while (it != end)
{
if (it->is_regular_file())
size += fs::file_size(*it);
else
size += getDirSize(it->path());
++it;
}
return size;
}
uint64_t KeeperDispatcher::getLogDirSize() const
{
return getDirSize(configuration_and_settings->log_storage_path);
}
uint64_t KeeperDispatcher::getSnapDirSize() const
{
return getDirSize(configuration_and_settings->snapshot_storage_path);
}
Keeper4LWInfo KeeperDispatcher::getKeeper4LWInfo() const
{
Keeper4LWInfo result;
result.is_follower = server->isFollower();
result.is_standalone = !result.is_follower && server->getFollowerCount() == 0;
result.is_leader = isLeader();
result.is_observer = server->isObserver();
result.has_leader = hasLeader();
{
std::lock_guard lock(push_request_mutex);
result.outstanding_requests_count = requests_queue->size();
}
{
std::lock_guard lock(session_to_response_callback_mutex);
result.alive_connections_count = session_to_response_callback.size();
}
if (result.is_leader)
{
result.follower_count = server->getFollowerCount();
result.synced_follower_count = server->getSyncedFollowerCount();
}
result.total_nodes_count = server->getKeeperStateMachine()->getNodesCount();
result.last_zxid = server->getKeeperStateMachine()->getLastProcessedZxid();
return result;
}
}

76
src/Coordination/KeeperDispatcher.h
View File

@ -13,22 +13,20 @@
#include <functional>
#include <Coordination/KeeperServer.h>
#include <Coordination/CoordinationSettings.h>
#include <Coordination/Keeper4LWInfo.h>
#include <Coordination/KeeperConnectionStats.h>
namespace DB
{
using ZooKeeperResponseCallback = std::function<void(const Coordination::ZooKeeperResponsePtr & response)>;
/// Highlevel wrapper for ClickHouse Keeper.
/// Process user requests via consensus and return responses.
class KeeperDispatcher
{
private:
std::mutex push_request_mutex;
mutable std::mutex push_request_mutex;
CoordinationSettingsPtr coordination_settings;
using RequestsQueue = ConcurrentBoundedQueue<KeeperStorage::RequestForSession>;
using SessionToResponseCallback = std::unordered_map<int64_t, ZooKeeperResponseCallback>;
using UpdateConfigurationQueue = ConcurrentBoundedQueue<ConfigUpdateAction>;
@ -43,7 +41,7 @@ private:
std::atomic<bool> shutdown_called{false};
std::mutex session_to_response_callback_mutex;
mutable std::mutex session_to_response_callback_mutex;
/// These two maps looks similar, but serves different purposes.
/// The first map is subscription map for normal responses like
/// (get, set, list, etc.). Dispatcher determines callback for each response
@ -70,6 +68,11 @@ private:
/// RAFT wrapper.
std::unique_ptr<KeeperServer> server;
mutable std::mutex keeper_stats_mutex;
KeeperConnectionStats keeper_stats;
KeeperConfigurationAndSettingsPtr configuration_and_settings;
Poco::Logger * log;
/// Counter for new session_id requests.
@ -123,6 +126,18 @@ public:
/// Put request to ClickHouse Keeper
bool putRequest(const Coordination::ZooKeeperRequestPtr & request, int64_t session_id);
/// Get new session ID
int64_t getSessionID(int64_t session_timeout_ms);
/// Register session and subscribe for responses with callback
void registerSession(int64_t session_id, ZooKeeperResponseCallback callback);
/// Call if we don't need any responses for this session no more (session was expired)
void finishSession(int64_t session_id);
/// Invoked when a request completes.
void updateKeeperStatLatency(uint64_t process_time_ms);
/// Are we leader
bool isLeader() const
{
@ -134,14 +149,51 @@ public:
return server->isLeaderAlive();
}
/// Get new session ID
int64_t getSessionID(int64_t session_timeout_ms);
bool isObserver() const
{
return server->isObserver();
}
/// Register session and subscribe for responses with callback
void registerSession(int64_t session_id, ZooKeeperResponseCallback callback);
uint64_t getLogDirSize() const;
/// Call if we don't need any responses for this session no more (session was expired)
void finishSession(int64_t session_id);
uint64_t getSnapDirSize() const;
/// Request statistics such as qps, latency etc.
KeeperConnectionStats getKeeperConnectionStats() const
{
std::lock_guard lock(keeper_stats_mutex);
return keeper_stats;
}
Keeper4LWInfo getKeeper4LWInfo() const;
const KeeperStateMachine & getStateMachine() const
{
return *server->getKeeperStateMachine();
}
const KeeperConfigurationAndSettingsPtr & getKeeperConfigurationAndSettings() const
{
return configuration_and_settings;
}
void incrementPacketsSent()
{
std::lock_guard lock(keeper_stats_mutex);
keeper_stats.incrementPacketsSent();
}
void incrementPacketsReceived()
{
std::lock_guard lock(keeper_stats_mutex);
keeper_stats.incrementPacketsReceived();
}
void resetConnectionStats()
{
std::lock_guard lock(keeper_stats_mutex);
keeper_stats.reset();
}
};
}

72
src/Coordination/KeeperServer.cpp
View File

@ -61,27 +61,12 @@ void setSSLParams(nuraft::asio_service::options & asio_opts)
}
#endif
std::string getSnapshotsPathFromConfig(const Poco::Util::AbstractConfiguration & config, bool standalone_keeper)
std::string checkAndGetSuperdigest(const String & user_and_digest)
{
/// the most specialized path
if (config.has("keeper_server.snapshot_storage_path"))
return config.getString("keeper_server.snapshot_storage_path");
if (config.has("keeper_server.storage_path"))
return std::filesystem::path{config.getString("keeper_server.storage_path")} / "snapshots";
if (standalone_keeper)
return std::filesystem::path{config.getString("path", KEEPER_DEFAULT_PATH)} / "snapshots";
else
return std::filesystem::path{config.getString("path", DBMS_DEFAULT_PATH)} / "coordination/snapshots";
}
std::string checkAndGetSuperdigest(const Poco::Util::AbstractConfiguration & config)
{
if (!config.has("keeper_server.superdigest"))
if (user_and_digest.empty())
return "";
auto user_and_digest = config.getString("keeper_server.superdigest");
std::vector<std::string> scheme_and_id;
boost::split(scheme_and_id, user_and_digest, [](char c) { return c == ':'; });
if (scheme_and_id.size() != 2 || scheme_and_id[0] != "super")
@ -93,20 +78,18 @@ std::string checkAndGetSuperdigest(const Poco::Util::AbstractConfiguration & con
}
KeeperServer::KeeperServer(
int server_id_,
const CoordinationSettingsPtr & coordination_settings_,
const KeeperConfigurationAndSettingsPtr & configuration_and_settings_,
const Poco::Util::AbstractConfiguration & config,
ResponsesQueue & responses_queue_,
SnapshotsQueue & snapshots_queue_,
bool standalone_keeper)
: server_id(server_id_)
, coordination_settings(coordination_settings_)
SnapshotsQueue & snapshots_queue_)
: server_id(configuration_and_settings_->server_id)
, coordination_settings(configuration_and_settings_->coordination_settings)
, state_machine(nuraft::cs_new<KeeperStateMachine>(
responses_queue_, snapshots_queue_,
getSnapshotsPathFromConfig(config, standalone_keeper),
configuration_and_settings_->snapshot_storage_path,
coordination_settings,
checkAndGetSuperdigest(config)))
, state_manager(nuraft::cs_new<KeeperStateManager>(server_id, "keeper_server", config, coordination_settings, standalone_keeper))
checkAndGetSuperdigest(configuration_and_settings_->super_digest)))
, state_manager(nuraft::cs_new<KeeperStateManager>(server_id, "keeper_server", configuration_and_settings_->log_storage_path, config, coordination_settings))
, log(&Poco::Logger::get("KeeperServer"))
{
if (coordination_settings->quorum_reads)
@ -302,11 +285,46 @@ bool KeeperServer::isLeader() const
return raft_instance->is_leader();
}
bool KeeperServer::isObserver() const
{
auto srv_config = state_manager->get_srv_config();
return srv_config->is_learner();
}
bool KeeperServer::isFollower() const
{
return !isLeader() && !isObserver();
}
bool KeeperServer::isLeaderAlive() const
{
return raft_instance->is_leader_alive();
}
/// TODO test whether taking failed peer in count
uint64_t KeeperServer::getFollowerCount() const
{
return raft_instance->get_peer_info_all().size();
}
uint64_t KeeperServer::getSyncedFollowerCount() const
{
uint64_t last_log_idx = raft_instance->get_last_log_idx();
const auto followers = raft_instance->get_peer_info_all();
uint64_t stale_followers = 0;
const uint64_t stale_follower_gap = raft_instance->get_current_params().stale_log_gap_;
for (const auto & fl : followers)
{
if (last_log_idx > fl.last_log_idx_ + stale_follower_gap)
stale_followers++;
}
return followers.size() - stale_followers;
}
nuraft::cb_func::ReturnCode KeeperServer::callbackFunc(nuraft::cb_func::Type type, nuraft::cb_func::Param * param)
{
if (initialized_flag)

24
src/Coordination/KeeperServer.h
View File

@ -6,7 +6,6 @@
#include <Coordination/KeeperStateMachine.h>
#include <Coordination/KeeperStorage.h>
#include <Coordination/CoordinationSettings.h>
#include <unordered_map>
#include <base/logger_useful.h>
namespace DB
@ -52,12 +51,10 @@ private:
public:
KeeperServer(
int server_id_,
const CoordinationSettingsPtr & coordination_settings_,
const Poco::Util::AbstractConfiguration & config,
const KeeperConfigurationAndSettingsPtr & settings_,
const Poco::Util::AbstractConfiguration & config_,
ResponsesQueue & responses_queue_,
SnapshotsQueue & snapshots_queue_,
bool standalone_keeper);
SnapshotsQueue & snapshots_queue_);
/// Load state machine from the latest snapshot and load log storage. Start NuRaft with required settings.
void startup();
@ -73,10 +70,25 @@ public:
/// Return set of the non-active sessions
std::vector<int64_t> getDeadSessions();
nuraft::ptr<KeeperStateMachine> getKeeperStateMachine() const
{
return state_machine;
}
bool isLeader() const;
bool isFollower() const;
bool isObserver() const;
bool isLeaderAlive() const;
/// @return follower count if node is not leader return 0
uint64_t getFollowerCount() const;
/// @return synced follower count if node is not leader return 0
uint64_t getSyncedFollowerCount() const;
/// Wait server initialization (see callbackFunc)
void waitInit();

70
src/Coordination/KeeperStateMachine.cpp
View File

@ -326,16 +326,82 @@ void KeeperStateMachine::processReadRequest(const KeeperStorage::RequestForSessi
throw Exception(ErrorCodes::SYSTEM_ERROR, "Could not push response with session id {} into responses queue", response.session_id);
}
void KeeperStateMachine::shutdownStorage()
{
std::lock_guard lock(storage_and_responses_lock);
storage->finalize();
}
std::vector<int64_t> KeeperStateMachine::getDeadSessions()
{
std::lock_guard lock(storage_and_responses_lock);
return storage->getDeadSessions();
}
void KeeperStateMachine::shutdownStorage()
uint64_t KeeperStateMachine::getLastProcessedZxid() const
{
std::lock_guard lock(storage_and_responses_lock);
storage->finalize();
return storage->getZXID();
}
uint64_t KeeperStateMachine::getNodesCount() const
{
std::lock_guard lock(storage_and_responses_lock);
return storage->getNodesCount();
}
uint64_t KeeperStateMachine::getTotalWatchesCount() const
{
std::lock_guard lock(storage_and_responses_lock);
return storage->getTotalWatchesCount();
}
uint64_t KeeperStateMachine::getWatchedPathsCount() const
{
std::lock_guard lock(storage_and_responses_lock);
return storage->getWatchedPathsCount();
}
uint64_t KeeperStateMachine::getSessionsWithWatchesCount() const
{
std::lock_guard lock(storage_and_responses_lock);
return storage->getSessionsWithWatchesCount();
}
uint64_t KeeperStateMachine::getTotalEphemeralNodesCount() const
{
std::lock_guard lock(storage_and_responses_lock);
return storage->getTotalEphemeralNodesCount();
}
uint64_t KeeperStateMachine::getSessionWithEphemeralNodesCount() const
{
std::lock_guard lock(storage_and_responses_lock);
return storage->getSessionWithEphemeralNodesCount();
}
void KeeperStateMachine::dumpWatches(WriteBufferFromOwnString & buf) const
{
std::lock_guard lock(storage_and_responses_lock);
storage->dumpWatches(buf);
}
void KeeperStateMachine::dumpWatchesByPath(WriteBufferFromOwnString & buf) const
{
std::lock_guard lock(storage_and_responses_lock);
storage->dumpWatchesByPath(buf);
}
void KeeperStateMachine::dumpSessionsAndEphemerals(WriteBufferFromOwnString & buf) const
{
std::lock_guard lock(storage_and_responses_lock);
storage->dumpSessionsAndEphemerals(buf);
}
uint64_t KeeperStateMachine::getApproximateDataSize() const
{
std::lock_guard lock(storage_and_responses_lock);
return storage->getApproximateDataSize();
}
ClusterConfigPtr KeeperStateMachine::getClusterConfig() const

23
src/Coordination/KeeperStateMachine.h
View File

@ -67,19 +67,36 @@ public:
nuraft::ptr<nuraft::buffer> & data_out,
bool & is_last_obj) override;
/// just for test
KeeperStorage & getStorage()
{
return *storage;
}
void shutdownStorage();
ClusterConfigPtr getClusterConfig() const;
/// Process local read request
void processReadRequest(const KeeperStorage::RequestForSession & request_for_session);
std::vector<int64_t> getDeadSessions();
void shutdownStorage();
/// Introspection functions for 4lw commands
uint64_t getLastProcessedZxid() const;
ClusterConfigPtr getClusterConfig() const;
uint64_t getNodesCount() const;
uint64_t getTotalWatchesCount() const;
uint64_t getWatchedPathsCount() const;
uint64_t getSessionsWithWatchesCount() const;
void dumpWatches(WriteBufferFromOwnString & buf) const;
void dumpWatchesByPath(WriteBufferFromOwnString & buf) const;
void dumpSessionsAndEphemerals(WriteBufferFromOwnString & buf) const;
uint64_t getSessionWithEphemeralNodesCount() const;
uint64_t getTotalEphemeralNodesCount() const;
uint64_t getApproximateDataSize() const;
private:
@ -110,7 +127,7 @@ private:
/// we can get strange cases when, for example client send read request with
/// watch and after that receive watch response and only receive response
/// for request.
std::mutex storage_and_responses_lock;
mutable std::mutex storage_and_responses_lock;
/// Last committed Raft log number.
std::atomic<uint64_t> last_committed_idx;

44
src/Coordination/KeeperStateManager.cpp
View File

@ -1,4 +1,5 @@
#include <Coordination/KeeperStateManager.h>
#include <Coordination/Defines.h>
#include <Common/Exception.h>
#include <filesystem>
@ -11,28 +12,7 @@ namespace ErrorCodes
extern const int RAFT_ERROR;
}
namespace
{
std::string getLogsPathFromConfig(
const std::string & config_prefix, const Poco::Util::AbstractConfiguration & config, bool standalone_keeper)
{
/// the most specialized path
if (config.has(config_prefix + ".log_storage_path"))
return config.getString(config_prefix + ".log_storage_path");
if (config.has(config_prefix + ".storage_path"))
return std::filesystem::path{config.getString(config_prefix + ".storage_path")} / "logs";
if (standalone_keeper)
return std::filesystem::path{config.getString("path", KEEPER_DEFAULT_PATH)} / "logs";
else
return std::filesystem::path{config.getString("path", DBMS_DEFAULT_PATH)} / "coordination/logs";
}
}
KeeperConfigurationWrapper KeeperStateManager::parseServersConfiguration(const Poco::Util::AbstractConfiguration & config, bool allow_without_us) const
KeeperStateManager::KeeperConfigurationWrapper KeeperStateManager::parseServersConfiguration(const Poco::Util::AbstractConfiguration & config, bool allow_without_us) const
{
KeeperConfigurationWrapper result;
result.cluster_config = std::make_shared<nuraft::cluster_config>();
@ -78,9 +58,9 @@ KeeperConfigurationWrapper KeeperStateManager::parseServersConfiguration(const P
}
KeeperStateManager::KeeperStateManager(int server_id_, const std::string & host, int port, const std::string & logs_path)
: my_server_id(server_id_)
, secure(false)
, log_store(nuraft::cs_new<KeeperLogStore>(logs_path, 5000, false, false))
: my_server_id(server_id_)
, secure(false)
, log_store(nuraft::cs_new<KeeperLogStore>(logs_path, 5000, false, false))
{
auto peer_config = nuraft::cs_new<nuraft::srv_config>(my_server_id, host + ":" + std::to_string(port));
configuration_wrapper.cluster_config = nuraft::cs_new<nuraft::cluster_config>();
@ -90,18 +70,20 @@ KeeperStateManager::KeeperStateManager(int server_id_, const std::string & host,
}
KeeperStateManager::KeeperStateManager(
int server_id_,
int my_server_id_,
const std::string & config_prefix_,
const std::string & log_storage_path,
const Poco::Util::AbstractConfiguration & config,
const CoordinationSettingsPtr & coordination_settings,
bool standalone_keeper)
: my_server_id(server_id_)
const CoordinationSettingsPtr & coordination_settings)
: my_server_id(my_server_id_)
, secure(config.getBool(config_prefix_ + ".raft_configuration.secure", false))
, config_prefix(config_prefix_)
, configuration_wrapper(parseServersConfiguration(config, false))
, log_store(nuraft::cs_new<KeeperLogStore>(
getLogsPathFromConfig(config_prefix_, config, standalone_keeper),
coordination_settings->rotate_log_storage_interval, coordination_settings->force_sync, coordination_settings->compress_logs))
log_storage_path,
coordination_settings->rotate_log_storage_interval,
coordination_settings->force_sync,
coordination_settings->compress_logs))
{
}

32
src/Coordination/KeeperStateManager.h
View File

@ -13,20 +13,6 @@ namespace DB
using KeeperServerConfigPtr = nuraft::ptr<nuraft::srv_config>;
/// Wrapper struct for Keeper cluster config. We parse this
/// info from XML files.
struct KeeperConfigurationWrapper
{
/// Our port
int port;
/// Our config
KeeperServerConfigPtr config;
/// Servers id's to start as followers
std::unordered_set<int> servers_start_as_followers;
/// Cluster config
ClusterConfigPtr cluster_config;
};
/// When our configuration changes the following action types
/// can happen
enum class ConfigUpdateActionType
@ -52,9 +38,9 @@ public:
KeeperStateManager(
int server_id_,
const std::string & config_prefix_,
const std::string & log_storage_path,
const Poco::Util::AbstractConfiguration & config,
const CoordinationSettingsPtr & coordination_settings,
bool standalone_keeper);
const CoordinationSettingsPtr & coordination_settings);
/// Constructor for tests
KeeperStateManager(
@ -121,6 +107,20 @@ public:
ConfigUpdateActions getConfigurationDiff(const Poco::Util::AbstractConfiguration & config) const;
private:
/// Wrapper struct for Keeper cluster config. We parse this
/// info from XML files.
struct KeeperConfigurationWrapper
{
/// Our port
int port;
/// Our config
KeeperServerConfigPtr config;
/// Servers id's to start as followers
std::unordered_set<int> servers_start_as_followers;
/// Cluster config
ClusterConfigPtr cluster_config;
};
int my_server_id;
bool secure;
std::string config_prefix;

108
src/Coordination/KeeperStorage.cpp
View File

@ -10,6 +10,7 @@
#include <Poco/SHA1Engine.h>
#include <Poco/Base64Encoder.h>
#include <boost/algorithm/string.hpp>
#include <Common/hex.h>
namespace DB
{
@ -132,6 +133,21 @@ static bool fixupACL(
return valid_found;
}
uint64_t KeeperStorage::Node::sizeInBytes() const
{
uint64_t total_size{0};
for (const auto & child : children)
total_size += child.size();
total_size += data.size();
total_size += sizeof(acl_id);
total_size += sizeof(is_sequental);
total_size += sizeof(stat);
total_size += sizeof(seq_num);
return total_size;
}
static KeeperStorage::ResponsesForSessions processWatchesImpl(const String & path, KeeperStorage::Watches & watches, KeeperStorage::Watches & list_watches, Coordination::Event event_type)
{
KeeperStorage::ResponsesForSessions result;
@ -1220,4 +1236,96 @@ void KeeperStorage::clearDeadWatches(int64_t session_id)
}
}
void KeeperStorage::dumpWatches(WriteBufferFromOwnString & buf) const
{
for (const auto & [session_id, watches_paths] : sessions_and_watchers)
{
buf << "0x" << getHexUIntLowercase(session_id) << "\n";
for (const String & path : watches_paths)
buf << "\t" << path << "\n";
}
}
void KeeperStorage::dumpWatchesByPath(WriteBufferFromOwnString & buf) const
{
auto write_int_vec = [&buf](const std::vector<int64_t> & session_ids)
{
for (int64_t session_id : session_ids)
{
buf << "\t0x" << getHexUIntLowercase(session_id) << "\n";
}
};
for (const auto & [watch_path, sessions] : watches)
{
buf << watch_path << "\n";
write_int_vec(sessions);
}
for (const auto & [watch_path, sessions] : list_watches)
{
buf << watch_path << "\n";
write_int_vec(sessions);
}
}
void KeeperStorage::dumpSessionsAndEphemerals(WriteBufferFromOwnString & buf) const
{
auto write_str_set = [&buf](const std::unordered_set<String> & ephemeral_paths)
{
for (const String & path : ephemeral_paths)
{
buf << "\t" << path << "\n";
}
};
buf << "Sessions dump (" << session_and_timeout.size() << "):\n";
for (const auto & [session_id, _] : session_and_timeout)
{
buf << "0x" << getHexUIntLowercase(session_id) << "\n";
}
buf << "Sessions with Ephemerals (" << getSessionWithEphemeralNodesCount() << "):\n";
for (const auto & [session_id, ephemeral_paths] : ephemerals)
{
buf << "0x" << getHexUIntLowercase(session_id) << "\n";
write_str_set(ephemeral_paths);
}
}
uint64_t KeeperStorage::getTotalWatchesCount() const
{
uint64_t ret = 0;
for (const auto & [path, subscribed_sessions] : watches)
ret += subscribed_sessions.size();
for (const auto & [path, subscribed_sessions] : list_watches)
ret += subscribed_sessions.size();
return ret;
}
uint64_t KeeperStorage::getSessionsWithWatchesCount() const
{
std::unordered_set<int64_t> counter;
for (const auto & [path, subscribed_sessions] : watches)
counter.insert(subscribed_sessions.begin(), subscribed_sessions.end());
for (const auto & [path, subscribed_sessions] : list_watches)
counter.insert(subscribed_sessions.begin(), subscribed_sessions.end());
return counter.size();
}
uint64_t KeeperStorage::getTotalEphemeralNodesCount() const
{
uint64_t ret = 0;
for (const auto & [session_id, nodes] : ephemerals)
ret += nodes.size();
return ret;
}
}

45
src/Coordination/KeeperStorage.h
View File

@ -1,6 +1,5 @@
#pragma once
#include <Common/ThreadPool.h>
#include <Common/ZooKeeper/IKeeper.h>
#include <Common/ConcurrentBoundedQueue.h>
#include <Common/ZooKeeper/ZooKeeperCommon.h>
@ -14,7 +13,6 @@
namespace DB
{
using namespace DB;
struct KeeperStorageRequestProcessor;
using KeeperStorageRequestProcessorPtr = std::shared_ptr<KeeperStorageRequestProcessor>;
using ResponseCallback = std::function<void(const Coordination::ZooKeeperResponsePtr &)>;
@ -29,8 +27,6 @@ struct KeeperStorageSnapshot;
class KeeperStorage
{
public:
int64_t session_id_counter{1};
struct Node
{
String data;
@ -39,6 +35,9 @@ public:
Coordination::Stat stat{};
int32_t seq_num = 0;
ChildrenSet children{};
/// Object memory size
uint64_t sizeInBytes() const;
};
struct ResponseForSession
@ -46,7 +45,6 @@ public:
int64_t session_id;
Coordination::ZooKeeperResponsePtr response;
};
using ResponsesForSessions = std::vector<ResponseForSession>;
struct RequestForSession
@ -76,10 +74,13 @@ public:
/// Just vector of SHA1 from user:password
using AuthIDs = std::vector<AuthID>;
using SessionAndAuth = std::unordered_map<int64_t, AuthIDs>;
SessionAndAuth session_and_auth;
using Watches = std::map<String /* path, relative of root_path */, SessionIDs>;
public:
int64_t session_id_counter{1};
SessionAndAuth session_and_auth;
/// Main hashtable with nodes. Contain all information about data.
/// All other structures expect session_and_timeout can be restored from
/// container.
@ -176,6 +177,36 @@ public:
{
return session_expiry_queue.getExpiredSessions();
}
/// Introspection functions mostly used in 4-letter commands
uint64_t getNodesCount() const
{
return container.size();
}
uint64_t getApproximateDataSize() const
{
return container.getApproximateDataSize();
}
uint64_t getTotalWatchesCount() const;
uint64_t getWatchedPathsCount() const
{
return watches.size() + list_watches.size();
}
uint64_t getSessionsWithWatchesCount() const;
uint64_t getSessionWithEphemeralNodesCount() const
{
return ephemerals.size();
}
uint64_t getTotalEphemeralNodesCount() const;
void dumpWatches(WriteBufferFromOwnString & buf) const;
void dumpWatchesByPath(WriteBufferFromOwnString & buf) const;
void dumpSessionsAndEphemerals(WriteBufferFromOwnString & buf) const;
};
using KeeperStoragePtr = std::unique_ptr<KeeperStorage>;

105
src/Coordination/SnapshotableHashTable.h
View File

@ -15,6 +15,7 @@ struct ListNode
bool active_in_map;
};
template <class V>
class SnapshotableHashTable
{
@ -28,6 +29,82 @@ private:
IndexMap map;
bool snapshot_mode{false};
uint64_t approximate_data_size{0};
enum OperationType
{
INSERT = 0,
INSERT_OR_REPLACE = 1,
ERASE = 2,
UPDATE_VALUE = 3,
GET_VALUE = 4,
FIND = 5,
CONTAINS = 6,
CLEAR = 7,
CLEAR_OUTDATED_NODES = 8
};
/// Update hash table approximate data size
/// op_type: operation type
/// key_size: key size
/// value_size: size of value to add
/// old_value_size: size of value to minus
/// old_value_size=0 means there is no old value with the same key.
void updateDataSize(OperationType op_type, uint64_t key_size, uint64_t value_size, uint64_t old_value_size)
{
switch (op_type)
{
case INSERT:
approximate_data_size += key_size;
approximate_data_size += value_size;
break;
case INSERT_OR_REPLACE:
/// replace
if (old_value_size != 0)
{
approximate_data_size += key_size;
approximate_data_size += value_size;
if (!snapshot_mode)
{
approximate_data_size += key_size;
approximate_data_size -= old_value_size;
}
}
/// insert
else
{
approximate_data_size += key_size;
approximate_data_size += value_size;
}
break;
case UPDATE_VALUE:
approximate_data_size += key_size;
approximate_data_size += value_size;
if (!snapshot_mode)
{
approximate_data_size -= key_size;
approximate_data_size -= old_value_size;
}
break;
case ERASE:
if (!snapshot_mode)
{
approximate_data_size -= key_size;
approximate_data_size -= old_value_size;
}
break;
case CLEAR:
approximate_data_size = 0;
break;
case CLEAR_OUTDATED_NODES:
approximate_data_size -= key_size;
approximate_data_size -= value_size;
break;
default:
break;
}
}
public:
using iterator = typename List::iterator;
@ -44,6 +121,7 @@ public:
ListElem elem{key, value, true};
auto itr = list.insert(list.end(), elem);
map.emplace(itr->key, itr);
updateDataSize(INSERT, key.size(), value.sizeInBytes(), 0);
return true;
}
@ -54,6 +132,8 @@ public:
void insertOrReplace(const std::string & key, const V & value)
{
auto it = map.find(key);
uint64_t old_value_size = it == map.end() ? 0 : it->second->value.sizeInBytes();
if (it == map.end())
{
ListElem elem{key, value, true};
@ -76,6 +156,7 @@ public:
list_itr->value = value;
}
}
updateDataSize(INSERT_OR_REPLACE, key.size(), value.sizeInBytes(), old_value_size);
}
bool erase(const std::string & key)
@ -85,6 +166,7 @@ public:
return false;
auto list_itr = it->second;
uint64_t old_data_size = list_itr->value.sizeInBytes();
if (snapshot_mode)
{
list_itr->active_in_map = false;
@ -96,6 +178,7 @@ public:
list.erase(list_itr);
}
updateDataSize(ERASE, key.size(), 0, old_data_size);
return true;
}
@ -108,23 +191,29 @@ public:
{
auto it = map.find(key);
assert(it != map.end());
auto list_itr = it->second;
uint64_t old_value_size = list_itr->value.sizeInBytes();
const_iterator ret;
if (snapshot_mode)
{
auto list_itr = it->second;
auto elem_copy = *(list_itr);
list_itr->active_in_map = false;
map.erase(it);
updater(elem_copy.value);
auto itr = list.insert(list.end(), elem_copy);
map.emplace(itr->key, itr);
return itr;
ret = itr;
}
else
{
auto list_itr = it->second;
updater(list_itr->value);
return list_itr;
ret = list_itr;
}
updateDataSize(UPDATE_VALUE, key.size(), ret->value.sizeInBytes(), old_value_size);
return ret;
}
const_iterator find(const std::string & key) const
@ -149,7 +238,10 @@ public:
for (auto itr = start; itr != end;)
{
if (!itr->active_in_map)
{
updateDataSize(CLEAR_OUTDATED_NODES, itr->key.size(), itr->value.sizeInBytes(), 0);
itr = list.erase(itr);
}
else
itr++;
}
@ -159,6 +251,7 @@ public:
{
list.clear();
map.clear();
updateDataSize(CLEAR, 0, 0, 0);
}
void enableSnapshotMode()
@ -181,6 +274,10 @@ public:
return list.size();
}
uint64_t getApproximateDataSize() const
{
return approximate_data_size;
}
iterator begin() { return list.begin(); }
const_iterator begin() const { return list.cbegin(); }

8
src/Coordination/WriteBufferFromNuraftBuffer.cpp
View File

@ -11,7 +11,7 @@ namespace ErrorCodes
void WriteBufferFromNuraftBuffer::nextImpl()
{
if (is_finished)
if (finalized)
throw Exception("WriteBufferFromNuraftBuffer is finished", ErrorCodes::CANNOT_WRITE_AFTER_END_OF_BUFFER);
/// pos may not be equal to vector.data() + old_size, because WriteBuffer::next() can be used to flush data
@ -35,12 +35,8 @@ WriteBufferFromNuraftBuffer::WriteBufferFromNuraftBuffer()
set(reinterpret_cast<Position>(buffer->data_begin()), buffer->size());
}
void WriteBufferFromNuraftBuffer::finalize()
void WriteBufferFromNuraftBuffer::finalizeImpl()
{
if (is_finished)
return;
is_finished = true;
size_t real_size = pos - reinterpret_cast<Position>(buffer->data_begin());
nuraft::ptr<nuraft::buffer> new_buffer = nuraft::buffer::alloc(real_size);
memcpy(new_buffer->data_begin(), buffer->data_begin(), real_size);

22
src/Coordination/WriteBufferFromNuraftBuffer.h
View File

@ -8,23 +8,23 @@ namespace DB
class WriteBufferFromNuraftBuffer : public WriteBuffer
{
private:
nuraft::ptr<nuraft::buffer> buffer;
bool is_finished = false;
static constexpr size_t initial_size = 32;
static constexpr size_t size_multiplier = 2;
void nextImpl() override;
public:
WriteBufferFromNuraftBuffer();
void finalize() override final;
nuraft::ptr<nuraft::buffer> getBuffer();
bool isFinished() const { return is_finished; }
bool isFinished() const { return finalized; }
~WriteBufferFromNuraftBuffer() override;
private:
void finalizeImpl() override final;
void nextImpl() override;
nuraft::ptr<nuraft::buffer> buffer;
static constexpr size_t initial_size = 32;
static constexpr size_t size_multiplier = 2;
};
}

89
src/Coordination/tests/gtest_coordination.cpp
View File

@ -829,15 +829,29 @@ TEST_P(CoordinationTest, ChangelogTestLostFiles)
EXPECT_FALSE(fs::exists("./logs/changelog_21_40.bin" + params.extension));
}
struct IntNode
{
int value;
IntNode(int value_) : value(value_) { } // NOLINT(google-explicit-constructor)
UInt64 sizeInBytes() const { return sizeof value; }
IntNode & operator=(int rhs)
{
this->value = rhs;
return *this;
}
bool operator==(const int & rhs) const { return value == rhs; }
bool operator!=(const int & rhs) const { return rhs != this->value; }
};
TEST_P(CoordinationTest, SnapshotableHashMapSimple)
{
DB::SnapshotableHashTable<int> hello;
DB::SnapshotableHashTable<IntNode> hello;
EXPECT_TRUE(hello.insert("hello", 5));
EXPECT_TRUE(hello.contains("hello"));
EXPECT_EQ(hello.getValue("hello"), 5);
EXPECT_FALSE(hello.insert("hello", 145));
EXPECT_EQ(hello.getValue("hello"), 5);
hello.updateValue("hello", [](int & value) { value = 7; });
hello.updateValue("hello", [](IntNode & value) { value = 7; });
EXPECT_EQ(hello.getValue("hello"), 7);
EXPECT_EQ(hello.size(), 1);
EXPECT_TRUE(hello.erase("hello"));
@ -846,12 +860,12 @@ TEST_P(CoordinationTest, SnapshotableHashMapSimple)
TEST_P(CoordinationTest, SnapshotableHashMapTrySnapshot)
{
DB::SnapshotableHashTable<int> map_snp;
DB::SnapshotableHashTable<IntNode> map_snp;
EXPECT_TRUE(map_snp.insert("/hello", 7));
EXPECT_FALSE(map_snp.insert("/hello", 145));
map_snp.enableSnapshotMode();
EXPECT_FALSE(map_snp.insert("/hello", 145));
map_snp.updateValue("/hello", [](int & value) { value = 554; });
map_snp.updateValue("/hello", [](IntNode & value) { value = 554; });
EXPECT_EQ(map_snp.getValue("/hello"), 554);
EXPECT_EQ(map_snp.snapshotSize(), 2);
EXPECT_EQ(map_snp.size(), 1);
@ -921,6 +935,73 @@ TEST_P(CoordinationTest, SnapshotableHashMapTrySnapshot)
map_snp.disableSnapshotMode();
}
TEST_P(CoordinationTest, SnapshotableHashMapDataSize)
{
/// int
DB::SnapshotableHashTable<IntNode> hello;
hello.disableSnapshotMode();
EXPECT_EQ(hello.getApproximateDataSize(), 0);
hello.insert("hello", 1);
EXPECT_EQ(hello.getApproximateDataSize(), 9);
hello.updateValue("hello", [](IntNode & value) { value = 2; });
EXPECT_EQ(hello.getApproximateDataSize(), 9);
hello.erase("hello");
EXPECT_EQ(hello.getApproximateDataSize(), 0);
hello.clear();
EXPECT_EQ(hello.getApproximateDataSize(), 0);
hello.enableSnapshotMode();
hello.insert("hello", 1);
EXPECT_EQ(hello.getApproximateDataSize(), 9);
hello.updateValue("hello", [](IntNode & value) { value = 2; });
EXPECT_EQ(hello.getApproximateDataSize(), 18);
hello.clearOutdatedNodes();
EXPECT_EQ(hello.getApproximateDataSize(), 9);
hello.erase("hello");
EXPECT_EQ(hello.getApproximateDataSize(), 9);
hello.clearOutdatedNodes();
EXPECT_EQ(hello.getApproximateDataSize(), 0);
/// Node
using Node = DB::KeeperStorage::Node;
DB::SnapshotableHashTable<Node> world;
Node n1;
n1.data = "1234";
Node n2;
n2.data = "123456";
n2.children.insert("");
world.disableSnapshotMode();
world.insert("world", n1);
EXPECT_EQ(world.getApproximateDataSize(), 94);
world.updateValue("world", [&](Node & value) { value = n2; });
EXPECT_EQ(world.getApproximateDataSize(), 96);
world.erase("world");
EXPECT_EQ(world.getApproximateDataSize(), 0);
world.enableSnapshotMode();
world.insert("world", n1);
EXPECT_EQ(world.getApproximateDataSize(), 94);
world.updateValue("world", [&](Node & value) { value = n2; });
EXPECT_EQ(world.getApproximateDataSize(), 190);
world.clearOutdatedNodes();
EXPECT_EQ(world.getApproximateDataSize(), 96);
world.erase("world");
EXPECT_EQ(world.getApproximateDataSize(), 96);
world.clear();
EXPECT_EQ(world.getApproximateDataSize(), 0);
}
void addNode(DB::KeeperStorage & storage, const std::string & path, const std::string & data, int64_t ephemeral_owner=0)
{
using Node = DB::KeeperStorage::Node;

9
src/Disks/DiskCacheWrapper.cpp
View File

@ -20,7 +20,7 @@ public:
{
try
{
CompletionAwareWriteBuffer::finalize();
finalize();
}
catch (...)
{
@ -28,12 +28,9 @@ public:
}
}
void finalize() override
void finalizeImpl() override
{
if (finalized)
return;
WriteBufferFromFileDecorator::finalize();
WriteBufferFromFileDecorator::finalizeImpl();
completion_callback();
}

2
src/Disks/DiskMemory.cpp
View File

@ -93,7 +93,7 @@ public:
}
}
void finalize() override
void finalizeImpl() override
{
if (impl.isFinished())
return;

9
src/Disks/DiskRestartProxy.cpp
View File

@ -41,7 +41,7 @@ public:
{
try
{
RestartAwareWriteBuffer::finalize();
finalize();
}
catch (...)
{
@ -49,12 +49,9 @@ public:
}
}
void finalize() override
void finalizeImpl() override
{
if (finalized)
return;
WriteBufferFromFileDecorator::finalize();
WriteBufferFromFileDecorator::finalizeImpl();
lock.unlock();
}

9
src/Disks/IO/WriteIndirectBufferFromRemoteFS.cpp
View File

@ -25,7 +25,7 @@ WriteIndirectBufferFromRemoteFS<T>::~WriteIndirectBufferFromRemoteFS()
{
try
{
WriteIndirectBufferFromRemoteFS::finalize();
finalize();
}
catch (...)
{
@ -35,12 +35,9 @@ WriteIndirectBufferFromRemoteFS<T>::~WriteIndirectBufferFromRemoteFS()
template <typename T>
void WriteIndirectBufferFromRemoteFS<T>::finalize()
void WriteIndirectBufferFromRemoteFS<T>::finalizeImpl()
{
if (finalized)
return;
WriteBufferFromFileDecorator::finalize();
WriteBufferFromFileDecorator::finalizeImpl();
metadata.addObject(remote_fs_path, count());
metadata.save();

4
src/Disks/IO/WriteIndirectBufferFromRemoteFS.h
View File

@ -21,13 +21,13 @@ public:
virtual ~WriteIndirectBufferFromRemoteFS() override;
void finalize() override;
void sync() override;
String getFileName() const override { return metadata.metadata_file_path; }
private:
void finalizeImpl() override;
IDiskRemote::Metadata metadata;
String remote_fs_path;

2
src/Functions/addressToLine.cpp
View File

@ -128,7 +128,7 @@ private:
writeChar(':', out);
writeIntText(location.line, out);
return out.finish();
return out.complete();
}
else
{

31
src/IO/BrotliWriteBuffer.cpp
View File

@ -4,8 +4,6 @@
# include <IO/BrotliWriteBuffer.h>
# include <brotli/encode.h>
#include <Common/MemoryTracker.h>
namespace DB
{
@ -32,13 +30,12 @@ public:
};
BrotliWriteBuffer::BrotliWriteBuffer(std::unique_ptr<WriteBuffer> out_, int compression_level, size_t buf_size, char * existing_memory, size_t alignment)
: BufferWithOwnMemory<WriteBuffer>(buf_size, existing_memory, alignment)
: WriteBufferWithOwnMemoryDecorator(std::move(out_), buf_size, existing_memory, alignment)
, brotli(std::make_unique<BrotliStateWrapper>())
, in_available(0)
, in_data(nullptr)
, out_capacity(0)
, out_data(nullptr)
, out(std::move(out_))
{
BrotliEncoderSetParameter(brotli->state, BROTLI_PARAM_QUALITY, static_cast<uint32_t>(compression_level));
// Set LZ77 window size. According to brotli sources default value is 24 (c/tools/brotli.c:81)
@ -47,9 +44,7 @@ BrotliWriteBuffer::BrotliWriteBuffer(std::unique_ptr<WriteBuffer> out_, int comp
BrotliWriteBuffer::~BrotliWriteBuffer()
{
/// FIXME move final flush into the caller
MemoryTracker::LockExceptionInThread lock(VariableContext::Global);
finish();
finalize();
}
void BrotliWriteBuffer::nextImpl()
@ -96,27 +91,7 @@ void BrotliWriteBuffer::nextImpl()
}
}
void BrotliWriteBuffer::finish()
{
if (finished)
return;
try
{
finishImpl();
out->finalize();
finished = true;
}
catch (...)
{
/// Do not try to flush next time after exception.
out->position() = out->buffer().begin();
finished = true;
throw;
}
}
void BrotliWriteBuffer::finishImpl()
void BrotliWriteBuffer::finalizeBefore()
{
next();

12
src/IO/BrotliWriteBuffer.h
View File

@ -2,11 +2,12 @@
#include <IO/WriteBuffer.h>
#include <IO/BufferWithOwnMemory.h>
#include <IO/WriteBufferDecorator.h>
namespace DB
{
class BrotliWriteBuffer : public BufferWithOwnMemory<WriteBuffer>
class BrotliWriteBuffer : public WriteBufferWithOwnMemoryDecorator
{
public:
BrotliWriteBuffer(
@ -18,13 +19,10 @@ public:
~BrotliWriteBuffer() override;
void finalize() override { finish(); }
private:
void nextImpl() override;
void finish();
void finishImpl();
void finalizeBefore() override;
class BrotliStateWrapper;
std::unique_ptr<BrotliStateWrapper> brotli;
@ -34,10 +32,6 @@ private:
size_t out_capacity;
uint8_t * out_data;
std::unique_ptr<WriteBuffer> out;
bool finished = false;
};
}

29
src/IO/Bzip2WriteBuffer.cpp
View File

@ -40,17 +40,14 @@ public:
};
Bzip2WriteBuffer::Bzip2WriteBuffer(std::unique_ptr<WriteBuffer> out_, int compression_level, size_t buf_size, char * existing_memory, size_t alignment)
: BufferWithOwnMemory<WriteBuffer>(buf_size, existing_memory, alignment)
: WriteBufferWithOwnMemoryDecorator(std::move(out_), buf_size, existing_memory, alignment)
, bz(std::make_unique<Bzip2StateWrapper>(compression_level))
, out(std::move(out_))
{
}
Bzip2WriteBuffer::~Bzip2WriteBuffer()
{
/// FIXME move final flush into the caller
MemoryTracker::LockExceptionInThread lock(VariableContext::Global);
finish();
finalize();
}
void Bzip2WriteBuffer::nextImpl()
@ -92,27 +89,7 @@ void Bzip2WriteBuffer::nextImpl()
}
}
void Bzip2WriteBuffer::finish()
{
if (finished)
return;
try
{
finishImpl();
out->finalize();
finished = true;
}
catch (...)
{
/// Do not try to flush next time after exception.
out->position() = out->buffer().begin();
finished = true;
throw;
}
}
void Bzip2WriteBuffer::finishImpl()
void Bzip2WriteBuffer::finalizeBefore()
{
next();

12
src/IO/Bzip2WriteBuffer.h
View File

@ -2,11 +2,12 @@
#include <IO/WriteBuffer.h>
#include <IO/BufferWithOwnMemory.h>
#include <IO/WriteBufferDecorator.h>
namespace DB
{
class Bzip2WriteBuffer : public BufferWithOwnMemory<WriteBuffer>
class Bzip2WriteBuffer : public WriteBufferWithOwnMemoryDecorator
{
public:
Bzip2WriteBuffer(
@ -18,20 +19,13 @@ public:
~Bzip2WriteBuffer() override;
void finalize() override { finish(); }
private:
void nextImpl() override;
void finish();
void finishImpl();
void finalizeBefore() override;
class Bzip2StateWrapper;
std::unique_ptr<Bzip2StateWrapper> bz;
std::unique_ptr<WriteBuffer> out;
bool finished = false;
};
}

38
src/IO/LZMADeflatingWriteBuffer.cpp
View File

@ -1,5 +1,4 @@
#include <IO/LZMADeflatingWriteBuffer.h>
#include <Common/MemoryTracker.h>
namespace DB
{
@ -10,7 +9,7 @@ namespace ErrorCodes
LZMADeflatingWriteBuffer::LZMADeflatingWriteBuffer(
std::unique_ptr<WriteBuffer> out_, int compression_level, size_t buf_size, char * existing_memory, size_t alignment)
: BufferWithOwnMemory<WriteBuffer>(buf_size, existing_memory, alignment), out(std::move(out_))
: WriteBufferWithOwnMemoryDecorator(std::move(out_), buf_size, existing_memory, alignment)
{
lstr = LZMA_STREAM_INIT;
@ -47,11 +46,7 @@ LZMADeflatingWriteBuffer::LZMADeflatingWriteBuffer(
LZMADeflatingWriteBuffer::~LZMADeflatingWriteBuffer()
{
/// FIXME move final flush into the caller
MemoryTracker::LockExceptionInThread lock(VariableContext::Global);
finish();
lzma_end(&lstr);
finalize();
}
void LZMADeflatingWriteBuffer::nextImpl()
@ -94,28 +89,7 @@ void LZMADeflatingWriteBuffer::nextImpl()
}
}
void LZMADeflatingWriteBuffer::finish()
{
if (finished)
return;
try
{
finishImpl();
out->finalize();
finished = true;
}
catch (...)
{
/// Do not try to flush next time after exception.
out->position() = out->buffer().begin();
finished = true;
throw;
}
}
void LZMADeflatingWriteBuffer::finishImpl()
void LZMADeflatingWriteBuffer::finalizeBefore()
{
next();
@ -142,5 +116,11 @@ void LZMADeflatingWriteBuffer::finishImpl()
} while (lstr.avail_out == 0);
}
void LZMADeflatingWriteBuffer::finalizeAfter()
{
lzma_end(&lstr);
}
}

11
src/IO/LZMADeflatingWriteBuffer.h
View File

@ -2,6 +2,7 @@
#include <IO/BufferWithOwnMemory.h>
#include <IO/WriteBuffer.h>
#include <IO/WriteBufferDecorator.h>
#include <lzma.h>
@ -10,7 +11,7 @@ namespace DB
{
/// Performs compression using lzma library and writes compressed data to out_ WriteBuffer.
class LZMADeflatingWriteBuffer : public BufferWithOwnMemory<WriteBuffer>
class LZMADeflatingWriteBuffer : public WriteBufferWithOwnMemoryDecorator
{
public:
LZMADeflatingWriteBuffer(
@ -20,19 +21,15 @@ public:
char * existing_memory = nullptr,
size_t alignment = 0);
void finalize() override { finish(); }
~LZMADeflatingWriteBuffer() override;
private:
void nextImpl() override;
void finish();
void finishImpl();
void finalizeBefore() override;
void finalizeAfter() override;
std::unique_ptr<WriteBuffer> out;
lzma_stream lstr;
bool finished = false;
};
}

35
src/IO/Lz4DeflatingWriteBuffer.cpp
View File

@ -1,6 +1,5 @@
#include <IO/Lz4DeflatingWriteBuffer.h>
#include <Common/Exception.h>
#include <Common/MemoryTracker.h>
namespace DB
@ -12,8 +11,7 @@ namespace ErrorCodes
Lz4DeflatingWriteBuffer::Lz4DeflatingWriteBuffer(
std::unique_ptr<WriteBuffer> out_, int compression_level, size_t buf_size, char * existing_memory, size_t alignment)
: BufferWithOwnMemory<WriteBuffer>(buf_size, existing_memory, alignment)
, out(std::move(out_))
: WriteBufferWithOwnMemoryDecorator(std::move(out_), buf_size, existing_memory, alignment)
, in_data(nullptr)
, out_data(nullptr)
, in_capacity(0)
@ -45,9 +43,7 @@ Lz4DeflatingWriteBuffer::Lz4DeflatingWriteBuffer(
Lz4DeflatingWriteBuffer::~Lz4DeflatingWriteBuffer()
{
MemoryTracker::LockExceptionInThread lock(VariableContext::Global);
finish();
LZ4F_freeCompressionContext(ctx);
finalize();
}
void Lz4DeflatingWriteBuffer::nextImpl()
@ -120,27 +116,7 @@ void Lz4DeflatingWriteBuffer::nextImpl()
}
}
void Lz4DeflatingWriteBuffer::finish()
{
if (finished)
return;
try
{
finishImpl();
out->finalize();
finished = true;
}
catch (...)
{
/// Do not try to flush next time after exception.
out->position() = out->buffer().begin();
finished = true;
throw;
}
}
void Lz4DeflatingWriteBuffer::finishImpl()
void Lz4DeflatingWriteBuffer::finalizeBefore()
{
next();
@ -165,4 +141,9 @@ void Lz4DeflatingWriteBuffer::finishImpl()
out->position() = out->buffer().end() - out_capacity;
}
void Lz4DeflatingWriteBuffer::finalizeAfter()
{
LZ4F_freeCompressionContext(ctx);
}
}

12
src/IO/Lz4DeflatingWriteBuffer.h
View File

@ -3,6 +3,7 @@
#include <IO/BufferWithOwnMemory.h>
#include <IO/CompressionMethod.h>
#include <IO/WriteBuffer.h>
#include <IO/WriteBufferDecorator.h>
#include <lz4.h>
#include <lz4frame.h>
@ -10,7 +11,7 @@
namespace DB
{
/// Performs compression using lz4 library and writes compressed data to out_ WriteBuffer.
class Lz4DeflatingWriteBuffer : public BufferWithOwnMemory<WriteBuffer>
class Lz4DeflatingWriteBuffer : public WriteBufferWithOwnMemoryDecorator
{
public:
Lz4DeflatingWriteBuffer(
@ -20,17 +21,13 @@ public:
char * existing_memory = nullptr,
size_t alignment = 0);
void finalize() override { finish(); }
~Lz4DeflatingWriteBuffer() override;
private:
void nextImpl() override;
void finish();
void finishImpl();
std::unique_ptr<WriteBuffer> out;
void finalizeBefore() override;
void finalizeAfter() override;
LZ4F_preferences_t kPrefs;
LZ4F_compressionContext_t ctx;
@ -42,6 +39,5 @@ private:
size_t out_capacity;
bool first_time = true;
bool finished = false;
};
}

38
src/IO/WriteBuffer.h
View File

@ -7,6 +7,7 @@
#include <cassert>
#include <Common/Exception.h>
#include <Common/MemoryTracker.h>
#include <IO/BufferBase.h>
@ -16,6 +17,7 @@ namespace DB
namespace ErrorCodes
{
extern const int CANNOT_WRITE_AFTER_END_OF_BUFFER;
extern const int LOGICAL_ERROR;
}
@ -58,8 +60,8 @@ public:
pos = working_buffer.begin();
}
/** it is desirable in the derived classes to place the next() call in the destructor,
* so that the last data is written
/** it is desirable in the derived classes to place the finalize() call in the destructor,
* so that the last data is written (if finalize() wasn't called explicitly)
*/
virtual ~WriteBuffer() = default;
@ -72,6 +74,9 @@ public:
void write(const char * from, size_t n)
{
if (finalized)
throw Exception{ErrorCodes::LOGICAL_ERROR, "Cannot write to finalized buffer"};
size_t bytes_copied = 0;
/// Produces endless loop
@ -90,6 +95,9 @@ public:
inline void write(char x)
{
if (finalized)
throw Exception{ErrorCodes::LOGICAL_ERROR, "Cannot write to finalized buffer"};
nextIfAtEnd();
*pos = x;
++pos;
@ -100,11 +108,35 @@ public:
next();
}
virtual void finalize()
/// Write the last data.
void finalize()
{
if (finalized)
return;
/// finalize() is often called from destructors.
MemoryTracker::LockExceptionInThread lock(VariableContext::Global);
try
{
finalizeImpl();
finalized = true;
}
catch (...)
{
pos = working_buffer.begin();
finalized = true;
throw;
}
}
protected:
virtual void finalizeImpl()
{
next();
}
bool finalized = false;
private:
/** Write the data in the buffer (from the beginning of the buffer to the current position).
* Throw an exception if something is wrong.

55
src/IO/WriteBufferDecorator.h Normal file
View File

@ -0,0 +1,55 @@
#pragma once
#include <IO/WriteBuffer.h>
#include <IO/BufferWithOwnMemory.h>
#include <utility>
#include <memory>
namespace DB
{
class WriteBuffer;
/// WriteBuffer that decorates data and delegates it to underlying buffer.
/// It's used for writing compressed and encrypted data
template <class Base>
class WriteBufferDecorator : public Base
{
public:
template <class ... BaseArgs>
explicit WriteBufferDecorator(std::unique_ptr<WriteBuffer> out_, BaseArgs && ... args)
: Base(std::forward<BaseArgs>(args)...), out(std::move(out_))
{
}
void finalizeImpl() override
{
try
{
finalizeBefore();
out->finalize();
finalizeAfter();
}
catch (...)
{
/// Do not try to flush next time after exception.
out->position() = out->buffer().begin();
throw;
}
}
WriteBuffer * getNestedBuffer() { return out.get(); }
protected:
/// Do some finalization before finalization of underlying buffer.
virtual void finalizeBefore() {}
/// Do some finalization after finalization of underlying buffer.
virtual void finalizeAfter() {}
std::unique_ptr<WriteBuffer> out;
};
using WriteBufferWithOwnMemoryDecorator = WriteBufferDecorator<BufferWithOwnMemory<WriteBuffer>>;
}

34
src/IO/WriteBufferFromArena.h
View File

@ -15,6 +15,23 @@ namespace DB
*/
class WriteBufferFromArena final : public WriteBuffer
{
public:
/// begin_ - start of previously used contiguous memory segment or nullptr (see Arena::allocContinue method).
WriteBufferFromArena(Arena & arena_, const char *& begin_)
: WriteBuffer(nullptr, 0), arena(arena_), begin(begin_)
{
nextImpl();
pos = working_buffer.begin();
}
StringRef complete()
{
/// Return over-allocated memory back into arena.
arena.rollback(buffer().end() - position());
/// Reference to written data.
return { position() - count(), count() };
}
private:
Arena & arena;
const char *& begin;
@ -46,23 +63,6 @@ private:
internalBuffer() = Buffer(const_cast<char *>(begin), end);
buffer() = Buffer(continuation, end);
}
public:
/// begin_ - start of previously used contiguous memory segment or nullptr (see Arena::allocContinue method).
WriteBufferFromArena(Arena & arena_, const char *& begin_)
: WriteBuffer(nullptr, 0), arena(arena_), begin(begin_)
{
nextImpl();
pos = working_buffer.begin();
}
StringRef finish()
{
/// Return over-allocated memory back into arena.
arena.rollback(buffer().end() - position());
/// Reference to written data.
return { position() - count(), count() };
}
};
}

32
src/IO/WriteBufferFromEncryptedFile.cpp
View File

@ -1,7 +1,6 @@
#include <IO/WriteBufferFromEncryptedFile.h>
#if USE_SSL
#include <Common/MemoryTracker.h>
namespace DB
{
@ -12,8 +11,7 @@ WriteBufferFromEncryptedFile::WriteBufferFromEncryptedFile(
const String & key_,
const FileEncryption::Header & header_,
size_t old_file_size)
: WriteBufferFromFileBase(buffer_size_, nullptr, 0)
, out(std::move(out_))
: WriteBufferDecorator<WriteBufferFromFileBase>(std::move(out_), buffer_size_, nullptr, 0)
, header(header_)
, flush_header(!old_file_size)
, encryptor(header.algorithm, key_, header.init_vector)
@ -23,32 +21,10 @@ WriteBufferFromEncryptedFile::WriteBufferFromEncryptedFile(
WriteBufferFromEncryptedFile::~WriteBufferFromEncryptedFile()
{
/// FIXME move final flush into the caller
MemoryTracker::LockExceptionInThread lock(VariableContext::Global);
finish();
finalize();
}
void WriteBufferFromEncryptedFile::finish()
{
if (finished)
return;
try
{
finishImpl();
out->finalize();
finished = true;
}
catch (...)
{
/// Do not try to flush next time after exception.
out->position() = out->buffer().begin();
finished = true;
throw;
}
}
void WriteBufferFromEncryptedFile::finishImpl()
void WriteBufferFromEncryptedFile::finalizeBefore()
{
/// If buffer has pending data - write it.
next();
@ -56,8 +32,6 @@ void WriteBufferFromEncryptedFile::finishImpl()
/// Note that if there is no data to write an empty file will be written, even without the initialization vector
/// (see nextImpl(): it writes the initialization vector only if there is some data ready to write).
/// That's fine because DiskEncrypted allows files without initialization vectors when they're empty.
out->finalize();
}
void WriteBufferFromEncryptedFile::sync()

14
src/IO/WriteBufferFromEncryptedFile.h
View File

@ -1,17 +1,19 @@
#pragma once
#include <Common/config.h>
#include <Common/assert_cast.h>
#if USE_SSL
#include <IO/WriteBufferFromFileBase.h>
#include <IO/FileEncryptionCommon.h>
#include <IO/WriteBufferDecorator.h>
namespace DB
{
/// Encrypts data and writes the encrypted data to the underlying write buffer.
class WriteBufferFromEncryptedFile : public WriteBufferFromFileBase
class WriteBufferFromEncryptedFile : public WriteBufferDecorator<WriteBufferFromFileBase>
{
public:
/// `old_file_size` should be set to non-zero if we're going to append an existing file.
@ -21,21 +23,17 @@ public:
const String & key_,
const FileEncryption::Header & header_,
size_t old_file_size = 0);
~WriteBufferFromEncryptedFile() override;
void sync() override;
void finalize() override { finish(); }
std::string getFileName() const override { return out->getFileName(); }
std::string getFileName() const override { return assert_cast<WriteBufferFromFileBase *>(out.get())->getFileName(); }
private:
void nextImpl() override;
void finish();
void finishImpl();
bool finished = false;
std::unique_ptr<WriteBufferFromFileBase> out;
void finalizeBefore() override;
FileEncryption::Header header;
bool flush_header = false;

13
src/IO/WriteBufferFromFile.cpp
View File

@ -3,7 +3,6 @@
#include <errno.h>
#include <Common/ProfileEvents.h>
#include <Common/MemoryTracker.h>
#include <IO/WriteBufferFromFile.h>
#include <IO/WriteHelpers.h>
@ -70,18 +69,18 @@ WriteBufferFromFile::WriteBufferFromFile(
fd_ = -1;
}
WriteBufferFromFile::~WriteBufferFromFile()
{
finalize();
::close(fd);
}
void WriteBufferFromFile::finalizeImpl()
{
if (fd < 0)
return;
/// FIXME move final flush into the caller
MemoryTracker::LockExceptionInThread lock(VariableContext::Global);
next();
::close(fd);
}

3
src/IO/WriteBufferFromFile.h
View File

@ -53,6 +53,9 @@ public:
{
return file_name;
}
private:
void finalizeImpl() override;
};
}

9
src/IO/WriteBufferFromFileDecorator.cpp
View File

@ -11,22 +11,17 @@ WriteBufferFromFileDecorator::WriteBufferFromFileDecorator(std::unique_ptr<Write
swap(*impl);
}
void WriteBufferFromFileDecorator::finalize()
void WriteBufferFromFileDecorator::finalizeImpl()
{
if (finalized)
return;
next();
impl->finalize();
finalized = true;
}
WriteBufferFromFileDecorator::~WriteBufferFromFileDecorator()
{
try
{
WriteBufferFromFileDecorator::finalize();
finalize();
}
catch (...)
{

5
src/IO/WriteBufferFromFileDecorator.h
View File

@ -13,15 +13,14 @@ public:
~WriteBufferFromFileDecorator() override;
void finalize() override;
void sync() override;
std::string getFileName() const override;
protected:
void finalizeImpl() override;
std::unique_ptr<WriteBuffer> impl;
bool finalized = false;
private:
void nextImpl() override;

10
src/IO/WriteBufferFromFileDescriptor.cpp
View File

@ -1,14 +1,12 @@
#include <unistd.h>
#include <errno.h>
#include <cassert>
#include <sys/types.h>
#include <sys/stat.h>
#include <Common/Exception.h>
#include <Common/ProfileEvents.h>
#include <Common/CurrentMetrics.h>
#include <Common/Stopwatch.h>
#include <Common/MemoryTracker.h>
#include <IO/WriteBufferFromFileDescriptor.h>
#include <IO/WriteHelpers.h>
@ -95,6 +93,11 @@ WriteBufferFromFileDescriptor::WriteBufferFromFileDescriptor(
WriteBufferFromFileDescriptor::~WriteBufferFromFileDescriptor()
{
finalize();
}
void WriteBufferFromFileDescriptor::finalizeImpl()
{
if (fd < 0)
{
@ -102,12 +105,9 @@ WriteBufferFromFileDescriptor::~WriteBufferFromFileDescriptor()
return;
}
/// FIXME move final flush into the caller
MemoryTracker::LockExceptionInThread lock(VariableContext::Global);
next();
}
void WriteBufferFromFileDescriptor::sync()
{
/// If buffer has pending data - write it.

17
src/IO/WriteBufferFromFileDescriptor.h
View File

@ -10,13 +10,6 @@ namespace DB
*/
class WriteBufferFromFileDescriptor : public WriteBufferFromFileBase
{
protected:
int fd;
/// If file has name contains filename, otherwise contains string "(fd=...)"
std::string file_name;
void nextImpl() override;
public:
WriteBufferFromFileDescriptor(
int fd_ = -1,
@ -51,6 +44,16 @@ public:
std::string getFileName() const override;
off_t size() const;
protected:
void nextImpl() override;
int fd;
/// If file has name contains filename, otherwise contains string "(fd=...)"
std::string file_name;
void finalizeImpl() override;
};
}

2
src/IO/WriteBufferFromHTTP.cpp
View File

@ -20,7 +20,7 @@ WriteBufferFromHTTP::WriteBufferFromHTTP(
ostr = &session->sendRequest(request);
}
void WriteBufferFromHTTP::finalize()
void WriteBufferFromHTTP::finalizeImpl()
{
receiveResponse(*session, request, response, false);
/// TODO: Response body is ignored.

18
src/IO/WriteBufferFromHTTP.h
View File

@ -17,19 +17,19 @@ namespace DB
*/
class WriteBufferFromHTTP : public WriteBufferFromOStream
{
public:
explicit WriteBufferFromHTTP(const Poco::URI & uri,
const std::string & method = Poco::Net::HTTPRequest::HTTP_POST, // POST or PUT only
const ConnectionTimeouts & timeouts = {},
size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE);
private:
/// Receives response from the server after sending all data.
void finalizeImpl() override;
HTTPSessionPtr session;
Poco::Net::HTTPRequest request;
Poco::Net::HTTPResponse response;
public:
explicit WriteBufferFromHTTP(const Poco::URI & uri,
const std::string & method = Poco::Net::HTTPRequest::HTTP_POST, // POST or PUT only
const ConnectionTimeouts & timeouts = {},
size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE);
/// Receives response from the server after sending all data.
void finalize() override;
};
}

5
src/IO/WriteBufferFromOStream.cpp
View File

@ -1,5 +1,4 @@
#include <IO/WriteBufferFromOStream.h>
#include <Common/MemoryTracker.h>
namespace DB
@ -42,9 +41,7 @@ WriteBufferFromOStream::WriteBufferFromOStream(
WriteBufferFromOStream::~WriteBufferFromOStream()
{
/// FIXME move final flush into the caller
MemoryTracker::LockExceptionInThread lock(VariableContext::Global);
next();
finalize();
}
}

14
src/IO/WriteBufferFromOStream.h
View File

@ -11,13 +11,6 @@ namespace DB
class WriteBufferFromOStream : public BufferWithOwnMemory<WriteBuffer>
{
protected:
std::ostream * ostr{};
void nextImpl() override;
WriteBufferFromOStream(size_t size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, size_t alignment = 0);
public:
WriteBufferFromOStream(
std::ostream & ostr_,
@ -26,6 +19,13 @@ public:
size_t alignment = 0);
~WriteBufferFromOStream() override;
protected:
WriteBufferFromOStream(size_t size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, size_t alignment = 0);
void nextImpl() override;
std::ostream * ostr{};
};
}

5
src/IO/WriteBufferFromPocoSocket.cpp
View File

@ -5,7 +5,6 @@
#include <Common/Exception.h>
#include <Common/NetException.h>
#include <Common/Stopwatch.h>
#include <Common/MemoryTracker.h>
#include <Common/ProfileEvents.h>
#include <Common/CurrentMetrics.h>
@ -83,9 +82,7 @@ WriteBufferFromPocoSocket::WriteBufferFromPocoSocket(Poco::Net::Socket & socket_
WriteBufferFromPocoSocket::~WriteBufferFromPocoSocket()
{
/// FIXME move final flush into the caller
MemoryTracker::LockExceptionInThread lock(VariableContext::Global);
next();
finalize();
}
}

15
src/IO/WriteBufferFromPocoSocket.h
View File

@ -13,7 +13,14 @@ namespace DB
*/
class WriteBufferFromPocoSocket : public BufferWithOwnMemory<WriteBuffer>
{
public:
WriteBufferFromPocoSocket(Poco::Net::Socket & socket_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE);
~WriteBufferFromPocoSocket() override;
protected:
void nextImpl() override;
Poco::Net::Socket & socket;
/** For error messages. It is necessary to receive this address in advance, because,
@ -21,14 +28,6 @@ protected:
* (getpeername will return an error).
*/
Poco::Net::SocketAddress peer_address;
void nextImpl() override;
public:
WriteBufferFromPocoSocket(Poco::Net::Socket & socket_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE);
~WriteBufferFromPocoSocket() override;
};
}

12
src/IO/WriteBufferFromS3.cpp
View File

@ -4,7 +4,6 @@
# include <IO/WriteBufferFromS3.h>
# include <IO/WriteHelpers.h>
# include <Common/MemoryTracker.h>
# include <aws/s3/S3Client.h>
# include <aws/s3/model/CreateMultipartUploadRequest.h>
@ -84,18 +83,13 @@ void WriteBufferFromS3::allocateBuffer()
last_part_size = 0;
}
void WriteBufferFromS3::finalize()
WriteBufferFromS3::~WriteBufferFromS3()
{
/// FIXME move final flush into the caller
MemoryTracker::LockExceptionInThread lock(VariableContext::Global);
finalizeImpl();
finalize();
}
void WriteBufferFromS3::finalizeImpl()
{
if (finalized)
return;
next();
if (multipart_upload_id.empty())
@ -108,8 +102,6 @@ void WriteBufferFromS3::finalizeImpl()
writePart();
completeMultipartUpload();
}
finalized = true;
}
void WriteBufferFromS3::createMultipartUpload()

55
src/IO/WriteBufferFromS3.h
View File

@ -31,7 +31,32 @@ namespace DB
*/
class WriteBufferFromS3 : public BufferWithOwnMemory<WriteBuffer>
{
public:
explicit WriteBufferFromS3(
std::shared_ptr<Aws::S3::S3Client> client_ptr_,
const String & bucket_,
const String & key_,
size_t minimum_upload_part_size_,
size_t max_single_part_upload_size_,
std::optional<std::map<String, String>> object_metadata_ = std::nullopt,
size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE);
~WriteBufferFromS3() override;
void nextImpl() override;
private:
void allocateBuffer();
void createMultipartUpload();
void writePart();
void completeMultipartUpload();
void makeSinglepartUpload();
/// Receives response from the server after sending all data.
void finalizeImpl() override;
String bucket;
String key;
std::optional<std::map<String, String>> object_metadata;
@ -43,39 +68,11 @@ private:
size_t last_part_size;
/// Upload in S3 is made in parts.
/// We initiate upload, then upload each part and get ETag as a response, and then finish upload with listing all our parts.
/// We initiate upload, then upload each part and get ETag as a response, and then finalizeImpl() upload with listing all our parts.
String multipart_upload_id;
std::vector<String> part_tags;
Poco::Logger * log = &Poco::Logger::get("WriteBufferFromS3");
public:
explicit WriteBufferFromS3(
std::shared_ptr<Aws::S3::S3Client> client_ptr_,
const String & bucket_,
const String & key_,
size_t minimum_upload_part_size_,
size_t max_single_part_upload_size_,
std::optional<std::map<String, String>> object_metadata_ = std::nullopt,
size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE);
void nextImpl() override;
/// Receives response from the server after sending all data.
void finalize() override;
private:
bool finalized = false;
void allocateBuffer();
void createMultipartUpload();
void writePart();
void completeMultipartUpload();
void makeSinglepartUpload();
void finalizeImpl();
};
}

5
src/IO/WriteBufferFromTemporaryFile.h
View File

@ -19,14 +19,11 @@ public:
~WriteBufferFromTemporaryFile() override;
protected:
private:
WriteBufferFromTemporaryFile(std::unique_ptr<TemporaryFile> && tmp_file);
std::shared_ptr<ReadBuffer> getReadBufferImpl() override;
protected:
std::unique_ptr<TemporaryFile> tmp_file;
friend class ReadBufferFromTemporaryWriteBuffer;

71
src/IO/WriteBufferFromVector.h
View File

@ -3,7 +3,6 @@
#include <vector>
#include <IO/WriteBuffer.h>
#include <Common/MemoryTracker.h>
namespace DB
@ -19,31 +18,11 @@ namespace ErrorCodes
* In destructor, vector is cut to the size of written data.
* You can call 'finalize' to resize earlier.
*
* The vector should live until this object is destroyed or until the 'finish' method is called.
* The vector should live until this object is destroyed or until the 'finalizeImpl()' method is called.
*/
template <typename VectorType>
class WriteBufferFromVector : public WriteBuffer
{
private:
VectorType & vector;
bool is_finished = false;
static constexpr size_t initial_size = 32;
static constexpr size_t size_multiplier = 2;
void nextImpl() override
{
if (is_finished)
throw Exception("WriteBufferFromVector is finished", ErrorCodes::CANNOT_WRITE_AFTER_END_OF_BUFFER);
size_t old_size = vector.size();
/// pos may not be equal to vector.data() + old_size, because WriteBuffer::next() can be used to flush data
size_t pos_offset = pos - reinterpret_cast<Position>(vector.data());
vector.resize(old_size * size_multiplier);
internal_buffer = Buffer(reinterpret_cast<Position>(vector.data() + pos_offset), reinterpret_cast<Position>(vector.data() + vector.size()));
working_buffer = internal_buffer;
}
public:
explicit WriteBufferFromVector(VectorType & vector_)
: WriteBuffer(reinterpret_cast<Position>(vector_.data()), vector_.size()), vector(vector_)
@ -68,11 +47,24 @@ public:
set(reinterpret_cast<Position>(vector.data() + old_size), (size - old_size) * sizeof(typename VectorType::value_type));
}
void finalize() override final
bool isFinished() const { return finalized; }
void restart()
{
if (vector.empty())
vector.resize(initial_size);
set(reinterpret_cast<Position>(vector.data()), vector.size());
finalized = false;
}
~WriteBufferFromVector() override
{
finalize();
}
private:
void finalizeImpl() override final
{
if (is_finished)
return;
is_finished = true;
vector.resize(
((position() - reinterpret_cast<Position>(vector.data()))
+ sizeof(typename VectorType::value_type) - 1) /// Align up.
@ -82,22 +74,23 @@ public:
set(nullptr, 0);
}
bool isFinished() const { return is_finished; }
void restart()
void nextImpl() override
{
if (vector.empty())
vector.resize(initial_size);
set(reinterpret_cast<Position>(vector.data()), vector.size());
is_finished = false;
if (finalized)
throw Exception("WriteBufferFromVector is finalized", ErrorCodes::CANNOT_WRITE_AFTER_END_OF_BUFFER);
size_t old_size = vector.size();
/// pos may not be equal to vector.data() + old_size, because WriteBuffer::next() can be used to flush data
size_t pos_offset = pos - reinterpret_cast<Position>(vector.data());
vector.resize(old_size * size_multiplier);
internal_buffer = Buffer(reinterpret_cast<Position>(vector.data() + pos_offset), reinterpret_cast<Position>(vector.data() + vector.size()));
working_buffer = internal_buffer;
}
~WriteBufferFromVector() override
{
/// FIXME move final flush into the caller
MemoryTracker::LockExceptionInThread lock(VariableContext::Global);
finalize();
}
VectorType & vector;
static constexpr size_t initial_size = 32;
static constexpr size_t size_multiplier = 2;
};
}

18
src/IO/WriteBufferValidUTF8.cpp
View File

@ -1,6 +1,5 @@
#include <Poco/UTF8Encoding.h>
#include <IO/WriteBufferValidUTF8.h>
#include <Common/MemoryTracker.h>
#include <base/types.h>
#ifdef __SSE2__
@ -118,13 +117,14 @@ void WriteBufferValidUTF8::nextImpl()
memory[i] = p[i];
working_buffer = Buffer(&memory[cnt], memory.data() + memory.size());
/// Propagate next() to the output buffer
output_buffer.next();
}
WriteBufferValidUTF8::~WriteBufferValidUTF8()
{
finalize();
}
void WriteBufferValidUTF8::finish()
void WriteBufferValidUTF8::finalizeImpl()
{
/// Write all complete sequences from buffer.
nextImpl();
@ -134,12 +134,4 @@ void WriteBufferValidUTF8::finish()
putReplacement();
}
WriteBufferValidUTF8::~WriteBufferValidUTF8()
{
/// FIXME move final flush into the caller
MemoryTracker::LockExceptionInThread lock(VariableContext::Global);
finish();
}
}

26
src/IO/WriteBufferValidUTF8.h
View File

@ -13,19 +13,6 @@ namespace DB
*/
class WriteBufferValidUTF8 final : public BufferWithOwnMemory<WriteBuffer>
{
private:
WriteBuffer & output_buffer;
bool group_replacements;
/// The last recorded character was `replacement`.
bool just_put_replacement = false;
std::string replacement;
void putReplacement();
void putValid(char * data, size_t len);
void nextImpl() override;
void finish();
public:
static const size_t DEFAULT_SIZE;
@ -36,6 +23,19 @@ public:
size_t size = DEFAULT_SIZE);
~WriteBufferValidUTF8() override;
private:
void putReplacement();
void putValid(char * data, size_t len);
void nextImpl() override;
void finalizeImpl() override;
WriteBuffer & output_buffer;
bool group_replacements;
/// The last recorded character was `replacement`.
bool just_put_replacement = false;
std::string replacement;
};
}

65
src/IO/ZlibDeflatingWriteBuffer.cpp
View File

@ -1,6 +1,4 @@
#include <IO/ZlibDeflatingWriteBuffer.h>
#include <Common/MemorySanitizer.h>
#include <Common/MemoryTracker.h>
#include <Common/Exception.h>
@ -20,8 +18,7 @@ ZlibDeflatingWriteBuffer::ZlibDeflatingWriteBuffer(
size_t buf_size,
char * existing_memory,
size_t alignment)
: BufferWithOwnMemory<WriteBuffer>(buf_size, existing_memory, alignment)
, out(std::move(out_))
: WriteBufferWithOwnMemoryDecorator(std::move(out_), buf_size, existing_memory, alignment)
{
zstr.zalloc = nullptr;
zstr.zfree = nullptr;
@ -46,27 +43,6 @@ ZlibDeflatingWriteBuffer::ZlibDeflatingWriteBuffer(
throw Exception(std::string("deflateInit2 failed: ") + zError(rc) + "; zlib version: " + ZLIB_VERSION, ErrorCodes::ZLIB_DEFLATE_FAILED);
}
ZlibDeflatingWriteBuffer::~ZlibDeflatingWriteBuffer()
{
/// FIXME move final flush into the caller
MemoryTracker::LockExceptionInThread lock(VariableContext::Global);
finish();
try
{
int rc = deflateEnd(&zstr);
if (rc != Z_OK)
throw Exception(std::string("deflateEnd failed: ") + zError(rc), ErrorCodes::ZLIB_DEFLATE_FAILED);
}
catch (...)
{
/// It is OK not to terminate under an error from deflateEnd()
/// since all data already written to the stream.
tryLogCurrentException(__PRETTY_FUNCTION__);
}
}
void ZlibDeflatingWriteBuffer::nextImpl()
{
if (!offset())
@ -99,27 +75,12 @@ void ZlibDeflatingWriteBuffer::nextImpl()
}
}
void ZlibDeflatingWriteBuffer::finish()
ZlibDeflatingWriteBuffer::~ZlibDeflatingWriteBuffer()
{
if (finished)
return;
try
{
finishImpl();
out->finalize();
finished = true;
}
catch (...)
{
/// Do not try to flush next time after exception.
out->position() = out->buffer().begin();
finished = true;
throw;
}
finalize();
}
void ZlibDeflatingWriteBuffer::finishImpl()
void ZlibDeflatingWriteBuffer::finalizeBefore()
{
next();
@ -153,7 +114,23 @@ void ZlibDeflatingWriteBuffer::finishImpl()
}
if (rc != Z_OK)
throw Exception(std::string("deflate finish failed: ") + zError(rc), ErrorCodes::ZLIB_DEFLATE_FAILED);
throw Exception(std::string("deflate finalizeImpl() failed: ") + zError(rc), ErrorCodes::ZLIB_DEFLATE_FAILED);
}
}
void ZlibDeflatingWriteBuffer::finalizeAfter()
{
try
{
int rc = deflateEnd(&zstr);
if (rc != Z_OK)
throw Exception(std::string("deflateEnd failed: ") + zError(rc), ErrorCodes::ZLIB_DEFLATE_FAILED);
}
catch (...)
{
/// It is OK not to terminate under an error from deflateEnd()
/// since all data already written to the stream.
tryLogCurrentException(__PRETTY_FUNCTION__);
}
}

11
src/IO/ZlibDeflatingWriteBuffer.h
View File

@ -3,6 +3,7 @@
#include <IO/WriteBuffer.h>
#include <IO/BufferWithOwnMemory.h>
#include <IO/CompressionMethod.h>
#include <IO/WriteBufferDecorator.h>
#include <zlib.h>
@ -12,7 +13,7 @@ namespace DB
{
/// Performs compression using zlib library and writes compressed data to out_ WriteBuffer.
class ZlibDeflatingWriteBuffer : public BufferWithOwnMemory<WriteBuffer>
class ZlibDeflatingWriteBuffer : public WriteBufferWithOwnMemoryDecorator
{
public:
ZlibDeflatingWriteBuffer(
@ -23,22 +24,18 @@ public:
char * existing_memory = nullptr,
size_t alignment = 0);
void finalize() override { finish(); }
~ZlibDeflatingWriteBuffer() override;
private:
void nextImpl() override;
void finishImpl();
/// Flush all pending data and write zlib footer to the underlying buffer.
/// After the first call to this function, subsequent calls will have no effect and
/// an attempt to write to this buffer will result in exception.
void finish();
virtual void finalizeBefore() override;
virtual void finalizeAfter() override;
std::unique_ptr<WriteBuffer> out;
z_stream zstr;
bool finished = false;
};
}

117
src/IO/ZstdDeflatingAppendableWriteBuffer.cpp
View File

@ -1,5 +1,4 @@
#include <IO/ZstdDeflatingAppendableWriteBuffer.h>
#include <Common/MemoryTracker.h>
#include <Common/Exception.h>
namespace DB
@ -11,10 +10,9 @@ namespace ErrorCodes
}
ZstdDeflatingAppendableWriteBuffer::ZstdDeflatingAppendableWriteBuffer(
WriteBuffer & out_, int compression_level, bool append_to_existing_stream_,
std::unique_ptr<WriteBuffer> out_, int compression_level, bool append_to_existing_stream_,
size_t buf_size, char * existing_memory, size_t alignment)
: BufferWithOwnMemory<WriteBuffer>(buf_size, existing_memory, alignment)
, out(out_)
: WriteBufferWithOwnMemoryDecorator(std::move(out_), buf_size, existing_memory, alignment)
, append_to_existing_stream(append_to_existing_stream_)
{
cctx = ZSTD_createCCtx();
@ -50,18 +48,18 @@ void ZstdDeflatingAppendableWriteBuffer::nextImpl()
bool ended = false;
do
{
out.nextIfAtEnd();
out->nextIfAtEnd();
output.dst = reinterpret_cast<unsigned char *>(out.buffer().begin());
output.size = out.buffer().size();
output.pos = out.offset();
output.dst = reinterpret_cast<unsigned char *>(out->buffer().begin());
output.size = out->buffer().size();
output.pos = out->offset();
size_t compression_result = ZSTD_compressStream2(cctx, &output, &input, mode);
if (ZSTD_isError(compression_result))
throw Exception(
ErrorCodes::ZSTD_ENCODER_FAILED, "Zstd stream encoding failed: error code: {}; zstd version: {}", ZSTD_getErrorName(compression_result), ZSTD_VERSION_STRING);
out.position() = out.buffer().begin() + output.pos;
out->position() = out->buffer().begin() + output.pos;
bool everything_was_compressed = (input.pos == input.size);
bool everything_was_flushed = compression_result == 0;
@ -72,42 +70,54 @@ void ZstdDeflatingAppendableWriteBuffer::nextImpl()
catch (...)
{
/// Do not try to write next time after exception.
out.position() = out.buffer().begin();
out->position() = out->buffer().begin();
throw;
}
}
void ZstdDeflatingAppendableWriteBuffer::finish()
{
if (finished || first_write)
{
/// Nothing was written or we have already finished
return;
}
try
{
finishImpl();
out.finalize();
finished = true;
}
catch (...)
{
/// Do not try to flush next time after exception.
out.position() = out.buffer().begin();
finished = true;
throw;
}
}
ZstdDeflatingAppendableWriteBuffer::~ZstdDeflatingAppendableWriteBuffer()
{
/// FIXME move final flush into the caller
MemoryTracker::LockExceptionInThread lock(VariableContext::Global);
finalize();
}
finish();
void ZstdDeflatingAppendableWriteBuffer::finalizeImpl()
{
if (first_write)
{
/// Nothing was written
return;
}
WriteBufferDecorator::finalizeImpl();
}
void ZstdDeflatingAppendableWriteBuffer::finalizeBefore()
{
next();
out->nextIfAtEnd();
input.src = reinterpret_cast<unsigned char *>(working_buffer.begin());
input.size = offset();
input.pos = 0;
output.dst = reinterpret_cast<unsigned char *>(out->buffer().begin());
output.size = out->buffer().size();
output.pos = out->offset();
size_t remaining = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end);
while (remaining != 0)
{
if (ZSTD_isError(remaining))
throw Exception(ErrorCodes::ZSTD_ENCODER_FAILED, "zstd stream encoder end failed: error: '{}' zstd version: {}", ZSTD_getErrorName(remaining), ZSTD_VERSION_STRING);
remaining = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end);
}
out->position() = out->buffer().begin() + output.pos;
}
void ZstdDeflatingAppendableWriteBuffer::finalizeAfter()
{
try
{
int err = ZSTD_freeCCtx(cctx);
@ -123,42 +133,17 @@ ZstdDeflatingAppendableWriteBuffer::~ZstdDeflatingAppendableWriteBuffer()
}
}
void ZstdDeflatingAppendableWriteBuffer::finishImpl()
{
next();
out.nextIfAtEnd();
input.src = reinterpret_cast<unsigned char *>(working_buffer.begin());
input.size = offset();
input.pos = 0;
output.dst = reinterpret_cast<unsigned char *>(out.buffer().begin());
output.size = out.buffer().size();
output.pos = out.offset();
size_t remaining = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end);
while (remaining != 0)
{
if (ZSTD_isError(remaining))
throw Exception(ErrorCodes::ZSTD_ENCODER_FAILED, "zstd stream encoder end failed: error: '{}' zstd version: {}", ZSTD_getErrorName(remaining), ZSTD_VERSION_STRING);
remaining = ZSTD_compressStream2(cctx, &output, &input, ZSTD_e_end);
}
out.position() = out.buffer().begin() + output.pos;
}
void ZstdDeflatingAppendableWriteBuffer::addEmptyBlock()
{
/// HACK: https://github.com/facebook/zstd/issues/2090#issuecomment-620158967
static const char empty_block[3] = {0x01, 0x00, 0x00};
if (out.buffer().size() - out.offset() < sizeof(empty_block))
out.next();
if (out->buffer().size() - out->offset() < sizeof(empty_block))
out->next();
std::memcpy(out.buffer().begin() + out.offset(), empty_block, sizeof(empty_block));
std::memcpy(out->buffer().begin() + out->offset(), empty_block, sizeof(empty_block));
out.position() = out.buffer().begin() + out.offset() + sizeof(empty_block);
out->position() = out->buffer().begin() + out->offset() + sizeof(empty_block);
}
}

23
src/IO/ZstdDeflatingAppendableWriteBuffer.h
View File

@ -3,6 +3,7 @@
#include <IO/BufferWithOwnMemory.h>
#include <IO/CompressionMethod.h>
#include <IO/WriteBuffer.h>
#include <IO/WriteBufferDecorator.h>
#include <zstd.h>
@ -19,51 +20,47 @@ namespace DB
/// said that there is no risks of compatibility issues https://github.com/facebook/zstd/issues/2090#issuecomment-620158967.
/// 2) Doesn't support internal ZSTD check-summing, because ZSTD checksums written at the end of frame (frame epilogue).
///
class ZstdDeflatingAppendableWriteBuffer : public BufferWithOwnMemory<WriteBuffer>
class ZstdDeflatingAppendableWriteBuffer : public WriteBufferWithOwnMemoryDecorator
{
public:
ZstdDeflatingAppendableWriteBuffer(
WriteBuffer & out_,
std::unique_ptr<WriteBuffer> out_,
int compression_level,
bool append_to_existing_stream_, /// if true then out mustn't be empty
size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
char * existing_memory = nullptr,
size_t alignment = 0);
/// Write terminating ZSTD_e_end: empty block + frame epilogue. BTW it
/// should be almost noop, because frame epilogue contains only checksums,
/// and they are disabled for this buffer.
void finalize() override { finish(); }
~ZstdDeflatingAppendableWriteBuffer() override;
void sync() override
{
next();
out.sync();
out->sync();
}
private:
/// NOTE: will fill compressed data to the out.working_buffer, but will not call out.next method until the buffer is full
void nextImpl() override;
/// Write terminating ZSTD_e_end: empty block + frame epilogue. BTW it
/// should be almost noop, because frame epilogue contains only checksums,
/// and they are disabled for this buffer.
/// Flush all pending data and write zstd footer to the underlying buffer.
/// After the first call to this function, subsequent calls will have no effect and
/// an attempt to write to this buffer will result in exception.
void finish();
void finishImpl();
void finalizeImpl() override;
void finalizeBefore() override;
void finalizeAfter() override;
/// Adding zstd empty block to out.working_buffer
void addEmptyBlock();
WriteBuffer & out;
/// We appending data to existing stream so on the first nextImpl call we
/// will append empty block.
bool append_to_existing_stream;
ZSTD_CCtx * cctx;
ZSTD_inBuffer input;
ZSTD_outBuffer output;
/// Flipped in finish call
bool finished = false;
/// Flipped on the first nextImpl call
bool first_write = true;
};

61
src/IO/ZstdDeflatingWriteBuffer.cpp
View File

@ -1,5 +1,4 @@
#include <IO/ZstdDeflatingWriteBuffer.h>
#include <Common/MemoryTracker.h>
#include <Common/Exception.h>
namespace DB
@ -11,7 +10,7 @@ namespace ErrorCodes
ZstdDeflatingWriteBuffer::ZstdDeflatingWriteBuffer(
std::unique_ptr<WriteBuffer> out_, int compression_level, size_t buf_size, char * existing_memory, size_t alignment)
: BufferWithOwnMemory<WriteBuffer>(buf_size, existing_memory, alignment), out(std::move(out_))
: WriteBufferWithOwnMemoryDecorator(std::move(out_), buf_size, existing_memory, alignment)
{
cctx = ZSTD_createCCtx();
if (cctx == nullptr)
@ -30,24 +29,7 @@ ZstdDeflatingWriteBuffer::ZstdDeflatingWriteBuffer(
ZstdDeflatingWriteBuffer::~ZstdDeflatingWriteBuffer()
{
/// FIXME move final flush into the caller
MemoryTracker::LockExceptionInThread lock(VariableContext::Global);
finish();
try
{
int err = ZSTD_freeCCtx(cctx);
/// This is just in case, since it is impossible to get an error by using this wrapper.
if (unlikely(err))
throw Exception(ErrorCodes::ZSTD_ENCODER_FAILED, "ZSTD_freeCCtx failed: error: '{}'; zstd version: {}", ZSTD_getErrorName(err), ZSTD_VERSION_STRING);
}
catch (...)
{
/// It is OK not to terminate under an error from ZSTD_freeCCtx()
/// since all data already written to the stream.
tryLogCurrentException(__PRETTY_FUNCTION__);
}
finalize();
}
void ZstdDeflatingWriteBuffer::nextImpl()
@ -94,27 +76,7 @@ void ZstdDeflatingWriteBuffer::nextImpl()
}
}
void ZstdDeflatingWriteBuffer::finish()
{
if (finished)
return;
try
{
finishImpl();
out->finalize();
finished = true;
}
catch (...)
{
/// Do not try to flush next time after exception.
out->position() = out->buffer().begin();
finished = true;
throw;
}
}
void ZstdDeflatingWriteBuffer::finishImpl()
void ZstdDeflatingWriteBuffer::finalizeBefore()
{
next();
@ -134,4 +96,21 @@ void ZstdDeflatingWriteBuffer::finishImpl()
out->position() = out->buffer().begin() + output.pos;
}
void ZstdDeflatingWriteBuffer::finalizeAfter()
{
try
{
int err = ZSTD_freeCCtx(cctx);
/// This is just in case, since it is impossible to get an error by using this wrapper.
if (unlikely(err))
throw Exception(ErrorCodes::ZSTD_ENCODER_FAILED, "ZSTD_freeCCtx failed: error: '{}'; zstd version: {}", ZSTD_getErrorName(err), ZSTD_VERSION_STRING);
}
catch (...)
{
/// It is OK not to terminate under an error from ZSTD_freeCCtx()
/// since all data already written to the stream.
tryLogCurrentException(__PRETTY_FUNCTION__);
}
}
}

11
src/IO/ZstdDeflatingWriteBuffer.h
View File

@ -3,6 +3,7 @@
#include <IO/BufferWithOwnMemory.h>
#include <IO/CompressionMethod.h>
#include <IO/WriteBuffer.h>
#include <IO/WriteBufferDecorator.h>
#include <zstd.h>
@ -10,7 +11,7 @@ namespace DB
{
/// Performs compression using zstd library and writes compressed data to out_ WriteBuffer.
class ZstdDeflatingWriteBuffer : public BufferWithOwnMemory<WriteBuffer>
class ZstdDeflatingWriteBuffer : public WriteBufferWithOwnMemoryDecorator
{
public:
ZstdDeflatingWriteBuffer(
@ -20,8 +21,6 @@ public:
char * existing_memory = nullptr,
size_t alignment = 0);
void finalize() override { finish(); }
~ZstdDeflatingWriteBuffer() override;
void sync() override
@ -35,14 +34,12 @@ private:
/// Flush all pending data and write zstd footer to the underlying buffer.
/// After the first call to this function, subsequent calls will have no effect and
/// an attempt to write to this buffer will result in exception.
void finish();
void finishImpl();
void finalizeBefore() override;
void finalizeAfter() override;
std::unique_ptr<WriteBuffer> out;
ZSTD_CCtx * cctx;
ZSTD_inBuffer input;
ZSTD_outBuffer output;
bool finished = false;
};
}

1
src/Processors/Executors/PushingPipelineExecutor.cpp
View File

@ -2,7 +2,6 @@
#include <Processors/Executors/PipelineExecutor.h>
#include <Processors/ISource.h>
#include <QueryPipeline/QueryPipeline.h>
#include <iostream>
namespace DB

1
src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h
View File

@ -11,7 +11,6 @@
#include <Poco/Event.h>
#include <IO/BufferWithOwnMemory.h>
#include <IO/WriteBuffer.h>
#include <IO/WriteBufferFromArena.h>
#include <deque>
#include <atomic>

6
src/QueryPipeline/RemoteQueryExecutorReadContext.cpp
View File

@ -172,10 +172,10 @@ bool RemoteQueryExecutorReadContext::resumeRoutine()
return false;
fiber = std::move(fiber).resume();
}
if (exception)
std::rethrow_exception(std::move(exception));
if (exception)
std::rethrow_exception(std::move(exception));
}
return true;
}

21
src/Server/HTTP/WriteBufferFromHTTPServerResponse.cpp
View File

@ -3,14 +3,6 @@
#include <IO/HTTPCommon.h>
#include <IO/Progress.h>
#include <IO/WriteBufferFromString.h>
#include <Common/Exception.h>
#include <Common/NetException.h>
#include <Common/Stopwatch.h>
#include <Common/MemoryTracker.h>
#include <Common/config.h>
#include <Poco/Version.h>
namespace DB
@ -168,8 +160,12 @@ void WriteBufferFromHTTPServerResponse::onProgress(const Progress & progress)
}
}
WriteBufferFromHTTPServerResponse::~WriteBufferFromHTTPServerResponse()
{
finalize();
}
void WriteBufferFromHTTPServerResponse::finalize()
void WriteBufferFromHTTPServerResponse::finalizeImpl()
{
try
{
@ -198,11 +194,4 @@ void WriteBufferFromHTTPServerResponse::finalize()
}
WriteBufferFromHTTPServerResponse::~WriteBufferFromHTTPServerResponse()
{
/// FIXME move final flush into the caller
MemoryTracker::LockExceptionInThread lock(VariableContext::Global);
finalize();
}
}

95
src/Server/HTTP/WriteBufferFromHTTPServerResponse.h
View File

@ -32,47 +32,6 @@ namespace DB
/// This allows to implement progress bar in HTTP clients.
class WriteBufferFromHTTPServerResponse final : public BufferWithOwnMemory<WriteBuffer>
{
private:
HTTPServerResponse & response;
bool is_http_method_head;
bool add_cors_header = false;
unsigned keep_alive_timeout = 0;
bool compress = false;
CompressionMethod compression_method;
int compression_level = 1;
std::shared_ptr<std::ostream> response_body_ostr;
std::shared_ptr<std::ostream> response_header_ostr;
std::unique_ptr<WriteBuffer> out;
bool initialized = false;
bool headers_started_sending = false;
bool headers_finished_sending = false; /// If true, you could not add any headers.
Progress accumulated_progress;
size_t send_progress_interval_ms = 100;
Stopwatch progress_watch;
std::mutex mutex; /// progress callback could be called from different threads.
/// Must be called under locked mutex.
/// This method send headers, if this was not done already,
/// but not finish them with \r\n, allowing to send more headers subsequently.
void startSendHeaders();
// Used for write the header X-ClickHouse-Progress
void writeHeaderProgress();
// Used for write the header X-ClickHouse-Summary
void writeHeaderSummary();
/// This method finish headers with \r\n, allowing to start to send body.
void finishSendHeaders();
void nextImpl() override;
public:
WriteBufferFromHTTPServerResponse(
HTTPServerResponse & response_,
@ -81,15 +40,11 @@ public:
bool compress_ = false, /// If true - set Content-Encoding header and compress the result.
CompressionMethod compression_method_ = CompressionMethod::None);
~WriteBufferFromHTTPServerResponse() override;
/// Writes progress in repeating HTTP headers.
void onProgress(const Progress & progress);
/// Send at least HTTP headers if no data has been sent yet.
/// Use after the data has possibly been sent and no error happened (and thus you do not plan
/// to change response HTTP code.
/// This method is idempotent.
void finalize() override;
/// Turn compression on or off.
/// The setting has any effect only if HTTP headers haven't been sent yet.
void setCompression(bool enable_compression)
@ -117,7 +72,51 @@ public:
send_progress_interval_ms = send_progress_interval_ms_;
}
~WriteBufferFromHTTPServerResponse() override;
private:
/// Send at least HTTP headers if no data has been sent yet.
/// Use after the data has possibly been sent and no error happened (and thus you do not plan
/// to change response HTTP code.
/// This method is idempotent.
void finalizeImpl() override;
/// Must be called under locked mutex.
/// This method send headers, if this was not done already,
/// but not finish them with \r\n, allowing to send more headers subsequently.
void startSendHeaders();
// Used for write the header X-ClickHouse-Progress
void writeHeaderProgress();
// Used for write the header X-ClickHouse-Summary
void writeHeaderSummary();
/// This method finish headers with \r\n, allowing to start to send body.
void finishSendHeaders();
void nextImpl() override;
HTTPServerResponse & response;
bool is_http_method_head;
bool add_cors_header = false;
unsigned keep_alive_timeout = 0;
bool compress = false;
CompressionMethod compression_method;
int compression_level = 1;
std::shared_ptr<std::ostream> response_body_ostr;
std::shared_ptr<std::ostream> response_header_ostr;
std::unique_ptr<WriteBuffer> out;
bool initialized = false;
bool headers_started_sending = false;
bool headers_finished_sending = false; /// If true, you could not add any headers.
Progress accumulated_progress;
size_t send_progress_interval_ms = 100;
Stopwatch progress_watch;
std::mutex mutex; /// progress callback could be called from different threads.
};
}

234
src/Server/KeeperTCPHandler.cpp
View File

@ -18,6 +18,9 @@
#include <IO/ReadBufferFromFileDescriptor.h>
#include <queue>
#include <mutex>
#include <Coordination/FourLetterCommand.h>
#include <Common/hex.h>
#ifdef POCO_HAVE_FD_EPOLL
#include <sys/epoll.h>
@ -29,6 +32,16 @@
namespace DB
{
struct LastOp
{
public:
String name{"NA"};
int64_t last_cxid{-1};
int64_t last_zxid{-1};
int64_t last_response_time{0};
};
static const LastOp EMPTY_LAST_OP {"NA", -1, -1, 0};
namespace ErrorCodes
{
@ -199,7 +212,9 @@ KeeperTCPHandler::KeeperTCPHandler(IServer & server_, const Poco::Net::StreamSoc
, session_timeout(0, global_context->getConfigRef().getUInt("keeper_server.session_timeout_ms", Coordination::DEFAULT_SESSION_TIMEOUT_MS) * 1000)
, poll_wrapper(std::make_unique<SocketInterruptablePollWrapper>(socket_))
, responses(std::make_unique<ThreadSafeResponseQueue>(std::numeric_limits<size_t>::max()))
, last_op(std::make_unique<LastOp>(EMPTY_LAST_OP))
{
KeeperTCPHandler::registerConnection(this);
}
void KeeperTCPHandler::sendHandshake(bool has_leader)
@ -222,16 +237,15 @@ void KeeperTCPHandler::run()
runImpl();
}
Poco::Timespan KeeperTCPHandler::receiveHandshake()
Poco::Timespan KeeperTCPHandler::receiveHandshake(int32_t handshake_length)
{
int32_t handshake_length;
int32_t protocol_version;
int64_t last_zxid_seen;
int32_t timeout_ms;
int64_t previous_session_id = 0; /// We don't support session restore. So previous session_id is always zero.
std::array<char, Coordination::PASSWORD_LENGTH> passwd {};
Coordination::read(handshake_length, *in);
if (handshake_length != Coordination::CLIENT_HANDSHAKE_LENGTH && handshake_length != Coordination::CLIENT_HANDSHAKE_LENGTH_WITH_READONLY)
if (!isHandShake(handshake_length))
throw Exception("Unexpected handshake length received: " + toString(handshake_length), ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT);
Coordination::read(protocol_version, *in);
@ -274,9 +288,32 @@ void KeeperTCPHandler::runImpl()
return;
}
int32_t header;
try
{
auto client_timeout = receiveHandshake();
Coordination::read(header, *in);
}
catch (const Exception & e)
{
LOG_WARNING(log, "Error while read connection header {}", e.displayText());
return;
}
/// All four letter word command code is larger than 2^24 or lower than 0.
/// Hand shake package length must be lower than 2^24 and larger than 0.
/// So collision never happens.
int32_t four_letter_cmd = header;
if (!isHandShake(four_letter_cmd))
{
tryExecuteFourLetterWordCmd(four_letter_cmd);
return;
}
try
{
int32_t handshake_length = header;
auto client_timeout = receiveHandshake(handshake_length);
if (client_timeout != 0)
session_timeout = std::min(client_timeout, session_timeout);
}
@ -345,6 +382,7 @@ void KeeperTCPHandler::runImpl()
session_stopwatch.start();
bool close_received = false;
try
{
while (true)
@ -356,6 +394,7 @@ void KeeperTCPHandler::runImpl()
if (result.has_requests && !close_received)
{
auto [received_op, received_xid] = receiveRequest();
packageReceived();
log_long_operation("Receiving request");
if (received_op == Coordination::OpNum::Close)
@ -368,6 +407,8 @@ void KeeperTCPHandler::runImpl()
{
LOG_TRACE(log, "Received heartbeat for session #{}", session_id);
}
else
operations[received_xid] = Poco::Timestamp();
/// Each request restarts session stopwatch
session_stopwatch.restart();
@ -390,6 +431,9 @@ void KeeperTCPHandler::runImpl()
return;
}
updateStats(response);
packageSent();
response->write(*out);
log_long_operation("Sending response");
if (response->error == Coordination::Error::ZSESSIONEXPIRED)
@ -422,6 +466,44 @@ void KeeperTCPHandler::runImpl()
}
}
bool KeeperTCPHandler::isHandShake(int32_t handshake_length)
{
return handshake_length == Coordination::CLIENT_HANDSHAKE_LENGTH
|| handshake_length == Coordination::CLIENT_HANDSHAKE_LENGTH_WITH_READONLY;
}
bool KeeperTCPHandler::tryExecuteFourLetterWordCmd(int32_t command)
{
if (!FourLetterCommandFactory::instance().isKnown(command))
{
LOG_WARNING(log, "invalid four letter command {}", IFourLetterCommand::toName(command));
return false;
}
else if (!FourLetterCommandFactory::instance().isEnabled(command))
{
LOG_WARNING(log, "Not enabled four letter command {}", IFourLetterCommand::toName(command));
return false;
}
else
{
auto command_ptr = FourLetterCommandFactory::instance().get(command);
LOG_DEBUG(log, "Receive four letter command {}", command_ptr->name());
try
{
String res = command_ptr->run();
out->write(res.data(), res.size());
out->next();
}
catch (...)
{
tryLogCurrentException(log, "Error when executing four letter command " + command_ptr->name());
}
return true;
}
}
std::pair<Coordination::OpNum, Coordination::XID> KeeperTCPHandler::receiveRequest()
{
int32_t length;
@ -441,6 +523,148 @@ std::pair<Coordination::OpNum, Coordination::XID> KeeperTCPHandler::receiveReque
return std::make_pair(opnum, xid);
}
void KeeperTCPHandler::packageSent()
{
{
std::lock_guard lock(conn_stats_mutex);
conn_stats.incrementPacketsSent();
}
keeper_dispatcher->incrementPacketsSent();
}
void KeeperTCPHandler::packageReceived()
{
{
std::lock_guard lock(conn_stats_mutex);
conn_stats.incrementPacketsReceived();
}
keeper_dispatcher->incrementPacketsReceived();
}
void KeeperTCPHandler::updateStats(Coordination::ZooKeeperResponsePtr & response)
{
/// update statistics ignoring watch response and heartbeat.
if (response->xid != Coordination::WATCH_XID && response->getOpNum() != Coordination::OpNum::Heartbeat)
{
Int64 elapsed = (Poco::Timestamp() - operations[response->xid]) / 1000;
{
std::lock_guard lock(conn_stats_mutex);
conn_stats.updateLatency(elapsed);
}
keeper_dispatcher->updateKeeperStatLatency(elapsed);
last_op.set(std::make_unique<LastOp>(LastOp{
.name = Coordination::toString(response->getOpNum()),
.last_cxid = response->xid,
.last_zxid = response->zxid,
.last_response_time = Poco::Timestamp().epochMicroseconds() / 1000,
}));
}
}
KeeperConnectionStats KeeperTCPHandler::getConnectionStats() const
{
std::lock_guard lock(conn_stats_mutex);
return conn_stats;
}
void KeeperTCPHandler::dumpStats(WriteBufferFromOwnString & buf, bool brief)
{
KeeperConnectionStats stats = getConnectionStats();
writeText(' ', buf);
writeText(socket().peerAddress().toString(), buf);
writeText("(recved=", buf);
writeIntText(stats.getPacketsReceived(), buf);
writeText(",sent=", buf);
writeIntText(stats.getPacketsSent(), buf);
if (!brief)
{
if (session_id != 0)
{
writeText(",sid=0x", buf);
writeText(getHexUIntLowercase(session_id), buf);
writeText(",lop=", buf);
LastOpPtr op = last_op.get();
writeText(op->name, buf);
writeText(",est=", buf);
writeIntText(established.epochMicroseconds() / 1000, buf);
writeText(",to=", buf);
writeIntText(session_timeout.totalMilliseconds(), buf);
int64_t last_cxid = op->last_cxid;
if (last_cxid >= 0)
{
writeText(",lcxid=0x", buf);
writeText(getHexUIntLowercase(last_cxid), buf);
}
writeText(",lzxid=0x", buf);
writeText(getHexUIntLowercase(op->last_zxid), buf);
writeText(",lresp=", buf);
writeIntText(op->last_response_time, buf);
writeText(",llat=", buf);
writeIntText(stats.getLastLatency(), buf);
writeText(",minlat=", buf);
writeIntText(stats.getMinLatency(), buf);
writeText(",avglat=", buf);
writeIntText(stats.getAvgLatency(), buf);
writeText(",maxlat=", buf);
writeIntText(stats.getMaxLatency(), buf);
}
}
writeText(')', buf);
writeText('\n', buf);
}
void KeeperTCPHandler::resetStats()
{
{
std::lock_guard lock(conn_stats_mutex);
conn_stats.reset();
}
last_op.set(std::make_unique<LastOp>(EMPTY_LAST_OP));
}
KeeperTCPHandler::~KeeperTCPHandler()
{
KeeperTCPHandler::unregisterConnection(this);
}
std::mutex KeeperTCPHandler::conns_mutex;
std::unordered_set<KeeperTCPHandler *> KeeperTCPHandler::connections;
void KeeperTCPHandler::registerConnection(KeeperTCPHandler * conn)
{
std::lock_guard lock(conns_mutex);
connections.insert(conn);
}
void KeeperTCPHandler::unregisterConnection(KeeperTCPHandler * conn)
{
std::lock_guard lock(conns_mutex);
connections.erase(conn);
}
void KeeperTCPHandler::dumpConnections(WriteBufferFromOwnString & buf, bool brief)
{
std::lock_guard lock(conns_mutex);
for (auto * conn : connections)
{
conn->dumpStats(buf, brief);
}
}
void KeeperTCPHandler::resetConnsStats()
{
std::lock_guard lock(conns_mutex);
for (auto * conn : connections)
{
conn->resetStats();
}
}
}
#endif

48
src/Server/KeeperTCPHandler.h
View File

@ -6,6 +6,7 @@
#if USE_NURAFT
#include <Poco/Net/TCPServerConnection.h>
#include <Common/MultiVersion.h>
#include "IServer.h"
#include <Common/Stopwatch.h>
#include <Interpreters/Context.h>
@ -16,6 +17,8 @@
#include <IO/WriteBufferFromPocoSocket.h>
#include <IO/ReadBufferFromPocoSocket.h>
#include <unordered_map>
#include <Coordination/KeeperConnectionStats.h>
#include <Poco/Timestamp.h>
namespace DB
{
@ -24,14 +27,36 @@ struct SocketInterruptablePollWrapper;
using SocketInterruptablePollWrapperPtr = std::unique_ptr<SocketInterruptablePollWrapper>;
using ThreadSafeResponseQueue = ConcurrentBoundedQueue<Coordination::ZooKeeperResponsePtr>;
using ThreadSafeResponseQueuePtr = std::unique_ptr<ThreadSafeResponseQueue>;
struct LastOp;
using LastOpMultiVersion = MultiVersion<LastOp>;
using LastOpPtr = LastOpMultiVersion::Version;
class KeeperTCPHandler : public Poco::Net::TCPServerConnection
{
public:
static void registerConnection(KeeperTCPHandler * conn);
static void unregisterConnection(KeeperTCPHandler * conn);
/// dump all connections statistics
static void dumpConnections(WriteBufferFromOwnString & buf, bool brief);
static void resetConnsStats();
private:
static std::mutex conns_mutex;
/// all connections
static std::unordered_set<KeeperTCPHandler *> connections;
public:
KeeperTCPHandler(IServer & server_, const Poco::Net::StreamSocket & socket_);
void run() override;
KeeperConnectionStats getConnectionStats() const;
void dumpStats(WriteBufferFromOwnString & buf, bool brief);
void resetStats();
~KeeperTCPHandler() override;
private:
IServer & server;
Poco::Logger * log;
@ -54,9 +79,28 @@ private:
void runImpl();
void sendHandshake(bool has_leader);
Poco::Timespan receiveHandshake();
Poco::Timespan receiveHandshake(int32_t handshake_length);
static bool isHandShake(int32_t handshake_length);
bool tryExecuteFourLetterWordCmd(int32_t command);
std::pair<Coordination::OpNum, Coordination::XID> receiveRequest();
void packageSent();
void packageReceived();
void updateStats(Coordination::ZooKeeperResponsePtr & response);
Poco::Timestamp established;
using Operations = std::map<Coordination::XID, Poco::Timestamp>;
Operations operations;
LastOpMultiVersion last_op;
mutable std::mutex conn_stats_mutex;
KeeperConnectionStats conn_stats;
};
}

2
src/Server/KeeperTCPHandlerFactory.h
View File

@ -21,6 +21,7 @@ private:
using Poco::Net::TCPServerConnection::TCPServerConnection;
void run() override {}
};
public:
KeeperTCPHandlerFactory(IServer & server_, bool secure)
: server(server_)
@ -41,6 +42,7 @@ public:
return new DummyTCPHandler(socket);
}
}
};
}

44
src/Server/TCPHandler.cpp
View File

@ -352,6 +352,13 @@ void TCPHandler::runImpl()
executor.setCancelCallback(callback, interactive_delay / 1000);
}
executor.execute();
/// Send final progress
///
/// NOTE: we cannot send Progress for regular INSERT (w/ VALUES)
/// w/o breaking protocol compatibility, but it can be done
/// by increasing revision.
sendProgress();
}
state.io.onFinish();
@ -589,8 +596,13 @@ void TCPHandler::processInsertQuery()
{
size_t num_threads = state.io.pipeline.getNumThreads();
auto send_table_columns = [&]()
auto run_executor = [&](auto & executor)
{
/// Made above the rest of the lines,
/// so that in case of `writePrefix` function throws an exception,
/// client receive exception before sending data.
executor.start();
/// Send ColumnsDescription for insertion table
if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_COLUMN_DEFAULTS_METADATA)
{
@ -604,17 +616,6 @@ void TCPHandler::processInsertQuery()
}
}
}
};
if (num_threads > 1)
{
PushingAsyncPipelineExecutor executor(state.io.pipeline);
/** Made above the rest of the lines, so that in case of `writePrefix` function throws an exception,
* client receive exception before sending data.
*/
executor.start();
send_table_columns();
/// Send block to the client - table structure.
sendData(executor.getHeader());
@ -625,22 +626,17 @@ void TCPHandler::processInsertQuery()
executor.push(std::move(state.block_for_insert));
executor.finish();
};
if (num_threads > 1)
{
PushingAsyncPipelineExecutor executor(state.io.pipeline);
run_executor(executor);
}
else
{
PushingPipelineExecutor executor(state.io.pipeline);
executor.start();
send_table_columns();
sendData(executor.getHeader());
sendLogs();
while (readDataNext())
executor.push(std::move(state.block_for_insert));
executor.finish();
run_executor(executor);
}
}

1
src/Storages/Distributed/DirectoryMonitor.cpp
View File

@ -1,7 +1,6 @@
#include <QueryPipeline/RemoteInserter.h>
#include <Formats/NativeReader.h>
#include <Processors/Sources/SourceWithProgress.h>
#include <Common/escapeForFileName.h>
#include <Common/CurrentMetrics.h>
#include <Common/StringUtils/StringUtils.h>
#include <Common/SipHash.h>

7
src/Storages/Distributed/DistributedSink.cpp
View File

@ -6,7 +6,6 @@
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTInsertQuery.h>
#include <Parsers/formatAST.h>
#include <Parsers/queryToString.h>
#include <IO/WriteBufferFromFile.h>
@ -21,11 +20,9 @@
#include <Interpreters/ExpressionActions.h>
#include <Interpreters/Context.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeLowCardinality.h>
#include <Common/setThreadName.h>
#include <Common/CurrentMetrics.h>
#include <Common/typeid_cast.h>
#include <Common/Exception.h>
#include <Common/ProfileEvents.h>
#include <Common/escapeForFileName.h>
@ -35,9 +32,6 @@
#include <base/range.h>
#include <base/scope_guard.h>
#include <future>
#include <condition_variable>
#include <mutex>
#include <filesystem>
@ -741,6 +735,7 @@ void DistributedSink::writeToShard(const Block & block, const std::vector<std::s
stream.write(block);
compress.finalize();
out.finalize();
if (fsync)
out.sync();

2
src/Storages/FileLog/StorageFileLog.cpp
View File

@ -470,7 +470,6 @@ void StorageFileLog::openFilesAndSetPos()
void StorageFileLog::closeFilesAndStoreMeta(size_t start, size_t end)
{
assert(start >= 0);
assert(start < end);
assert(end <= file_infos.file_names.size());
@ -491,7 +490,6 @@ void StorageFileLog::closeFilesAndStoreMeta(size_t start, size_t end)
void StorageFileLog::storeMetas(size_t start, size_t end)
{
assert(start >= 0);
assert(start < end);
assert(end <= file_infos.file_names.size());

3
src/Storages/HDFS/WriteBufferFromHDFS.cpp
View File

@ -2,7 +2,6 @@
#if USE_HDFS
#include <Interpreters/Context.h>
#include <Storages/HDFS/WriteBufferFromHDFS.h>
#include <Storages/HDFS/HDFSCommon.h>
#include <hdfs/hdfs.h>
@ -109,7 +108,7 @@ void WriteBufferFromHDFS::sync()
}
void WriteBufferFromHDFS::finalize()
void WriteBufferFromHDFS::finalizeImpl()
{
try
{

6
src/Storages/HDFS/WriteBufferFromHDFS.h
View File

@ -5,6 +5,8 @@
#if USE_HDFS
#include <IO/WriteBuffer.h>
#include <IO/BufferWithOwnMemory.h>
#include <Poco/Util/AbstractConfiguration.h>
#include <fcntl.h>
#include <string>
#include <memory>
@ -32,9 +34,9 @@ public:
void sync() override;
void finalize() override;
private:
void finalizeImpl() override;
struct WriteBufferFromHDFSImpl;
std::unique_ptr<WriteBufferFromHDFSImpl> impl;
};

Some files were not shown because too many files have changed in this diff Show More