From 1f9331198d4885b9c0f495868ec6e066fee6f411 Mon Sep 17 00:00:00 2001 From: John Skopis Date: Thu, 8 Oct 2020 16:07:58 +0000 Subject: [PATCH 001/149] [WIP] Introduce dynamic TLS server context This is a first pass at setting an x509 keypair dynamically. I'm not so sure it should be a singleton (especaially because it depends on the SSLManager singleton). Needs more better error handling. Functionally, it works. Simply change the cert/key path in config, or touch cert; touch config. --- programs/server/Server.cpp | 5 ++ src/Server/CertReloader.cpp | 99 +++++++++++++++++++++++++++++++ src/Server/CertReloader.h | 112 ++++++++++++++++++++++++++++++++++++ src/Server/ya.make | 1 + 4 files changed, 217 insertions(+) create mode 100644 src/Server/CertReloader.cpp create mode 100644 src/Server/CertReloader.h diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index da5760acc09..55136bdc9c0 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -63,6 +63,7 @@ #include #include #include +#include #if !defined(ARCADIA_BUILD) @@ -567,6 +568,7 @@ int Server::main(const std::vector & /*args*/) global_context->reloadZooKeeperIfChanged(config); global_context->updateStorageConfiguration(*config); + CertReloader::instance().reload(*config); }, /* already_loaded = */ true); @@ -970,6 +972,7 @@ int Server::main(const std::vector & /*args*/) #endif }); + /// Interserver IO HTTP create_server("interserver_http_port", [&](UInt16 port) { @@ -1049,6 +1052,8 @@ int Server::main(const std::vector & /*args*/) throw Exception("No servers started (add valid listen_host and 'tcp_port' or 'http_port' to configuration file.)", ErrorCodes::NO_ELEMENTS_IN_CONFIG); + CertReloader::instance().init(config()); + global_context->enableNamedSessions(); for (auto & server : servers) diff --git a/src/Server/CertReloader.cpp b/src/Server/CertReloader.cpp new file mode 100644 index 00000000000..e3f91ab9c0e --- /dev/null +++ b/src/Server/CertReloader.cpp @@ -0,0 +1,99 @@ +#include "CertReloader.h" + +namespace DB +{ +#if USE_SSL +int cert_reloader_dispatch_set_cert(SSL * ssl, [[maybe_unused]] void * arg) +{ + return CertReloader::instance().SetCert(ssl); +} + +int CertReloader::SetCert(SSL * ssl) +{ + std::shared_lock lock(lck); + SSL_use_certificate(ssl, const_cast(cert->certificate())); + SSL_use_RSAPrivateKey(ssl, key->impl()->getRSA()); + + int err = SSL_check_private_key(ssl); + if (err != 1) + { + std::string msg = Poco::Net::Utility::getLastError(); + LOG_ERROR(log, "Unusable keypair {}", msg); + return -1; + } + return 1; +} + +void CertReloader::init(const Poco::Util::AbstractConfiguration & config) +{ + LOG_DEBUG(log, "Initializing config reloader."); + + auto & ssl_manager = Poco::Net::SSLManager::instance(); + const auto ssl_ctx_ptr = ssl_manager.defaultServerContext(); + auto ctx = ssl_ctx_ptr->sslContext(); + SSL_CTX_set_cert_cb(ctx, cert_reloader_dispatch_set_cert, nullptr); + + reload(config); +} + +void CertReloader::reload(const Poco::Util::AbstractConfiguration & config) +{ + LOG_DEBUG(log, "Handling cert reload."); + const auto cert_file_ = config.getString("openSSL.server.certificateFile", ""); + const auto key_file_ = config.getString("openSSL.server.privateKeyFile", ""); + + if (setKeyFile(key_file_) || setCertFile(cert_file_)) + { + LOG_INFO(log, "Reloading cert({}), key({})", cert_file, key_file); + { + std::unique_lock lock(lck); + key.reset(new Poco::Crypto::RSAKey("", key_file)); + cert.reset(new Poco::Crypto::X509Certificate(cert_file)); + } + LOG_INFO(log, "Reloaded cert {}", cert_file); + } +} + +bool CertReloader::setKeyFile(const std::string key_file_) +{ + if (key_file_.empty()) + return false; + + stat_t st; + int res = stat(key_file_.c_str(), &st); + if (res == -1) + return false; + + if (st.st_mtime != key_file_st.st_mtime || key_file != key_file_) + { + key_file = key_file_; + key_file_st = st; + return true; + } + + return false; +} + +bool CertReloader::setCertFile(const std::string cert_file_) +{ + if (cert_file_.empty()) + return false; + + stat_t st; + int res = stat(cert_file_.c_str(), &st); + if (res == -1) + return false; + + if (st.st_mtime != cert_file_st.st_mtime || cert_file != cert_file_) + { + cert_file = cert_file_; + cert_file_st = st; + return true; + } + + return false; +} + +#endif + +} diff --git a/src/Server/CertReloader.h b/src/Server/CertReloader.h new file mode 100644 index 00000000000..013692f5e15 --- /dev/null +++ b/src/Server/CertReloader.h @@ -0,0 +1,112 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include + +#if !defined(ARCADIA_BUILD) +# include +#endif + +#if USE_SSL +# include +# include +# include +# include +# include +# include +# include +#endif + +namespace DB +{ +#if !USE_SSL +class CertReloader +{ +public: + /// Singleton + CertReloader(CertReloader const &) = delete; + void operator=(CertReloader const &) = delete; + static CertReloader & instance() + { + static CertReloader instance; + return instance; + } + + /// Initialize the callback and perfom the initial cert loading + void init([[maybe_unused]] const ::Poco::Util::AbstractConfiguration & config) { LOG_WARNING(log, "Not reloading (SSL is disabled)."); } + + /// Handle configuration reload + void reload([[maybe_unused]] const Poco::Util::AbstractConfiguration & config){}; + +private: + CertReloader() : log(&Poco::Logger::get("CertReloader")){}; + Poco::Logger * log; +}; + +#else + +/// SSL_CTX_set_cert_cb function +extern "C" int cert_reloader_dispatch(SSL * ssl, void * arg); + +/// The CertReloader singleton performs 2 functions: +/// 1. Dynamic reloading of TLS keypair when requested by main: +/// Main notifies CertReloader when the config changes. On changed config, +/// CertReloader reloads certs from disk. +/// 2. Implement `SSL_CTX_set_cert_cb` to set certificate for a new connection: +/// OpenSSL invokes `cert_reloader_dispatch_set_cert` to setup a connection. +class CertReloader +{ +public: + using stat_t = struct stat; + + /// Singleton + CertReloader(CertReloader const &) = delete; + void operator=(CertReloader const &) = delete; + static CertReloader & instance() + { + static CertReloader instance; + return instance; + } + + /// Initialize the callback and perfom the initial cert loading + void init(const ::Poco::Util::AbstractConfiguration & config); + + /// Handle configuration reload + void reload(const Poco::Util::AbstractConfiguration & config); + + /// Add cert, key to SSL* connection. SetCert runs in an IO thread during + /// connection setup. SetCert is + /// establishing a new TLS connection. + int SetCert(SSL * ssl); + +private: + CertReloader() : log(&Poco::Logger::get("CertReloader")), cert_file(""), cert(nullptr), key_file(""), key(nullptr) { } + + mutable std::shared_mutex lck; + Poco::Logger * log; + + std::string cert_file; + stat_t cert_file_st; + std::unique_ptr cert; + bool setCertFile(std::string cert_file); + + std::string key_file; + stat_t key_file_st; + std::unique_ptr key; + bool setKeyFile(std::string key_file); +}; + +#endif +} diff --git a/src/Server/ya.make b/src/Server/ya.make index 011aec19a15..9d6bfc3419d 100644 --- a/src/Server/ya.make +++ b/src/Server/ya.make @@ -9,6 +9,7 @@ PEERDIR( CFLAGS(-g0) SRCS( + CertReloader.cpp HTTPHandler.cpp HTTPHandlerFactory.cpp InterserverIOHTTPHandler.cpp From 2f2b584b4d139e0e3d45f57de5756ca17957661e Mon Sep 17 00:00:00 2001 From: John Skopis Date: Thu, 8 Oct 2020 20:15:04 +0000 Subject: [PATCH 002/149] Revert some cleanup (to be squashed) When I was cleaning up this up for review I factored out this change but it's required. --- src/Server/CertReloader.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Server/CertReloader.cpp b/src/Server/CertReloader.cpp index e3f91ab9c0e..183cb0f24a4 100644 --- a/src/Server/CertReloader.cpp +++ b/src/Server/CertReloader.cpp @@ -42,7 +42,11 @@ void CertReloader::reload(const Poco::Util::AbstractConfiguration & config) const auto cert_file_ = config.getString("openSSL.server.certificateFile", ""); const auto key_file_ = config.getString("openSSL.server.privateKeyFile", ""); - if (setKeyFile(key_file_) || setCertFile(cert_file_)) + bool changed = false; + changed |= setKeyFile(key_file_); + changed |= setCertFile(cert_file_); + + if (changed) { LOG_INFO(log, "Reloading cert({}), key({})", cert_file, key_file); { From 9290182112a3c9d598d63191317d7ec9376dc31a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 7 Jul 2021 01:14:00 +0300 Subject: [PATCH 003/149] Update Server.cpp --- programs/server/Server.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 9749ff9b47f..7bb158fa3bd 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1261,7 +1261,6 @@ int Server::main(const std::vector & /*args*/) #endif }); - /// Interserver IO HTTP port_name = "interserver_http_port"; createServer(listen_host, port_name, listen_try, [&](UInt16 port) From e0b98cfa7628d62880c9f3caff8e54e011de4fea Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 7 Jul 2021 01:18:41 +0300 Subject: [PATCH 004/149] Update CertReloader.h --- src/Server/CertReloader.h | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Server/CertReloader.h b/src/Server/CertReloader.h index 013692f5e15..92ea6c83e51 100644 --- a/src/Server/CertReloader.h +++ b/src/Server/CertReloader.h @@ -92,10 +92,12 @@ public: int SetCert(SSL * ssl); private: - CertReloader() : log(&Poco::Logger::get("CertReloader")), cert_file(""), cert(nullptr), key_file(""), key(nullptr) { } + CertReloader() + { + } - mutable std::shared_mutex lck; - Poco::Logger * log; + mutable std::shared_mutex lock; + Poco::Logger * log = &Poco::Logger::get("CertReloader"); std::string cert_file; stat_t cert_file_st; From 019edee013ec6460558dd944d1c41bab06928036 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 7 Jul 2021 01:19:05 +0300 Subject: [PATCH 005/149] Update CertReloader.h --- src/Server/CertReloader.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Server/CertReloader.h b/src/Server/CertReloader.h index 92ea6c83e51..bc3165b1230 100644 --- a/src/Server/CertReloader.h +++ b/src/Server/CertReloader.h @@ -96,7 +96,7 @@ private: { } - mutable std::shared_mutex lock; + mutable std::shared_mutex mutex; Poco::Logger * log = &Poco::Logger::get("CertReloader"); std::string cert_file; From f3c88219199d90563080f1f4108f0613ef5f6e05 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 7 Jul 2021 01:19:26 +0300 Subject: [PATCH 006/149] Update CertReloader.cpp --- src/Server/CertReloader.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Server/CertReloader.cpp b/src/Server/CertReloader.cpp index 183cb0f24a4..6f49bf1388d 100644 --- a/src/Server/CertReloader.cpp +++ b/src/Server/CertReloader.cpp @@ -10,7 +10,7 @@ int cert_reloader_dispatch_set_cert(SSL * ssl, [[maybe_unused]] void * arg) int CertReloader::SetCert(SSL * ssl) { - std::shared_lock lock(lck); + std::shared_lock lock(mutex); SSL_use_certificate(ssl, const_cast(cert->certificate())); SSL_use_RSAPrivateKey(ssl, key->impl()->getRSA()); @@ -50,7 +50,7 @@ void CertReloader::reload(const Poco::Util::AbstractConfiguration & config) { LOG_INFO(log, "Reloading cert({}), key({})", cert_file, key_file); { - std::unique_lock lock(lck); + std::unique_lock lock(mutex); key.reset(new Poco::Crypto::RSAKey("", key_file)); cert.reset(new Poco::Crypto::X509Certificate(cert_file)); } From 6007ea9423a74f1c32e11f20b08472f99682888d Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 7 Jul 2021 01:20:14 +0300 Subject: [PATCH 007/149] Update CertReloader.cpp --- src/Server/CertReloader.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Server/CertReloader.cpp b/src/Server/CertReloader.cpp index 6f49bf1388d..587d4dbac7a 100644 --- a/src/Server/CertReloader.cpp +++ b/src/Server/CertReloader.cpp @@ -51,8 +51,8 @@ void CertReloader::reload(const Poco::Util::AbstractConfiguration & config) LOG_INFO(log, "Reloading cert({}), key({})", cert_file, key_file); { std::unique_lock lock(mutex); - key.reset(new Poco::Crypto::RSAKey("", key_file)); - cert.reset(new Poco::Crypto::X509Certificate(cert_file)); + key = std::make_unique("", key_file); + cert = std::make_unique(cert_file); } LOG_INFO(log, "Reloaded cert {}", cert_file); } From e52680479e93fba2dcafee84cf68ef70bf88d1a8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 7 Jul 2021 02:15:30 +0300 Subject: [PATCH 008/149] Twiddling --- programs/server/Server.cpp | 6 +- ...rtReloader.cpp => CertificateReloader.cpp} | 24 ++-- .../{CertReloader.h => CertificateReloader.h} | 125 +++++++++--------- src/Server/ya.make | 3 +- 4 files changed, 81 insertions(+), 77 deletions(-) rename src/Server/{CertReloader.cpp => CertificateReloader.cpp} (81%) rename src/Server/{CertReloader.h => CertificateReloader.h} (66%) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 7bb158fa3bd..27723b38a1b 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -72,7 +72,7 @@ #include #include #include -#include +#include #include #include #include @@ -839,7 +839,7 @@ int Server::main(const std::vector & /*args*/) global_context->updateStorageConfiguration(*config); global_context->updateInterserverCredentials(*config); - CertReloader::instance().reload(*config); + CertificateReloader::instance().reload(*config); }, /* already_loaded = */ false); /// Reload it right now (initial loading) @@ -1373,7 +1373,7 @@ int Server::main(const std::vector & /*args*/) throw Exception("No servers started (add valid listen_host and 'tcp_port' or 'http_port' to configuration file.)", ErrorCodes::NO_ELEMENTS_IN_CONFIG); - CertReloader::instance().init(config()); + CertificateReloader::instance().init(config()); /// Must be done after initialization of `servers`, because async_metrics will access `servers` variable from its thread. async_metrics.start(); diff --git a/src/Server/CertReloader.cpp b/src/Server/CertificateReloader.cpp similarity index 81% rename from src/Server/CertReloader.cpp rename to src/Server/CertificateReloader.cpp index 587d4dbac7a..72a0d5e1ba1 100644 --- a/src/Server/CertReloader.cpp +++ b/src/Server/CertificateReloader.cpp @@ -1,14 +1,16 @@ -#include "CertReloader.h" +#include "CertificateReloader.h" + +#if USE_SSL namespace DB { -#if USE_SSL + int cert_reloader_dispatch_set_cert(SSL * ssl, [[maybe_unused]] void * arg) { - return CertReloader::instance().SetCert(ssl); + return CertificateReloader::instance().setCertificate(ssl); } -int CertReloader::SetCert(SSL * ssl) +int CertificateReloader::setCertificate(SSL * ssl) { std::shared_lock lock(mutex); SSL_use_certificate(ssl, const_cast(cert->certificate())); @@ -24,7 +26,7 @@ int CertReloader::SetCert(SSL * ssl) return 1; } -void CertReloader::init(const Poco::Util::AbstractConfiguration & config) +void CertificateReloader::init(const Poco::Util::AbstractConfiguration & config) { LOG_DEBUG(log, "Initializing config reloader."); @@ -36,7 +38,7 @@ void CertReloader::init(const Poco::Util::AbstractConfiguration & config) reload(config); } -void CertReloader::reload(const Poco::Util::AbstractConfiguration & config) +void CertificateReloader::reload(const Poco::Util::AbstractConfiguration & config) { LOG_DEBUG(log, "Handling cert reload."); const auto cert_file_ = config.getString("openSSL.server.certificateFile", ""); @@ -44,7 +46,7 @@ void CertReloader::reload(const Poco::Util::AbstractConfiguration & config) bool changed = false; changed |= setKeyFile(key_file_); - changed |= setCertFile(cert_file_); + changed |= setCertificateFile(cert_file_); if (changed) { @@ -58,7 +60,7 @@ void CertReloader::reload(const Poco::Util::AbstractConfiguration & config) } } -bool CertReloader::setKeyFile(const std::string key_file_) +bool CertificateReloader::setKeyFile(const std::string key_file_) { if (key_file_.empty()) return false; @@ -78,7 +80,7 @@ bool CertReloader::setKeyFile(const std::string key_file_) return false; } -bool CertReloader::setCertFile(const std::string cert_file_) +bool CertificateReloader::setCertificateFile(const std::string cert_file_) { if (cert_file_.empty()) return false; @@ -98,6 +100,6 @@ bool CertReloader::setCertFile(const std::string cert_file_) return false; } -#endif - } + +#endif diff --git a/src/Server/CertReloader.h b/src/Server/CertificateReloader.h similarity index 66% rename from src/Server/CertReloader.h rename to src/Server/CertificateReloader.h index bc3165b1230..3bbad94e68a 100644 --- a/src/Server/CertReloader.h +++ b/src/Server/CertificateReloader.h @@ -31,16 +31,72 @@ namespace DB { -#if !USE_SSL -class CertReloader +#if USE_SSL + +/// SSL_CTX_set_cert_cb function +extern "C" int cert_reloader_dispatch(SSL * ssl, void * arg); + +/// The CertificateReloader singleton performs 2 functions: +/// 1. Dynamic reloading of TLS keypair when requested by main: +/// Main notifies CertificateReloader when the config changes. On changed config, +/// CertificateReloader reloads certs from disk. +/// 2. Implement `SSL_CTX_set_cert_cb` to set certificate for a new connection: +/// OpenSSL invokes `cert_reloader_dispatch_set_cert` to setup a connection. +class CertificateReloader +{ +public: + using stat_t = struct stat; + + /// Singleton + CertificateReloader(CertificateReloader const &) = delete; + void operator=(CertificateReloader const &) = delete; + static CertificateReloader & instance() + { + static CertificateReloader instance; + return instance; + } + + /// Initialize the callback and perfom the initial cert loading + void init(const ::Poco::Util::AbstractConfiguration & config); + + /// Handle configuration reload + void reload(const Poco::Util::AbstractConfiguration & config); + + /// Add cert, key to SSL* connection. SetCertificate runs in an IO thread during + /// connection setup. SetCertificate is + /// establishing a new TLS connection. + int setCertificate(SSL * ssl); + +private: + CertificateReloader() + { + } + + mutable std::shared_mutex mutex; + Poco::Logger * log = &Poco::Logger::get("CertificateReloader"); + + std::string cert_file; + stat_t cert_file_st; + std::unique_ptr cert; + bool setCertificateFile(std::string cert_file); + + std::string key_file; + stat_t key_file_st; + std::unique_ptr key; + bool setKeyFile(std::string key_file); +}; + +#else + +class CertificateReloader { public: /// Singleton - CertReloader(CertReloader const &) = delete; - void operator=(CertReloader const &) = delete; - static CertReloader & instance() + CertificateReloader(CertificateReloader const &) = delete; + void operator=(CertificateReloader const &) = delete; + static CertificateReloader & instance() { - static CertReloader instance; + static CertificateReloader instance; return instance; } @@ -51,64 +107,9 @@ public: void reload([[maybe_unused]] const Poco::Util::AbstractConfiguration & config){}; private: - CertReloader() : log(&Poco::Logger::get("CertReloader")){}; + CertificateReloader() : log(&Poco::Logger::get("CertificateReloader")){}; Poco::Logger * log; }; -#else - -/// SSL_CTX_set_cert_cb function -extern "C" int cert_reloader_dispatch(SSL * ssl, void * arg); - -/// The CertReloader singleton performs 2 functions: -/// 1. Dynamic reloading of TLS keypair when requested by main: -/// Main notifies CertReloader when the config changes. On changed config, -/// CertReloader reloads certs from disk. -/// 2. Implement `SSL_CTX_set_cert_cb` to set certificate for a new connection: -/// OpenSSL invokes `cert_reloader_dispatch_set_cert` to setup a connection. -class CertReloader -{ -public: - using stat_t = struct stat; - - /// Singleton - CertReloader(CertReloader const &) = delete; - void operator=(CertReloader const &) = delete; - static CertReloader & instance() - { - static CertReloader instance; - return instance; - } - - /// Initialize the callback and perfom the initial cert loading - void init(const ::Poco::Util::AbstractConfiguration & config); - - /// Handle configuration reload - void reload(const Poco::Util::AbstractConfiguration & config); - - /// Add cert, key to SSL* connection. SetCert runs in an IO thread during - /// connection setup. SetCert is - /// establishing a new TLS connection. - int SetCert(SSL * ssl); - -private: - CertReloader() - { - } - - mutable std::shared_mutex mutex; - Poco::Logger * log = &Poco::Logger::get("CertReloader"); - - std::string cert_file; - stat_t cert_file_st; - std::unique_ptr cert; - bool setCertFile(std::string cert_file); - - std::string key_file; - stat_t key_file_st; - std::unique_ptr key; - bool setKeyFile(std::string key_file); -}; - #endif } diff --git a/src/Server/ya.make b/src/Server/ya.make index 9f1d941e4b8..36f5c13c97b 100644 --- a/src/Server/ya.make +++ b/src/Server/ya.make @@ -10,7 +10,7 @@ PEERDIR( SRCS( - CertReloader.cpp + CertificateReloader.cpp GRPCServer.cpp HTTP/HTMLForm.cpp HTTP/HTTPServer.cpp @@ -36,6 +36,7 @@ SRCS( StaticRequestHandler.cpp TCPHandler.cpp WebUIRequestHandler.cpp + ) END() From 3351b135e7364b91a307044fa904e85397b4923d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 7 Jul 2021 02:34:43 +0300 Subject: [PATCH 009/149] More twiddling --- src/Server/CertificateReloader.cpp | 33 ++++++++++++++++++++++-------- src/Server/CertificateReloader.h | 3 --- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/Server/CertificateReloader.cpp b/src/Server/CertificateReloader.cpp index 72a0d5e1ba1..f5a539654e8 100644 --- a/src/Server/CertificateReloader.cpp +++ b/src/Server/CertificateReloader.cpp @@ -1,10 +1,17 @@ #include "CertificateReloader.h" +#include + #if USE_SSL namespace DB { +namespace ErrorCodes +{ + extern const int CANNOT_STAT; +} + int cert_reloader_dispatch_set_cert(SSL * ssl, [[maybe_unused]] void * arg) { return CertificateReloader::instance().setCertificate(ssl); @@ -28,19 +35,19 @@ int CertificateReloader::setCertificate(SSL * ssl) void CertificateReloader::init(const Poco::Util::AbstractConfiguration & config) { - LOG_DEBUG(log, "Initializing config reloader."); + LOG_DEBUG(log, "Initializing certificate reloader."); - auto & ssl_manager = Poco::Net::SSLManager::instance(); - const auto ssl_ctx_ptr = ssl_manager.defaultServerContext(); - auto ctx = ssl_ctx_ptr->sslContext(); - SSL_CTX_set_cert_cb(ctx, cert_reloader_dispatch_set_cert, nullptr); + SSL_CTX_set_cert_cb( + Poco::Net::SSLManager::instance().defaultClientContext()->sslContext(), + [](SSL * ssl, void * arg) { return reinterpret_cast(arg)->setCertificate(ssl); }, + static_cast(this)); reload(config); } void CertificateReloader::reload(const Poco::Util::AbstractConfiguration & config) { - LOG_DEBUG(log, "Handling cert reload."); + LOG_DEBUG(log, "Handling certificate reload."); const auto cert_file_ = config.getString("openSSL.server.certificateFile", ""); const auto key_file_ = config.getString("openSSL.server.privateKeyFile", ""); @@ -50,13 +57,13 @@ void CertificateReloader::reload(const Poco::Util::AbstractConfiguration & confi if (changed) { - LOG_INFO(log, "Reloading cert({}), key({})", cert_file, key_file); + LOG_INFO(log, "Reloading certificate ({}) and key ({}).", cert_file, key_file); { std::unique_lock lock(mutex); - key = std::make_unique("", key_file); + key = std::make_unique(/* public key */ "", /* private key */ key_file); cert = std::make_unique(cert_file); } - LOG_INFO(log, "Reloaded cert {}", cert_file); + LOG_INFO(log, "Reloaded certificate ({}).", cert_file); } } @@ -68,7 +75,12 @@ bool CertificateReloader::setKeyFile(const std::string key_file_) stat_t st; int res = stat(key_file_.c_str(), &st); if (res == -1) + { + LOG_ERROR(log, "Cannot obtain stat for key file {}, skipping update. {}", key_file_, errnoToString(ErrorCodes::CANNOT_STAT)); return false; + } + + /// NOTE: if file changed twice in a second, the update will be missed. if (st.st_mtime != key_file_st.st_mtime || key_file != key_file_) { @@ -88,7 +100,10 @@ bool CertificateReloader::setCertificateFile(const std::string cert_file_) stat_t st; int res = stat(cert_file_.c_str(), &st); if (res == -1) + { + LOG_ERROR(log, "Cannot obtain stat for certificate file {}, skipping update. {}", cert_file_, errnoToString(ErrorCodes::CANNOT_STAT)); return false; + } if (st.st_mtime != cert_file_st.st_mtime || cert_file != cert_file_) { diff --git a/src/Server/CertificateReloader.h b/src/Server/CertificateReloader.h index 3bbad94e68a..2b639a435ba 100644 --- a/src/Server/CertificateReloader.h +++ b/src/Server/CertificateReloader.h @@ -33,9 +33,6 @@ namespace DB { #if USE_SSL -/// SSL_CTX_set_cert_cb function -extern "C" int cert_reloader_dispatch(SSL * ssl, void * arg); - /// The CertificateReloader singleton performs 2 functions: /// 1. Dynamic reloading of TLS keypair when requested by main: /// Main notifies CertificateReloader when the config changes. On changed config, From 0b8729370cb57740166bbd9d78e7646a8d8d2736 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 7 Jul 2021 02:52:24 +0300 Subject: [PATCH 010/149] More twiddling --- src/Server/CertificateReloader.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Server/CertificateReloader.cpp b/src/Server/CertificateReloader.cpp index f5a539654e8..2a800d97f85 100644 --- a/src/Server/CertificateReloader.cpp +++ b/src/Server/CertificateReloader.cpp @@ -12,11 +12,6 @@ namespace ErrorCodes extern const int CANNOT_STAT; } -int cert_reloader_dispatch_set_cert(SSL * ssl, [[maybe_unused]] void * arg) -{ - return CertificateReloader::instance().setCertificate(ssl); -} - int CertificateReloader::setCertificate(SSL * ssl) { std::shared_lock lock(mutex); From 7c218b51e6b412c56ba4e1e5eeee8f55bde2660c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 7 Jul 2021 03:13:37 +0300 Subject: [PATCH 011/149] Fix build --- src/Server/CertificateReloader.h | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/Server/CertificateReloader.h b/src/Server/CertificateReloader.h index 2b639a435ba..15a9eb464b3 100644 --- a/src/Server/CertificateReloader.h +++ b/src/Server/CertificateReloader.h @@ -54,7 +54,7 @@ public: } /// Initialize the callback and perfom the initial cert loading - void init(const ::Poco::Util::AbstractConfiguration & config); + void init(const Poco::Util::AbstractConfiguration & config); /// Handle configuration reload void reload(const Poco::Util::AbstractConfiguration & config); @@ -88,24 +88,22 @@ private: class CertificateReloader { public: - /// Singleton CertificateReloader(CertificateReloader const &) = delete; void operator=(CertificateReloader const &) = delete; + static CertificateReloader & instance() { static CertificateReloader instance; return instance; } - /// Initialize the callback and perfom the initial cert loading - void init([[maybe_unused]] const ::Poco::Util::AbstractConfiguration & config) { LOG_WARNING(log, "Not reloading (SSL is disabled)."); } + void init(const Poco::Util::AbstractConfiguration &) + { + } - /// Handle configuration reload - void reload([[maybe_unused]] const Poco::Util::AbstractConfiguration & config){}; - -private: - CertificateReloader() : log(&Poco::Logger::get("CertificateReloader")){}; - Poco::Logger * log; + void reload(const Poco::Util::AbstractConfiguration &) + { + } }; #endif From f73605365c7e575dd5d9602403ce08aeb71b1013 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 9 Jul 2021 03:13:04 +0300 Subject: [PATCH 012/149] Fix build --- src/Server/CertificateReloader.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Server/CertificateReloader.h b/src/Server/CertificateReloader.h index 15a9eb464b3..f4d442edc26 100644 --- a/src/Server/CertificateReloader.h +++ b/src/Server/CertificateReloader.h @@ -88,9 +88,6 @@ private: class CertificateReloader { public: - CertificateReloader(CertificateReloader const &) = delete; - void operator=(CertificateReloader const &) = delete; - static CertificateReloader & instance() { static CertificateReloader instance; From 12b2a77ba3de9a198b04b99a4f8a72bfb8e330bb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 9 Jul 2021 05:27:01 +0300 Subject: [PATCH 013/149] More normal --- src/Server/CertificateReloader.cpp | 99 +++++++++++++----------------- src/Server/CertificateReloader.h | 79 +++++++++++++----------- 2 files changed, 87 insertions(+), 91 deletions(-) diff --git a/src/Server/CertificateReloader.cpp b/src/Server/CertificateReloader.cpp index 2a800d97f85..6ee77ae84b3 100644 --- a/src/Server/CertificateReloader.cpp +++ b/src/Server/CertificateReloader.cpp @@ -1,9 +1,14 @@ #include "CertificateReloader.h" -#include - #if USE_SSL +#include +#include +#include +#include +#include + + namespace DB { @@ -12,98 +17,82 @@ namespace ErrorCodes extern const int CANNOT_STAT; } + +/// This is callback for OpenSSL. It will be called on every connection to obtain a certificate and private key. int CertificateReloader::setCertificate(SSL * ssl) { - std::shared_lock lock(mutex); - SSL_use_certificate(ssl, const_cast(cert->certificate())); - SSL_use_RSAPrivateKey(ssl, key->impl()->getRSA()); + auto current = data.get(); + if (!current) + return -1; + + SSL_use_certificate(ssl, const_cast(current->cert.certificate())); + SSL_use_RSAPrivateKey(ssl, current->key.impl()->getRSA()); int err = SSL_check_private_key(ssl); if (err != 1) { std::string msg = Poco::Net::Utility::getLastError(); - LOG_ERROR(log, "Unusable keypair {}", msg); + LOG_ERROR(log, "Unusable key-pair {}", msg); return -1; } + return 1; } + void CertificateReloader::init(const Poco::Util::AbstractConfiguration & config) { LOG_DEBUG(log, "Initializing certificate reloader."); + reload(config); + + /// Set a callback for OpenSSL to allow get the updated cert and key. + SSL_CTX_set_cert_cb( Poco::Net::SSLManager::instance().defaultClientContext()->sslContext(), [](SSL * ssl, void * arg) { return reinterpret_cast(arg)->setCertificate(ssl); }, static_cast(this)); - - reload(config); } + void CertificateReloader::reload(const Poco::Util::AbstractConfiguration & config) { - LOG_DEBUG(log, "Handling certificate reload."); - const auto cert_file_ = config.getString("openSSL.server.certificateFile", ""); - const auto key_file_ = config.getString("openSSL.server.privateKeyFile", ""); + /// If at least one of the files is modified - recreate - bool changed = false; - changed |= setKeyFile(key_file_); - changed |= setCertificateFile(cert_file_); + std::string new_cert_path = config.getString("openSSL.server.certificateFile", ""); + std::string new_key_path = config.getString("openSSL.server.privateKeyFile", ""); - if (changed) + if (cert_file.changeIfModified(std::move(new_cert_path), log) + || key_file.changeIfModified(std::move(new_key_path), log)) { - LOG_INFO(log, "Reloading certificate ({}) and key ({}).", cert_file, key_file); - { - std::unique_lock lock(mutex); - key = std::make_unique(/* public key */ "", /* private key */ key_file); - cert = std::make_unique(cert_file); - } - LOG_INFO(log, "Reloaded certificate ({}).", cert_file); + LOG_DEBUG(log, "Reloading certificate ({}) and key ({}).", cert_file.path, key_file.path); + data.set(std::make_unique(cert_file.path, key_file.path)); + LOG_INFO(log, "Reloaded certificate ({}) and key ({}).", cert_file.path, key_file.path); } } -bool CertificateReloader::setKeyFile(const std::string key_file_) + +CertificateReloader::Data::Data(std::string cert_path, std::string key_path) + : cert(cert_path), key(/* public key */ "", /* private key */ key_path) { - if (key_file_.empty()) - return false; - - stat_t st; - int res = stat(key_file_.c_str(), &st); - if (res == -1) - { - LOG_ERROR(log, "Cannot obtain stat for key file {}, skipping update. {}", key_file_, errnoToString(ErrorCodes::CANNOT_STAT)); - return false; - } - - /// NOTE: if file changed twice in a second, the update will be missed. - - if (st.st_mtime != key_file_st.st_mtime || key_file != key_file_) - { - key_file = key_file_; - key_file_st = st; - return true; - } - - return false; } -bool CertificateReloader::setCertificateFile(const std::string cert_file_) -{ - if (cert_file_.empty()) - return false; - stat_t st; - int res = stat(cert_file_.c_str(), &st); - if (res == -1) +bool CertificateReloader::File::changeIfModified(std::string new_path, Poco::Logger * logger) +{ + std::error_code ec; + std::filesystem::file_time_type new_modification_time = std::filesystem::last_write_time(new_path, ec); + if (ec) { - LOG_ERROR(log, "Cannot obtain stat for certificate file {}, skipping update. {}", cert_file_, errnoToString(ErrorCodes::CANNOT_STAT)); + LOG_ERROR(logger, "Cannot obtain modification time for {} file {}, skipping update. {}", + description, new_path, errnoToString(ErrorCodes::CANNOT_STAT, ec.value())); return false; } - if (st.st_mtime != cert_file_st.st_mtime || cert_file != cert_file_) + if (new_path != path || new_modification_time != modification_time) { - cert_file = cert_file_; - cert_file_st = st; + path = new_path; + modification_time = new_modification_time; return true; } diff --git a/src/Server/CertificateReloader.h b/src/Server/CertificateReloader.h index f4d442edc26..0b1817211cf 100644 --- a/src/Server/CertificateReloader.h +++ b/src/Server/CertificateReloader.h @@ -1,40 +1,28 @@ #pragma once -#include -#include -#include - -#include -#include -#include -#include -#include - -#include -#include - -#include - #if !defined(ARCADIA_BUILD) # include #endif #if USE_SSL -# include -# include -# include -# include -# include -# include -# include -#endif + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + namespace DB { -#if USE_SSL /// The CertificateReloader singleton performs 2 functions: -/// 1. Dynamic reloading of TLS keypair when requested by main: +/// 1. Dynamic reloading of TLS key-pair when requested by main: /// Main notifies CertificateReloader when the config changes. On changed config, /// CertificateReloader reloads certs from disk. /// 2. Implement `SSL_CTX_set_cert_cb` to set certificate for a new connection: @@ -53,7 +41,7 @@ public: return instance; } - /// Initialize the callback and perfom the initial cert loading + /// Initialize the callback and perform the initial cert loading void init(const Poco::Util::AbstractConfiguration & config); /// Handle configuration reload @@ -69,22 +57,40 @@ private: { } - mutable std::shared_mutex mutex; Poco::Logger * log = &Poco::Logger::get("CertificateReloader"); - std::string cert_file; - stat_t cert_file_st; - std::unique_ptr cert; - bool setCertificateFile(std::string cert_file); + struct File + { + const char * description; + File(const char * description_) : description(description_) {} - std::string key_file; - stat_t key_file_st; - std::unique_ptr key; - bool setKeyFile(std::string key_file); + std::string path; + std::filesystem::file_time_type modification_time; + + bool changeIfModified(std::string new_path, Poco::Logger * logger); + }; + + File cert_file{"certificate"}; + File key_file{"key"}; + + struct Data + { + Poco::Crypto::X509Certificate cert; + Poco::Crypto::RSAKey key; + + Data(std::string cert_path, std::string key_path); + }; + + MultiVersion data; }; +} + #else +namespace DB +{ + class CertificateReloader { public: @@ -103,5 +109,6 @@ public: } }; -#endif } + +#endif From 9b299fbb56d915f683e31e81a8b81003cb0438bf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 10 Jul 2021 05:57:44 +0300 Subject: [PATCH 014/149] Simplification --- programs/server/Server.cpp | 6 +++--- src/Server/CertificateReloader.h | 25 ------------------------- 2 files changed, 3 insertions(+), 28 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 27723b38a1b..c83af36c9fa 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -81,9 +81,6 @@ #if !defined(ARCADIA_BUILD) # include "config_core.h" # include "Common/config_version.h" -# if USE_OPENCL -# include "Common/BitonicSort.h" // Y_IGNORE -# endif #endif #if defined(OS_LINUX) @@ -839,7 +836,10 @@ int Server::main(const std::vector & /*args*/) global_context->updateStorageConfiguration(*config); global_context->updateInterserverCredentials(*config); + +#if USE_SSL CertificateReloader::instance().reload(*config); +#endif }, /* already_loaded = */ false); /// Reload it right now (initial loading) diff --git a/src/Server/CertificateReloader.h b/src/Server/CertificateReloader.h index 0b1817211cf..cfe96025bf5 100644 --- a/src/Server/CertificateReloader.h +++ b/src/Server/CertificateReloader.h @@ -86,29 +86,4 @@ private: } -#else - -namespace DB -{ - -class CertificateReloader -{ -public: - static CertificateReloader & instance() - { - static CertificateReloader instance; - return instance; - } - - void init(const Poco::Util::AbstractConfiguration &) - { - } - - void reload(const Poco::Util::AbstractConfiguration &) - { - } -}; - -} - #endif From dfe7f2df8b2f0c4099897e920a3233d80653dd78 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 10 Jul 2021 05:59:11 +0300 Subject: [PATCH 015/149] Simplification --- programs/server/Server.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 14e33a44389..5b6e882eabc 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1373,7 +1373,9 @@ int Server::main(const std::vector & /*args*/) throw Exception("No servers started (add valid listen_host and 'tcp_port' or 'http_port' to configuration file.)", ErrorCodes::NO_ELEMENTS_IN_CONFIG); +#if USE_SSL CertificateReloader::instance().init(config()); +#endif /// Must be done after initialization of `servers`, because async_metrics will access `servers` variable from its thread. async_metrics.start(); From 21774d6700fcc11e0ac2784a376b173757c4ea1c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 10 Jul 2021 06:01:24 +0300 Subject: [PATCH 016/149] Correct misleading comments --- src/Server/CertificateReloader.h | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/Server/CertificateReloader.h b/src/Server/CertificateReloader.h index cfe96025bf5..a3eb258acd2 100644 --- a/src/Server/CertificateReloader.h +++ b/src/Server/CertificateReloader.h @@ -22,11 +22,11 @@ namespace DB { /// The CertificateReloader singleton performs 2 functions: -/// 1. Dynamic reloading of TLS key-pair when requested by main: -/// Main notifies CertificateReloader when the config changes. On changed config, -/// CertificateReloader reloads certs from disk. +/// 1. Dynamic reloading of TLS key-pair when requested by server: +/// Server config reloader notifies CertificateReloader when the config changes. +/// On changed config, CertificateReloader reloads certs from disk. /// 2. Implement `SSL_CTX_set_cert_cb` to set certificate for a new connection: -/// OpenSSL invokes `cert_reloader_dispatch_set_cert` to setup a connection. +/// OpenSSL invokes a callback to setup a connection. class CertificateReloader { public: @@ -47,9 +47,7 @@ public: /// Handle configuration reload void reload(const Poco::Util::AbstractConfiguration & config); - /// Add cert, key to SSL* connection. SetCertificate runs in an IO thread during - /// connection setup. SetCertificate is - /// establishing a new TLS connection. + /// A callback for OpenSSL int setCertificate(SSL * ssl); private: From 68bc8d63a1c15acb3396a30d42886a8eb0cf641c Mon Sep 17 00:00:00 2001 From: Artur Filatenkov <613623@mail.ru> Date: Wed, 10 Nov 2021 15:55:57 +0300 Subject: [PATCH 017/149] Working version --- src/Server/CertificateReloader.cpp | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/src/Server/CertificateReloader.cpp b/src/Server/CertificateReloader.cpp index 6ee77ae84b3..02b50d4d951 100644 --- a/src/Server/CertificateReloader.cpp +++ b/src/Server/CertificateReloader.cpp @@ -2,8 +2,8 @@ #if USE_SSL -#include -#include +#include +#include #include #include #include @@ -12,6 +12,12 @@ namespace DB { +int cert_reloader_dispatch_set_cert(SSL * ssl, [[maybe_unused]] void * arg) +{ + return CertificateReloader::instance().setCertificate(ssl); +} + + namespace ErrorCodes { extern const int CANNOT_STAT; @@ -48,10 +54,15 @@ void CertificateReloader::init(const Poco::Util::AbstractConfiguration & config) /// Set a callback for OpenSSL to allow get the updated cert and key. - SSL_CTX_set_cert_cb( - Poco::Net::SSLManager::instance().defaultClientContext()->sslContext(), - [](SSL * ssl, void * arg) { return reinterpret_cast(arg)->setCertificate(ssl); }, - static_cast(this)); + // SSL_CTX_set_cert_cb( + // Poco::Net::SSLManager::instance().defaultClientContext()->sslContext(), + // [](SSL * ssl, void * arg) { return reinterpret_cast(arg)->setCertificate(ssl); }, + // static_cast(this)); + + auto & ssl_manager = Poco::Net::SSLManager::instance(); + const auto ssl_ctx_ptr = ssl_manager.defaultServerContext(); + auto ctx = ssl_ctx_ptr->sslContext(); + SSL_CTX_set_cert_cb(ctx, cert_reloader_dispatch_set_cert, nullptr); } @@ -62,8 +73,10 @@ void CertificateReloader::reload(const Poco::Util::AbstractConfiguration & confi std::string new_cert_path = config.getString("openSSL.server.certificateFile", ""); std::string new_key_path = config.getString("openSSL.server.privateKeyFile", ""); - if (cert_file.changeIfModified(std::move(new_cert_path), log) - || key_file.changeIfModified(std::move(new_key_path), log)) + bool cert_file_changed = cert_file.changeIfModified(std::move(new_cert_path), log); + bool key_file_changed = key_file.changeIfModified(std::move(new_key_path), log); + + if (cert_file_changed || key_file_changed) { LOG_DEBUG(log, "Reloading certificate ({}) and key ({}).", cert_file.path, key_file.path); data.set(std::make_unique(cert_file.path, key_file.path)); From 4aaa034c8ef7760c29e0d6d6a35cb29b556be2bb Mon Sep 17 00:00:00 2001 From: Artur <613623@mail.ru> Date: Wed, 10 Nov 2021 16:56:07 +0000 Subject: [PATCH 018/149] Correct behaviour if certificate or key was not set --- programs/server/Server.cpp | 4 ++-- src/Server/CertificateReloader.cpp | 17 ++++++++++++----- src/Server/CertificateReloader.h | 5 +++-- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index bcc5cdef6e6..1159de3751e 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -878,7 +878,7 @@ if (ThreadFuzzer::instance().isEffective()) CompressionCodecEncrypted::Configuration::instance().tryLoad(*config, "encryption_codecs"); #if USE_SSL - CertificateReloader::instance().reload(*config); + CertificateReloader::instance().tryLoad(*config); #endif }, /* already_loaded = */ false); /// Reload it right now (initial loading) @@ -1468,7 +1468,7 @@ if (ThreadFuzzer::instance().isEffective()) ErrorCodes::NO_ELEMENTS_IN_CONFIG); #if USE_SSL - CertificateReloader::instance().init(config()); + CertificateReloader::instance().tryLoad(config()); #endif /// Must be done after initialization of `servers`, because async_metrics will access `servers` variable from its thread. diff --git a/src/Server/CertificateReloader.cpp b/src/Server/CertificateReloader.cpp index 06fa668ead1..4889e794d47 100644 --- a/src/Server/CertificateReloader.cpp +++ b/src/Server/CertificateReloader.cpp @@ -51,26 +51,28 @@ int CertificateReloader::setCertificate(SSL * ssl) } -void CertificateReloader::init(const Poco::Util::AbstractConfiguration & config) +void CertificateReloader::init() { LOG_DEBUG(log, "Initializing certificate reloader."); - reload(config); - /// Set a callback for OpenSSL to allow get the updated cert and key. - auto* ctx = Poco::Net::SSLManager::instance().defaultServerContext()->sslContext(); + auto* ctx = Poco::Net::SSLManager::instance().defaultServerContext()->sslContext(); SSL_CTX_set_cert_cb(ctx, callSetCertificate, nullptr); + first_load = false; } -void CertificateReloader::reload(const Poco::Util::AbstractConfiguration & config) +void CertificateReloader::tryLoad(const Poco::Util::AbstractConfiguration & config) { /// If at least one of the files is modified - recreate std::string new_cert_path = config.getString("openSSL.server.certificateFile", ""); std::string new_key_path = config.getString("openSSL.server.privateKeyFile", ""); + /// For empty paths (that means, that user doesn't want to use certificates) + /// no processing required + if (new_cert_path.empty() || new_key_path.empty()) { LOG_INFO(log, "One of paths is empty. Cannot apply new configuration for certificates. Fill all paths and try again."); @@ -86,6 +88,11 @@ void CertificateReloader::reload(const Poco::Util::AbstractConfiguration & confi data.set(std::make_unique(cert_file.path, key_file.path)); LOG_INFO(log, "Reloaded certificate ({}) and key ({}).", cert_file.path, key_file.path); } + + /// If callback is not set yet + + if (first_load) + init(); } } diff --git a/src/Server/CertificateReloader.h b/src/Server/CertificateReloader.h index a3eb258acd2..a078e16b45c 100644 --- a/src/Server/CertificateReloader.h +++ b/src/Server/CertificateReloader.h @@ -42,10 +42,10 @@ public: } /// Initialize the callback and perform the initial cert loading - void init(const Poco::Util::AbstractConfiguration & config); + void init(); /// Handle configuration reload - void reload(const Poco::Util::AbstractConfiguration & config); + void tryLoad(const Poco::Util::AbstractConfiguration & config); /// A callback for OpenSSL int setCertificate(SSL * ssl); @@ -80,6 +80,7 @@ private: }; MultiVersion data; + bool first_load = true; }; } From 96070b414de8dd582c6c983940b87faa8b343820 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 14 Dec 2021 15:54:20 +0300 Subject: [PATCH 019/149] support read_in_order optimization if prefix of sorting key is fixed --- src/Interpreters/InterpreterSelectQuery.cpp | 12 +- src/Interpreters/TreeCNFConverter.cpp | 2 +- src/Interpreters/TreeCNFConverter.h | 9 + src/Storages/ReadInOrderOptimizer.cpp | 303 ++++++++++++------ src/Storages/ReadInOrderOptimizer.h | 8 + ...02149_read_in_order_fixed_prefix.reference | 79 +++++ .../02149_read_in_order_fixed_prefix.sql | 21 ++ ...d_in_order_fixed_prefix_negative.reference | 5 + ...149_read_in_order_fixed_prefix_negative.sh | 20 ++ 9 files changed, 351 insertions(+), 108 deletions(-) create mode 100644 tests/queries/0_stateless/02149_read_in_order_fixed_prefix.reference create mode 100644 tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql create mode 100644 tests/queries/0_stateless/02149_read_in_order_fixed_prefix_negative.reference create mode 100755 tests/queries/0_stateless/02149_read_in_order_fixed_prefix_negative.sh diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index c8f48f2ed1f..48a41b65b6e 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1924,6 +1924,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc { query_info.projection->order_optimizer = std::make_shared( // TODO Do we need a projection variant for this field? + query, analysis_result.order_by_elements_actions, getSortDescription(query, context), query_info.syntax_analyzer_result); @@ -1931,7 +1932,10 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc else { query_info.order_optimizer = std::make_shared( - analysis_result.order_by_elements_actions, getSortDescription(query, context), query_info.syntax_analyzer_result); + query, + analysis_result.order_by_elements_actions, + getSortDescription(query, context), + query_info.syntax_analyzer_result); } } else @@ -1939,6 +1943,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc if (query_info.projection) { query_info.projection->order_optimizer = std::make_shared( + query, query_info.projection->group_by_elements_actions, getSortDescriptionFromGroupBy(query), query_info.syntax_analyzer_result); @@ -1946,7 +1951,10 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc else { query_info.order_optimizer = std::make_shared( - analysis_result.group_by_elements_actions, getSortDescriptionFromGroupBy(query), query_info.syntax_analyzer_result); + query, + analysis_result.group_by_elements_actions, + getSortDescriptionFromGroupBy(query), + query_info.syntax_analyzer_result); } } diff --git a/src/Interpreters/TreeCNFConverter.cpp b/src/Interpreters/TreeCNFConverter.cpp index a55bacee5fa..149bddab11f 100644 --- a/src/Interpreters/TreeCNFConverter.cpp +++ b/src/Interpreters/TreeCNFConverter.cpp @@ -26,7 +26,7 @@ bool isLogicalFunction(const ASTFunction & func) size_t countAtoms(const ASTPtr & node) { checkStackSize(); - if (node->as()) + if (node->as() || node->as()) return 1; const auto * func = node->as(); diff --git a/src/Interpreters/TreeCNFConverter.h b/src/Interpreters/TreeCNFConverter.h index 7ea5fa54680..0ec5368ba71 100644 --- a/src/Interpreters/TreeCNFConverter.h +++ b/src/Interpreters/TreeCNFConverter.h @@ -89,6 +89,15 @@ public: return *this; } + template + const CNFQuery & iterateAtoms(F func) const + { + for (const auto & group : statements) + for (const auto & atom : group) + func(atom); + return *this; + } + CNFQuery & appendGroup(AndGroup&& and_group) { for (auto && or_group : and_group) diff --git a/src/Storages/ReadInOrderOptimizer.cpp b/src/Storages/ReadInOrderOptimizer.cpp index bae24f97b28..ea67c6c70bd 100644 --- a/src/Storages/ReadInOrderOptimizer.cpp +++ b/src/Storages/ReadInOrderOptimizer.cpp @@ -6,6 +6,8 @@ #include #include #include +#include +#include namespace DB { @@ -15,13 +17,152 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +namespace +{ + +ASTPtr getFixedPoint(const ASTPtr & ast) +{ + const auto * func = ast->as(); + if (!func || func->name != "equals") + return nullptr; + + const auto & lhs = func->arguments->children[0]; + const auto & rhs = func->arguments->children[1]; + + if (lhs->as()) + return rhs; + + if (rhs->as()) + return lhs; + + return nullptr; +} + +size_t calculateFixedPrefixSize( + const ASTSelectQuery & query, const Names & sorting_key_columns) +{ + ASTPtr condition; + if (query.where() && query.prewhere()) + condition = makeASTFunction("and", query.where(), query.prewhere()); + else if (query.where()) + condition = query.where(); + else if (query.prewhere()) + condition = query.prewhere(); + + if (!condition) + return 0; + + /// Convert condition to CNF for more convinient analysis. + auto cnf = TreeCNFConverter::tryConvertToCNF(condition); + if (!cnf) + return 0; + + NameSet fixed_points; + + /// If we met expression like 'column = x', where 'x' is literal, + /// in clause of size 1 in CNF, then we can guarantee + /// that in all filtered rows 'column' will be equal to 'x'. + cnf->iterateGroups([&](const auto & group) + { + if (group.size() == 1 && !group.begin()->negative) + { + auto fixed_point = getFixedPoint(group.begin()->ast); + if (fixed_point) + fixed_points.insert(fixed_point->getColumnName()); + } + }); + + size_t prefix_size = 0; + for (const auto & column_name : sorting_key_columns) + { + if (!fixed_points.contains(column_name)) + break; + + ++prefix_size; + } + + return prefix_size; +} + +/// Optimize in case of exact match with order key element +/// or in some simple cases when order key element is wrapped into monotonic function. +/// Returns on of {-1, 0, 1} - direction of the match. 0 means - doesn't match. +int matchSortDescriptionAndKey( + const ExpressionActions::Actions & actions, + const SortColumnDescription & sort_column, + const String & sorting_key_column) +{ + /// If required order depend on collation, it cannot be matched with primary key order. + /// Because primary keys cannot have collations. + if (sort_column.collator) + return 0; + + int current_direction = sort_column.direction; + /// For the path: order by (sort_column, ...) + if (sort_column.column_name == sorting_key_column) + return current_direction; + + /// For the path: order by (function(sort_column), ...) + /// Allow only one simple monotonic functions with one argument + /// Why not allow multi monotonic functions? + bool found_function = false; + + for (const auto & action : actions) + { + if (action.node->type != ActionsDAG::ActionType::FUNCTION) + continue; + + if (found_function) + { + current_direction = 0; + break; + } + else + { + found_function = true; + } + + if (action.node->children.size() != 1 || action.node->children.at(0)->result_name != sorting_key_column) + { + current_direction = 0; + break; + } + + const auto & func = *action.node->function_base; + if (!func.hasInformationAboutMonotonicity()) + { + current_direction = 0; + break; + } + + auto monotonicity = func.getMonotonicityForRange(*func.getArgumentTypes().at(0), {}, {}); + if (!monotonicity.is_monotonic) + { + current_direction = 0; + break; + } + else if (!monotonicity.is_positive) + { + current_direction *= -1; + } + } + + if (!found_function) + current_direction = 0; + + return current_direction; +} + +} ReadInOrderOptimizer::ReadInOrderOptimizer( + const ASTSelectQuery & query_, const ManyExpressionActions & elements_actions_, const SortDescription & required_sort_description_, const TreeRewriterResultPtr & syntax_result) : elements_actions(elements_actions_) , required_sort_description(required_sort_description_) + , query(query_) { if (elements_actions.size() != required_sort_description.size()) throw Exception("Sizes of sort description and actions are mismatched", ErrorCodes::LOGICAL_ERROR); @@ -35,120 +176,34 @@ ReadInOrderOptimizer::ReadInOrderOptimizer( array_join_result_to_source = syntax_result->array_join_result_to_source; } -InputOrderInfoPtr ReadInOrderOptimizer::getInputOrder(const StorageMetadataPtr & metadata_snapshot, ContextPtr context, UInt64 limit) const +InputOrderInfoPtr ReadInOrderOptimizer::getInputOrderImpl( + const StorageMetadataPtr & metadata_snapshot, + const SortDescription & description, + const ManyExpressionActions & actions, + UInt64 limit) const { - Names sorting_key_columns = metadata_snapshot->getSortingKeyColumns(); - if (!metadata_snapshot->hasSortingKey()) - return {}; - + auto sorting_key_columns = metadata_snapshot->getSortingKeyColumns(); SortDescription order_key_prefix_descr; - int read_direction = required_sort_description.at(0).direction; + int read_direction = description.at(0).direction; - size_t prefix_size = std::min(required_sort_description.size(), sorting_key_columns.size()); - auto aliased_columns = metadata_snapshot->getColumns().getAliases(); + size_t fixed_prefix_size = calculateFixedPrefixSize(query, sorting_key_columns); + size_t descr_prefix_size = std::min(description.size(), sorting_key_columns.size() - fixed_prefix_size); - for (size_t i = 0; i < prefix_size; ++i) + for (size_t i = 0; i < descr_prefix_size; ++i) { - if (forbidden_columns.count(required_sort_description[i].column_name)) + if (forbidden_columns.count(description[i].column_name)) break; - /// Optimize in case of exact match with order key element - /// or in some simple cases when order key element is wrapped into monotonic function. - auto apply_order_judge = [&] (const ExpressionActions::Actions & actions, const String & sort_column) - { - /// If required order depend on collation, it cannot be matched with primary key order. - /// Because primary keys cannot have collations. - if (required_sort_description[i].collator) - return false; + int current_direction = matchSortDescriptionAndKey( + actions[i]->getActions(), description[i], sorting_key_columns[i + fixed_prefix_size]); - int current_direction = required_sort_description[i].direction; - /// For the path: order by (sort_column, ...) - if (sort_column == sorting_key_columns[i] && current_direction == read_direction) - { - return true; - } - /// For the path: order by (function(sort_column), ...) - /// Allow only one simple monotonic functions with one argument - /// Why not allow multi monotonic functions? - else - { - bool found_function = false; - - for (const auto & action : actions) - { - if (action.node->type != ActionsDAG::ActionType::FUNCTION) - { - continue; - } - - if (found_function) - { - current_direction = 0; - break; - } - else - found_function = true; - - if (action.node->children.size() != 1 || action.node->children.at(0)->result_name != sorting_key_columns[i]) - { - current_direction = 0; - break; - } - - const auto & func = *action.node->function_base; - if (!func.hasInformationAboutMonotonicity()) - { - current_direction = 0; - break; - } - - auto monotonicity = func.getMonotonicityForRange(*func.getArgumentTypes().at(0), {}, {}); - if (!monotonicity.is_monotonic) - { - current_direction = 0; - break; - } - else if (!monotonicity.is_positive) - current_direction *= -1; - } - - if (!found_function) - current_direction = 0; - - if (!current_direction || (i > 0 && current_direction != read_direction)) - return false; - - if (i == 0) - read_direction = current_direction; - - return true; - } - }; - - const auto & actions = elements_actions[i]->getActions(); - bool ok; - /// check if it's alias column - /// currently we only support alias column without any function wrapper - /// ie: `order by aliased_column` can have this optimization, but `order by function(aliased_column)` can not. - /// This suits most cases. - if (context->getSettingsRef().optimize_respect_aliases && aliased_columns.contains(required_sort_description[i].column_name)) - { - auto column_expr = metadata_snapshot->getColumns().get(required_sort_description[i].column_name).default_desc.expression->clone(); - replaceAliasColumnsInQuery(column_expr, metadata_snapshot->getColumns(), array_join_result_to_source, context); - - auto syntax_analyzer_result = TreeRewriter(context).analyze(column_expr, metadata_snapshot->getColumns().getAll()); - const auto expression_analyzer = ExpressionAnalyzer(column_expr, syntax_analyzer_result, context).getActions(true); - const auto & alias_actions = expression_analyzer->getActions(); - - ok = apply_order_judge(alias_actions, column_expr->getColumnName()); - } - else - ok = apply_order_judge(actions, required_sort_description[i].column_name); - - if (ok) - order_key_prefix_descr.push_back(required_sort_description[i]); - else + if (!current_direction || (i > 0 && current_direction != read_direction)) break; + + if (i == 0) + read_direction = current_direction; + + order_key_prefix_descr.push_back(required_sort_description[i]); } if (order_key_prefix_descr.empty()) @@ -157,4 +212,42 @@ InputOrderInfoPtr ReadInOrderOptimizer::getInputOrder(const StorageMetadataPtr & return std::make_shared(std::move(order_key_prefix_descr), read_direction, limit); } +InputOrderInfoPtr ReadInOrderOptimizer::getInputOrder( + const StorageMetadataPtr & metadata_snapshot, ContextPtr context, UInt64 limit) const +{ + if (!metadata_snapshot->hasSortingKey()) + return {}; + + auto aliased_columns = metadata_snapshot->getColumns().getAliases(); + + /// Replace alias column with proper expressions. + /// Currently we only support alias column without any function wrapper, + /// i.e.: `order by aliased_column` can have this optimization, but `order by function(aliased_column)` can not. + /// This suits most cases. + if (context->getSettingsRef().optimize_respect_aliases && !aliased_columns.empty()) + { + SortDescription aliases_sort_description = required_sort_description; + ManyExpressionActions aliases_actions = elements_actions; + + for (size_t i = 0; i < required_sort_description.size(); ++i) + { + if (!aliased_columns.contains(required_sort_description[i].column_name)) + continue; + + auto column_expr = metadata_snapshot->getColumns().get(required_sort_description[i].column_name).default_desc.expression->clone(); + replaceAliasColumnsInQuery(column_expr, metadata_snapshot->getColumns(), array_join_result_to_source, context); + + auto syntax_analyzer_result = TreeRewriter(context).analyze(column_expr, metadata_snapshot->getColumns().getAll()); + auto expression_analyzer = ExpressionAnalyzer(column_expr, syntax_analyzer_result, context); + + aliases_sort_description[i].column_name = column_expr->getColumnName(); + aliases_actions[i] = expression_analyzer.getActions(true); + } + + return getInputOrderImpl(metadata_snapshot, aliases_sort_description, aliases_actions, limit); + } + + return getInputOrderImpl(metadata_snapshot, required_sort_description, elements_actions, limit); +} + } diff --git a/src/Storages/ReadInOrderOptimizer.h b/src/Storages/ReadInOrderOptimizer.h index 2686d081855..fd8c9187ddb 100644 --- a/src/Storages/ReadInOrderOptimizer.h +++ b/src/Storages/ReadInOrderOptimizer.h @@ -18,6 +18,7 @@ class ReadInOrderOptimizer { public: ReadInOrderOptimizer( + const ASTSelectQuery & query, const ManyExpressionActions & elements_actions, const SortDescription & required_sort_description, const TreeRewriterResultPtr & syntax_result); @@ -25,10 +26,17 @@ public: InputOrderInfoPtr getInputOrder(const StorageMetadataPtr & metadata_snapshot, ContextPtr context, UInt64 limit = 0) const; private: + InputOrderInfoPtr getInputOrderImpl( + const StorageMetadataPtr & metadata_snapshot, + const SortDescription & description, + const ManyExpressionActions & actions, + UInt64 limit) const; + /// Actions for every element of order expression to analyze functions for monotonicity ManyExpressionActions elements_actions; NameSet forbidden_columns; NameToNameMap array_join_result_to_source; SortDescription required_sort_description; + const ASTSelectQuery & query; }; } diff --git a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.reference b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.reference new file mode 100644 index 00000000000..981491baccb --- /dev/null +++ b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.reference @@ -0,0 +1,79 @@ +2020-10-11 0 +2020-10-11 0 +2020-10-11 0 +2020-10-11 0 +2020-10-11 0 +2020-10-11 0 +2020-10-11 0 +2020-10-11 0 +2020-10-11 0 +2020-10-11 0 +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + (Expression) + ExpressionTransform + (Filter) + FilterTransform + (SettingQuotaAndLimits) + (ReadFromMergeTree) + MergeTreeInOrder 0 → 1 +2020-10-11 0 0 +2020-10-11 0 10 +2020-10-11 0 20 +2020-10-11 0 30 +2020-10-11 0 40 +2020-10-11 0 50 +2020-10-11 0 60 +2020-10-11 0 70 +2020-10-11 0 80 +2020-10-11 0 90 +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + FinishSortingTransform + PartialSortingTransform + (Expression) + ExpressionTransform + (Filter) + FilterTransform + (SettingQuotaAndLimits) + (ReadFromMergeTree) + MergeTreeInOrder 0 → 1 +2020-10-12 0 +2020-10-12 1 +2020-10-12 2 +2020-10-12 3 +2020-10-12 4 +2020-10-12 5 +2020-10-12 6 +2020-10-12 7 +2020-10-12 8 +2020-10-12 9 +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + (Expression) + ExpressionTransform + (Filter) + FilterTransform + (SettingQuotaAndLimits) + (ReadFromMergeTree) + ReverseTransform + MergeTreeReverse 0 → 1 +2020-10-12 99999 +2020-10-12 99998 +2020-10-12 99997 +2020-10-12 99996 +2020-10-12 99995 +2020-10-12 99994 +2020-10-12 99993 +2020-10-12 99992 +2020-10-12 99991 +2020-10-12 99990 diff --git a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql new file mode 100644 index 00000000000..316a01e1542 --- /dev/null +++ b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql @@ -0,0 +1,21 @@ +DROP TABLE IF EXISTS t_read_in_order; + +CREATE TABLE t_read_in_order(date Date, i UInt64, v UInt64) +ENGINE = MergeTree ORDER BY (date, i); + +INSERT INTO t_read_in_order SELECT '2020-10-10', number % 10, number FROM numbers(100000); +INSERT INTO t_read_in_order SELECT '2020-10-11', number % 10, number FROM numbers(100000); + +SELECT date, i FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i LIMIT 10; +EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i LIMIT 10; + +SELECT * FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i, v LIMIT 10; +EXPLAIN PIPELINE SELECT * FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i, v LIMIT 10; + +INSERT INTO t_read_in_order SELECT '2020-10-12', number, number FROM numbers(100000); +SELECT date, i FROM t_read_in_order WHERE date = '2020-10-12' ORDER BY i LIMIT 10; + +EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order WHERE date = '2020-10-12' ORDER BY i DESC LIMIT 10; +SELECT date, i FROM t_read_in_order WHERE date = '2020-10-12' ORDER BY i DESC LIMIT 10; + +DROP TABLE t_read_in_order; diff --git a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix_negative.reference b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix_negative.reference new file mode 100644 index 00000000000..4a3cbec8ce3 --- /dev/null +++ b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix_negative.reference @@ -0,0 +1,5 @@ +MergeSorting +MergeSorting +MergeSorting +MergeSorting +MergeSorting diff --git a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix_negative.sh b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix_negative.sh new file mode 100755 index 00000000000..a2f297da5ad --- /dev/null +++ b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix_negative.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_read_in_order_2"; + +$CLICKHOUSE_CLIENT -q "CREATE TABLE t_read_in_order_2(date Date, i UInt64, v UInt64) ENGINE = MergeTree ORDER BY (date, i)" + +$CLICKHOUSE_CLIENT -q "INSERT INTO t_read_in_order_2 SELECT '2020-10-10', number % 10, number FROM numbers(100000)" +$CLICKHOUSE_CLIENT -q "INSERT INTO t_read_in_order_2 SELECT '2020-10-11', number % 10, number FROM numbers(100000)" + +$CLICKHOUSE_CLIENT -q "EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order_2 WHERE date = '2020-10-11' OR date = '2020-10-12' ORDER BY i DESC LIMIT 10" | grep -o "MergeSorting" +$CLICKHOUSE_CLIENT -q "EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order_2 WHERE date >= '2020-10-11' ORDER BY i DESC LIMIT 10" | grep -o "MergeSorting" +$CLICKHOUSE_CLIENT -q "EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order_2 WHERE date = '2020-10-11' OR v = 100 ORDER BY i DESC LIMIT 10" | grep -o "MergeSorting" +$CLICKHOUSE_CLIENT -q "EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order_2 WHERE date != '2020-10-11' ORDER BY i DESC LIMIT 10" | grep -o "MergeSorting" +$CLICKHOUSE_CLIENT -q "EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order_2 WHERE NOT (date = '2020-10-11') ORDER BY i DESC LIMIT 10" | grep -o "MergeSorting" + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_read_in_order_2"; From 1f67177d4d7bf4e8830cd2dffc3bbe1befb8171c Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 23 Dec 2021 18:02:32 +0300 Subject: [PATCH 020/149] fix optimize_read_in_order with preliminary merge and add some tests --- .../QueryPlan/ReadFromMergeTree.cpp | 24 ++-- src/Processors/QueryPlan/ReadFromMergeTree.h | 1 - src/Storages/ReadInOrderOptimizer.cpp | 16 ++- src/Storages/SelectQueryInfo.h | 15 ++- ...02149_read_in_order_fixed_prefix.reference | 116 +++++++++++++++--- .../02149_read_in_order_fixed_prefix.sql | 51 ++++++-- 6 files changed, 174 insertions(+), 49 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index eddbbb9138c..3df9dc127f3 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -365,7 +365,6 @@ static ActionsDAGPtr createProjection(const Block & header) Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( RangesInDataParts && parts_with_ranges, const Names & column_names, - const ActionsDAGPtr & sorting_key_prefix_expr, ActionsDAGPtr & out_projection, const InputOrderInfoPtr & input_order_info) { @@ -507,10 +506,19 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( if (need_preliminary_merge) { + size_t fixed_prefix_size = input_order_info->order_key_fixed_prefix_descr.size(); + size_t prefix_size = fixed_prefix_size + input_order_info->order_key_prefix_descr.size(); + + auto order_key_prefix_ast = metadata_snapshot->getSortingKey().expression_list_ast->clone(); + order_key_prefix_ast->children.resize(prefix_size); + + auto syntax_result = TreeRewriter(context).analyze(order_key_prefix_ast, metadata_snapshot->getColumns().getAllPhysical()); + auto sorting_key_prefix_expr = ExpressionAnalyzer(order_key_prefix_ast, syntax_result, context).getActionsDAG(false); + const auto & sorting_columns = metadata_snapshot->getSortingKey().column_names; + SortDescription sort_description; - for (size_t j = 0; j < input_order_info->order_key_prefix_descr.size(); ++j) - sort_description.emplace_back(metadata_snapshot->getSortingKey().column_names[j], - input_order_info->direction, 1); + for (size_t j = 0; j < prefix_size; ++j) + sort_description.emplace_back(sorting_columns[j], input_order_info->direction); auto sorting_key_expr = std::make_shared(sorting_key_prefix_expr); @@ -1048,17 +1056,9 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons } else if ((settings.optimize_read_in_order || settings.optimize_aggregation_in_order) && input_order_info) { - size_t prefix_size = input_order_info->order_key_prefix_descr.size(); - auto order_key_prefix_ast = metadata_snapshot->getSortingKey().expression_list_ast->clone(); - order_key_prefix_ast->children.resize(prefix_size); - - auto syntax_result = TreeRewriter(context).analyze(order_key_prefix_ast, metadata_snapshot->getColumns().getAllPhysical()); - auto sorting_key_prefix_expr = ExpressionAnalyzer(order_key_prefix_ast, syntax_result, context).getActionsDAG(false); - pipe = spreadMarkRangesAmongStreamsWithOrder( std::move(result.parts_with_ranges), column_names_to_read, - sorting_key_prefix_expr, result_projection, input_order_info); } diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index 0bdfa66bcc7..0d07a3e2ea2 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -173,7 +173,6 @@ private: Pipe spreadMarkRangesAmongStreamsWithOrder( RangesInDataParts && parts_with_ranges, const Names & column_names, - const ActionsDAGPtr & sorting_key_prefix_expr, ActionsDAGPtr & out_projection, const InputOrderInfoPtr & input_order_info); diff --git a/src/Storages/ReadInOrderOptimizer.cpp b/src/Storages/ReadInOrderOptimizer.cpp index ea67c6c70bd..9e46595e0fc 100644 --- a/src/Storages/ReadInOrderOptimizer.cpp +++ b/src/Storages/ReadInOrderOptimizer.cpp @@ -52,7 +52,7 @@ size_t calculateFixedPrefixSize( if (!condition) return 0; - /// Convert condition to CNF for more convinient analysis. + /// Convert condition to CNF for more convenient analysis. auto cnf = TreeCNFConverter::tryConvertToCNF(condition); if (!cnf) return 0; @@ -183,12 +183,14 @@ InputOrderInfoPtr ReadInOrderOptimizer::getInputOrderImpl( UInt64 limit) const { auto sorting_key_columns = metadata_snapshot->getSortingKeyColumns(); - SortDescription order_key_prefix_descr; int read_direction = description.at(0).direction; size_t fixed_prefix_size = calculateFixedPrefixSize(query, sorting_key_columns); size_t descr_prefix_size = std::min(description.size(), sorting_key_columns.size() - fixed_prefix_size); + SortDescription order_key_prefix_descr; + order_key_prefix_descr.reserve(descr_prefix_size); + for (size_t i = 0; i < descr_prefix_size; ++i) { if (forbidden_columns.count(description[i].column_name)) @@ -209,7 +211,15 @@ InputOrderInfoPtr ReadInOrderOptimizer::getInputOrderImpl( if (order_key_prefix_descr.empty()) return {}; - return std::make_shared(std::move(order_key_prefix_descr), read_direction, limit); + SortDescription order_key_fixed_prefix_descr; + order_key_fixed_prefix_descr.reserve(fixed_prefix_size); + for (size_t i = 0; i < fixed_prefix_size; ++i) + order_key_fixed_prefix_descr.emplace_back(sorting_key_columns[i], read_direction); + + return std::make_shared( + std::move(order_key_fixed_prefix_descr), + std::move(order_key_prefix_descr), + read_direction, limit); } InputOrderInfoPtr ReadInOrderOptimizer::getInputOrder( diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index e53f5adec52..a40bd7955d3 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -87,19 +87,22 @@ struct FilterDAGInfo struct InputOrderInfo { + SortDescription order_key_fixed_prefix_descr; SortDescription order_key_prefix_descr; int direction; UInt64 limit; - InputOrderInfo(const SortDescription & order_key_prefix_descr_, int direction_, UInt64 limit_) - : order_key_prefix_descr(order_key_prefix_descr_), direction(direction_), limit(limit_) {} - - bool operator ==(const InputOrderInfo & other) const + InputOrderInfo( + const SortDescription & order_key_fixed_prefix_descr_, + const SortDescription & order_key_prefix_descr_, + int direction_, UInt64 limit_) + : order_key_fixed_prefix_descr(order_key_fixed_prefix_descr_) + , order_key_prefix_descr(order_key_prefix_descr_) + , direction(direction_), limit(limit_) { - return order_key_prefix_descr == other.order_key_prefix_descr && direction == other.direction; } - bool operator !=(const InputOrderInfo & other) const { return !(*this == other); } + bool operator==(const InputOrderInfo &) const = default; }; class IMergeTreeDataPart; diff --git a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.reference b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.reference index 981491baccb..9e24b7c6ea6 100644 --- a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.reference +++ b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.reference @@ -1,8 +1,56 @@ -2020-10-11 0 -2020-10-11 0 -2020-10-11 0 -2020-10-11 0 -2020-10-11 0 +2020-10-01 0 +2020-10-01 0 +2020-10-01 0 +2020-10-01 0 +2020-10-01 0 +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + MergingSortedTransform 2 → 1 + (Expression) + ExpressionTransform × 2 + (SettingQuotaAndLimits) + (ReadFromMergeTree) + MergeTreeInOrder × 2 0 → 1 +2020-10-01 9 +2020-10-01 9 +2020-10-01 9 +2020-10-01 9 +2020-10-01 9 +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + MergingSortedTransform 2 → 1 + (Expression) + ExpressionTransform × 2 + (SettingQuotaAndLimits) + (ReadFromMergeTree) + ReverseTransform + MergeTreeReverse 0 → 1 + ReverseTransform + MergeTreeReverse 0 → 1 +2020-10-01 9 +2020-10-01 9 +2020-10-01 9 +2020-10-01 9 +2020-10-01 9 +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + FinishSortingTransform + PartialSortingTransform + MergingSortedTransform 2 → 1 + (Expression) + ExpressionTransform × 2 + (SettingQuotaAndLimits) + (ReadFromMergeTree) + MergeTreeInOrder × 2 0 → 1 2020-10-11 0 2020-10-11 0 2020-10-11 0 @@ -25,11 +73,6 @@ ExpressionTransform 2020-10-11 0 20 2020-10-11 0 30 2020-10-11 0 40 -2020-10-11 0 50 -2020-10-11 0 60 -2020-10-11 0 70 -2020-10-11 0 80 -2020-10-11 0 90 (Expression) ExpressionTransform (Limit) @@ -49,11 +92,6 @@ ExpressionTransform 2020-10-12 2 2020-10-12 3 2020-10-12 4 -2020-10-12 5 -2020-10-12 6 -2020-10-12 7 -2020-10-12 8 -2020-10-12 9 (Expression) ExpressionTransform (Limit) @@ -72,8 +110,46 @@ ExpressionTransform 2020-10-12 99997 2020-10-12 99996 2020-10-12 99995 -2020-10-12 99994 -2020-10-12 99993 -2020-10-12 99992 -2020-10-12 99991 -2020-10-12 99990 +1 2 +1 2 +1 3 +1 3 +1 4 +1 4 +======== +1 4 +1 4 +1 3 +1 3 +1 2 +1 2 +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + (Expression) + ExpressionTransform + (SettingQuotaAndLimits) + (ReadFromMergeTree) + MergeTreeInOrder 0 → 1 +2020-10-10 00:00:00 0.01 +2020-10-10 00:00:00 0.01 +2020-10-10 00:00:00 0.01 +2020-10-10 00:00:00 0.01 +2020-10-10 00:00:00 0.01 +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + (Expression) + ExpressionTransform + (SettingQuotaAndLimits) + (ReadFromMergeTree) + MergeTreeInOrder 0 → 1 +2020-10-10 00:00:00 0.01 +2020-10-10 00:00:00 0.01 +2020-10-10 00:00:00 0.01 +2020-10-10 00:00:00 0.01 +2020-10-10 00:00:00 0.01 diff --git a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql index 316a01e1542..7d0e9111d9c 100644 --- a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql +++ b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql @@ -6,16 +6,53 @@ ENGINE = MergeTree ORDER BY (date, i); INSERT INTO t_read_in_order SELECT '2020-10-10', number % 10, number FROM numbers(100000); INSERT INTO t_read_in_order SELECT '2020-10-11', number % 10, number FROM numbers(100000); -SELECT date, i FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i LIMIT 10; -EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i LIMIT 10; +SELECT toStartOfMonth(date) as d, i FROM t_read_in_order ORDER BY d, i LIMIT 5; +EXPLAIN PIPELINE SELECT toStartOfMonth(date) as d, i FROM t_read_in_order ORDER BY d, i LIMIT 5; -SELECT * FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i, v LIMIT 10; -EXPLAIN PIPELINE SELECT * FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i, v LIMIT 10; +SELECT toStartOfMonth(date) as d, i FROM t_read_in_order ORDER BY d DESC, -i LIMIT 5; +EXPLAIN PIPELINE SELECT toStartOfMonth(date) as d, i FROM t_read_in_order ORDER BY d DESC, -i LIMIT 5; + +-- Here FinishSorting is used, because directions don't match. +SELECT toStartOfMonth(date) as d, i FROM t_read_in_order ORDER BY d, -i LIMIT 5; +EXPLAIN PIPELINE SELECT toStartOfMonth(date) as d, i FROM t_read_in_order ORDER BY d, -i LIMIT 5; + +SELECT date, i FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i LIMIT 5; +EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i LIMIT 5; + +SELECT * FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i, v LIMIT 5; +EXPLAIN PIPELINE SELECT * FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i, v LIMIT 5; INSERT INTO t_read_in_order SELECT '2020-10-12', number, number FROM numbers(100000); -SELECT date, i FROM t_read_in_order WHERE date = '2020-10-12' ORDER BY i LIMIT 10; -EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order WHERE date = '2020-10-12' ORDER BY i DESC LIMIT 10; -SELECT date, i FROM t_read_in_order WHERE date = '2020-10-12' ORDER BY i DESC LIMIT 10; +SELECT date, i FROM t_read_in_order WHERE date = '2020-10-12' ORDER BY i LIMIT 5; + +EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order WHERE date = '2020-10-12' ORDER BY i DESC LIMIT 5; +SELECT date, i FROM t_read_in_order WHERE date = '2020-10-12' ORDER BY i DESC LIMIT 5; + +DROP TABLE IF EXISTS t_read_in_order; + +CREATE TABLE t_read_in_order(a UInt32, b UInt32) +ENGINE = MergeTree ORDER BY (a, b) +SETTINGS index_granularity = 3; + +SYSTEM STOP MERGES t_read_in_order; + +INSERT INTO t_read_in_order VALUES (0, 100), (1, 2), (1, 3), (1, 4), (2, 5); +INSERT INTO t_read_in_order VALUES (0, 100), (1, 2), (1, 3), (1, 4), (2, 5); + +SELECT a, b FROM t_read_in_order WHERE a = 1 ORDER BY b SETTINGS read_in_order_two_level_merge_threshold = 1; +SELECT '========'; +SELECT a, b FROM t_read_in_order WHERE a = 1 ORDER BY b DESC SETTINGS read_in_order_two_level_merge_threshold = 1; DROP TABLE t_read_in_order; + +CREATE TABLE t_read_in_order(dt DateTime, d Decimal64(5), v UInt64) +ENGINE = MergeTree ORDER BY (toStartOfDay(dt), d); + +INSERT INTO t_read_in_order SELECT toDateTime('2020-10-10 00:00:00') + number, 1 / (number % 100 + 1), number FROM numbers(1000); + +EXPLAIN PIPELINE SELECT toStartOfDay(dt) as date, d FROM t_read_in_order ORDER BY date, round(d) LIMIT 5; +SELECT toStartOfDay(dt) as date, d FROM t_read_in_order ORDER BY date, round(d) LIMIT 5; + +EXPLAIN PIPELINE SELECT toStartOfDay(dt) as date, d FROM t_read_in_order ORDER BY date, round(d) LIMIT 5; +SELECT toStartOfDay(dt) as date, d FROM t_read_in_order WHERE date = '2020-10-10' ORDER BY round(d) LIMIT 5; From 760288590b69132e5fd0956f3b4bed4815ec4c90 Mon Sep 17 00:00:00 2001 From: achimbab <07c00h@gmail.com> Date: Tue, 9 Nov 2021 12:08:13 +0900 Subject: [PATCH 021/149] Add confidence intervals to ttests --- CMakeLists.txt | 1 + cmake/find/stats.cmake | 20 +++++ contrib/CMakeLists.txt | 4 + .../AggregateFunctionStudentTTest.cpp | 20 +++-- .../AggregateFunctionTTest.h | 74 +++++++++++++++---- .../AggregateFunctionWelchTTest.cpp | 38 +++++----- src/AggregateFunctions/CMakeLists.txt | 4 + src/AggregateFunctions/Moments.h | 46 ++++++++++++ src/Common/config.h.in | 1 + .../queries/0_stateless/01558_ttest.reference | 2 + tests/queries/0_stateless/01558_ttest.sql | 2 + 11 files changed, 173 insertions(+), 39 deletions(-) create mode 100644 cmake/find/stats.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index fdc9cfcd303..18eaa9d48c5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -546,6 +546,7 @@ include (cmake/find/avro.cmake) include (cmake/find/msgpack.cmake) include (cmake/find/cassandra.cmake) include (cmake/find/sentry.cmake) +include (cmake/find/stats.cmake) include (cmake/find/datasketches.cmake) include (cmake/find/libprotobuf-mutator.cmake) diff --git a/cmake/find/stats.cmake b/cmake/find/stats.cmake new file mode 100644 index 00000000000..ef5b1a73659 --- /dev/null +++ b/cmake/find/stats.cmake @@ -0,0 +1,20 @@ +option(ENABLE_STATS "Enalbe StatsLib library" ${ENABLE_LIBRARIES}) + +if (ENABLE_STATS) + if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/stats") + message (WARNING "submodule contrib/stats is missing. to fix try run: \n git submodule update --init --recursive") + set (ENABLE_STATS 0) + set (USE_STATS 0) + elseif (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/gcem") + message (WARNING "submodule contrib/gcem is missing. to fix try run: \n git submodule update --init --recursive") + set (ENABLE_STATS 0) + set (USE_STATS 0) + else() + set(STATS_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/stats/include) + set(GCEM_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/gcem/include) + set (USE_STATS 1) + endif() +endif() + +message (STATUS "Using stats=${USE_STATS} : ${STATS_INCLUDE_DIR}") +message (STATUS "Using gcem=${USE_STATS}: ${GCEM_INCLUDE_DIR}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 9eb3b145c85..8f394709d2a 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -299,6 +299,10 @@ if (USE_S2_GEOMETRY) add_subdirectory(s2geometry-cmake) endif() +if (USE_STATS) + add_subdirectory (stats-cmake) +endif() + # Put all targets defined here and in subdirectories under "contrib/" folders in GUI-based IDEs. # Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear # in "contrib/..." as originally planned, so we workaround this by fixing FOLDER properties of all targets manually, diff --git a/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp b/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp index c6e7029d283..ec993f4b54b 100644 --- a/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp @@ -7,6 +7,7 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } @@ -27,14 +28,19 @@ struct StudentTTestData : public TTestMoments { static constexpr auto name = "studentTTest"; - std::pair getResult() const + Float64 getDegreesOfFreedom() const { - Float64 mean_x = x1 / nx; - Float64 mean_y = y1 / ny; + return nx + ny - 2; + } + + std::tuple getResult() const + { + Float64 mean_x = getMeanX(); + Float64 mean_y = getMeanY(); /// To estimate the variance we first estimate two means. /// That's why the number of degrees of freedom is the total number of values of both samples minus 2. - Float64 degrees_of_freedom = nx + ny - 2; + Float64 degrees_of_freedom = getDegreesOfFreedom(); /// Calculate s^2 /// The original formulae looks like @@ -59,12 +65,14 @@ AggregateFunctionPtr createAggregateFunctionStudentTTest( const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings *) { assertBinary(name, argument_types); - assertNoParameters(name, parameters); + + if (parameters.size() > 1) + throw Exception("Aggregate function " + name + " requires zero or one parameter.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); if (!isNumber(argument_types[0]) || !isNumber(argument_types[1])) throw Exception("Aggregate function " + name + " only supports numerical types", ErrorCodes::BAD_ARGUMENTS); - return std::make_shared>(argument_types); + return std::make_shared>(argument_types, parameters); } } diff --git a/src/AggregateFunctions/AggregateFunctionTTest.h b/src/AggregateFunctions/AggregateFunctionTTest.h index a91ce16c3ff..21348e45372 100644 --- a/src/AggregateFunctions/AggregateFunctionTTest.h +++ b/src/AggregateFunctions/AggregateFunctionTTest.h @@ -79,10 +79,18 @@ template class AggregateFunctionTTest : public IAggregateFunctionDataHelper> { +private: + bool need_ci = false; + Float64 confidence_level; public: - AggregateFunctionTTest(const DataTypes & arguments) - : IAggregateFunctionDataHelper>({arguments}, {}) + AggregateFunctionTTest(const DataTypes & arguments, [[maybe_unused]] const Array & params) + : IAggregateFunctionDataHelper>({arguments}, params) { + if (params.size() > 0) + { + need_ci = true; + confidence_level = params.at(0).safeGet(); + } } String getName() const override @@ -92,22 +100,48 @@ public: DataTypePtr getReturnType() const override { - DataTypes types + if (need_ci) { - std::make_shared>(), - std::make_shared>(), - }; + DataTypes types + { + std::make_shared>(), + std::make_shared>(), + std::make_shared>(), + std::make_shared>(), + }; - Strings names + Strings names + { + "t_statistic", + "p_value", + "confidence_interval_low", + "confidence_interval_high", + }; + + return std::make_shared( + std::move(types), + std::move(names) + ); + } + else { - "t_statistic", - "p_value" - }; + DataTypes types + { + std::make_shared>(), + std::make_shared>(), + }; - return std::make_shared( - std::move(types), - std::move(names) - ); + Strings names + { + "t_statistic", + "p_value", + }; + + return std::make_shared( + std::move(types), + std::move(names) + ); + } } bool allocatesMemoryInArena() const override { return false; } @@ -140,7 +174,8 @@ public: void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override { - auto [t_statistic, p_value] = this->data(place).getResult(); + auto & data = this->data(place); + auto [t_statistic, p_value] = data.getResult(); /// Because p-value is a probability. p_value = std::min(1.0, std::max(0.0, p_value)); @@ -151,6 +186,15 @@ public: column_stat.getData().push_back(t_statistic); column_value.getData().push_back(p_value); + + if (need_ci) + { + auto [ci_low, ci_high] = data.getConfidenceIntervals(confidence_level, data.getDegreesOfFreedom()); + auto & column_ci_low = assert_cast &>(column_tuple.getColumn(2)); + auto & column_ci_high = assert_cast &>(column_tuple.getColumn(3)); + column_ci_low.getData().push_back(ci_low); + column_ci_high.getData().push_back(ci_high); + } } }; diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index c7c56da79f6..4b43bb73e6b 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -7,6 +7,7 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } @@ -21,34 +22,33 @@ struct WelchTTestData : public TTestMoments { static constexpr auto name = "welchTTest"; - std::pair getResult() const + Float64 getDegreesOfFreedom() const { - Float64 mean_x = x1 / nx; - Float64 mean_y = y1 / ny; - - /// s_x^2, s_y^2 - - /// The original formulae looks like \frac{1}{size_x - 1} \sum_{i = 1}^{size_x}{(x_i - \bar{x}) ^ 2} - /// But we made some mathematical transformations not to store original sequences. - /// Also we dropped sqrt, because later it will be squared later. + Float64 mean_x = getMeanX(); + Float64 mean_y = getMeanY(); Float64 sx2 = (x2 + nx * mean_x * mean_x - 2 * mean_x * x1) / (nx - 1); Float64 sy2 = (y2 + ny * mean_y * mean_y - 2 * mean_y * y1) / (ny - 1); - /// t-statistic - Float64 t_stat = (mean_x - mean_y) / sqrt(sx2 / nx + sy2 / ny); - - /// degrees of freedom - Float64 numerator_sqrt = sx2 / nx + sy2 / ny; Float64 numerator = numerator_sqrt * numerator_sqrt; Float64 denominator_x = sx2 * sx2 / (nx * nx * (nx - 1)); Float64 denominator_y = sy2 * sy2 / (ny * ny * (ny - 1)); - Float64 degrees_of_freedom = numerator / (denominator_x + denominator_y); + return numerator / (denominator_x + denominator_y); + } - return {t_stat, getPValue(degrees_of_freedom, t_stat * t_stat)}; + std::tuple getResult() const + { + Float64 mean_x = getMeanX(); + Float64 mean_y = getMeanY(); + + /// t-statistic + Float64 se = getStandardError(); + Float64 t_stat = (mean_x - mean_y) / se; + + return {t_stat, getPValue(getDegreesOfFreedom(), t_stat * t_stat)}; } }; @@ -56,12 +56,14 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest( const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings *) { assertBinary(name, argument_types); - assertNoParameters(name, parameters); + + if (parameters.size() > 1) + throw Exception("Aggregate function " + name + " requires zero or one parameter.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); if (!isNumber(argument_types[0]) || !isNumber(argument_types[1])) throw Exception("Aggregate function " + name + " only supports numerical types", ErrorCodes::BAD_ARGUMENTS); - return std::make_shared>(argument_types); + return std::make_shared>(argument_types, parameters); } } diff --git a/src/AggregateFunctions/CMakeLists.txt b/src/AggregateFunctions/CMakeLists.txt index 64f6eed9a6c..9c08c733511 100644 --- a/src/AggregateFunctions/CMakeLists.txt +++ b/src/AggregateFunctions/CMakeLists.txt @@ -28,3 +28,7 @@ target_link_libraries(clickhouse_aggregate_functions PRIVATE dbms PUBLIC ${CITYH if(ENABLE_EXAMPLES) add_subdirectory(examples) endif() + +if (USE_STATS) + target_link_libraries(clickhouse_aggregate_functions PRIVATE stats) +endif() diff --git a/src/AggregateFunctions/Moments.h b/src/AggregateFunctions/Moments.h index 6f51e76607f..140257bf812 100644 --- a/src/AggregateFunctions/Moments.h +++ b/src/AggregateFunctions/Moments.h @@ -2,6 +2,12 @@ #include #include +#include + +#if !defined(ARCADIA_BUILD) && USE_STATS + +#define STATS_ENABLE_STDVEC_WRAPPERS +#include namespace DB @@ -357,6 +363,46 @@ struct TTestMoments { readPODBinary(*this, buf); } + + Float64 getMeanX() const + { + return x1 / nx; + } + + Float64 getMeanY() const + { + return y1 / ny; + } + + Float64 getStandardError() const + { + /// The original formulae looks like \frac{1}{size_x - 1} \sum_{i = 1}^{size_x}{(x_i - \bar{x}) ^ 2} + /// But we made some mathematical transformations not to store original sequences. + /// Also we dropped sqrt, because later it will be squared later. + Float64 mean_x = getMeanX(); + Float64 mean_y = getMeanY(); + + Float64 sx2 = (x2 + nx * mean_x * mean_x - 2 * mean_x * x1) / (nx - 1); + Float64 sy2 = (y2 + ny * mean_y * mean_y - 2 * mean_y * y1) / (ny - 1); + + return sqrt(sx2 / nx + sy2 / ny); + } + + std::pair getConfidenceIntervals(Float64 confidence_level, Float64 degrees_of_freedom) const + { + Float64 mean_x = getMeanX(); + Float64 mean_y = getMeanY(); + Float64 se = getStandardError(); + + Float64 t = stats::qt(1.0f - (1.0f - confidence_level) / 2.0f, degrees_of_freedom); + Float64 mean_diff = mean_x - mean_y; + Float64 ci_low = mean_diff - t * se; + Float64 ci_high = mean_diff + t * se; + + return {ci_low, ci_high}; + } }; } + +#endif diff --git a/src/Common/config.h.in b/src/Common/config.h.in index c53abd17bc2..1c14cddff28 100644 --- a/src/Common/config.h.in +++ b/src/Common/config.h.in @@ -20,3 +20,4 @@ #cmakedefine01 USE_YAML_CPP #cmakedefine01 CLICKHOUSE_SPLIT_BINARY #cmakedefine01 USE_BZIP2 +#cmakedefine01 USE_STATS diff --git a/tests/queries/0_stateless/01558_ttest.reference b/tests/queries/0_stateless/01558_ttest.reference index 473960c0e7c..967e5864446 100644 --- a/tests/queries/0_stateless/01558_ttest.reference +++ b/tests/queries/0_stateless/01558_ttest.reference @@ -8,7 +8,9 @@ -0.5028215369187079 0.6152361677171103 14.971190998235835 5.898143508382202e-44 14.971190998235837 0 +7.6505301757756445 9.960200184224346 -2.610898982580138 0.00916587538237954 -2.610898982580134 0.0091658753823834 -28.740781574102936 7.667329672103986e-133 -28.74078157410298 0 +-9.625938422387687 -8.395483817612316 diff --git a/tests/queries/0_stateless/01558_ttest.sql b/tests/queries/0_stateless/01558_ttest.sql index 7a66bba9d2a..94a48c38fcc 100644 --- a/tests/queries/0_stateless/01558_ttest.sql +++ b/tests/queries/0_stateless/01558_ttest.sql @@ -30,6 +30,7 @@ CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (4.82025, 0), (6.13896, 0), (15.20277, 0), (14.15351, 0), (7.21338, 0), (8.55506, 0), (13.80816, 0), (11.28411, 0), (7.4612, 0), (7.43759, 0), (12.9832, 0), (-5.74783, 0), (12.47114, 0), (15.14223, 0), (3.40603, 0), (9.27323, 0), (7.88547, 0), (8.56456, 0), (4.59731, 0), (7.91213, 0), (7.33894, 0), (21.74811, 0), (11.92111, 0), (0.18828, 0), (10.47314, 0), (20.37396, 0), (11.04991, 0), (13.30083, 0), (14.28065, 0), (2.86942, 0), (24.96072, 0), (14.20164, 0), (18.28769, 0), (10.50949, 0), (9.22273, 0), (11.77608, 0), (8.56872, 0), (13.74535, 0), (11.65209, 0), (12.51894, 0), (17.76256, 0), (13.52122, 0), (8.70796, 0), (6.04749, 0), (16.33064, 0), (8.35636, 0), (14.03496, 0), (11.05834, 0), (14.49261, 0), (2.59383, 0), (8.01022, 0), (4.05458, 0), (13.26384, 0), (14.62058, 0), (10.52489, 0), (8.46357, 0), (6.4147, 0), (9.70071, 0), (12.47581, 0), (4.38333, 0), (17.54172, 0), (10.12109, 0), (7.73186, 0), (14.0279, 0), (11.6621, 0), (17.47045, 0), (15.50223, 0), (15.46034, 0), (13.39964, 0), (14.98025, 0), (15.87912, 0), (17.67374, 0), (9.64073, 0), (12.84904, 0), (7.70278, 0), (13.03156, 0), (9.04512, 0), (15.97014, 0), (8.96389, 0), (11.48009, 0), (9.71153, 0), (13.00084, 0), (12.39803, 0), (13.08188, 0), (5.82244, 0), (10.81871, 0), (8.2539, 0), (7.52114, 0), (9.11488, 0), (8.37482, 0), (14.48652, 0), (11.42152, 0), (16.03111, 0), (13.14057, 0), (-2.26351, 0), (15.50394, 0), (14.88603, 0), (13.37257, 0), (11.84026, 0), (7.66558, 0), (6.24584, 0), (3.6312, 0), (2.7018, 0), (5.63656, 0), (5.82643, 0), (10.06745, 0), (-0.5831, 0), (14.84202, 0), (9.5524, 0), (19.71713, 0), (14.23109, 0), (8.69105, 0), (5.33742, 0), (7.30372, 0), (7.93342, 0), (15.20884, 0), (7.53839, 0), (13.45311, 0), (11.04473, 0), (10.76673, 0), (15.44145, 0), (14.06596, 0), (9.14873, 0), (12.88372, 0), (8.74994, 0), (10.53263, 0), (16.16694, 0), (8.37197, 0), (3.43739, 0), (4.72799, 0), (9.08802, 0), (11.2531, 0), (5.16115, 0), (10.20895, 0), (18.70884, 0), (15.88924, 0), (3.38758, 0), (6.46449, 0), (10.21088, 0), (14.08458, 0), (15.74508, 0), (19.31896, 0), (13.19641, 0), (11.95409, 0), (10.70718, 0), (1.05245, 0), (10.04772, 0), (17.01369, 0), (10.2286, 0), (19.58323, 0), (7.02892, 0), (4.16866, 0), (8.94326, 0), (4.99854, 0), (8.88352, 0), (18.65422, 0), (17.32328, 0), (9.33492, 0), (14.94788, 0), (8.05863, 0), (14.6737, 0), (10.93801, 0), (0.54036, 0), (-0.34242, 0), (5.89076, 0), (3.15189, 0), (1.94421, 0), (6.38698, 0), (10.50654, 0), (8.95362, 0), (6.23711, 0), (11.75359, 0), (12.42155, 0), (-1.55472, 0), (4.6688, 0), (10.48087, 0), (11.74615, 0), (9.26822, 0), (7.55517, 0), (12.76005, 0), (16.47102, 0), (11.31297, 0), (14.37437, 0), (2.38799, 0), (6.44577, 0), (5.07471, 0), (11.55123, 0), (7.76795, 0), (10.60116, 0), (14.40885, 0), (11.58158, 0), (8.81648, 0), (12.92299, 0), (11.26939, 0), (17.95014, 0), (2.95002, 0), (17.41959, 0), (11.12455, 0), (8.78541, 0), (14.36413, 0), (12.98554, 0), (12.58505, 0), (15.49789, 0), (11.70999, 0), (0.65596, 0), (11.08202, 0), (14.75752, 0), (6.84385, 0), (9.27245, 0), (13.78243, 0), (17.4863, 0), (4.01777, 0), (11.82861, 0), (13.86551, 0), (6.16591, 0), (8.71589, 0), (16.77195, 0), (17.23243, 0), (-2.12941, 0), (5.66629, 0), (12.45153, 0), (1.63971, 0), (13.84031, 0), (4.6144, 0), (5.26169, 0), (9.27769, 0), (9.14288, 0), (9.71953, 0), (9.38446, 0), (1.64788, 0), (11.72922, 0), (13.68926, 0), (9.42952, 0), (12.05574, 0), (9.09148, 0), (5.32273, 0), (20.25258, 0), (10.14599, 0), (10.82156, 0), (5.75736, 0), (7.13567, 0), (9.29746, 0), (5.1618, 0), (10.076, 0), (21.65669, 0), (13.35486, 0), (6.79957, 0), (8.76243, 0), (14.59294, 0), (16.90609, 0), (10.50337, 0), (-0.07923, 0), (13.51648, 0), (12.0676, 0), (0.86482, 0), (9.03563, 0), (5.38751, 0), (17.16866, 0), (2.78702, 0), (11.15548, 0), (12.30843, 0), (8.04897, 0), (9.95814, 0), (11.29308, 0), (14.13032, 0), (21.05877, 0), (3.57386, 0), (7.96631, 0), (3.30484, 0), (18.61856, 0), (16.35184, 0), (7.65236, 0), (18.02895, 0), (9.79458, 0), (16.7274, 0), (8.84453, 0), (13.05709, 0), (10.91447, 0), (8.40171, 0), (16.95211, 0), (11.82194, 0), (19.87978, 0), (12.88455, 0), (-0.00947, 0), (12.28109, 0), (6.96462, 0), (13.75282, 0), (14.39141, 0), (11.07193, 0), (12.88039, 0), (11.38253, 0), (21.02707, 0), (7.51955, 0), (6.31984, 0), (15.6543, 0), (14.80315, 0), (8.38024, 0), (21.7516, 0), (14.31336, 0), (15.04703, 0), (5.73787, 0), (13.16911, 0), (12.40695, 0), (9.88968, 0), (8.46703, 0), (8.70637, 0), (8.03551, 0), (5.9757, 0), (12.22951, 0), (3.14736, 0), (10.51266, 0), (18.593, 0), (10.82213, 0), (7.14216, 0), (6.81154, 0), (-0.6486, 0), (20.56136, 0), (11.35367, 0), (11.38205, 0), (17.14, 0), (14.91215, 0), (15.50207, 0), (5.93162, 0), (3.74869, 0), (14.11532, 0), (7.38954, 0), (5.45764, 0), (18.33733, 0), (9.91923, 0), (2.38991, 0), (14.16756, 0), (2.39791, 0), (6.92586, 0), (5.32474, 0), (2.28812, 0), (5.71718, 0), (5.84197, 0), (2.76206, 0), (19.05928, 0), (11.51788, 0), (6.56648, 0), (3.35735, 0), (7.55948, 0), (19.99908, 0), (13.00634, 0), (18.36886, 0), (11.14675, 0), (16.72931, 0), (12.50106, 0), (6.00605, 0), (23.06653, 0), (5.39694, 0), (9.53167, 0), (12.76944, 0), (7.20604, 0), (13.25391, 0), (13.7341, 0), (10.85292, 0), (-7.75835, 0), (10.29728, 0), (13.70099, 0), (10.17959, 0), (9.98399, 0), (12.69389, 0), (-0.28848, 0), (-2.18319, 0), (13.36378, 0), (10.09232, 0), (5.49489, 0), (5.46156, 0), (0.94225, 0), (12.79205, 0), (10.09593, 0), (6.06218, 0), (0.89463, 0), (11.88986, 0), (10.79733, 0), (1.51371, 0), (2.20967, 0), (15.45732, 0), (16.5262, 0), (5.99724, 0), (8.3613, 0), (15.68183, 0), (15.32117, 0), (14.15674, 0), (6.64553, 0), (4.20777, 0), (-0.10521, 0), (-0.88169, 0), (1.85913, 0), (9.73673, 0), (0.30926, 0), (6.17559, 0), (11.76602, 0), (5.68385, 0), (14.57088, 0), (12.81509, 0), (9.85682, 0), (12.06376, 0), (6.08874, 0), (11.63921, 0), (14.86722, 0), (10.41035, 0), (2.93794, 0), (12.21841, 0), (0.23804, 0), (3.14845, 0), (7.29748, 0), (3.06134, 0), (13.77684, 0), (16.21992, 0), (5.33511, 0), (9.68959, 0), (9.44169, 0), (18.08012, 0), (4.04224, 0), (8.77918, 0), (10.18324, 0), (9.38914, 0), (11.76995, 0), (14.19963, 0), (6.88817, 0), (16.56123, 0), (15.39885, 0), (5.21241, 0), (4.44408, 0), (17.87587, 0), (12.53337, 0), (13.60916, 0), (6.60104, 0), (7.35453, 0), (18.61572, 0), (6.10437, 0), (13.08682, 0), (12.15404, 0), (4.90789, 0), (2.13353, 0), (12.49593, 0), (11.93056, 0), (13.29408, 0), (5.70038, 0), (8.40271, 0), (5.19456, 0), (-5.51028, 0), (14.0329, 0), (10.38365, 0), (6.56812, 0), (4.21129, 0), (9.7157, 0), (9.88553, 0), (13.45346, 0), (4.97752, 0), (12.77595, 0), (8.56465, 0), (4.27703, 0), (18.12502, 0), (12.45735, 0), (12.42912, 0), (12.08125, 0), (10.85779, 0), (4.36013, 0), (11.85062, 0), (8.47776, 0), (9.60822, 0), (11.3069, 0), (14.25525, 0), (1.55168, 0), (14.57782, 0), (7.84786, 0), (9.87774, 0), (14.75575, 0), (3.68774, 0), (9.37667, 0), (20.28676, 0), (12.10027, 0), (8.01819, 0), (18.78158, 0), (20.85402, 0), (18.98069, 0), (16.1429, 0), (9.24047, 0), (14.12487, 0), (10.18841, 0), (-3.04478, 0), (5.7552, 0), (9.30376, 0), (11.42837, 0), (6.02364, 0), (8.86984, 0), (10.91177, 0), (10.04418, 0), (18.10774, 0), (7.49384, 0), (9.11556, 0), (9.7051, 0), (5.23268, 0), (9.04647, 0), (8.81547, 0), (2.65098, 0), (-2.69857, 1), (15.80943, 1), (7.31555, 1), (3.96517, 1), (4.77809, 1), (9.6472, 1), (-26.41717, 1), (-10.85635, 1), (-1.4376, 1), (-0.96308, 1), (2.84315, 1), (5.79467, 1), (-3.06091, 1), (-14.62902, 1), (22.08022, 1), (-2.11982, 1), (-4.84824, 1), (-10.50447, 1), (2.4891, 1), (9.90324, 1), (-22.66866, 1), (-0.97103, 1), (-16.57608, 1), (-3.78749, 1), (25.84511, 1), (5.30797, 1), (-18.19466, 1), (11.72708, 1), (0.2891, 1), (-9.83474, 1), (6.69942, 1), (18.09604, 1), (18.52651, 1), (1.38201, 1), (7.64615, 1), (17.66598, 1), (-2.44141, 1), (-9.01598, 1), (27.69142, 1), (4.06946, 1), (-15.0077, 1), (-10.49648, 1), (-4.88322, 1), (-25.09805, 1), (-4.64024, 1), (20.94434, 1), (24.12126, 1), (-14.10962, 1), (10.6512, 1), (14.50687, 1), (-19.88081, 1), (-11.55271, 1), (13.16921, 1), (16.63864, 1), (-24.08114, 1), (-9.09949, 1), (-10.54702, 1), (0.20813, 1), (8.19066, 1), (-2.70523, 1), (-0.23954, 1), (7.19398, 1), (-7.1618, 1), (-7.44322, 1), (-17.92031, 1), (-1.58146, 1), (9.18338, 1), (3.25838, 1), (-14.30234, 1), (1.84695, 1), (31.13794, 1), (-0.85067, 1), (19.02787, 1), (-3.09594, 1), (13.45584, 1), (-5.48104, 1), (-22.74928, 1), (-8.03697, 1), (17.31143, 1), (-16.65231, 1), (-18.58713, 1), (-16.52641, 1), (14.95261, 1), (12.56762, 1), (15.00188, 1), (1.85858, 1), (2.1926, 1), (-2.4095, 1), (21.56873, 1), (3.35509, 1), (-4.98672, 1), (35.08603, 1), (-10.01602, 1), (-3.85153, 1), (-6.81974, 1), (19.56525, 1), (-9.35488, 1), (0.24268, 1), (-3.51488, 1), (-0.37066, 1), (24.20888, 1), (-11.73537, 1), (0.01282, 1), (0.03963, 1), (-9.65589, 1), (-0.37429, 1), (5.61255, 1), (0.49984, 1), (-10.15066, 1), (-14.54314, 1), (16.56889, 1), (-7.73873, 1), (-3.76422, 1), (1.40722, 1), (2.28818, 1), (-13.12643, 1), (5.17082, 1), (4.79089, 1), (-17.42643, 1), (8.72548, 1), (-3.70285, 1), (16.77893, 1), (13.382, 1), (19.98418, 1), (0.00483, 1), (-4.75951, 1), (2.35391, 1), (21.65809, 1), (-9.2714, 1), (-18.38253, 1), (7.23097, 1), (14.97927, 1), (-4.02197, 1), (-29.8189, 1), (-12.8554, 1), (-7.60124, 1), (-14.90158, 1), (-3.31486, 1), (31.38144, 1), (-8.61288, 1), (15.31895, 1), (-10.19488, 1), (13.796, 1), (-0.32912, 1), (-0.0684, 1), (-30.06834, 1), (24.93912, 1), (-3.26506, 1), (-8.29751, 1), (-5.39189, 1), (-25.08603, 1), (-1.45318, 1), (16.72724, 1), (-3.38467, 1), (-26.00478, 1), (7.28369, 1), (16.96226, 1), (16.5858, 1), (10.46583, 1), (3.84345, 1), (-2.99382, 1), (1.42078, 1), (-11.0123, 1), (2.09909, 1), (1.21064, 1), (15.36079, 1), (-21.61349, 1), (22.7726, 1), (10.50512, 1), (-6.95825, 1), (9.20036, 1), (15.66902, 1), (3.28098, 1), (-9.05692, 1), (0.32882, 1), (-1.64934, 1), (-4.81406, 1), (-5.06006, 1), (19.97493, 1), (2.88646, 1), (-0.34552, 1), (7.55186, 1), (-22.96115, 1), (31.29166, 1), (6.18798, 1), (-2.52715, 1), (-11.58799, 1), (14.13596, 1), (13.45069, 1), (12.15179, 1), (3.44491, 1), (-8.78006, 1), (18.32087, 1), (11.91757, 1), (-2.00179, 1), (10.88411, 1), (9.09327, 1), (6.62484, 1), (8.87178, 1), (11.52254, 1), (-14.15988, 1), (-17.19515, 1), (14.03089, 1), (-2.4095, 1), (-16.83575, 1), (2.71469, 1), (4.84351, 1), (-1.17651, 1), (-3.37529, 1), (-19.92137, 1), (4.48952, 1), (-12.4906, 1), (-5.65277, 1), (8.50819, 1), (-19.61261, 1), (12.54156, 1), (11.06784, 1), (-12.59285, 1), (3.43683, 1), (-3.00325, 1), (12.49082, 1), (7.20955, 1), (17.6547, 1), (15.8619, 1), (24.3048, 1), (-8.05434, 1), (-6.06901, 1), (-15.69515, 1), (-11.13917, 1), (-3.90757, 1), (-2.57038, 1), (5.14065, 1), (17.8497, 1), (-8.64665, 1), (-18.68331, 1), (5.8567, 1), (-20.93884, 1), (4.40583, 1), (14.35985, 1), (4.18134, 1), (4.3635, 1), (9.35428, 1), (2.8908, 1), (16.01017, 1), (-1.48499, 1), (-9.97949, 1), (1.03055, 1), (-2.79697, 1), (6.85977, 1), (4.73213, 1), (2.7815, 1), (-2.46866, 1), (18.39425, 1), (-0.80378, 1), (-0.22982, 1), (-16.11608, 1), (3.0862, 1), (3.20779, 1), (10.50146, 1), (-0.21305, 1), (11.21012, 1), (-0.99825, 1), (18.39633, 1), (-3.39003, 1), (-0.64411, 1), (-1.39932, 1), (15.45319, 1), (-0.66044, 1), (-15.2223, 1), (-34.39907, 1), (-3.57836, 1), (16.82828, 1), (1.66624, 1), (15.43475, 1), (8.17776, 1), (5.50486, 1), (10.43082, 1), (-6.63332, 1), (2.28008, 1), (16.37203, 1), (5.16313, 1), (-8.85281, 1), (13.26692, 1), (-7.46842, 1), (8.43091, 1), (-13.18172, 1), (-0.72401, 1), (22.3881, 1), (10.65448, 1), (2.81289, 1), (10.92405, 1), (-8.95358, 1), (19.80653, 1), (-12.86527, 1), (5.38826, 1), (-6.83501, 1), (-15.7647, 1), (-27.67412, 1), (8.6499, 1), (-4.89542, 1), (16.76167, 1), (12.84284, 1), (-17.27324, 1), (-4.18726, 1), (-14.62366, 1), (-5.49863, 1), (-16.22846, 1), (10.60329, 1), (6.46781, 1), (1.70458, 1), (10.77448, 1), (0.8463, 1), (13.0482, 1), (-4.36264, 1), (3.22647, 1), (2.38828, 1), (6.7946, 1), (-0.25254, 1), (1.2497, 1), (1.6544, 1), (4.1019, 1), (11.27839, 1), (-5.04127, 1), (18.11674, 1), (0.51231, 1), (-0.51029, 1), (13.52556, 1), (16.10171, 1), (5.68197, 1), (-2.85904, 1), (-8.89167, 1), (6.24489, 1), (10.85319, 1), (-0.39816, 1), (3.87079, 1), (-3.1867, 1), (1.55322, 1), (16.86779, 1), (-14.60321, 1), (-1.81952, 1), (-3.11624, 1), (1.24193, 1), (10.18179, 1), (4.69796, 1), (0.69032, 1), (11.7723, 1), (7.62896, 1), (9.89741, 1), (9.11484, 1), (-3.84676, 1), (-0.4777, 1), (0.95958, 1), (-7.95056, 1), (-10.97474, 1), (-6.54861, 1), (34.74933, 1), (27.39463, 1), (4.18299, 1), (6.02476, 1), (-1.99397, 1), (1.26478, 1), (23.37106, 1), (10.49682, 1), (-11.04354, 1), (-12.22284, 1), (-9.87635, 1), (28.90511, 1), (6.77613, 1), (0.55352, 1), (0.37031, 1), (7.1418, 1), (3.24897, 1), (-1.60918, 1), (3.1675, 1), (-17.97072, 1), (-5.61743, 1), (14.1422, 1), (14.87695, 1), (-4.65961, 1), (-0.99174, 1), (-2.96623, 1), (-9.02263, 1), (-17.2088, 1), (2.78608, 1), (6.74239, 1), (4.8524, 1), (7.46731, 1), (1.04894, 1), (-12.8023, 1), (-17.18188, 1), (-5.08801, 1), (22.13942, 1), (-0.36384, 1), (17.80564, 1), (7.67504, 1), (1.59779, 1), (4.10942, 1), (0.61074, 1), (-14.40767, 1), (10.59906, 1), (16.57017, 1), (-15.17526, 1), (-6.98549, 1), (-0.64548, 1), (3.23756, 1), (14.65504, 1), (4.583, 1), (12.72378, 1), (5.26547, 1), (0.81781, 1), (9.38273, 1), (10.37636, 1), (10.70325, 1), (-0.83043, 1), (-7.53149, 1), (-9.09147, 1), (-19.51381, 1), (-28.44508, 1), (6.44392, 1), (11.10201, 1), (-2.86184, 1), (8.30673, 1), (8.8797, 1), (10.68053, 1), (15.62919, 1), (8.00579, 1), (6.4651, 1), (-4.50029, 1), (18.04514, 1), (11.12996, 1), (-5.14007, 1), (9.43857, 1), (3.13476, 1), (4.9772, 1), (-17.45782, 1), (0.05552, 1), (-1.90283, 1), (2.67908, 1), (-2.62243, 1), (-3.22767, 1), (-8.70222, 1), (-23.11605, 1), (21.6757, 1), (12.70076, 1), (4.4322, 1), (11.69344, 1), (9.18052, 1), (-2.2549, 1), (-2.15615, 1), (20.29765, 1), (-0.29536, 1), (15.50109, 1), (8.79187, 1), (5.11533, 1), (-20.44436, 1), (-3.00909, 1), (-4.48291, 1), (21.84462, 1), (1.94225, 1), (-2.81908, 1), (17.19418, 1), (-9.33528, 1), (-0.17346, 1), (0.03958, 1), (-35.17786, 1), (8.36887, 1), (-9.02292, 1), (-10.98804, 1), (0.29335, 1), (4.29634, 1), (3.87718, 1), (-9.08532, 1), (7.13922, 1), (-7.62463, 1), (-10.5666, 1), (4.68165, 1), (-3.30172, 1), (13.04852, 1), (13.45616, 1), (2.41043, 1), (-0.36501, 1), (-15.67383, 1), (17.92217, 1), (8.42106, 1), (3.22063, 1), (-7.31753, 1), (21.99596, 1), (-36.8273, 1), (-20.46391, 1), (5.74179, 1), (-15.83178, 1), (14.90454, 1), (-8.84645, 1), (3.72036, 1), (4.6877, 1), (16.35418, 1), (3.15441, 1), (2.39907, 1), (-17.58664, 1), (-13.18269, 1); SELECT '14.971190998235835', '5.898143508382202e-44'; SELECT roundBankers(welchTTest(left, right).1, 16) as t_stat, roundBankers(welchTTest(left, right).2, 16) as p_value from welch_ttest; +SELECT roundBankers(welchTTest(0.95)(left, right).3, 16) as t_stat, roundBankers(welchTTest(0.95)(left, right).4, 16) as p_value from welch_ttest; DROP TABLE IF EXISTS welch_ttest; @@ -52,4 +53,5 @@ CREATE TABLE student_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO student_ttest VALUES (4.52546, 0), (8.69444, 1), (3.73628, 0), (3.81414, 1), (-0.39478, 0), (12.38442, 1), (5.15633, 0), (8.9738, 1), (0.50539, 0), (9.19594, 1), (-5.34036, 0), (7.21009, 1), (0.19336, 0), (4.97743, 1), (8.35729, 0), (4.94756, 1), (6.95818, 0), (19.80911, 1), (-2.93812, 0), (13.75358, 1), (8.30807, 0), (16.56373, 1), (-3.3517, 0), (9.72882, 1), (4.16279, 0), (4.64509, 1), (-3.17231, 0), (17.76854, 1), (1.93545, 0), (4.80693, 1), (11.06606, 0), (8.79505, 1), (-4.22678, 0), (10.88868, 1), (-1.99975, 0), (6.21932, 1), (-4.51178, 0), (15.11614, 1), (-4.50711, 0), (13.24703, 1), (1.89786, 0), (14.76476, 1), (-6.19638, 0), (-0.6117, 1), (-3.70188, 0), (17.48993, 1), (5.01334, 0), (12.11847, 1), (1.79036, 0), (4.87439, 1), (2.14435, 0), (18.56479, 1), (3.0282, 0), (1.23712, 1), (2.35528, 0), (5.41596, 1), (-12.18535, 0), (4.54994, 1), (5.59709, 0), (11.37668, 1), (-12.92336, 0), (9.5982, 1), (-0.04281, 0), (6.59822, 1), (-0.16923, 0), (1.16703, 1), (0.88924, 0), (8.88418, 1), (-4.68414, 0), (10.95047, 1), (8.01099, 0), (5.52787, 1), (2.61686, 0), (-1.11647, 1), (-2.76895, 0), (14.49946, 1), (3.32165, 0), (3.27585, 1), (-0.85135, 0), (-0.42025, 1), (1.21368, 0), (6.37906, 1), (4.38673, 0), (2.5242, 1), (6.20964, 0), (8.1405, 1), (-1.23172, 0), (6.46732, 1), (4.65516, 0), (9.89332, 1), (-1.87143, 0), (10.4374, 1), (0.86429, 0), (-1.06465, 1), (2.51184, 0), (6.84902, 1), (-1.88822, 0), (10.96576, 1), (-1.61802, 0), (7.83319, 1), (1.93653, 0), (14.39823, 1), (-3.66631, 0), (7.02594, 1), (-1.05294, 0), (13.46629, 1), (-10.74718, 0), (10.39531, 1), (16.49295, 0), (11.27348, 1), (-7.65494, 0), (9.32187, 1), (-3.39303, 0), (12.32667, 1), (-4.89418, 0), (8.98905, 1), (3.2521, 0), (9.54757, 1), (0.05831, 0), (5.98325, 1), (-3.00409, 0), (3.47248, 1), (5.76702, 0), (9.26966, 1), (2.67674, 0), (5.77816, 1), (10.52623, 0), (6.32966, 1), (-0.54501, 0), (9.49313, 1), (-4.89835, 0), (6.21337, 1), (3.52457, 0), (10.00242, 1), (-0.0451, 0), (6.25167, 1), (-6.61226, 0), (15.64671, 1), (9.02391, 0), (2.78968, 1), (5.52571, 0), (6.55442, 1), (4.54352, 0), (3.68819, 1), (-3.8394, 0), (9.55934, 1), (-7.75295, 0), (4.166, 1), (5.91167, 0), (12.32471, 1), (1.38897, 0), (7.10969, 1), (6.24166, 0), (16.31723, 1), (5.58536, 0), (12.99482, 1), (4.7591, 0), (10.11585, 1), (-2.58336, 0), (10.29455, 1), (-1.91263, 0), (18.27524, 1), (3.31575, 0), (12.84435, 1), (5.3507, 0), (13.11954, 1), (-15.22081, 0), (12.84147, 1), (-0.84775, 0), (15.55658, 1), (-4.538, 0), (11.45329, 1), (6.71177, 0), (7.50912, 1), (0.52882, 0), (8.56226, 1), (2.0242, 0), (8.63104, 1), (5.69146, 0), (15.68026, 1), (4.63328, 0), (21.6361, 1), (0.22984, 0), (6.23925, 1), (-2.84052, 0), (8.65714, 1), (7.91867, 0), (9.9423, 1), (1.11001, 0), (12.28213, 1), (-0.11251, 0), (3.11279, 1), (-0.20905, 0), (13.58128, 1), (0.03287, 0), (16.51407, 1), (-1.59397, 0), (16.60476, 1), (-5.39405, 0), (12.02022, 1), (-7.1233, 0), (12.11035, 1), (4.51517, 0), (9.47832, 1), (-0.70967, 0), (6.40742, 1), (5.67299, 0), (8.87252, 1), (-0.33835, 0), (15.14265, 1), (-1.83047, 0), (2.23572, 1), (-0.62877, 0), (11.57144, 1), (-7.23148, 0), (18.87737, 1), (0.1802, 0), (12.1833, 1), (11.73325, 0), (11.17519, 1), (2.17603, 0), (16.80422, 1), (-0.11683, 0), (6.81423, 1), (-1.29102, 0), (12.12546, 1), (-0.23201, 0), (8.06153, 1), (-6.8643, 0), (10.97228, 1), (-6.85153, 0), (7.30596, 1), (-4.77163, 0), (15.44026, 1), (6.11721, 0), (8.00993, 1), (5.96406, 0), (12.60196, 1), (3.59135, 0), (13.96832, 1), (-0.60095, 0), (14.03207, 1), (3.11163, 0), (4.53758, 1), (-0.18831, 0), (8.08297, 1), (0.67657, 0), (4.90451, 1), (-3.16117, 0), (8.14253, 1), (0.26957, 0), (19.88605, 1), (2.18653, 0), (13.85254, 1), (-5.94611, 0), (23.01839, 1), (-4.39352, 0), (6.02084, 1), (-3.71525, 0), (9.60319, 1), (5.11103, 0), (1.90511, 1), (1.33998, 0), (10.35237, 1), (1.01629, 0), (16.27082, 1), (-3.36917, 0), (12.52379, 1), (-3.99661, 0), (11.37435, 1), (8.19336, 0), (13.61823, 1), (2.89168, 0), (15.77622, 1), (-11.10373, 0), (15.17254, 1), (11.68005, 0), (6.711, 1), (3.08282, 0), (4.74205, 1), (-6.81506, 0), (10.09812, 1), (-2.34587, 0), (6.61722, 1), (-2.68725, 0), (10.34164, 1), (0.3577, 0), (8.96602, 1), (-3.05682, 0), (12.32157, 1), (9.08062, 0), (11.75711, 1), (-0.77913, 0), (13.49499, 1), (10.35215, 0), (8.57713, 1), (6.82565, 0), (11.50313, 1), (-1.24674, 0), (1.13097, 1), (5.18822, 0), (7.83205, 1), (-3.70743, 0), (5.77957, 1), (1.40319, 0), (15.5519, 1), (5.89432, 0), (10.82676, 1), (1.43152, 0), (11.51218, 1), (6.70638, 0), (9.29779, 1), (9.76613, 0), (9.77021, 1), (4.27604, 0), (9.94114, 1), (-2.63141, 0), (15.54513, 1), (-7.8133, 0), (19.10736, 1), (-0.06668, 0), (15.04205, 1), (1.05391, 0), (9.03114, 1), (4.41797, 0), (24.0104, 1), (0.09337, 0), (9.94205, 1), (6.16075, 0), (2.5925, 1), (7.49413, 0), (8.82726, 1), (-3.52872, 0), (10.0209, 1), (-2.17126, 0), (8.1635, 1), (-3.87605, 0), (4.24074, 1), (3.26607, 0), (7.67291, 1), (-3.28045, 0), (5.21642, 1), (2.1429, 0), (11.2808, 1), (1.53386, 0), (6.88172, 1), (0.21169, 0), (5.98743, 1), (-0.63674, 0), (17.97249, 1), (5.84893, 0), (6.46323, 1), (-0.63498, 0), (15.37416, 1), (8.29526, 0), (2.89957, 1), (-1.08358, 0), (17.13044, 1), (-2.306, 0), (11.06355, 1), (2.86991, 0), (3.09625, 1), (-0.76074, 0), (-2.33019, 1), (5.49191, 0), (7.42675, 1), (1.82883, 0), (15.06792, 1), (-3.70497, 0), (8.81116, 1), (-0.53232, 0), (19.17446, 1), (-11.49722, 0), (18.77181, 1), (3.44877, 0), (14.06443, 1), (-1.8596, 0), (12.81241, 1), (-10.34851, 0), (2.72299, 1), (1.13093, 0), (18.67739, 1), (-10.93389, 0), (11.63275, 1), (-3.39703, 0), (2.23891, 1), (0.19749, 0), (13.01195, 1), (-3.68389, 0), (7.43402, 1), (-4.67863, 0), (8.14599, 1), (10.78916, 0), (16.65328, 1), (0.37675, 0), (1.362, 1), (3.98094, 0), (3.87957, 1), (-3.64775, 0), (11.16134, 1), (-4.8443, 0), (6.25357, 1), (1.102, 0), (4.21945, 1), (8.72112, 0), (12.50047, 1), (-1.47361, 0), (6.45486, 1), (6.24183, 0), (18.99924, 1), (6.83569, 0), (18.09508, 1), (-3.11684, 0), (13.59528, 1), (4.91306, 0), (3.39681, 1), (-0.03628, 0), (13.33157, 1), (5.1282, 0), (5.8945, 1), (-2.38558, 0), (5.61212, 1), (2.33351, 0), (8.41149, 1), (-0.97191, 0), (13.78608, 1), (-0.05588, 0), (6.08609, 1), (-4.70019, 0), (12.76962, 1), (-5.12371, 0), (3.26206, 1), (0.65606, 0), (0.25528, 1), (-0.11574, 0), (11.9083, 1), (4.4238, 0), (4.35071, 1), (6.93399, 0), (11.19855, 1), (3.68712, 0), (13.87404, 1), (-0.01187, 0), (6.87986, 1), (1.8332, 0), (8.32566, 1), (5.81322, 0), (22.51334, 1), (-4.04709, 0), (2.5226, 1), (-8.26397, 0), (16.84498, 1), (-2.11273, 0), (6.26108, 1), (5.28396, 0), (13.84824, 1), (0.73054, 0), (6.03262, 1), (6.43559, 0), (14.12668, 1), (4.35565, 0), (16.01939, 1), (-1.05545, 0), (8.19237, 1), (5.00087, 0), (18.01595, 1), (-2.72239, 0), (9.45609, 1), (7.32313, 0), (6.90459, 1), (2.11548, 0), (12.83115, 1), (-3.40953, 0), (10.603, 1), (6.97051, 0), (13.70439, 1), (-0.45567, 0), (6.1633, 1), (1.31699, 0), (4.1151, 1), (-1.49871, 0), (8.20499, 1), (7.14772, 0), (11.67903, 1), (0.79277, 0), (7.30851, 1), (6.9698, 0), (6.50941, 1), (2.08733, 0), (7.3949, 1), (-3.55962, 0), (12.80075, 1), (0.75601, 0), (5.62043, 1), (1.21, 0), (18.2542, 1), (-2.17877, 0), (17.9393, 1), (1.83206, 0), (16.4569, 1), (5.72463, 0), (8.78811, 1), (7.42257, 0), (4.85949, 1), (0.97829, 0), (-3.36394, 1), (7.54238, 0), (5.38683, 1), (9.91081, 0), (12.26083, 1), (-4.61743, 0), (10.27907, 1), (-4.40799, 0), (11.5144, 1), (9.99854, 0), (11.57335, 1), (8.53725, 0), (1.94203, 1), (3.2905, 0), (7.78228, 1), (0.38634, 0), (11.79385, 1), (-2.53374, 0), (10.18415, 1), (4.94758, 0), (14.67613, 1), (4.79624, 0), (4.70301, 1), (5.57664, 0), (12.72151, 1), (-6.44871, 0), (-3.35508, 1), (3.34431, 0), (17.63775, 1), (0.14209, 0), (2.53883, 1), (10.88431, 0), (14.01483, 1), (0.31846, 0), (12.4387, 1), (-0.54703, 0), (11.15408, 1), (-4.67791, 0), (7.74882, 1), (-5.68011, 0), (13.60956, 1), (-4.93362, 0), (7.81991, 1), (1.2271, 0), (10.90969, 1), (5.27512, 0), (8.19828, 1), (-3.84611, 0), (-1.18523, 1), (6.81706, 0), (0.5916, 1), (10.33033, 0), (0.35805, 1), (5.13979, 0), (12.98364, 1), (3.66534, 0), (11.38628, 1), (-2.07219, 0), (13.94644, 1), (10.65442, 0), (2.03781, 1), (-3.31751, 0), (10.74447, 1), (-1.82011, 0), (12.35656, 1), (-0.39886, 0), (7.08701, 1), (1.77052, 0), (2.69871, 1), (1.29049, 0), (19.66653, 1), (7.92344, 0), (7.88636, 1), (-2.92595, 0), (10.36916, 1), (-2.67107, 0), (1.632, 1), (5.64708, 0), (11.86081, 1), (0.34639, 0), (13.47602, 1), (-3.04356, 0), (6.60204, 1), (3.98828, 0), (7.01303, 1), (-1.36695, 0), (20.19992, 1), (-8.48462, 0), (18.88249, 1), (-4.04669, 0), (11.34367, 1), (9.84561, 0), (12.97305, 1), (-6.1537, 0), (9.5776, 1), (0.82433, 0), (17.91364, 1), (1.92449, 0), (18.3247, 1), (2.51288, 0), (9.9211, 1), (0.40965, 0), (7.14257, 1), (2.89183, 0), (6.59133, 1), (3.84347, 0), (12.35274, 1), (0.66829, 0), (10.57523, 1), (-3.45094, 0), (12.12859, 1), (1.3544, 0), (9.47177, 1), (-9.85456, 0), (0.60659, 1), (5.25689, 0), (4.72996, 1), (-5.26018, 0), (4.51121, 1), (-6.16912, 0), (13.28893, 1), (-1.77163, 0), (8.09014, 1), (3.96687, 0), (8.02511, 1), (0.70893, 0), (13.85406, 1), (-5.45342, 0), (1.75412, 1), (-3.89706, 0), (6.00641, 1), (3.11868, 0), (6.35554, 1), (4.41714, 0), (7.11293, 1), (7.64841, 0), (8.30442, 1), (0.00489, 0), (12.63024, 1), (3.2263, 0), (12.38966, 1), (-5.33042, 0), (7.6801, 1), (2.52189, 0), (11.33744, 1), (-7.40308, 0), (4.67713, 1), (0.67891, 0), (7.62276, 1), (2.49343, 0), (2.14478, 1), (5.43133, 0), (15.32988, 1), (-0.67541, 0), (1.52299, 1), (-0.60299, 0), (17.00017, 1), (-6.32903, 0), (8.29701, 1), (-3.44336, 0), (10.92961, 1), (-0.23963, 0), (6.78449, 1), (6.94686, 0), (7.02698, 1), (6.59442, 0), (11.51719, 1), (-4.18532, 0), (9.97926, 1), (-1.8228, 0), (7.44251, 1), (-0.29443, 0), (7.58541, 1), (2.99821, 0), (4.76058, 1), (2.51942, 0), (12.88959, 1), (-3.49176, 0), (9.974, 1), (-0.57979, 0), (17.03689, 1), (8.69471, 0), (11.14554, 1), (-1.19427, 0), (11.7392, 1), (-3.17119, 0), (11.50029, 1), (-2.99566, 0), (19.41759, 1), (-3.34493, 0), (9.65127, 1), (-2.33826, 0), (9.87673, 1), (-5.04164, 0), (14.13485, 1), (-0.48214, 0), (9.78034, 1), (7.45097, 0), (1.57826, 1), (3.04787, 0), (3.72091, 1), (2.92632, 0), (9.4054, 1), (1.39694, 0), (23.22816, 1), (4.38686, 0), (-0.12571, 1), (3.25753, 0), (6.97343, 1), (7.14218, 0), (10.09049, 1), (-4.04341, 0), (11.78393, 1), (-9.19352, 0), (3.01909, 1), (2.78473, 0), (16.09448, 1), (0.33331, 0), (6.25485, 1), (9.89238, 0), (7.13164, 1), (6.00566, 0), (7.75879, 1), (-1.7511, 0), (9.56834, 1), (4.77815, 0), (6.14824, 1), (5.07457, 0), (13.53454, 1), (2.56132, 0), (8.26364, 1), (2.38317, 0), (8.7095, 1), (-1.63486, 0), (10.61607, 1), (-1.46871, 0), (10.64418, 1), (-5.8681, 0), (23.9106, 1), (-2.96227, 0), (11.38978, 1), (-1.90638, 0), (11.4383, 1), (-13.3052, 0), (18.41498, 1), (-2.14705, 0), (3.70959, 1), (-9.62069, 0), (19.95918, 1), (2.29313, 0), (9.53847, 1), (0.22162, 0), (14.04957, 1), (-1.83956, 0), (13.70151, 1), (4.1853, 0), (5.45046, 1), (6.05965, 0), (10.95061, 1), (-0.23737, 0), (9.55156, 1), (6.07452, 0), (17.92345, 1), (4.34629, 0), (6.23976, 1), (4.02922, 0), (8.71029, 1), (3.62622, 0), (13.58736, 1), (-3.95825, 0), (8.78527, 1), (-1.63412, 0), (11.14213, 1), (-1.25727, 0), (12.23717, 1), (5.06323, 0), (16.44557, 1), (-0.66176, 0), (0.47144, 1), (2.36606, 0), (9.7198, 1), (-5.77792, 0), (13.50981, 1), (4.535, 0), (14.27806, 1), (1.02031, 0), (13.50793, 1), (4.49345, 0), (7.47381, 1), (-4.99791, 0), (11.07844, 1), (2.46716, 0), (9.89844, 1), (3.65471, 0), (21.48548, 1), (11.2283, 0), (6.92085, 1), (6.69743, 0), (4.44074, 1), (-5.60375, 0), (19.98074, 1), (0.28683, 0), (7.92826, 1), (-0.85737, 0), (16.6313, 1), (4.26726, 0), (17.17618, 1), (-3.4322, 0), (13.80807, 1), (-2.07039, 0), (5.37083, 1), (-2.26798, 0), (9.73962, 1), (-0.99818, 0), (10.66273, 1), (0.41335, 0), (8.90639, 1), (5.18124, 0), (12.24596, 1), (-5.01858, 0), (16.89203, 1), (2.05561, 0), (12.69184, 1), (-0.12117, 0), (15.59077, 1), (0.99471, 0), (6.94287, 1), (6.89979, 0), (-0.1801, 1), (-4.18527, 0), (3.25318, 1), (-6.35104, 0), (8.08804, 1), (3.89734, 0), (13.78384, 1), (-1.979, 0), (0.46434, 1), (3.15404, 0), (7.78224, 1), (3.52672, 0), (9.10987, 1), (2.48372, 0), (-0.89391, 1), (-6.13089, 0), (14.3696, 1), (2.2968, 0), (3.01763, 1), (-2.74324, 0), (8.03559, 1), (-0.12876, 0), (7.24609, 1), (-1.51135, 0), (11.86271, 1), (-3.92434, 0), (6.28196, 1), (-1.71254, 0), (8.9725, 1), (-1.25878, 0), (14.46114, 1), (2.03021, 0), (9.50216, 1), (4.31726, 0), (16.30413, 1), (-3.02908, 0), (1.02795, 1), (9.7093, 0), (1.88717, 1), (-3.36284, 0), (9.80106, 1), (6.70938, 0), (4.53487, 1), (0.42762, 0), (16.34543, 1), (5.04726, 0), (7.71098, 1), (2.78386, 0), (2.74639, 1), (6.83022, 0), (6.51875, 1), (-3.02109, 0), (10.42308, 1), (-0.65382, 0), (13.57901, 1), (-15.58675, 0), (0.52784, 1), (5.89746, 0), (4.4708, 1), (-4.11598, 0), (6.39619, 1), (-1.37208, 0), (14.57666, 1), (10.08082, 0), (2.71602, 1), (5.35686, 0), (12.53905, 1), (1.93331, 0), (11.4292, 1), (10.47444, 0), (12.44641, 1), (-2.36872, 0), (14.50894, 1), (6.50752, 0), (17.64374, 1), (2.54603, 0), (11.03218, 1), (-0.4332, 0), (9.82789, 1), (5.26572, 0), (10.11104, 1), (2.09016, 0), (2.16137, 1), (1.15513, 0), (10.24054, 1), (14.95941, 0), (12.86909, 1), (-3.85505, 0), (15.22845, 1), (-2.36239, 0), (5.05411, 1), (1.64338, 0), (10.84836, 1), (-4.25074, 0), (11.15717, 1), (7.29744, 0), (0.91782, 1), (-1.18964, 0), (13.29961, 1), (5.60612, 0), (15.11314, 1), (-3.77011, 0), (11.54004, 1), (6.67642, 0), (-0.94238, 1), (-0.06862, 0), (19.32581, 1), (5.60514, 0), (10.20744, 1), (3.7341, 0), (6.54857, 1), (9.59001, 0), (8.69108, 1), (3.30093, 0), (8.2296, 1), (-2.75658, 0), (8.4474, 1), (4.71994, 0), (6.81178, 1), (0.74699, 0), (5.99415, 1), (2.91095, 0), (13.99336, 1), (-7.36829, 0), (8.7469, 1), (-5.29487, 0), (8.62349, 1), (3.31079, 0), (1.84212, 1), (1.06974, 0), (4.4762, 1), (-1.18424, 0), (9.25421, 1), (-7.415, 0), (10.44229, 1), (3.40595, 0), (12.21649, 1), (-7.63085, 0), (10.45968, 1), (1.13336, 0), (15.34722, 1), (-0.0096, 0), (5.50868, 1), (0.8928, 0), (10.93609, 1), (-0.5943, 0), (2.78631, 1), (7.48306, 0), (11.86145, 1), (10.11943, 0), (18.67385, 1), (5.60459, 0), (10.64051, 1), (4.00189, 0), (12.75565, 1), (2.35823, 0), (6.63666, 1), (0.33475, 0), (12.19343, 1), (3.47072, 0), (9.08636, 1), (-6.68867, 0), (11.67256, 1), (3.31031, 0), (20.31392, 1), (2.17159, 0), (11.66443, 1); SELECT '-28.740781574102936', '7.667329672103986e-133'; SELECT roundBankers(studentTTest(left, right).1, 16) as t_stat, roundBankers(studentTTest(left, right).2, 16) as p_value from student_ttest; +SELECT roundBankers(studentTTest(0.95)(left, right).3, 16) as t_stat, roundBankers(studentTTest(0.95)(left, right).4, 16) as p_value from student_ttest; DROP TABLE IF EXISTS student_ttest; From f31ba5ad1946f78dfcaee846aaad9b4ba727374a Mon Sep 17 00:00:00 2001 From: achimbab <07c00h@gmail.com> Date: Wed, 29 Dec 2021 19:29:49 +0900 Subject: [PATCH 022/149] Fix build error --- .gitmodules | 6 ++++++ contrib/CMakeLists.txt | 1 + contrib/gcem | 1 + contrib/stats | 1 + 4 files changed, 9 insertions(+) create mode 160000 contrib/gcem create mode 160000 contrib/stats diff --git a/.gitmodules b/.gitmodules index 5321712f1f1..3f470d587f8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -250,3 +250,9 @@ [submodule "contrib/azure"] path = contrib/azure url = https://github.com/ClickHouse-Extras/azure-sdk-for-cpp.git +[submodule "contrib/stats"] + path = contrib/stats + url = https://github.com/kthohr/stats.git +[submodule "contrib/gcem"] + path = contrib/gcem + url = https://github.com/kthohr/gcem.git diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 8f394709d2a..db510c740c0 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -301,6 +301,7 @@ endif() if (USE_STATS) add_subdirectory (stats-cmake) + add_subdirectory (gcem) endif() # Put all targets defined here and in subdirectories under "contrib/" folders in GUI-based IDEs. diff --git a/contrib/gcem b/contrib/gcem new file mode 160000 index 00000000000..8d0e62d9c9c --- /dev/null +++ b/contrib/gcem @@ -0,0 +1 @@ +Subproject commit 8d0e62d9c9c47dfad2cb2d800281908cf7ff16c6 diff --git a/contrib/stats b/contrib/stats new file mode 160000 index 00000000000..309c0e92d23 --- /dev/null +++ b/contrib/stats @@ -0,0 +1 @@ +Subproject commit 309c0e92d2326a58cad4544124c614e6a8a46079 From 316f6986cf6fc9db025e35fa950a927a8ebd5860 Mon Sep 17 00:00:00 2001 From: achimbab <07c00h@gmail.com> Date: Thu, 30 Dec 2021 07:17:46 +0900 Subject: [PATCH 023/149] Replaced the stat library with the boost library. --- .gitmodules | 6 ------ CMakeLists.txt | 1 - cmake/find/stats.cmake | 20 ------------------- contrib/CMakeLists.txt | 5 ----- contrib/gcem | 1 - contrib/stats | 1 - .../AggregateFunctionTTest.h | 2 +- src/AggregateFunctions/CMakeLists.txt | 4 ---- src/AggregateFunctions/Moments.h | 12 +++-------- src/Common/config.h.in | 1 - .../queries/0_stateless/01558_ttest.reference | 4 ++-- 11 files changed, 6 insertions(+), 51 deletions(-) delete mode 100644 cmake/find/stats.cmake delete mode 160000 contrib/gcem delete mode 160000 contrib/stats diff --git a/.gitmodules b/.gitmodules index 3f470d587f8..5321712f1f1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -250,9 +250,3 @@ [submodule "contrib/azure"] path = contrib/azure url = https://github.com/ClickHouse-Extras/azure-sdk-for-cpp.git -[submodule "contrib/stats"] - path = contrib/stats - url = https://github.com/kthohr/stats.git -[submodule "contrib/gcem"] - path = contrib/gcem - url = https://github.com/kthohr/gcem.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 18eaa9d48c5..fdc9cfcd303 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -546,7 +546,6 @@ include (cmake/find/avro.cmake) include (cmake/find/msgpack.cmake) include (cmake/find/cassandra.cmake) include (cmake/find/sentry.cmake) -include (cmake/find/stats.cmake) include (cmake/find/datasketches.cmake) include (cmake/find/libprotobuf-mutator.cmake) diff --git a/cmake/find/stats.cmake b/cmake/find/stats.cmake deleted file mode 100644 index ef5b1a73659..00000000000 --- a/cmake/find/stats.cmake +++ /dev/null @@ -1,20 +0,0 @@ -option(ENABLE_STATS "Enalbe StatsLib library" ${ENABLE_LIBRARIES}) - -if (ENABLE_STATS) - if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/stats") - message (WARNING "submodule contrib/stats is missing. to fix try run: \n git submodule update --init --recursive") - set (ENABLE_STATS 0) - set (USE_STATS 0) - elseif (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/gcem") - message (WARNING "submodule contrib/gcem is missing. to fix try run: \n git submodule update --init --recursive") - set (ENABLE_STATS 0) - set (USE_STATS 0) - else() - set(STATS_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/stats/include) - set(GCEM_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/gcem/include) - set (USE_STATS 1) - endif() -endif() - -message (STATUS "Using stats=${USE_STATS} : ${STATS_INCLUDE_DIR}") -message (STATUS "Using gcem=${USE_STATS}: ${GCEM_INCLUDE_DIR}") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index db510c740c0..9eb3b145c85 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -299,11 +299,6 @@ if (USE_S2_GEOMETRY) add_subdirectory(s2geometry-cmake) endif() -if (USE_STATS) - add_subdirectory (stats-cmake) - add_subdirectory (gcem) -endif() - # Put all targets defined here and in subdirectories under "contrib/" folders in GUI-based IDEs. # Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear # in "contrib/..." as originally planned, so we workaround this by fixing FOLDER properties of all targets manually, diff --git a/contrib/gcem b/contrib/gcem deleted file mode 160000 index 8d0e62d9c9c..00000000000 --- a/contrib/gcem +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8d0e62d9c9c47dfad2cb2d800281908cf7ff16c6 diff --git a/contrib/stats b/contrib/stats deleted file mode 160000 index 309c0e92d23..00000000000 --- a/contrib/stats +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 309c0e92d2326a58cad4544124c614e6a8a46079 diff --git a/src/AggregateFunctions/AggregateFunctionTTest.h b/src/AggregateFunctions/AggregateFunctionTTest.h index 21348e45372..15a6269a6d4 100644 --- a/src/AggregateFunctions/AggregateFunctionTTest.h +++ b/src/AggregateFunctions/AggregateFunctionTTest.h @@ -83,7 +83,7 @@ private: bool need_ci = false; Float64 confidence_level; public: - AggregateFunctionTTest(const DataTypes & arguments, [[maybe_unused]] const Array & params) + AggregateFunctionTTest(const DataTypes & arguments, const Array & params) : IAggregateFunctionDataHelper>({arguments}, params) { if (params.size() > 0) diff --git a/src/AggregateFunctions/CMakeLists.txt b/src/AggregateFunctions/CMakeLists.txt index 9c08c733511..64f6eed9a6c 100644 --- a/src/AggregateFunctions/CMakeLists.txt +++ b/src/AggregateFunctions/CMakeLists.txt @@ -28,7 +28,3 @@ target_link_libraries(clickhouse_aggregate_functions PRIVATE dbms PUBLIC ${CITYH if(ENABLE_EXAMPLES) add_subdirectory(examples) endif() - -if (USE_STATS) - target_link_libraries(clickhouse_aggregate_functions PRIVATE stats) -endif() diff --git a/src/AggregateFunctions/Moments.h b/src/AggregateFunctions/Moments.h index 140257bf812..4fca93527c0 100644 --- a/src/AggregateFunctions/Moments.h +++ b/src/AggregateFunctions/Moments.h @@ -2,12 +2,7 @@ #include #include -#include - -#if !defined(ARCADIA_BUILD) && USE_STATS - -#define STATS_ENABLE_STDVEC_WRAPPERS -#include +#include namespace DB @@ -394,7 +389,8 @@ struct TTestMoments Float64 mean_y = getMeanY(); Float64 se = getStandardError(); - Float64 t = stats::qt(1.0f - (1.0f - confidence_level) / 2.0f, degrees_of_freedom); + boost::math::students_t dist(degrees_of_freedom); + Float64 t = boost::math::quantile(boost::math::complement(dist, (1.0f - confidence_level) / 2.0f)); Float64 mean_diff = mean_x - mean_y; Float64 ci_low = mean_diff - t * se; Float64 ci_high = mean_diff + t * se; @@ -404,5 +400,3 @@ struct TTestMoments }; } - -#endif diff --git a/src/Common/config.h.in b/src/Common/config.h.in index 1c14cddff28..c53abd17bc2 100644 --- a/src/Common/config.h.in +++ b/src/Common/config.h.in @@ -20,4 +20,3 @@ #cmakedefine01 USE_YAML_CPP #cmakedefine01 CLICKHOUSE_SPLIT_BINARY #cmakedefine01 USE_BZIP2 -#cmakedefine01 USE_STATS diff --git a/tests/queries/0_stateless/01558_ttest.reference b/tests/queries/0_stateless/01558_ttest.reference index 967e5864446..5cbc86038c3 100644 --- a/tests/queries/0_stateless/01558_ttest.reference +++ b/tests/queries/0_stateless/01558_ttest.reference @@ -8,9 +8,9 @@ -0.5028215369187079 0.6152361677171103 14.971190998235835 5.898143508382202e-44 14.971190998235837 0 -7.6505301757756445 9.960200184224346 +7.650530175770567 9.960200184229425 -2.610898982580138 0.00916587538237954 -2.610898982580134 0.0091658753823834 -28.740781574102936 7.667329672103986e-133 -28.74078157410298 0 --9.625938422387687 -8.395483817612316 +-9.625938422388245 -8.395483817611758 From 87e6ba050ffa6154ff0fed3b4aef5f7be3d66acd Mon Sep 17 00:00:00 2001 From: achimbab <07c00h@gmail.com> Date: Sat, 1 Jan 2022 15:37:18 +0900 Subject: [PATCH 024/149] Fix docs for studenTTtest and welchTTest. --- .../aggregate-functions/reference/studentttest.md | 10 +++++++++- .../aggregate-functions/reference/welchttest.md | 9 ++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/studentttest.md b/docs/en/sql-reference/aggregate-functions/reference/studentttest.md index fd391298bc3..0489569146c 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/studentttest.md +++ b/docs/en/sql-reference/aggregate-functions/reference/studentttest.md @@ -10,7 +10,7 @@ Applies Student's t-test to samples from two populations. **Syntax** ``` sql -studentTTest(sample_data, sample_index) +studentTTest([confidence_level])(sample_data, sample_index) ``` Values of both samples are in the `sample_data` column. If `sample_index` equals to 0 then the value in that row belongs to the sample from the first population. Otherwise it belongs to the sample from the second population. @@ -21,12 +21,20 @@ The null hypothesis is that means of populations are equal. Normal distribution - `sample_data` — Sample data. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md). - `sample_index` — Sample index. [Integer](../../../sql-reference/data-types/int-uint.md). +**Parameters** + +- `confidence_level` — Confidence level in order to calculate confidence intervals. [Float](../../../sql-reference/data-types/float.md). + + **Returned values** [Tuple](../../../sql-reference/data-types/tuple.md) with two elements: +If the optional confidence_level is specified, it returns a [Tuple](../../../sql-reference/data-types/tuple.md) with four elements. - calculated t-statistic. [Float64](../../../sql-reference/data-types/float.md). - calculated p-value. [Float64](../../../sql-reference/data-types/float.md). +- [calculated confidence-interval-low.] [Float64](../../../sql-reference/data-types/float.md). +- [calculated confidence-interval-high.] [Float64](../../../sql-reference/data-types/float.md). **Example** diff --git a/docs/en/sql-reference/aggregate-functions/reference/welchttest.md b/docs/en/sql-reference/aggregate-functions/reference/welchttest.md index 62f5761b32e..89a6afffc2c 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/welchttest.md +++ b/docs/en/sql-reference/aggregate-functions/reference/welchttest.md @@ -10,7 +10,7 @@ Applies Welch's t-test to samples from two populations. **Syntax** ``` sql -welchTTest(sample_data, sample_index) +welchTTest([confidence_level])(sample_data, sample_index) ``` Values of both samples are in the `sample_data` column. If `sample_index` equals to 0 then the value in that row belongs to the sample from the first population. Otherwise it belongs to the sample from the second population. @@ -21,12 +21,19 @@ The null hypothesis is that means of populations are equal. Normal distribution - `sample_data` — Sample data. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md). - `sample_index` — Sample index. [Integer](../../../sql-reference/data-types/int-uint.md). +**Parameters** + +- `confidence_level` — Confidence level in order to calculate confidence intervals. [Float](../../../sql-reference/data-types/float.md). + **Returned values** [Tuple](../../../sql-reference/data-types/tuple.md) with two elements: +If the optional confidence_level is specified, it returns a [Tuple](../../../sql-reference/data-types/tuple.md) with four elements. - calculated t-statistic. [Float64](../../../sql-reference/data-types/float.md). - calculated p-value. [Float64](../../../sql-reference/data-types/float.md). +- [calculated confidence-interval-low.] [Float64](../../../sql-reference/data-types/float.md). +- [calculated confidence-interval-high.] [Float64](../../../sql-reference/data-types/float.md). **Example** From 9359a4abd4ce3a446aa44423deffb566fd0e0f57 Mon Sep 17 00:00:00 2001 From: achimbab <07c00h@gmail.com> Date: Sat, 1 Jan 2022 19:45:55 +0900 Subject: [PATCH 025/149] Validate the parameter --- src/AggregateFunctions/AggregateFunctionTTest.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/AggregateFunctions/AggregateFunctionTTest.h b/src/AggregateFunctions/AggregateFunctionTTest.h index 15a6269a6d4..ccd81b6aba7 100644 --- a/src/AggregateFunctions/AggregateFunctionTTest.h +++ b/src/AggregateFunctions/AggregateFunctionTTest.h @@ -28,6 +28,11 @@ struct Settings; class ReadBuffer; class WriteBuffer; +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + /** * If you have a cumulative distribution function F, then calculating the p-value for given statistic T is simply 1−F(T) * In our case p-value is two-sided, so we multiply it by 2. @@ -90,6 +95,17 @@ public: { need_ci = true; confidence_level = params.at(0).safeGet(); + + if (!std::isfinite(confidence_level)) + { + throw Exception("Aggregate function " + getName() + " requires finite parameter values.", ErrorCodes::BAD_ARGUMENTS); + } + + if (confidence_level <= 0.0f || confidence_level >= 1.0f) + { + throw Exception("Confidence level parameter must be between 0 and 1 in aggregate function " + getName(), ErrorCodes::BAD_ARGUMENTS); + } + } } From eae38c0ac0378edd39c5dadf4d9afabeeec995bf Mon Sep 17 00:00:00 2001 From: achimbab <07c00h@gmail.com> Date: Mon, 3 Jan 2022 10:48:47 +0900 Subject: [PATCH 026/149] Remove virtual dispatch in constructor --- src/AggregateFunctions/AggregateFunctionTTest.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionTTest.h b/src/AggregateFunctions/AggregateFunctionTTest.h index ccd81b6aba7..440fa465a80 100644 --- a/src/AggregateFunctions/AggregateFunctionTTest.h +++ b/src/AggregateFunctions/AggregateFunctionTTest.h @@ -98,12 +98,12 @@ public: if (!std::isfinite(confidence_level)) { - throw Exception("Aggregate function " + getName() + " requires finite parameter values.", ErrorCodes::BAD_ARGUMENTS); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Aggregate function {} requires finite parameter values.", Data::name); } if (confidence_level <= 0.0f || confidence_level >= 1.0f) { - throw Exception("Confidence level parameter must be between 0 and 1 in aggregate function " + getName(), ErrorCodes::BAD_ARGUMENTS); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Confidence level parameter must be between 0 and 1 in aggregate function {}.", Data::name); } } From 57441fdb96d57ad9963d9cd368cf37249ba4cbc1 Mon Sep 17 00:00:00 2001 From: achimbab <07c00h@gmail.com> Date: Thu, 6 Jan 2022 13:32:09 +0900 Subject: [PATCH 027/149] Minor fixes --- src/AggregateFunctions/AggregateFunctionTTest.h | 2 +- src/AggregateFunctions/Moments.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionTTest.h b/src/AggregateFunctions/AggregateFunctionTTest.h index 440fa465a80..18874292f6f 100644 --- a/src/AggregateFunctions/AggregateFunctionTTest.h +++ b/src/AggregateFunctions/AggregateFunctionTTest.h @@ -101,7 +101,7 @@ public: throw Exception(ErrorCodes::BAD_ARGUMENTS, "Aggregate function {} requires finite parameter values.", Data::name); } - if (confidence_level <= 0.0f || confidence_level >= 1.0f) + if (confidence_level <= 0.0 || confidence_level >= 1.0) { throw Exception(ErrorCodes::BAD_ARGUMENTS, "Confidence level parameter must be between 0 and 1 in aggregate function {}.", Data::name); } diff --git a/src/AggregateFunctions/Moments.h b/src/AggregateFunctions/Moments.h index 4fca93527c0..add9df74737 100644 --- a/src/AggregateFunctions/Moments.h +++ b/src/AggregateFunctions/Moments.h @@ -390,7 +390,7 @@ struct TTestMoments Float64 se = getStandardError(); boost::math::students_t dist(degrees_of_freedom); - Float64 t = boost::math::quantile(boost::math::complement(dist, (1.0f - confidence_level) / 2.0f)); + Float64 t = boost::math::quantile(boost::math::complement(dist, (1.0 - confidence_level) / 2.0)); Float64 mean_diff = mean_x - mean_y; Float64 ci_low = mean_diff - t * se; Float64 ci_high = mean_diff + t * se; From 5bad86a2f4f0332d6cdebaf80f4c33ffdb178cbe Mon Sep 17 00:00:00 2001 From: achimbab <07c00h@gmail.com> Date: Sat, 15 Jan 2022 19:33:27 +0900 Subject: [PATCH 028/149] Validation of the number of observations in the t-test --- .../AggregateFunctionStudentTTest.cpp | 5 +++++ .../AggregateFunctionTTest.h | 21 ++++++++++++++++++- .../AggregateFunctionWelchTTest.cpp | 5 +++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp b/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp index ec993f4b54b..83a91ef06fc 100644 --- a/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp @@ -28,6 +28,11 @@ struct StudentTTestData : public TTestMoments { static constexpr auto name = "studentTTest"; + bool hasEnoughObservations() const + { + return nx > 0 && ny > 0 && nx + ny > 2; + } + Float64 getDegreesOfFreedom() const { return nx + ny - 2; diff --git a/src/AggregateFunctions/AggregateFunctionTTest.h b/src/AggregateFunctions/AggregateFunctionTTest.h index 18874292f6f..40319e8c845 100644 --- a/src/AggregateFunctions/AggregateFunctionTTest.h +++ b/src/AggregateFunctions/AggregateFunctionTTest.h @@ -191,12 +191,31 @@ public: void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override { auto & data = this->data(place); + auto & column_tuple = assert_cast(to); + + if (!data.hasEnoughObservations()) + { + auto & column_stat = assert_cast &>(column_tuple.getColumn(0)); + auto & column_value = assert_cast &>(column_tuple.getColumn(1)); + column_stat.getData().push_back(std::numeric_limits::quiet_NaN()); + column_value.getData().push_back(std::numeric_limits::quiet_NaN()); + + if (need_ci) + { + auto & column_ci_low = assert_cast &>(column_tuple.getColumn(2)); + auto & column_ci_high = assert_cast &>(column_tuple.getColumn(3)); + column_ci_low.getData().push_back(std::numeric_limits::quiet_NaN()); + column_ci_high.getData().push_back(std::numeric_limits::quiet_NaN()); + } + + return; + } + auto [t_statistic, p_value] = data.getResult(); /// Because p-value is a probability. p_value = std::min(1.0, std::max(0.0, p_value)); - auto & column_tuple = assert_cast(to); auto & column_stat = assert_cast &>(column_tuple.getColumn(0)); auto & column_value = assert_cast &>(column_tuple.getColumn(1)); diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index 4b43bb73e6b..fe5cf83c509 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -22,6 +22,11 @@ struct WelchTTestData : public TTestMoments { static constexpr auto name = "welchTTest"; + bool hasEnoughObservations() const + { + return nx > 1 && ny > 1; + } + Float64 getDegreesOfFreedom() const { Float64 mean_x = getMeanX(); From da0608f3652abaed675477244d204142d1bd1b2f Mon Sep 17 00:00:00 2001 From: frank chen Date: Tue, 18 Jan 2022 13:24:56 +0800 Subject: [PATCH 029/149] Parse trace-id in big-endian order Signed-off-by: frank chen --- src/Interpreters/OpenTelemetrySpanLog.cpp | 34 ++++++++--------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/src/Interpreters/OpenTelemetrySpanLog.cpp b/src/Interpreters/OpenTelemetrySpanLog.cpp index 89cce890555..a666c6f16b6 100644 --- a/src/Interpreters/OpenTelemetrySpanLog.cpp +++ b/src/Interpreters/OpenTelemetrySpanLog.cpp @@ -148,24 +148,6 @@ OpenTelemetrySpanHolder::~OpenTelemetrySpanHolder() } } - -template -static T readHex(const char * data) -{ - T x{}; - - const char * end = data + sizeof(T) * 2; - while (data < end) - { - x *= 16; - x += unhex(*data); - ++data; - } - - return x; -} - - bool OpenTelemetryTraceContext::parseTraceparentHeader(const std::string & traceparent, std::string & error) { @@ -183,7 +165,7 @@ bool OpenTelemetryTraceContext::parseTraceparentHeader(const std::string & trace const char * data = traceparent.data(); - uint8_t version = readHex(data); + uint8_t version = unhex2(data); data += 2; if (version != 0) @@ -199,10 +181,15 @@ bool OpenTelemetryTraceContext::parseTraceparentHeader(const std::string & trace } ++data; - UInt128 trace_id_128 = readHex(data); - trace_id = trace_id_128; + UInt64 trace_id_higher_64 = unhexUInt(data); + UInt64 trace_id_lower_64 = unhexUInt(data + 16); data += 32; + // store the 128-bit trace id in big-endian order + trace_id.toUnderType().items[0] = trace_id_higher_64; + trace_id.toUnderType().items[1] = trace_id_lower_64; + + if (*data != '-') { error = fmt::format("Malformed traceparant header: {}", traceparent); @@ -210,7 +197,7 @@ bool OpenTelemetryTraceContext::parseTraceparentHeader(const std::string & trace } ++data; - span_id = readHex(data); + span_id = unhexUInt(data); data += 16; if (*data != '-') @@ -218,9 +205,10 @@ bool OpenTelemetryTraceContext::parseTraceparentHeader(const std::string & trace error = fmt::format("Malformed traceparant header: {}", traceparent); return false; } + ++data; - trace_flags = readHex(data); + trace_flags = unhex2(data); return true; } From 962bb9cd9c9aec22ae9fb973b4651043061d6b03 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Thu, 20 Jan 2022 21:23:15 +0800 Subject: [PATCH 030/149] Update test case --- .../0_stateless/01455_opentelemetry_distributed.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh index fbbf74bbba2..95d99449837 100755 --- a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh @@ -20,7 +20,7 @@ select attribute['db.statement'] as query, attribute['clickhouse.tracestate'] as tracestate, 1 as sorted_by_start_time from system.opentelemetry_span_log - where trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) + where trace_id = UUIDNumToString(toFixedString(unhex('$trace_id'), 16)) and operation_name = 'query' order by start_time_us ; @@ -31,7 +31,7 @@ select attribute['db.statement'] as query, attribute['clickhouse.tracestate'] as tracestate, 1 as sorted_by_finish_time from system.opentelemetry_span_log - where trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) + where trace_id = UUIDNumToString(toFixedString(unhex('$trace_id'), 16)) and operation_name = 'query' order by finish_time_us ; @@ -43,7 +43,7 @@ select count(*) "'"'"total spans"'"'", uniqExactIf(parent_span_id, parent_span_id != 0) "'"'"unique non-zero parent spans"'"'" from system.opentelemetry_span_log - where trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) + where trace_id = UUIDNumToString(toFixedString(unhex('$trace_id'), 16)) and operation_name = 'query' ; @@ -56,7 +56,7 @@ select count(*) "'"'"initial query spans with proper parent"'"'" mapValues(attribute) as attribute_value) o join system.query_log on query_id = o.attribute_value where - trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) + trace_id = UUIDNumToString(toFixedString(unhex('$trace_id'), 16)) and current_database = currentDatabase() and operation_name = 'query' and parent_span_id = reinterpretAsUInt64(unhex('73')) @@ -71,7 +71,7 @@ select uniqExact(value) "'"'"unique non-empty tracestate values"'"'" from system.opentelemetry_span_log array join mapKeys(attribute) as name, mapValues(attribute) as value where - trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) + trace_id = UUIDNumToString(toFixedString(unhex('$trace_id'), 16)) and operation_name = 'query' and name = 'clickhouse.tracestate' and length(value) > 0 From c1f9d6cd6f470dc3ca30e92953bf205f50ae5946 Mon Sep 17 00:00:00 2001 From: Frank Chen Date: Thu, 20 Jan 2022 23:12:23 +0800 Subject: [PATCH 031/149] Fix traceparent propagation out of ClickHouse via URL engine --- src/Interpreters/OpenTelemetrySpanLog.cpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/OpenTelemetrySpanLog.cpp b/src/Interpreters/OpenTelemetrySpanLog.cpp index dfe20f19b24..40f31e4976c 100644 --- a/src/Interpreters/OpenTelemetrySpanLog.cpp +++ b/src/Interpreters/OpenTelemetrySpanLog.cpp @@ -218,11 +218,14 @@ std::string OpenTelemetryTraceContext::composeTraceparentHeader() const { // This span is a parent for its children, so we specify this span_id as a // parent id. - return fmt::format("00-{:032x}-{:016x}-{:02x}", __uint128_t(trace_id.toUnderType()), - span_id, - // This cast is needed because fmt is being weird and complaining that - // "mixing character types is not allowed". - static_cast(trace_flags)); + return fmt::format("00-{:016x}{:016x}-{:016x}-{:02x}", + // Output the trace id in network byte order + trace_id.toUnderType().items[0], + trace_id.toUnderType().items[1], + span_id, + // This cast is needed because fmt is being weird and complaining that + // "mixing character types is not allowed". + static_cast(trace_flags)); } From 42eb6b9d38001bd0aac66b43e6c1f332a9f8042d Mon Sep 17 00:00:00 2001 From: Tiaonmmn Date: Sat, 22 Jan 2022 20:53:20 +0800 Subject: [PATCH 032/149] Update ansi.md --- docs/zh/sql-reference/ansi.md | 343 +++++++++++++++++----------------- 1 file changed, 176 insertions(+), 167 deletions(-) diff --git a/docs/zh/sql-reference/ansi.md b/docs/zh/sql-reference/ansi.md index 0e7fa1d06c3..5aad2cf52a8 100644 --- a/docs/zh/sql-reference/ansi.md +++ b/docs/zh/sql-reference/ansi.md @@ -1,180 +1,189 @@ --- -machine_translated: true -machine_translated_rev: ad252bbb4f7e2899c448eb42ecc39ff195c8faa1 toc_priority: 40 toc_title: "ANSI\u517C\u5BB9\u6027" --- -# Ansi Sql兼容性的ClickHouse SQL方言 {#ansi-sql-compatibility-of-clickhouse-sql-dialect} +# ClickHouse SQL方言 与ANSI SQL的兼容性{#ansi-sql-compatibility-of-clickhouse-sql-dialect} !!! note "注" - 本文依赖于表38, “Feature taxonomy and definition for mandatory features”, Annex F of ISO/IEC CD 9075-2:2013. + 本文参考Annex G所著的[ISO/IEC CD 9075-2:2011](https://www.iso.org/obp/ui/#iso:std:iso-iec:9075:-2:ed-4:v1:en:sec:8)标准. ## 行为差异 {#differences-in-behaviour} -下表列出了查询功能在ClickHouse中有效但不符合ANSI SQL标准的情况。 +下表列出了ClickHouse能够使用，但与ANSI SQL规定有差异的查询特性。 -| Feature ID | 功能名称 | 差异 | -|------------|--------------------|---------------------------------------------------------------------| -| E011 | 数值（Numeric）数据类型 | 带小数点的数值文字被解释为近似值 (`Float64`）而不是精确值 (`Decimal`) | -| E051-05 | SELECT字段可以重命名 | 字段不仅仅在SELECT结果中可被重命名 | -| E141-01 | 非空约束 | 表中每一列默认为`NOT NULL` | -| E011-04 | 算术运算符 | ClickHouse不会检查算法，并根据自定义规则更改结果数据类型，而是会溢出 | +| 功能ID | 功能名称 | 差异 | +| ------- | --------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| E011 | 数值型数据类型 | 带小数点的数字被视为近似值 (`Float64`）而不是精确值 (`Decimal`) | +| E051-05 | SELECT 的列可以重命名 | 字段重命名的作用范围不限于进行重命名的SELECT子查询（参考[表达式别名](https://clickhouse.com/docs/zh/sql-reference/syntax/#notes-on-usage)） | +| E141-01 | NOT NULL（非空）约束 | ClickHouse表中每一列默认为`NOT NULL` | +| E011-04 | 算术运算符 | ClickHouse在运算时会进行溢出，而不是四舍五入。此外会根据自定义规则修改结果数据类型（参考[溢出检查](https://clickhouse.com/docs/zh/sql-reference/data-types/decimal/#yi-chu-jian-cha)） | -## 功能匹配 {#feature-status} +## 功能状态 {#feature-status} -| Feature ID | 功能名称 | 匹配 | 评论 | -|------------|----------------------------------------------------------------|--------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **E011** | **数字数据类型** | **部分**{.text-warning} | | -| E011-01 | 整型和小型数据类型 | 是 {.text-success} | | -| E011-02 | 真实、双精度和浮点数据类型数据类型 | 部分 {.text-warning} | `FLOAT()`, `REAL` 和 `DOUBLE PRECISION` 不支持 | -| E011-03 | 十进制和数值数据类型 | 部分 {.text-warning} | 只有 `DECIMAL(p,s)` 支持，而不是 `NUMERIC` | -| E011-04 | 算术运算符 | 是 {.text-success} | | -| E011-05 | 数字比较 | 是 {.text-success} | | -| E011-06 | 数字数据类型之间的隐式转换 | 否。 {.text-danger} | ANSI SQL允许在数值类型之间进行任意隐式转换，而ClickHouse依赖于具有多个重载的函数而不是隐式转换 | -| **E021** | **字符串类型** | **部分**{.text-warning} | | -| E021-01 | 字符数据类型 | 否。 {.text-danger} | | -| E021-02 | 字符变化数据类型 | 否。 {.text-danger} | `String` 行为类似，但括号中没有长度限制 | -| E021-03 | 字符文字 | 部分 {.text-warning} | 不自动连接连续文字和字符集支持 | -| E021-04 | 字符长度函数 | 部分 {.text-warning} | 非也。 `USING` 条款 | -| E021-05 | OCTET_LENGTH函数 | 非也。 {.text-danger} | `LENGTH` 表现类似 | -| E021-06 | SUBSTRING | 部分 {.text-warning} | 不支持 `SIMILAR` 和 `ESCAPE` 条款，否 `SUBSTRING_REGEX` 备选案文 | -| E021-07 | 字符串联 | 部分 {.text-warning} | 非也。 `COLLATE` 条款 | -| E021-08 | 上下功能 | 是 {.text-success} | | -| E021-09 | 修剪功能 | 是 {.text-success} | | -| E021-10 | 固定长度和可变长度字符串类型之间的隐式转换 | 否。 {.text-danger} | ANSI SQL允许在字符串类型之间进行任意隐式转换，而ClickHouse依赖于具有多个重载的函数而不是隐式转换 | -| E021-11 | 职位功能 | 部分 {.text-warning} | 不支持 `IN` 和 `USING` 条款，否 `POSITION_REGEX` 备选案文 | -| E021-12 | 字符比较 | 是 {.text-success} | | -| **E031** | **标识符** | **部分**{.text-warning} | | -| E031-01 | 分隔标识符 | 部分 {.text-warning} | Unicode文字支持有限 | -| E031-02 | 小写标识符 | 是 {.text-success} | | -| E031-03 | 尾部下划线 | 是 {.text-success} | | -| **E051** | **基本查询规范** | **部分**{.text-warning} | | -| E051-01 | SELECT DISTINCT | 是 {.text-success} | | -| E051-02 | GROUP BY子句 | 是 {.text-success} | | -| E051-04 | 分组依据可以包含不在列 ``中出现的列 | 是 {.text-success} | | +| E051-05 | SELECT 的列可以重命名 | 是 {.text-success} | | +| E051-06 | HAVING 从句 | 是 {.text-success} | | +| E051-07 | SELECT 选择的列中允许出现\* | 是 {.text-success} | | +| E051-08 | FROM 从句中的关联名称 | 是 {.text-success} | | +| E051-09 | 重命名 FROM 从句中的列 | 否 {.text-danger} | | +| **E061** | **基本谓词和搜索条件** | **部分**{.text-warning} | | +| E061-01 | 比较谓词 | 是 {.text-success} | | +| E061-02 | BETWEEN 谓词 | 部分 {.text-warning} | 不支持 `SYMMETRIC` 和 `ASYMMETRIC` 从句 | +| E061-03 | IN 谓词后可接值列表 | 是 {.text-success} | | +| E061-04 | LIKE 谓词 | 是 {.text-success} | | +| E061-05 | LIKE 谓词后接 ESCAPE 从句 | 否 {.text-danger} | | +| E061-06 | NULL 谓词 | 是 {.text-success} | | +| E061-07 | 量化比较谓词（ALL、SOME、ANY） | 否 {.text-danger} | | +| E061-08 | EXISTS 谓词 | 否 {.text-danger} | | +| E061-09 | 比较谓词中的子查询 | 是 {.text-success} | | +| E061-11 | IN 谓词中的子查询 | 是 {.text-success} | | +| E061-12 | 量化比较谓词（BETWEEN、IN、LIKE）中的子查询 | 否 {.text-danger} | | +| E061-13 | 相关子查询 | 否 {.text-danger} | | +| E061-14 | 搜索条件 | 是 {.text-success} | | +| **E071** | **基本查询表达式** | **部分**{.text-warning} | | +| E071-01 | UNION DISTINCT 表运算符 | 是 {.text-success} | | +| E071-02 | UNION ALL 表运算符 | 是 {.text-success} | | +| E071-03 | EXCEPT DISTINCT 表运算符 | 否 {.text-danger} | | +| E071-05 | 通过表运算符组合的列不必具有完全相同的数据类型 | 是 {.text-success} | | +| E071-06 | 子查询中的表运算符 | 是 {.text-success} | | +| **E081** | **基本权限** | **是**{.text-success} | | +| E081-01 | 表级别的SELECT（查询）权限 | 是 {.text-success} | | +| E081-02 | DELETE（删除）权限 | 是 {.text-success} | | +| E081-03 | 表级别的INSERT（插入）权限 | 是 {.text-success} | | +| E081-04 | 表级别的UPDATE（更新）权限 | 是 {.text-success} | | +| E081-05 | 列级别的UPDATE（更新）权限 | 是 {.text-success} | | +| E081-06 | 表级别的REFERENCES（引用）权限 | 是 {.text-success} | | +| E081-07 | 列级别的REFERENCES（引用）权限 | 是 {.text-success} | | +| E081-08 | WITH GRANT OPTION | 是 {.text-success} | | +| E081-09 | USAGE（使用）权限 | 是 {.text-success} | | +| E081-10 | EXECUTE（执行）权限 | 是 {.text-success} | | +| **E091** | **集合函数** | **是**{.text-success} | | +| E091-01 | AVG | 是 {.text-success} | | +| E091-02 | COUNT | 是 {.text-success} | | +| E091-03 | MAX | 是 {.text-success} | | +| E091-04 | MIN | 是 {.text-success} | | +| E091-05 | SUM | 是 {.text-success} | | +| E091-06 | ALL修饰词 | 否。 {.text-danger} | | +| E091-07 | DISTINCT修饰词 | 是 {.text-success} | 并非所有聚合函数都支持该修饰词 | +| **E101** | **基本数据操作** | **部分**{.text-warning} | | +| E101-01 | INSERT（插入）语句 | 是 {.text-success} | 注：ClickHouse中的主键并不隐含`UNIQUE` 约束 | +| E101-03 | 可指定范围的UPDATE（更新）语句 | 部分 {.text-warning} | `ALTER UPDATE` 语句用来批量更新数据 | +| E101-04 | 可指定范围的DELETE（删除）语句 | 部分 {.text-warning} | `ALTER DELETE` 语句用来批量删除数据 | +| **E111** | **返回一行的SELECT语句** | **否**{.text-danger} | | +| **E121** | **基本游标支持** | **否**{.text-danger} | | +| E121-01 | DECLARE CURSOR | 否 {.text-danger} | | +| E121-02 | ORDER BY 涉及的列不需要出现在SELECT的列中 | 是 {.text-success} | | +| E121-03 | ORDER BY 从句中的表达式 | 是 {.text-success} | | +| E121-04 | OPEN 语句 | 否 {.text-danger} | | +| E121-06 | 受游标位置控制的 UPDATE 语句 | 否 {.text-danger} | | +| E121-07 | 受游标位置控制的 DELETE 语句 | 否 {.text-danger} | | +| E121-08 | CLOSE 语句 | 否 {.text-danger} | | +| E121-10 | FETCH 语句中包含隐式NEXT | 否 {.text-danger} | | +| E121-17 | WITH HOLD 游标 | 否 {.text-danger} | | +| **E131** | **空值支持** | **是**{.text-success} | 有部分限制 | +| **E141** | **基本完整性约束** | **部分**{.text-warning} | | +| E141-01 | NOT NULL（非空）约束 | 是 {.text-success} | 注: 默认情况下ClickHouse表中的列隐含`NOT NULL`约束 | +| E141-02 | NOT NULL（非空）列的UNIQUE（唯一）约束 | 否 {.text-danger} | | +| E141-03 | 主键约束 | 部分 {.text-warning} | | +| E141-04 | 对于引用删除和引用更新操作，基本的FOREIGN KEY（外键）约束默认不进行任何操作（NO ACTION） | 否 {.text-danger} | | +| E141-06 | CHECK（检查）约束 | 是 {.text-success} | | +| E141-07 | 列默认值 | 是 {.text-success} | | +| E141-08 | 在主键上推断非空 | 是 {.text-success} | | +| E141-10 | 可以按任何顺序指定外键中的名称 | 否 {.text-danger} | | +| **E151** | **事务支持** | **否**{.text-danger} | | +| E151-01 | COMMIT（提交）语句 | 否 {.text-danger} | | +| E151-02 | ROLLBACK（回滚）语句 | 否 {.text-danger} | | +| **E152** | **基本的SET TRANSACTION（设置事务隔离级别）语句** | **否**{.text-danger} | | +| E152-01 | SET TRANSACTION语句：ISOLATION LEVEL SERIALIZABLE（隔离级别为串行化）从句 | 否 {.text-danger} | | +| E152-02 | SET TRANSACTION语句：READ ONLY（只读）和READ WRITE（读写）从句 | 否 {.text-danger} | | +| **E153** | **具有子查询的可更新查询** | **是**{.text-success} | | +| **E161** | **使用“--”符号作为SQL注释** | **是**{.text-success} | | +| **E171** | **SQLSTATE支持** | **否**{.text-danger} | | +| **E182** | **主机语言绑定** | **否**{.text-danger} | | +| **F031** | **基本架构操作** | **部分**{.text-warning} | | +| F031-01 | 使用 CREATE TABLE 语句创建持久表 | 部分 {.text-warning} | 不支持 `SYSTEM VERSIONING`, `ON COMMIT`, `GLOBAL`, `LOCAL`, `PRESERVE`, `DELETE`, `REF IS`, `WITH OPTIONS`, `UNDER`, `LIKE`, `PERIOD FOR` 从句，不支持用户解析的数据类型 | +| F031-02 | CREATE VIEW（创建视图）语句 | 部分 {.text-warning} | 不支持 `RECURSIVE`, `CHECK`, `UNDER`, `WITH OPTIONS` 从句，不支持用户解析的数据类型 | +| F031-03 | GRANT（授权）语句 | 是 {.text-success} | | +| F031-04 | ALTER TABLE语句：ADD COLUMN从句 | 是 {.text-success} | 不支持 `GENERATED` 从句和以系统时间做参数 | +| F031-13 | DROP TABLE语句：RESTRICT从句 | 否 {.text-danger} | | +| F031-16 | DROP VIEW语句：RESTRICT子句 | 否 {.text-danger} | | +| F031-19 | REVOKE语句：RESTRICT子句 | 否 {.text-danger} | | +| **F041** | **基本连接关系** | **部分**{.text-warning} | | +| F041-01 | Inner join（但不一定是INNER关键字) | 是 {.text-success} | | +| F041-02 | INNER 关键字 | 是 {.text-success} | | +| F041-03 | LEFT OUTER JOIN | 是 {.text-success} | | +| F041-04 | RIGHT OUTER JOIN | 是 {.text-success} | | +| F041-05 | 外连接可嵌套 | 是 {.text-success} | | +| F041-07 | 左外部连接或右外连接中的内部表也可用于内部联接 | 是 {.text-success} | | +| F041-08 | 支持所有比较运算符（而不仅仅是=) | 否 {.text-danger} | | +| **F051** | **基本日期和时间** | **部分**{.text-warning} | | +| F051-01 | DATE（日期）数据类型（并支持用于表达日期的字面量) | 是 {.text-success} | | +| F051-02 | TIME（时间）数据类型（并支持用于表达时间的字面量），小数秒精度至少为0 | 否 {.text-danger} | | +| F051-03 | 时间戳数据类型（并支持用于表达时间戳的字面量），小数秒精度至少为0和6 | 是 {.text-danger} | | +| F051-04 | 日期、时间和时间戳数据类型的比较谓词 | 是 {.text-success} | | +| F051-05 | Datetime 类型和字符串形式表达的时间之间的显式转换 | 是 {.text-success} | | +| F051-06 | CURRENT_DATE | 否 {.text-danger} | 使用`today()`替代 | +| F051-07 | LOCALTIME | 否 {.text-danger} | 使用`now()`替代 | +| F051-08 | LOCALTIMESTAMP | 否 {.text-danger} | | +| **F081** | **视图的UNION和EXCEPT操作** | **部分**{.text-warning} | | +| **F131** | **分组操作** | **部分**{.text-warning} | | +| F131-01 | 在具有分组视图的查询中支持 WHERE、GROUP BY 和 HAVING 子句 | 是 {.text-success} | | +| F131-02 | 在分组视图中支持多张表 | 是 {.text-success} | | +| F131-03 | 分组视图的查询中支持集合函数 | 是 {.text-success} | | +| F131-04 | 带有 `GROUP BY` 和 `HAVING` 从句，以及分组视图的子查询 | 是 {.text-success} | | +| F131-05 | 带有 `GROUP BY` 和 `HAVING` 从句，以及分组视图的仅返回1条记录的SELECT查询 | 否 {.text-danger} | | +| **F181** | **多模块支持** | **否**{.text-danger} | | +| **F201** | **CAST 函数** | **是**{.text-success} | | +| **F221** | **显式默认值** | **否**{.text-danger} | | +| **F261** | **CASE 表达式** | **是**{.text-success} | | +| F261-01 | 简单 CASE 表达式 | 是 {.text-success} | | +| F261-02 | 搜索型 CASE 表达式 | 是 {.text-success} | | +| F261-03 | NULLIF | 是 {.text-success} | | +| F261-04 | COALESCE | 是 {.text-success} | | +| **F311** | **架构定义语句** | **部分**{.text-warning} | | +| F311-01 | CREATE SCHEMA | 部分 {.text-warning} | 见`CREATE DATABASE` | +| F311-02 | 用于创建持久表的 CREATE TABLE | 是 {.text-success} | | +| F311-03 | CREATE VIEW | 是 {.text-success} | | +| F311-04 | CREATE VIEW: WITH CHECK OPTION | 否 {.text-danger} | | +| F311-05 | GRANT 语句 | 是 {.text-success} | | +| **F471** | **标量子查询** | **是**{.text-success} | | +| **F481** | **扩展 NULL 谓词** | **是**{.text-success} | | +| **F812** | **基本标志位** | **否**{.text-danger} | +| **S011** | **用于不重复数据的数据类型** | **否**{.text-danger} | +| **T321** | **基本的SQL调用例程** | **否**{.text-danger} | | +| T321-01 | 没有重载的用户定义函数 | 否{.text-danger} | | +| T321-02 | 没有重载的用户定义存储过程 | 否{.text-danger} | | +| T321-03 | 功能调用 | 否 {.text-danger} | | +| T321-04 | CALL 语句 | 否 {.text-danger} | | +| T321-05 | RETURN 语句 | 否 {.text-danger} | | +| **T631** | **IN 谓词后接一个列表** | **是**{.text-success} | | From 3a55f7a7d67b8593dd65955d6c0778c95e648acd Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Mon, 24 Jan 2022 11:53:42 +0000 Subject: [PATCH 033/149] Do not use clickhouse-test.deb in stateless --- docker/test/stateless/run.sh | 6 +++++- tests/ci/functional_test_check.py | 7 +++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index d6d9f189e89..bd0fc494826 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -12,7 +12,11 @@ dpkg -i package_folder/clickhouse-common-static_*.deb dpkg -i package_folder/clickhouse-common-static-dbg_*.deb dpkg -i package_folder/clickhouse-server_*.deb dpkg -i package_folder/clickhouse-client_*.deb -dpkg -i package_folder/clickhouse-test_*.deb +if [[ -n "$TEST_CASES_FROM_DEB" ]] && [[ "$TEST_CASES_FROM_DEB" -eq 1 ]]; then + dpkg -i package_folder/clickhouse-test_*.deb +else + ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test +fi # install test configs /usr/share/clickhouse-test/config/install.sh diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 7220b86a482..3a38ef38abd 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -44,7 +44,7 @@ def get_image_name(check_name): else: raise Exception(f"Cannot deduce image name based on check name {check_name}") -def get_run_command(builds_path, result_path, server_log_path, kill_timeout, additional_envs, image, flaky_check, tests_to_run): +def get_run_command(builds_path, repo_tests_path, result_path, server_log_path, kill_timeout, additional_envs, image, flaky_check, tests_to_run): additional_options = ['--hung-check'] additional_options.append('--print-time') @@ -63,6 +63,7 @@ def get_run_command(builds_path, result_path, server_log_path, kill_timeout, add env_str = ' '.join(envs) return f"docker run --volume={builds_path}:/package_folder " \ + f"--volume={repo_tests_path}:/usr/share/clickhouse-test" \ f"--volume={result_path}:/test_output --volume={server_log_path}:/var/log/clickhouse-server " \ f"--cap-add=SYS_PTRACE {env_str} {additional_options_str} {image}" @@ -167,6 +168,8 @@ if __name__ == "__main__": image_name = get_image_name(check_name) docker_image = get_image_with_version(reports_path, image_name) + repo_tests_path = os.path.join(repo_path, "tests") + packages_path = os.path.join(temp_path, "packages") if not os.path.exists(packages_path): os.makedirs(packages_path) @@ -184,7 +187,7 @@ if __name__ == "__main__": run_log_path = os.path.join(result_path, "runlog.log") additional_envs = get_additional_envs(check_name, run_by_hash_num, run_by_hash_total) - run_command = get_run_command(packages_path, result_path, server_log_path, kill_timeout, additional_envs, docker_image, flaky_check, tests_to_run) + run_command = get_run_command(packages_path, repo_tests_path, result_path, server_log_path, kill_timeout, additional_envs, docker_image, flaky_check, tests_to_run) logging.info("Going to run func tests: %s", run_command) with TeePopen(run_command, run_log_path) as process: From bc10e08fb4dbeb7b8ed0ef8dbc87d800a4c48dd1 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 25 Jan 2022 09:06:11 +0000 Subject: [PATCH 034/149] add missing space --- tests/ci/functional_test_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 3a38ef38abd..7328db26926 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -63,7 +63,7 @@ def get_run_command(builds_path, repo_tests_path, result_path, server_log_path, env_str = ' '.join(envs) return f"docker run --volume={builds_path}:/package_folder " \ - f"--volume={repo_tests_path}:/usr/share/clickhouse-test" \ + f"--volume={repo_tests_path}:/usr/share/clickhouse-test " \ f"--volume={result_path}:/test_output --volume={server_log_path}:/var/log/clickhouse-server " \ f"--cap-add=SYS_PTRACE {env_str} {additional_options_str} {image}" From cb4ad97d4556c11bf68f320e92a876dac0476b8c Mon Sep 17 00:00:00 2001 From: FArthur-cmd <613623@mail.ru> Date: Fri, 28 Jan 2022 17:37:52 +0000 Subject: [PATCH 035/149] add test --- programs/server/Server.cpp | 2 +- programs/server/config.xml | 7 +- src/Server/CertificateReloader.cpp | 2 +- .../test_reload_certificate/__init__.py | 0 .../test_reload_certificate/configs/cert.xml | 14 ++++ .../test_reload_certificate/configs/first.crt | 19 +++++ .../test_reload_certificate/configs/first.key | 28 +++++++ .../configs/second.crt | 19 +++++ .../configs/second.key | 28 +++++++ .../test_reload_certificate/test.py | 75 +++++++++++++++++++ 10 files changed, 189 insertions(+), 5 deletions(-) create mode 100644 tests/integration/test_reload_certificate/__init__.py create mode 100644 tests/integration/test_reload_certificate/configs/cert.xml create mode 100644 tests/integration/test_reload_certificate/configs/first.crt create mode 100644 tests/integration/test_reload_certificate/configs/first.key create mode 100644 tests/integration/test_reload_certificate/configs/second.crt create mode 100644 tests/integration/test_reload_certificate/configs/second.key create mode 100644 tests/integration/test_reload_certificate/test.py diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 8e9594a9a10..9eaad1f41d6 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1355,7 +1355,7 @@ if (ThreadFuzzer::instance().isEffective()) ErrorCodes::NO_ELEMENTS_IN_CONFIG); } - if (servers->empty()) + if (servers.empty()) throw Exception("No servers started (add valid listen_host and 'tcp_port' or 'http_port' to configuration file.)", ErrorCodes::NO_ELEMENTS_IN_CONFIG); diff --git a/programs/server/config.xml b/programs/server/config.xml index d88773a3fc4..713fe9f4f67 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -237,14 +237,15 @@ - /etc/clickhouse-server/server.crt - /etc/clickhouse-server/server.key + + + - /etc/clickhouse-server/dhparam.pem + none true true diff --git a/src/Server/CertificateReloader.cpp b/src/Server/CertificateReloader.cpp index 4889e794d47..64947d45bc4 100644 --- a/src/Server/CertificateReloader.cpp +++ b/src/Server/CertificateReloader.cpp @@ -90,7 +90,7 @@ void CertificateReloader::tryLoad(const Poco::Util::AbstractConfiguration & conf } /// If callback is not set yet - + if (first_load) init(); } diff --git a/tests/integration/test_reload_certificate/__init__.py b/tests/integration/test_reload_certificate/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_reload_certificate/configs/cert.xml b/tests/integration/test_reload_certificate/configs/cert.xml new file mode 100644 index 00000000000..91fe8156d1e --- /dev/null +++ b/tests/integration/test_reload_certificate/configs/cert.xml @@ -0,0 +1,14 @@ + + + 8443 + + + /etc/clickhouse-server/config.d/first.crt + /etc/clickhouse-server/config.d/first.key + true + true + sslv2,sslv3 + true + + + diff --git a/tests/integration/test_reload_certificate/configs/first.crt b/tests/integration/test_reload_certificate/configs/first.crt new file mode 100644 index 00000000000..b0c4c3e5adf --- /dev/null +++ b/tests/integration/test_reload_certificate/configs/first.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDCzCCAfOgAwIBAgIUcA+y3LQyfpxlBzL7IQVKUfnhRncwDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTIyMDEyODExNTA1NloYDzIzMTIw +NDE4MTE1MDU2WjAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQCumguFo+M0AQ6SgyL8K2Kep3YFYaJUjU3Mm7rrYXP7 +aGnm0Cvh3dqituSHF1ZoLgThqdBR4e/e5SZvS7ShCnFCBZpWSuhodp7qZy4ETqa8 +1/TJUr2hQLH+GpldAeGxPuDJwsdEEk100l4UHWQYg0+kqAmkijWlXDrxJYzeZ5Q5 +r/qxJN1kGxFnGYtlFjM3IpunXsREjcxjJmE4pDHp+Bkvkp0znajPJo8AE4pZ0zEQ +K/LfQSHh5BWSLw3SwGzHTTsHkn7KduaIppbYCG1j8/VEGKJIZMWUP4UbBfZ5Pl1+ +tm7sPOKho+pu35pA/7keYEP1XxGSbHy4e8xO2DkIDSBRAgMBAAGjUzBRMB0GA1Ud +DgQWBBQQ1RwP+LO8L9WmeW9xijBbG93jTjAfBgNVHSMEGDAWgBQQ1RwP+LO8L9Wm +eW9xijBbG93jTjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAV +hqUMF75V8dEqCF61GsHizcL0R+KN36AjFckzZfMCk/H5UFIQ2L6vCsm8tmi6DlZQ +4VXlq+gCuAXuKWDe5yGNBxveKgdIltCJ85ZcySU1KON0OJIICW0lmOsFflqjD6cZ +tnP0FvFgoRRHW+lEQcgklXDF4taV9cF40xhNQ+TnzXSEmeg9fJQeC2rKLdwbpSAe +xfFPUvaKTF6w4caUqBojvTk0eKahva+kEtoeJ6KKPoBxkJsUzdL63V7P6FlktWut +0O3H4oFTZ6fNYKBKRwYeSZY30o8N6Lib0LihScLF7HWQNs0eyQ6m6v93KcJyK42F +wweXxFoWbdsITFYOdBmD +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_certificate/configs/first.key b/tests/integration/test_reload_certificate/configs/first.key new file mode 100644 index 00000000000..a66bef9ad98 --- /dev/null +++ b/tests/integration/test_reload_certificate/configs/first.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCumguFo+M0AQ6S +gyL8K2Kep3YFYaJUjU3Mm7rrYXP7aGnm0Cvh3dqituSHF1ZoLgThqdBR4e/e5SZv +S7ShCnFCBZpWSuhodp7qZy4ETqa81/TJUr2hQLH+GpldAeGxPuDJwsdEEk100l4U +HWQYg0+kqAmkijWlXDrxJYzeZ5Q5r/qxJN1kGxFnGYtlFjM3IpunXsREjcxjJmE4 +pDHp+Bkvkp0znajPJo8AE4pZ0zEQK/LfQSHh5BWSLw3SwGzHTTsHkn7KduaIppbY +CG1j8/VEGKJIZMWUP4UbBfZ5Pl1+tm7sPOKho+pu35pA/7keYEP1XxGSbHy4e8xO +2DkIDSBRAgMBAAECggEAZSiW2Fy1fCHIoZYcpOE2CBmZxVBlznr3wj3PtCQIIHbE +NJgTdI8m5vLzwFkDFOTkqyHJskcmxIsbE4xXIJ5+M/QvESPhNvTS6ZfSD2jKLcso +5aNsfoqPFVuv0zUN37VAY2TYMlYwTii7nQfSQGmDsTAyNgRlRGMFO0W4Mfrs4+Zd +ysoxdb+562DfKnqzTaqWIGXB7kW4bdUmQwK5dCmuj4m5yh0TknPM2w+jtI/O5mA9 +pTG8p/te8b8qkrVaPyrApVNuBEonIOBfesFjnvjqIMquCutysii/hMkP6LbkXE+0 +bpcVV8Rs1W0I1zU6veAh3ValDYpiWRGX/9IALfIMAQKBgQDjEqK0Qzh8kMhO2g2E ++zh32ZsMmt7/wmhYhfmunhYKgjFyLVmAsDVHDoC/wtimItljSdKJdPenBKloTzxh +L/EuP5Fqt6BIlkrwdiXurTXEVWrntNenzpUBxGeXSmJ4B4BFJhNpQj1ewASuKjrM +CrIwwhIJRq0MjsG8aOwWHOYNMQKBgQDE2DU+jjHN12rvF+68+8HE6Gx/op6POSiW +Jb+IJRFGSrntLEXRQgvj6Tybm+dbrTy+OtXAZRo7W9hUjf5eYav1lnLj/YguvAhy +/9x97edZ9CJjvW/fEpkRBXdNkvKfqaR8qQaQSlAJRu6syreXJbPgAQOwQWVIXGa8 +N+TGIhz9IQKBgQCfLvY+to0HzhuOI5Css8yPQE5QlNVVqHyr6ifyAMLk1QZCy4Xe +ECkZShJ52+cy+GU7FIpycDwYqszz4fArFYfW6xtPG7FSkYGxdrH60xRJMbRDAOTZ +r5mH5p7UUYIcMO38C8g51wTcwnHFgrc7SRhH1BT+ybwQfJdWNJukmNexUQKBgQCI +eRHpLfKvsMNtwtz9X1qHZ1EZ6KgfylQuTTuOa4yffF2NZt186FqQB/vCMwPjVqc/ +iFD8E9xs/Q9uCAgsbXEoUseS9Ar/w9PjzyqSkGeOwSk6l3NBaIaA+5YsTU4zjg0B +dLqdPThiRjBh0iYY/8XG700cXSqYUZ/UrLfK+om4oQKBgBWSH0guEZyx37kzrN9z +bIsjSgkYxuoD18fpifAyVDmJGhi3JK4fSNz9gvsCEpPTVYMh2++HB2Pm45oNeO/m +XxqJQS+/tOcFxLC3Goy274jH6LK+3AXE++W2jV/G9Rkgv3et7KiKXbwWnYW3qIA0 +Sgm0PSmXIcMhsb03LCqfsQa1 +-----END PRIVATE KEY----- diff --git a/tests/integration/test_reload_certificate/configs/second.crt b/tests/integration/test_reload_certificate/configs/second.crt new file mode 100644 index 00000000000..165ddc1959d --- /dev/null +++ b/tests/integration/test_reload_certificate/configs/second.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDCzCCAfOgAwIBAgIUc5pIrYIv905gvuYtD/Y4LPDeXKswDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTIyMDEyODExNTE0NVoYDzIzMTIw +NDE4MTE1MTQ1WjAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDrQQCvvoSpNjl1HvPgI9ST+UbjZGEYHLVPlLTVh/Q/ +RIQg/zxVOWmuHOnFXRAyb2b7B8uuNBC8wzHaubLkP6wk8nBbhc5kK4ohhiFA3DD8 +6DGXpxfZhlZ/x/mnQnb8T+PFSPXNfJxTer1RttBBzHSiRcG0cTkCPH0oIYBRbNAO +Ig7/76EGHhNNBLDgU7CaMpvOeefMVJ1qd5SYRDgLRvZa4Y3KWtA9WrTQmDTH6YzH ++bLnVBfUV5rs9nyM4B8pGNrezb/deFlzB9c8+FhFxxm8UjeOZZhzyro+7eToVpDf +btuYcmjgju7O3142/s2P29RTogCteA8PP4KHc6oekztDAgMBAAGjUzBRMB0GA1Ud +DgQWBBQ5/fotxXkvhLkBUMfzjurEbwukOzAfBgNVHSMEGDAWgBQ5/fotxXkvhLkB +UMfzjurEbwukOzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQB+ +PWMgxHoN0xqDdLq/3cqGoO+4iG4CZGUIG7VspnojjGrHKTA/0jvC2lu62q/Dxwsv +EXL/0Gt3ycfP0pbtVIcPjB2j3u+c9iJJ2R47Q/6FRwSiN7LpUohJvCMnTDpK19wj +p5TNZL5DCFzrOqINewOZBaAn4TpkdSfmZUA65KZe8qrOmw7YbVxX8HOP0RPPtA96 +zbUtc2VjHT0lNMZmiTuuLYtKxV+dyyAqX48KxthNCCs0zP414nUXxymdE0MCSUss +565FXf+FrMA+owe1SlacX/IjCkgN2oauRMDXN+JDLXJUwDKVKOb3yObXyIJ0/b6E ++cGmwo/7m6CE5hoCSncE +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_certificate/configs/second.key b/tests/integration/test_reload_certificate/configs/second.key new file mode 100644 index 00000000000..007aba24fb6 --- /dev/null +++ b/tests/integration/test_reload_certificate/configs/second.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDrQQCvvoSpNjl1 +HvPgI9ST+UbjZGEYHLVPlLTVh/Q/RIQg/zxVOWmuHOnFXRAyb2b7B8uuNBC8wzHa +ubLkP6wk8nBbhc5kK4ohhiFA3DD86DGXpxfZhlZ/x/mnQnb8T+PFSPXNfJxTer1R +ttBBzHSiRcG0cTkCPH0oIYBRbNAOIg7/76EGHhNNBLDgU7CaMpvOeefMVJ1qd5SY +RDgLRvZa4Y3KWtA9WrTQmDTH6YzH+bLnVBfUV5rs9nyM4B8pGNrezb/deFlzB9c8 ++FhFxxm8UjeOZZhzyro+7eToVpDfbtuYcmjgju7O3142/s2P29RTogCteA8PP4KH +c6oekztDAgMBAAECggEANQeZFQSYOOB9QTZx+ON6xsRZQ2bcMChAgqjdvoh/+UcD +lcCTJA7mEJZ558Bbp1LPXuTZ9/HKmBJUCZ70gVkM/+Mairb12ESsRXRLyKgZ7tiU +XUAQMzuCAhnc3+QumB+WE2Gn7uMZBgRT6riP51UkMXQR/w/KrwNdnw82MqSZnaWH +G2U+VWzrwloGDn4wdTm0egPMUCoF9kTW04HCTRZBg2a1DGCIOd2QcnYrsLiRQHmK +J8hfVVHDk+aaDmkYhJqyruTan51ifWziMOAcxTxuhDDtGOGXtMMlJOF9sskVT8CF +ToCSMJ8Rs+TPHqdBTy9l9n6kSki9Z/JFylMBClQsAQKBgQD6/FF5EF7WaVjs6gJt +DcOCPBAo8Se1YxFQ+lHIflswJs9O8u/wOej06o3c2S0zqj8v6aomhw4hYds2RaSa +fJ1jOS/7l+b3A4QndmZ6ZPBXc4mdWeUwcJ1snzi8of7D+BInAoXa9mwmOYp4u8DB +x+udumDr9zorR3aI6LXARF8+AQKBgQDv9DlBKXO7iqf2n3peb5QaLuxLvaiQM2+I +kA5xTakrODWE8+nZ0fTjrq5gVsYoIGaci6FlvX1yKsWiwy79ng+XBPNcz14FEhrn +xQFDWa/t2xEMruar3cQ0eqotlwe4yanM+/5SYu5kgm8qs9CUdr14vrV3MYB0moKb +YAg2qmgBQwKBgQDavGPU+qtsecuCTj9nA4PMUMRUqjdNIdXJmR8FePnH8UrjJ15t +Iksgh/qy6qM2T71Z6G7dvP5XoY0Gs5NNACW6f/CNeElWJb5bFhkhui6sSIk6lUnk ++YB5VhqAaz45VE2dqdk2h2Shu6wupJLNT4rMn84wV/peFZ38m7MqqWvIAQKBgCh4 +CxP3VsKBfxR0DyJQNS05TrbzdLNlSWFB0n2/eFGGuFgE/yKya1ffBR/QYrkvxb6P +Ohg7niWcGxr5SjqR5tU0i4rSmmvGgu0l57GhNa+q67Q050iDLW0gZwUrXK0Ire+Z +bGoer1AaQ39zNjFj2U6880P4AE8qI+7qglgd406bAoGAVhnd9sA+hU6pG+WJ7JbX +TQoUUmO2BF3M2C3F6dje2LNsgtuZg+YkhG4RyBFuKOCZKdN6IhfoMhBw08qskB8J +a9vFyzygqEH4X2yhgmsMpb1dHOyCIvyPwlZIeGkzzJcXlcdx6cOQvmt1NLXPwrAz +GJbH6utej4bup+u4gDT5wEk= +-----END PRIVATE KEY----- diff --git a/tests/integration/test_reload_certificate/test.py b/tests/integration/test_reload_certificate/test.py new file mode 100644 index 00000000000..dc0c391d6f0 --- /dev/null +++ b/tests/integration/test_reload_certificate/test.py @@ -0,0 +1,75 @@ +import pytest +import os +from helpers.cluster import ClickHouseCluster + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance('node', main_configs=["configs/first.crt", "configs/first.key", + "configs/second.crt", "configs/second.key", + "configs/cert.xml"]) + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + +def change_config_to_key(name): + ''' + * Generate config with certificate/key name from args. + * Reload config. + ''' + node.exec_in_container(["bash", "-c" , """cat > /etc/clickhouse-server/config.d/cert.xml << EOF + + + 8443 + + + /etc/clickhouse-server/config.d/{cur_name}.crt + /etc/clickhouse-server/config.d/{cur_name}.key + true + true + sslv2,sslv3 + true + + + +EOF""".format(cur_name=name)]) + node.query("SYSTEM RELOAD CONFIG") + +def test_first_than_second_cert(): + ''' Consistently set first key and check that only it will be accepted, then repeat same for second key. ''' + # Set first key + change_config_to_key('first') + + # Command with correct certificate + assert node.exec_in_container(['curl', '--silent', '--cacert', '/etc/clickhouse-server/config.d/{cur_name}.crt'.format(cur_name='first'), + 'https://localhost:8443/']) == 'Ok.\n' + + # Command with wrong certificate + # This command don't use option '-k', so it will lead to error while execution. + # That's why except will always work + try: + node.exec_in_container(['curl', '--silent', '--cacert', '/etc/clickhouse-server/config.d/{cur_name}.crt'.format(cur_name='second'), + 'https://localhost:8443/']) + assert False + except: + assert True + + # Change to other key + change_config_to_key('second') + + # Command with correct certificate + assert node.exec_in_container(['curl', '--silent', '--cacert', '/etc/clickhouse-server/config.d/{cur_name}.crt'.format(cur_name='second'), + 'https://localhost:8443/']) == 'Ok.\n' + + # Command with wrong certificate + # Same as previous + try: + node.exec_in_container(['curl', '--silent', '--cacert', '/etc/clickhouse-server/config.d/{cur_name}.crt'.format(cur_name='first'), + 'https://localhost:8443/']) + assert False + except: + assert True From 54517753d782273cafffd3702b802e8ee8733ab7 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Fri, 28 Jan 2022 23:14:54 +0800 Subject: [PATCH 036/149] Combining our lexer with replxx --- base/base/ReplxxLineReader.cpp | 23 +++-------------------- src/Client/ClientBaseHelpers.cpp | 9 +++++++++ src/Parsers/Lexer.cpp | 7 +++++++ src/Parsers/Lexer.h | 1 + 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/base/base/ReplxxLineReader.cpp b/base/base/ReplxxLineReader.cpp index 6ba63a00d01..66cbad99a54 100644 --- a/base/base/ReplxxLineReader.cpp +++ b/base/base/ReplxxLineReader.cpp @@ -25,13 +25,6 @@ void trim(String & s) s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) { return !std::isspace(ch); }).base(), s.end()); } -/// Check if string ends with given character after skipping whitespaces. -bool ends_with(const std::string_view & s, const std::string_view & p) -{ - auto ss = std::string_view(s.data(), s.rend() - std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) { return !std::isspace(ch); })); - return ss.ends_with(p); -} - std::string getEditor() { const char * editor = std::getenv("EDITOR"); @@ -132,6 +125,7 @@ void convertHistoryFile(const std::string & path, replxx::Replxx & rx) } +bool replxx_last_is_delimiter = false; ReplxxLineReader::ReplxxLineReader( Suggest & suggest, const String & history_file_path_, @@ -196,21 +190,10 @@ ReplxxLineReader::ReplxxLineReader( auto commit_action = [this](char32_t code) { - std::string_view str = rx.get_state().text(); - - /// Always commit line when we see extender at the end. It will start a new prompt. - for (const auto * extender : extenders) - if (ends_with(str, extender)) - return rx.invoke(Replxx::ACTION::COMMIT_LINE, code); - - /// If we see an delimiter at the end, commit right away. - for (const auto * delimiter : delimiters) - if (ends_with(str, delimiter)) - return rx.invoke(Replxx::ACTION::COMMIT_LINE, code); - /// If we allow multiline and there is already something in the input, start a newline. - if (multiline && !input.empty()) + if (multiline && !replxx_last_is_delimiter) return rx.invoke(Replxx::ACTION::NEW_LINE, code); + replxx_last_is_delimiter = false; return rx.invoke(Replxx::ACTION::COMMIT_LINE, code); }; /// bind C-j to ENTER action. diff --git a/src/Client/ClientBaseHelpers.cpp b/src/Client/ClientBaseHelpers.cpp index 3a5d4f4cf33..1d558159139 100644 --- a/src/Client/ClientBaseHelpers.cpp +++ b/src/Client/ClientBaseHelpers.cpp @@ -6,6 +6,9 @@ #include #include +#if USE_REPLXX +extern bool replxx_last_is_delimiter; +#endif namespace DB { @@ -114,6 +117,7 @@ void highlight(const String & query, std::vector & colors {TokenType::Comma, replxx::color::bold(Replxx::Color::DEFAULT)}, {TokenType::Semicolon, replxx::color::bold(Replxx::Color::DEFAULT)}, + {TokenType::VerticalDelimiter, replxx::color::bold(Replxx::Color::DEFAULT)}, {TokenType::Dot, replxx::color::bold(Replxx::Color::DEFAULT)}, {TokenType::Asterisk, replxx::color::bold(Replxx::Color::DEFAULT)}, {TokenType::HereDoc, Replxx::Color::CYAN}, @@ -151,6 +155,11 @@ void highlight(const String & query, std::vector & colors for (Token token = lexer.nextToken(); !token.isEnd(); token = lexer.nextToken()) { + if (token.type == TokenType::Semicolon || token.type == TokenType::VerticalDelimiter) + replxx_last_is_delimiter = true; + else if (token.type != TokenType::Whitespace) + replxx_last_is_delimiter = false; + size_t utf8_len = UTF8::countCodePoints(reinterpret_cast(token.begin), token.size()); for (size_t code_point_index = 0; code_point_index < utf8_len; ++code_point_index) { diff --git a/src/Parsers/Lexer.cpp b/src/Parsers/Lexer.cpp index 3ac9d417b46..654174c18f7 100644 --- a/src/Parsers/Lexer.cpp +++ b/src/Parsers/Lexer.cpp @@ -335,6 +335,13 @@ Token Lexer::nextTokenImpl() return Token(TokenType::DoubleAt, token_begin, ++pos); return Token(TokenType::At, token_begin, pos); } + case '\\': + { + ++pos; + if (pos < end && *pos == 'G') + return Token(TokenType::VerticalDelimiter, token_begin, ++pos); + return Token(TokenType::Error, token_begin, pos); + } default: if (*pos == '$') diff --git a/src/Parsers/Lexer.h b/src/Parsers/Lexer.h index f41e05147e5..ec472fb1a36 100644 --- a/src/Parsers/Lexer.h +++ b/src/Parsers/Lexer.h @@ -28,6 +28,7 @@ namespace DB \ M(Comma) \ M(Semicolon) \ + M(VerticalDelimiter) /** Vertical delimiter \G */ \ M(Dot) /** Compound identifiers, like a.b or tuple access operator a.1, (x, y).2. */ \ /** Need to be distinguished from floating point number with omitted integer part: .1 */ \ \ From 666313b51ad15673228658963f8ad18320b473cf Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Fri, 28 Jan 2022 23:14:54 +0800 Subject: [PATCH 037/149] Bump replxx --- contrib/replxx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/replxx b/contrib/replxx index f019cba7ea1..54a9b16a278 160000 --- a/contrib/replxx +++ b/contrib/replxx @@ -1 +1 @@ -Subproject commit f019cba7ea1bcd1b4feb7826f28ed57fb581b04c +Subproject commit 54a9b16a278b85db58094a5bd495dca3b136dffc From 556cce9f3c36dda73d52f76de0def159484b0afa Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Sat, 29 Jan 2022 03:13:01 +0800 Subject: [PATCH 038/149] We need highlight to use Lexer --- base/base/ReplxxLineReader.cpp | 3 ++- .../01293_client_interactive_vertical_multiline.expect | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/base/base/ReplxxLineReader.cpp b/base/base/ReplxxLineReader.cpp index 66cbad99a54..a08acfcfb8a 100644 --- a/base/base/ReplxxLineReader.cpp +++ b/base/base/ReplxxLineReader.cpp @@ -191,7 +191,8 @@ ReplxxLineReader::ReplxxLineReader( auto commit_action = [this](char32_t code) { /// If we allow multiline and there is already something in the input, start a newline. - if (multiline && !replxx_last_is_delimiter) + /// NOTE: Lexer is only available if we use highlighter. + if (highlighter && multiline && !replxx_last_is_delimiter) return rx.invoke(Replxx::ACTION::NEW_LINE, code); replxx_last_is_delimiter = false; return rx.invoke(Replxx::ACTION::COMMIT_LINE, code); diff --git a/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect b/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect index e4442047c87..5057ec44e8a 100755 --- a/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect +++ b/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect @@ -1,7 +1,7 @@ #!/usr/bin/expect -f log_user 0 -set timeout 60 +set timeout 10 match_max 100000 expect_after { @@ -11,6 +11,9 @@ expect_after { timeout { exit 1 } } +# useful debugging configuration +# exp_internal 1 + set basedir [file dirname $argv0] spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" expect ":) " @@ -41,7 +44,7 @@ expect ":) " send -- "" expect eof -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --multiline" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --highlight 0 --multiline" expect ":) " send -- "SELECT 1;\r" From 62e89a64459f62b60b974c071f81628af389ea43 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Sat, 29 Jan 2022 05:05:14 +0800 Subject: [PATCH 039/149] Ignore case for history search and completion --- base/base/ReplxxLineReader.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/base/base/ReplxxLineReader.cpp b/base/base/ReplxxLineReader.cpp index a08acfcfb8a..1f75eed479b 100644 --- a/base/base/ReplxxLineReader.cpp +++ b/base/base/ReplxxLineReader.cpp @@ -179,6 +179,7 @@ ReplxxLineReader::ReplxxLineReader( rx.set_completion_callback(callback); rx.set_complete_on_empty(false); rx.set_word_break_characters(word_break_characters); + rx.set_ignore_case(true); if (highlighter) rx.set_highlighter_callback(highlighter); From bb34435928a80b61a1b9bdea41a32a8ac435149f Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Sat, 29 Jan 2022 19:44:27 +0800 Subject: [PATCH 040/149] Better code --- base/base/ReplxxLineReader.cpp | 7 ++++++- base/base/ReplxxLineReader.h | 3 +++ src/Client/ClientBaseHelpers.cpp | 8 ++------ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/base/base/ReplxxLineReader.cpp b/base/base/ReplxxLineReader.cpp index 1f75eed479b..9ea53bb132b 100644 --- a/base/base/ReplxxLineReader.cpp +++ b/base/base/ReplxxLineReader.cpp @@ -125,7 +125,12 @@ void convertHistoryFile(const std::string & path, replxx::Replxx & rx) } -bool replxx_last_is_delimiter = false; +static bool replxx_last_is_delimiter = false; +void ReplxxLineReader::setLastIsDelimiter(bool flag) +{ + replxx_last_is_delimiter = flag; +} + ReplxxLineReader::ReplxxLineReader( Suggest & suggest, const String & history_file_path_, diff --git a/base/base/ReplxxLineReader.h b/base/base/ReplxxLineReader.h index 4a975d2975d..b9ec214d02c 100644 --- a/base/base/ReplxxLineReader.h +++ b/base/base/ReplxxLineReader.h @@ -19,6 +19,9 @@ public: void enableBracketedPaste() override; + /// If highlight is on, we will set a flag to denote whether the last token is a delimiter. + /// This is useful to determine the behavior of key when multiline is enabled. + static void setLastIsDelimiter(bool flag); private: InputStatus readOneLine(const String & prompt) override; void addToHistory(const String & line) override; diff --git a/src/Client/ClientBaseHelpers.cpp b/src/Client/ClientBaseHelpers.cpp index 1d558159139..5ad34ba8e81 100644 --- a/src/Client/ClientBaseHelpers.cpp +++ b/src/Client/ClientBaseHelpers.cpp @@ -6,10 +6,6 @@ #include #include -#if USE_REPLXX -extern bool replxx_last_is_delimiter; -#endif - namespace DB { @@ -156,9 +152,9 @@ void highlight(const String & query, std::vector & colors for (Token token = lexer.nextToken(); !token.isEnd(); token = lexer.nextToken()) { if (token.type == TokenType::Semicolon || token.type == TokenType::VerticalDelimiter) - replxx_last_is_delimiter = true; + ReplxxLineReader::setLastIsDelimiter(true); else if (token.type != TokenType::Whitespace) - replxx_last_is_delimiter = false; + ReplxxLineReader::setLastIsDelimiter(false); size_t utf8_len = UTF8::countCodePoints(reinterpret_cast(token.begin), token.size()); for (size_t code_point_index = 0; code_point_index < utf8_len; ++code_point_index) From ae16b362d39b3ca699c8c2b07c53059b62553327 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Sun, 30 Jan 2022 19:46:16 +0800 Subject: [PATCH 041/149] Better case-insensitive completion --- base/base/LineReader.cpp | 2 +- contrib/replxx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base/base/LineReader.cpp b/base/base/LineReader.cpp index d028dace52a..686d70f247d 100644 --- a/base/base/LineReader.cpp +++ b/base/base/LineReader.cpp @@ -81,7 +81,7 @@ replxx::Replxx::completions_t LineReader::Suggest::getCompletions(const String & std::lock_guard lock(mutex); /// Only perform case sensitive completion when the prefix string contains any uppercase characters - if (std::none_of(prefix.begin(), prefix.end(), [&](auto c) { return c >= 'A' && c <= 'Z'; })) + if (std::none_of(prefix.begin(), prefix.end(), [](char32_t x) { return iswupper(static_cast(x)); })) range = std::equal_range( words_no_case.begin(), words_no_case.end(), last_word, [prefix_length](std::string_view s, std::string_view prefix_searched) { diff --git a/contrib/replxx b/contrib/replxx index 54a9b16a278..c745b3fb012 160000 --- a/contrib/replxx +++ b/contrib/replxx @@ -1 +1 @@ -Subproject commit 54a9b16a278b85db58094a5bd495dca3b136dffc +Subproject commit c745b3fb012ee5ae762fbc8cd7a40c4dc3fe15df From 3f878b9cf6a175e7e42ee4b4accd370fd6605e0a Mon Sep 17 00:00:00 2001 From: achimbab <07c00h@gmail.com> Date: Sun, 30 Jan 2022 16:44:15 +0900 Subject: [PATCH 042/149] Fix floating point comparison --- src/AggregateFunctions/AggregateFunctionTTest.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/AggregateFunctions/AggregateFunctionTTest.h b/src/AggregateFunctions/AggregateFunctionTTest.h index 40319e8c845..40e8cbbf2ea 100644 --- a/src/AggregateFunctions/AggregateFunctionTTest.h +++ b/src/AggregateFunctions/AggregateFunctionTTest.h @@ -9,6 +9,7 @@ #include #include #include +#include /// This function is used in implementations of different T-Tests. @@ -101,7 +102,7 @@ public: throw Exception(ErrorCodes::BAD_ARGUMENTS, "Aggregate function {} requires finite parameter values.", Data::name); } - if (confidence_level <= 0.0 || confidence_level >= 1.0) + if (confidence_level <= 0.0 || confidence_level >= 1.0 || fabs(confidence_level - 0.0) < FLT_EPSILON || fabs(confidence_level - 1.0) < FLT_EPSILON) { throw Exception(ErrorCodes::BAD_ARGUMENTS, "Confidence level parameter must be between 0 and 1 in aggregate function {}.", Data::name); } From 9822b7dfc948dd77f66c436d52a99f9588b82b7c Mon Sep 17 00:00:00 2001 From: achimbab <07c00h@gmail.com> Date: Mon, 31 Jan 2022 02:25:51 +0900 Subject: [PATCH 043/149] Handle exceptional case where data are essentially constant. --- src/AggregateFunctions/AggregateFunctionTTest.h | 4 ++-- src/AggregateFunctions/Moments.h | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionTTest.h b/src/AggregateFunctions/AggregateFunctionTTest.h index 40e8cbbf2ea..36605196951 100644 --- a/src/AggregateFunctions/AggregateFunctionTTest.h +++ b/src/AggregateFunctions/AggregateFunctionTTest.h @@ -102,7 +102,7 @@ public: throw Exception(ErrorCodes::BAD_ARGUMENTS, "Aggregate function {} requires finite parameter values.", Data::name); } - if (confidence_level <= 0.0 || confidence_level >= 1.0 || fabs(confidence_level - 0.0) < FLT_EPSILON || fabs(confidence_level - 1.0) < FLT_EPSILON) + if (confidence_level <= 0.0 || confidence_level >= 1.0 || fabs(confidence_level - 0.0) < DBL_EPSILON || fabs(confidence_level - 1.0) < DBL_EPSILON) { throw Exception(ErrorCodes::BAD_ARGUMENTS, "Confidence level parameter must be between 0 and 1 in aggregate function {}.", Data::name); } @@ -194,7 +194,7 @@ public: auto & data = this->data(place); auto & column_tuple = assert_cast(to); - if (!data.hasEnoughObservations()) + if (!data.hasEnoughObservations() || data.isEssentiallyConstant()) { auto & column_stat = assert_cast &>(column_tuple.getColumn(0)); auto & column_value = assert_cast &>(column_tuple.getColumn(1)); diff --git a/src/AggregateFunctions/Moments.h b/src/AggregateFunctions/Moments.h index 4c18e076082..45a77e9cfdb 100644 --- a/src/AggregateFunctions/Moments.h +++ b/src/AggregateFunctions/Moments.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB @@ -398,6 +399,11 @@ struct TTestMoments return {ci_low, ci_high}; } + + bool isEssentiallyConstant() const + { + return getStandardError() < 10 * DBL_EPSILON * std::max(std::abs(getMeanX()), std::abs(getMeanY())); + } }; template From 61ef73a6fcba44bb9b7b84338e2e892395120c3b Mon Sep 17 00:00:00 2001 From: FArthur-cmd <613623@mail.ru> Date: Mon, 31 Jan 2022 09:31:38 +0000 Subject: [PATCH 044/149] return grpc to correct version --- contrib/grpc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/grpc b/contrib/grpc index 60c986e15ca..7eac189a6ba 160000 --- a/contrib/grpc +++ b/contrib/grpc @@ -1 +1 @@ -Subproject commit 60c986e15cae70aade721d26badabab1f822fdd6 +Subproject commit 7eac189a6badddac593580ec2ad1478bd2656fc7 From 7811f5d5995323a403e46ed7c453497f928f3896 Mon Sep 17 00:00:00 2001 From: FArthur-cmd <613623@mail.ru> Date: Mon, 31 Jan 2022 11:18:57 +0000 Subject: [PATCH 045/149] try uncomment config --- programs/server/config.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/programs/server/config.xml b/programs/server/config.xml index 713fe9f4f67..49e10679165 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -238,14 +238,14 @@ - - + /etc/clickhouse-server/server.crt + /etc/clickhouse-server/server.key - + /etc/clickhouse-server/dhparam.pem none true true From 9ac99bbfdd0ce7859c36ea7f6a87dc08d466d3cb Mon Sep 17 00:00:00 2001 From: zhifeng Date: Mon, 31 Jan 2022 19:43:33 +0800 Subject: [PATCH 046/149] Translate zh/faq/operations/multi-region-replication: rename old file --- docs/zh/faq/operations/multi-region-replication.md | 1 - docs/zh/faq/operations/multi-region-replication.md.bak | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 120000 docs/zh/faq/operations/multi-region-replication.md create mode 100644 docs/zh/faq/operations/multi-region-replication.md.bak diff --git a/docs/zh/faq/operations/multi-region-replication.md b/docs/zh/faq/operations/multi-region-replication.md deleted file mode 120000 index dbc985ee1fb..00000000000 --- a/docs/zh/faq/operations/multi-region-replication.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/faq/operations/multi-region-replication.md \ No newline at end of file diff --git a/docs/zh/faq/operations/multi-region-replication.md.bak b/docs/zh/faq/operations/multi-region-replication.md.bak new file mode 100644 index 00000000000..dbc985ee1fb --- /dev/null +++ b/docs/zh/faq/operations/multi-region-replication.md.bak @@ -0,0 +1 @@ +../../../en/faq/operations/multi-region-replication.md \ No newline at end of file From af2cd2a3e72e62a203628167a94c13a438599387 Mon Sep 17 00:00:00 2001 From: zhifeng Date: Mon, 31 Jan 2022 19:44:19 +0800 Subject: [PATCH 047/149] Translate zh/faq/operations/multi-region-replication: reimport files --- docs/zh/faq/operations/multi-region-replication.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 docs/zh/faq/operations/multi-region-replication.md diff --git a/docs/zh/faq/operations/multi-region-replication.md b/docs/zh/faq/operations/multi-region-replication.md new file mode 100644 index 00000000000..506dad87435 --- /dev/null +++ b/docs/zh/faq/operations/multi-region-replication.md @@ -0,0 +1,13 @@ +--- +title: Does ClickHouse support multi-region replication? +toc_hidden: true +toc_priority: 30 +--- + +# Does ClickHouse support multi-region replication? {#does-clickhouse-support-multi-region-replication} + +The short answer is "yes". However, we recommend keeping latency between all regions/datacenters in two-digit range, otherwise write performance will suffer as it goes through distributed consensus protocol. For example, replication between US coasts will likely work fine, but between the US and Europe won't. + +Configuration-wise there's no difference compared to single-region replication, simply use hosts that are located in different locations for replicas. + +For more information, see [full article on data replication](../../engines/table-engines/mergetree-family/replication.md). From e4e633f5d6f1db9aeb4c04963e354e8395b79be1 Mon Sep 17 00:00:00 2001 From: zhifeng Date: Mon, 31 Jan 2022 19:46:28 +0800 Subject: [PATCH 048/149] Translate zh/faq/operations/multi-region-replication: translate zh --- docs/zh/faq/operations/multi-region-replication.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/zh/faq/operations/multi-region-replication.md b/docs/zh/faq/operations/multi-region-replication.md index 506dad87435..f5ab147bde6 100644 --- a/docs/zh/faq/operations/multi-region-replication.md +++ b/docs/zh/faq/operations/multi-region-replication.md @@ -1,13 +1,14 @@ --- -title: Does ClickHouse support multi-region replication? +title: ClickHouse支持多区域复制吗? toc_hidden: true toc_priority: 30 --- -# Does ClickHouse support multi-region replication? {#does-clickhouse-support-multi-region-replication} +# ClickHouse支持多区域复制吗? {#does-clickhouse-support-multi-region-replication} -The short answer is "yes". However, we recommend keeping latency between all regions/datacenters in two-digit range, otherwise write performance will suffer as it goes through distributed consensus protocol. For example, replication between US coasts will likely work fine, but between the US and Europe won't. +简短的回答是“是的”。然而，我们建议将所有区域/数据中心之间的延迟保持在两位数字范围内，否则，在通过分布式共识协议时，写性能将受到影响。例如，美国海岸之间的复制可能会很好，但美国和欧洲之间就不行。 -Configuration-wise there's no difference compared to single-region replication, simply use hosts that are located in different locations for replicas. +在配置方面，这与单区域复制没有区别，只是使用位于不同位置的主机作为副本。 + +更多信息，请参见[关于数据复制的完整文章](../../engines/table-engines/mergetree-family/replication.md)。 -For more information, see [full article on data replication](../../engines/table-engines/mergetree-family/replication.md). From 2d3adfea76953c86517fba08a6f845eac15bc221 Mon Sep 17 00:00:00 2001 From: zhifeng Date: Mon, 31 Jan 2022 19:47:17 +0800 Subject: [PATCH 049/149] Translate zh/faq/operations/multi-region-replication: remove bak file --- docs/zh/faq/operations/multi-region-replication.md.bak | 1 - 1 file changed, 1 deletion(-) delete mode 100644 docs/zh/faq/operations/multi-region-replication.md.bak diff --git a/docs/zh/faq/operations/multi-region-replication.md.bak b/docs/zh/faq/operations/multi-region-replication.md.bak deleted file mode 100644 index dbc985ee1fb..00000000000 --- a/docs/zh/faq/operations/multi-region-replication.md.bak +++ /dev/null @@ -1 +0,0 @@ -../../../en/faq/operations/multi-region-replication.md \ No newline at end of file From 891411a2beb373af34158d15ddf91ed787d67f99 Mon Sep 17 00:00:00 2001 From: FArthur-cmd <613623@mail.ru> Date: Mon, 31 Jan 2022 12:13:28 +0000 Subject: [PATCH 050/149] add try catch to avoid errors on start --- src/Server/CertificateReloader.cpp | 13 +++++++++---- src/Server/CertificateReloader.h | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/Server/CertificateReloader.cpp b/src/Server/CertificateReloader.cpp index 64947d45bc4..a8527af7dd0 100644 --- a/src/Server/CertificateReloader.cpp +++ b/src/Server/CertificateReloader.cpp @@ -59,7 +59,7 @@ void CertificateReloader::init() auto* ctx = Poco::Net::SSLManager::instance().defaultServerContext()->sslContext(); SSL_CTX_set_cert_cb(ctx, callSetCertificate, nullptr); - first_load = false; + init_was_not_made = false; } @@ -90,9 +90,14 @@ void CertificateReloader::tryLoad(const Poco::Util::AbstractConfiguration & conf } /// If callback is not set yet - - if (first_load) - init(); + try + { + if (init_was_not_made) + init(); + } catch (...) { + init_was_not_made = true; + LOG_ERROR(log, getCurrentExceptionMessage(false)); + } } } diff --git a/src/Server/CertificateReloader.h b/src/Server/CertificateReloader.h index a078e16b45c..7f93b006875 100644 --- a/src/Server/CertificateReloader.h +++ b/src/Server/CertificateReloader.h @@ -80,7 +80,7 @@ private: }; MultiVersion data; - bool first_load = true; + bool init_was_not_made = true; }; } From 3dc64f031a077280b6190f5ed8923a551152550a Mon Sep 17 00:00:00 2001 From: FArthur-cmd <613623@mail.ru> Date: Mon, 31 Jan 2022 12:21:22 +0000 Subject: [PATCH 051/149] correct code style --- src/Server/CertificateReloader.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Server/CertificateReloader.cpp b/src/Server/CertificateReloader.cpp index a8527af7dd0..48d4063dbfb 100644 --- a/src/Server/CertificateReloader.cpp +++ b/src/Server/CertificateReloader.cpp @@ -94,7 +94,9 @@ void CertificateReloader::tryLoad(const Poco::Util::AbstractConfiguration & conf { if (init_was_not_made) init(); - } catch (...) { + } + catch (...) + { init_was_not_made = true; LOG_ERROR(log, getCurrentExceptionMessage(false)); } From 6fe5a0c051a8255cd81eab8bf64dfc5d513c51cb Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 31 Jan 2022 09:53:17 +0300 Subject: [PATCH 052/149] rocksdb: cleanup include directories leftovers (after cmake cleanup) Signed-off-by: Azat Khuzhin --- contrib/rocksdb-cmake/CMakeLists.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/contrib/rocksdb-cmake/CMakeLists.txt b/contrib/rocksdb-cmake/CMakeLists.txt index 902d29a9630..c35009ba10a 100644 --- a/contrib/rocksdb-cmake/CMakeLists.txt +++ b/contrib/rocksdb-cmake/CMakeLists.txt @@ -72,11 +72,6 @@ else() if(WITH_ZSTD) add_definitions(-DZSTD) - include_directories(${ZSTD_INCLUDE_DIR}) - include_directories("${ZSTD_INCLUDE_DIR}/common") - include_directories("${ZSTD_INCLUDE_DIR}/dictBuilder") - include_directories("${ZSTD_INCLUDE_DIR}/deprecated") - list(APPEND THIRDPARTY_LIBS ch_contrib::zstd) endif() endif() From 9b8ee0fabec8e28c54a93ae3a4482a0d5feb8160 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 31 Jan 2022 10:00:51 +0300 Subject: [PATCH 053/149] Cleanup cassandra contrib linking (linking with dictionraies is enough) Signed-off-by: Azat Khuzhin --- src/CMakeLists.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 57d4bf29491..bb359b427c7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -475,11 +475,6 @@ if (TARGET ch_contrib::sqlite) dbms_target_link_libraries(PUBLIC ch_contrib::sqlite) endif() -if (USE_CASSANDRA) - dbms_target_link_libraries(PUBLIC ${CASSANDRA_LIBRARY}) - dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${CASS_INCLUDE_DIR}) -endif() - if (TARGET ch_contrib::msgpack) target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::msgpack) endif() From 3e5ad1a138b94e11e3732abbf60daf7f3f06426b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 31 Jan 2022 10:26:07 +0300 Subject: [PATCH 054/149] Move cmake/find/cxx.cmake -> cmake/cxx.cmake Signed-off-by: Azat Khuzhin --- cmake/{find => }/cxx.cmake | 0 cmake/darwin/default_libs.cmake | 2 +- cmake/freebsd/default_libs.cmake | 2 +- cmake/linux/default_libs.cmake | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename cmake/{find => }/cxx.cmake (100%) diff --git a/cmake/find/cxx.cmake b/cmake/cxx.cmake similarity index 100% rename from cmake/find/cxx.cmake rename to cmake/cxx.cmake diff --git a/cmake/darwin/default_libs.cmake b/cmake/darwin/default_libs.cmake index ca4beaea8b6..1f92663a4b9 100644 --- a/cmake/darwin/default_libs.cmake +++ b/cmake/darwin/default_libs.cmake @@ -22,7 +22,7 @@ set(CMAKE_OSX_DEPLOYMENT_TARGET 10.15) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -include (cmake/find/cxx.cmake) +include (cmake/cxx.cmake) target_link_libraries(global-group INTERFACE $ diff --git a/cmake/freebsd/default_libs.cmake b/cmake/freebsd/default_libs.cmake index f7a333df6e6..17ca29f08af 100644 --- a/cmake/freebsd/default_libs.cmake +++ b/cmake/freebsd/default_libs.cmake @@ -23,7 +23,7 @@ set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) include (cmake/find/unwind.cmake) -include (cmake/find/cxx.cmake) +include (cmake/cxx.cmake) target_link_libraries(global-group INTERFACE $ diff --git a/cmake/linux/default_libs.cmake b/cmake/linux/default_libs.cmake index 98951822015..8fb82ec8b82 100644 --- a/cmake/linux/default_libs.cmake +++ b/cmake/linux/default_libs.cmake @@ -43,7 +43,7 @@ if (NOT OS_ANDROID) endif () include (cmake/find/unwind.cmake) -include (cmake/find/cxx.cmake) +include (cmake/cxx.cmake) target_link_libraries(global-group INTERFACE -Wl,--start-group From 116af7dfb977d40fd5b31eddfa7451d01e951172 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 31 Jan 2022 10:27:00 +0300 Subject: [PATCH 055/149] Move cmake/find/unwind.cmake -> cmake/unwind.cmake Signed-off-by: Azat Khuzhin --- cmake/freebsd/default_libs.cmake | 2 +- cmake/linux/default_libs.cmake | 2 +- cmake/{find => }/unwind.cmake | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename cmake/{find => }/unwind.cmake (100%) diff --git a/cmake/freebsd/default_libs.cmake b/cmake/freebsd/default_libs.cmake index 17ca29f08af..65d5f0511d9 100644 --- a/cmake/freebsd/default_libs.cmake +++ b/cmake/freebsd/default_libs.cmake @@ -22,7 +22,7 @@ set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS}) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -include (cmake/find/unwind.cmake) +include (cmake/unwind.cmake) include (cmake/cxx.cmake) target_link_libraries(global-group INTERFACE diff --git a/cmake/linux/default_libs.cmake b/cmake/linux/default_libs.cmake index 8fb82ec8b82..21bead7020c 100644 --- a/cmake/linux/default_libs.cmake +++ b/cmake/linux/default_libs.cmake @@ -42,7 +42,7 @@ if (NOT OS_ANDROID) add_subdirectory(base/harmful) endif () -include (cmake/find/unwind.cmake) +include (cmake/unwind.cmake) include (cmake/cxx.cmake) target_link_libraries(global-group INTERFACE diff --git a/cmake/find/unwind.cmake b/cmake/unwind.cmake similarity index 100% rename from cmake/find/unwind.cmake rename to cmake/unwind.cmake From b90cf4e7ff7acd0e0369ca063ede7d1cfd5ce958 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 31 Jan 2022 10:29:00 +0300 Subject: [PATCH 056/149] Move cmake/find/ccache.cmake -> cmake/ccache.cmake Signed-off-by: Azat Khuzhin --- CMakeLists.txt | 2 +- cmake/{find => }/ccache.cmake | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename cmake/{find => }/ccache.cmake (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c846cdd51e..08ccfef3324 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,7 +67,7 @@ if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git" AND NOT EXISTS "${ClickHouse_SOURC message (FATAL_ERROR "Submodules are not initialized. Run\n\tgit submodule update --init --recursive") endif () -include (cmake/find/ccache.cmake) +include (cmake/ccache.cmake) # Take care to add prlimit in command line before ccache, or else ccache thinks that # prlimit is compiler, and clang++ is its input file, and refuses to work with diff --git a/cmake/find/ccache.cmake b/cmake/ccache.cmake similarity index 100% rename from cmake/find/ccache.cmake rename to cmake/ccache.cmake From 09d7bc3ed4b0cf0ded0e47368f2796d95d0d5493 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 31 Jan 2022 10:28:27 +0300 Subject: [PATCH 057/149] Update mentions about cmake/find Signed-off-by: Azat Khuzhin --- docs/_includes/cmake_in_clickhouse_header.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_includes/cmake_in_clickhouse_header.md b/docs/_includes/cmake_in_clickhouse_header.md index f950cdcc6db..02019f13964 100644 --- a/docs/_includes/cmake_in_clickhouse_header.md +++ b/docs/_includes/cmake_in_clickhouse_header.md @@ -22,7 +22,7 @@ cmake .. \ 1. ClickHouse's source CMake files (located in the root directory and in `/src`). 2. Arch-dependent CMake files (located in `/cmake/*os_name*`). -3. Libraries finders (search for contrib libraries, located in `/cmake/find`). +3. Libraries finders (search for contrib libraries, located in `/contrib/*/CMakeLists.txt`). 3. Contrib build CMake files (used instead of libraries' own CMake files, located in `/cmake/modules`) ## List of CMake flags From 9d59d4a5d3457c60f302e6db63bec44d745c626b Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 31 Jan 2022 12:42:34 +0300 Subject: [PATCH 058/149] arrow: add a note about hdfs Signed-off-by: Azat Khuzhin --- contrib/arrow-cmake/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 54bfead6da7..b65fb9974cd 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -98,6 +98,7 @@ add_subdirectory(${FLATBUFFERS_SRC_DIR} "${FLATBUFFERS_BINARY_DIR}") message(STATUS "FLATBUFFERS_LIBRARY: ${FLATBUFFERS_LIBRARY}") # === hdfs +# NOTE: cannot use ch_contrib::hdfs since it's INCLUDE_DIRECTORIES does not includes trailing "hdfs/" set(HDFS_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include/hdfs/") # arrow-cmake cmake file calling orc cmake subroutine which detects certain compiler features. From 0051ed5196b719681eef3588559301f044dcd569 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 31 Jan 2022 12:34:22 +0300 Subject: [PATCH 059/149] arrow: separate out flatbuffers library Signed-off-by: Azat Khuzhin --- contrib/arrow-cmake/CMakeLists.txt | 37 ++++++++++++++++-------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index b65fb9974cd..1d725fba5f4 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -29,12 +29,6 @@ if (OS_FREEBSD) message (FATAL_ERROR "Using internal parquet library on FreeBSD is not supported") endif() -if(USE_STATIC_LIBRARIES) - set(FLATBUFFERS_LIBRARY flatbuffers) -else() - set(FLATBUFFERS_LIBRARY flatbuffers_shared) -endif() - set (CMAKE_CXX_STANDARD 17) set(ARROW_VERSION "6.0.1") @@ -95,7 +89,13 @@ set(FLATBUFFERS_BUILD_TESTS OFF CACHE BOOL "Skip flatbuffers tests") add_subdirectory(${FLATBUFFERS_SRC_DIR} "${FLATBUFFERS_BINARY_DIR}") -message(STATUS "FLATBUFFERS_LIBRARY: ${FLATBUFFERS_LIBRARY}") +add_library(_flatbuffers INTERFACE) +if(USE_STATIC_LIBRARIES) + target_link_libraries(_flatbuffers INTERFACE flatbuffers) +else() + target_link_libraries(_flatbuffers INTERFACE flatbuffers_shared) +endif() +target_include_directories(_flatbuffers INTERFACE ${FLATBUFFERS_INCLUDE_DIR}) # === hdfs # NOTE: cannot use ch_contrib::hdfs since it's INCLUDE_DIRECTORIES does not includes trailing "hdfs/" @@ -357,21 +357,25 @@ endif () add_library(_arrow ${ARROW_SRCS}) -# Arrow dependencies -add_dependencies(_arrow ${FLATBUFFERS_LIBRARY}) +target_link_libraries(_arrow PRIVATE + boost::filesystem -target_link_libraries(_arrow PRIVATE ${FLATBUFFERS_LIBRARY} boost::filesystem) + _flatbuffers + ch_contrib::protobuf + + ch_contrib::double_conversion + + ch_contrib::lz4 + ch_contrib::snappy + ch_contrib::zlib + ch_contrib::zstd + ch_contrib::zstd +) add_dependencies(_arrow protoc) target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ARROW_SRC_DIR}) target_include_directories(_arrow SYSTEM BEFORE PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/cpp/src") -target_link_libraries(_arrow PRIVATE ch_contrib::double_conversion) -target_link_libraries(_arrow PRIVATE ch_contrib::protobuf) -target_link_libraries(_arrow PRIVATE ch_contrib::lz4) -target_link_libraries(_arrow PRIVATE ch_contrib::snappy) -target_link_libraries(_arrow PRIVATE ch_contrib::zlib) -target_link_libraries(_arrow PRIVATE ch_contrib::zstd) target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ORC_INCLUDE_DIR}) target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ORC_BUILD_INCLUDE_DIR}) @@ -380,7 +384,6 @@ target_include_directories(_arrow SYSTEM PRIVATE ${ORC_SOURCE_WRAP_DIR}) target_include_directories(_arrow SYSTEM PRIVATE ${ORC_BUILD_SRC_DIR}) target_include_directories(_arrow SYSTEM PRIVATE ${ORC_ADDITION_SOURCE_DIR}) target_include_directories(_arrow SYSTEM PRIVATE ${ARROW_SRC_DIR}) -target_include_directories(_arrow SYSTEM PRIVATE ${FLATBUFFERS_INCLUDE_DIR}) target_include_directories(_arrow SYSTEM PRIVATE ${HDFS_INCLUDE_DIR}) # === parquet From 681f4117e0a269e1ad753093898e8508476b0381 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 31 Jan 2022 15:55:40 +0300 Subject: [PATCH 060/149] arrow: separate out orc library v2: exclude adaptors [1] [1]: https://s3.amazonaws.com/clickhouse-builds/34167/6986f34741c534565621080c1742e2dd747c9792/binary_splitted/build_log.log Signed-off-by: Azat Khuzhin --- contrib/arrow-cmake/CMakeLists.txt | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 1d725fba5f4..2e4059efc17 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -124,8 +124,6 @@ configure_file("${ORC_SOURCE_SRC_DIR}/Adaptor.hh.in" "${ORC_BUILD_INCLUDE_DIR}/A # ARROW_ORC + adapters/orc/CMakefiles set(ORC_SRCS - "${ARROW_SRC_DIR}/arrow/adapters/orc/adapter.cc" - "${ARROW_SRC_DIR}/arrow/adapters/orc/adapter_util.cc" "${ORC_SOURCE_SRC_DIR}/Exceptions.cc" "${ORC_SOURCE_SRC_DIR}/OrcFile.cc" "${ORC_SOURCE_SRC_DIR}/Reader.cc" @@ -152,6 +150,22 @@ set(ORC_SRCS "${ORC_ADDITION_SOURCE_DIR}/orc_proto.pb.cc" ) +add_library(_orc ${ORC_SRCS}) +target_link_libraries(_orc PRIVATE + ch_contrib::protobuf + ch_contrib::lz4 + ch_contrib::snappy + ch_contrib::zlib + ch_contrib::zstd) +target_include_directories(_orc SYSTEM BEFORE PUBLIC ${ORC_INCLUDE_DIR}) +target_include_directories(_orc SYSTEM BEFORE PUBLIC ${ORC_BUILD_INCLUDE_DIR}) +target_include_directories(_orc SYSTEM PRIVATE + ${ORC_SOURCE_SRC_DIR} + ${ORC_SOURCE_WRAP_DIR} + ${ORC_BUILD_SRC_DIR} + ${ORC_ADDITION_SOURCE_DIR} + ${ARROW_SRC_DIR}) + # === arrow @@ -337,7 +351,8 @@ set(ARROW_SRCS "${LIBRARY_DIR}/ipc/reader.cc" "${LIBRARY_DIR}/ipc/writer.cc" - ${ORC_SRCS} + "${ARROW_SRC_DIR}/arrow/adapters/orc/adapter.cc" + "${ARROW_SRC_DIR}/arrow/adapters/orc/adapter_util.cc" ) add_definitions(-DARROW_WITH_LZ4) @@ -361,7 +376,6 @@ target_link_libraries(_arrow PRIVATE boost::filesystem _flatbuffers - ch_contrib::protobuf ch_contrib::double_conversion @@ -371,18 +385,13 @@ target_link_libraries(_arrow PRIVATE ch_contrib::zstd ch_contrib::zstd ) +target_link_libraries(_arrow PUBLIC _orc) add_dependencies(_arrow protoc) target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ARROW_SRC_DIR}) target_include_directories(_arrow SYSTEM BEFORE PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/cpp/src") -target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ORC_INCLUDE_DIR}) -target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ORC_BUILD_INCLUDE_DIR}) -target_include_directories(_arrow SYSTEM PRIVATE ${ORC_SOURCE_SRC_DIR}) -target_include_directories(_arrow SYSTEM PRIVATE ${ORC_SOURCE_WRAP_DIR}) -target_include_directories(_arrow SYSTEM PRIVATE ${ORC_BUILD_SRC_DIR}) -target_include_directories(_arrow SYSTEM PRIVATE ${ORC_ADDITION_SOURCE_DIR}) target_include_directories(_arrow SYSTEM PRIVATE ${ARROW_SRC_DIR}) target_include_directories(_arrow SYSTEM PRIVATE ${HDFS_INCLUDE_DIR}) From 210784c51d0c36e6f986db174c07c88d1c9da5f9 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 31 Jan 2022 16:04:25 +0300 Subject: [PATCH 061/149] docs/tols/single_page.py: fix relative link check Signed-off-by: Azat Khuzhin --- docs/tools/single_page.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/tools/single_page.py b/docs/tools/single_page.py index cf41e2b78c2..3d32ba30a21 100644 --- a/docs/tools/single_page.py +++ b/docs/tools/single_page.py @@ -90,7 +90,10 @@ def concatenate(lang, docs_path, single_page_file, nav): line) # If failed to replace the relative link, print to log - if '../' in line: + # But with some exceptions: + # - "../src/" -- for cmake-in-clickhouse.md (link to sources) + # - "../usr/share" -- changelog entry that has "../usr/share/zoneinfo" + if '../' in line and (not '../usr/share' in line) and (not '../src/' in line): logging.info('Failed to resolve relative link:') logging.info(path) logging.info(line) From f20f2f6beb862d3c8e366b74bba4610b18cde87d Mon Sep 17 00:00:00 2001 From: FArthur-cmd <613623@mail.ru> Date: Mon, 31 Jan 2022 13:39:39 +0000 Subject: [PATCH 062/149] fix test --- programs/server/config.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/server/config.xml b/programs/server/config.xml index 49e10679165..f306195993f 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -245,7 +245,7 @@ openssl dhparam -out /etc/clickhouse-server/dhparam.pem 4096 Only file format with BEGIN DH PARAMETERS is supported. --> - /etc/clickhouse-server/dhparam.pem + none true true From 424900bf1f094459d1304a0863ec50d6771bbe22 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 31 Jan 2022 15:49:24 +0100 Subject: [PATCH 063/149] Add authorisation for dockerhub proxy container --- tests/ci/worker/dockerhub_proxy_template.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/ci/worker/dockerhub_proxy_template.sh b/tests/ci/worker/dockerhub_proxy_template.sh index 5ee63a05125..add15158d94 100644 --- a/tests/ci/worker/dockerhub_proxy_template.sh +++ b/tests/ci/worker/dockerhub_proxy_template.sh @@ -5,4 +5,7 @@ mkdir /home/ubuntu/registrystorage sed -i 's/preserve_hostname: false/preserve_hostname: true/g' /etc/cloud/cloud.cfg -docker run -d --network=host -p 5000:5000 -v /home/ubuntu/registrystorage:/var/lib/registry -e REGISTRY_HTTP_ADDR=0.0.0.0:5000 -e REGISTRY_STORAGE_DELETE_ENABLED=true -e REGISTRY_PROXY_REMOTEURL=https://registry-1.docker.io --restart=always --name registry registry:2 +REGISTRY_PROXY_USERNAME=robotclickhouse +REGISTRY_PROXY_PASSWORD=$(aws ssm get-parameter --name dockerhub_robot_password --with-decryption | jq '.Parameter.Value' -r) + +docker run -d --network=host -p 5000:5000 -v /home/ubuntu/registrystorage:/var/lib/registry -e REGISTRY_HTTP_ADDR=0.0.0.0:5000 -e REGISTRY_STORAGE_DELETE_ENABLED=true -e REGISTRY_PROXY_REMOTEURL=https://registry-1.docker.io -e REGISTRY_PROXY_PASSWORD="$REGISTRY_PROXY_PASSWORD" -e REGISTRY_PROXY_USERNAME="$REGISTRY_PROXY_USERNAME" --restart=always --name registry registry:2 From 43d9a19f9dc56ea2d7a1e12ed3a5e4b7defbabc3 Mon Sep 17 00:00:00 2001 From: tesw yew isal <278153+cnmade@users.noreply.github.com> Date: Tue, 1 Feb 2022 06:38:25 +0800 Subject: [PATCH 064/149] sync index from operations: doese clickhouse support multi-region replication: to zh --- docs/zh/faq/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/zh/faq/index.md b/docs/zh/faq/index.md index 9887d2c6c0a..1ba1b792fbd 100644 --- a/docs/zh/faq/index.md +++ b/docs/zh/faq/index.md @@ -26,6 +26,7 @@ toc_priority: 76 - **[运维操作](../faq/operations/index.md)** - [如果想在生产环境部署，需要用哪个版本的 ClickHouse 呢?](../faq/operations/production.md) - [是否可能从 ClickHouse 数据表中删除所有旧的数据记录?](../faq/operations/delete-old-data.md) + - [ClickHouse支持多区域复制吗?](../faq/operations/multi-region-replication.md) - **[集成开发](../faq/integration/index.md)** - [如何从 ClickHouse 导出数据到一个文件?](../faq/integration/file-export.md) - [如果我用ODBC链接Oracle数据库出现编码问题该怎么办?](../faq/integration/oracle-odbc.md) From 83136f3515b25678246e72f6aa52ad56c70f572b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 1 Feb 2022 02:49:26 +0300 Subject: [PATCH 065/149] Allow \r in the middle of the line in format `Regexp` --- .../Formats/Impl/RegexpRowInputFormat.cpp | 26 +++++++++++-------- ...0_format_regexp_cr_in_the_middle.reference | 3 +++ .../02190_format_regexp_cr_in_the_middle.sh | 8 ++++++ 3 files changed, 26 insertions(+), 11 deletions(-) create mode 100644 tests/queries/0_stateless/02190_format_regexp_cr_in_the_middle.reference create mode 100755 tests/queries/0_stateless/02190_format_regexp_cr_in_the_middle.sh diff --git a/src/Processors/Formats/Impl/RegexpRowInputFormat.cpp b/src/Processors/Formats/Impl/RegexpRowInputFormat.cpp index d793d2e0491..379abc1b7fe 100644 --- a/src/Processors/Formats/Impl/RegexpRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/RegexpRowInputFormat.cpp @@ -37,7 +37,7 @@ bool RegexpFieldExtractor::parseRow(PeekableReadBuffer & buf) do { - char * pos = find_first_symbols<'\n', '\r'>(buf.position(), buf.buffer().end()); + char * pos = find_first_symbols<'\n'>(buf.position(), buf.buffer().end()); line_size += pos - buf.position(); buf.position() = pos; } while (buf.position() == buf.buffer().end() && !buf.eof()); @@ -45,15 +45,19 @@ bool RegexpFieldExtractor::parseRow(PeekableReadBuffer & buf) buf.makeContinuousMemoryFromCheckpointToPos(); buf.rollbackToCheckpoint(); - bool match = re2_st::RE2::FullMatchN(re2_st::StringPiece(buf.position(), line_size), regexp, re2_arguments_ptrs.data(), re2_arguments_ptrs.size()); + /// Allow DOS line endings. + size_t line_to_match = line_size; + if (line_size > 0 && buf.position()[line_size - 1] == '\r') + --line_to_match; + + bool match = re2_st::RE2::FullMatchN(re2_st::StringPiece(buf.position(), line_to_match), regexp, re2_arguments_ptrs.data(), re2_arguments_ptrs.size()); if (!match && !skip_unmatched) - throw Exception("Line \"" + std::string(buf.position(), line_size) + "\" doesn't match the regexp.", ErrorCodes::INCORRECT_DATA); + throw Exception("Line \"" + std::string(buf.position(), line_to_match) + "\" doesn't match the regexp.", ErrorCodes::INCORRECT_DATA); buf.position() += line_size; - checkChar('\r', buf); if (!buf.eof() && !checkChar('\n', buf)) - throw Exception("No \\n after \\r at the end of line.", ErrorCodes::INCORRECT_DATA); + throw Exception("No \\n at the end of line.", ErrorCodes::LOGICAL_ERROR); return match; } @@ -65,12 +69,12 @@ RegexpRowInputFormat::RegexpRowInputFormat( } RegexpRowInputFormat::RegexpRowInputFormat( - std::unique_ptr buf_, const Block & header_, Params params_, const FormatSettings & format_settings_) - : IRowInputFormat(header_, *buf_, std::move(params_)) - , buf(std::move(buf_)) - , format_settings(format_settings_) - , escaping_rule(format_settings_.regexp.escaping_rule) - , field_extractor(RegexpFieldExtractor(format_settings_)) + std::unique_ptr buf_, const Block & header_, Params params_, const FormatSettings & format_settings_) + : IRowInputFormat(header_, *buf_, std::move(params_)) + , buf(std::move(buf_)) + , format_settings(format_settings_) + , escaping_rule(format_settings_.regexp.escaping_rule) + , field_extractor(RegexpFieldExtractor(format_settings_)) { } diff --git a/tests/queries/0_stateless/02190_format_regexp_cr_in_the_middle.reference b/tests/queries/0_stateless/02190_format_regexp_cr_in_the_middle.reference new file mode 100644 index 00000000000..858b670b9d9 --- /dev/null +++ b/tests/queries/0_stateless/02190_format_regexp_cr_in_the_middle.reference @@ -0,0 +1,3 @@ +xyz\rabc +Hello, world +End diff --git a/tests/queries/0_stateless/02190_format_regexp_cr_in_the_middle.sh b/tests/queries/0_stateless/02190_format_regexp_cr_in_the_middle.sh new file mode 100755 index 00000000000..1f6c040a34b --- /dev/null +++ b/tests/queries/0_stateless/02190_format_regexp_cr_in_the_middle.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, long + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +echo -ne 'xyz\rabc\nHello, world\r\nEnd' | ${CLICKHOUSE_LOCAL} --structure 's String' --input-format Regexp --format_regexp '(.*)' --query 'SELECT * FROM table' From e4e71692775b27c41fa0867b413244aebf55f86a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 1 Feb 2022 02:52:36 +0300 Subject: [PATCH 066/149] Remove some strange code --- src/Processors/Formats/Impl/RegexpRowInputFormat.cpp | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/Processors/Formats/Impl/RegexpRowInputFormat.cpp b/src/Processors/Formats/Impl/RegexpRowInputFormat.cpp index 379abc1b7fe..4754b70d375 100644 --- a/src/Processors/Formats/Impl/RegexpRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/RegexpRowInputFormat.cpp @@ -178,20 +178,12 @@ static std::pair fileSegmentationEngineRegexpImpl(ReadBuffer & in, while (loadAtPosition(in, memory, pos) && need_more_data) { - pos = find_first_symbols<'\n', '\r'>(pos, in.buffer().end()); + pos = find_first_symbols<'\n'>(pos, in.buffer().end()); if (pos > in.buffer().end()) - throw Exception("Position in buffer is out of bounds. There must be a bug.", ErrorCodes::LOGICAL_ERROR); + throw Exception("Position in buffer is out of bounds. There must be a bug.", ErrorCodes::LOGICAL_ERROR); else if (pos == in.buffer().end()) continue; - // Support DOS-style newline ("\r\n") - if (*pos == '\r') - { - ++pos; - if (pos == in.buffer().end()) - loadAtPosition(in, memory, pos); - } - if (memory.size() + static_cast(pos - in.position()) >= min_chunk_size) need_more_data = false; From 989a5ad35e49212f7606d55e8ad4b0fe433f04e0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 1 Feb 2022 03:29:27 +0300 Subject: [PATCH 067/149] More cases for parseDateTimeBestEffort --- src/IO/parseDateTimeBestEffort.cpp | 70 +++++++++++-------- ...date_time_best_effort_more_cases.reference | 10 +++ ...parse_date_time_best_effort_more_cases.sql | 10 +++ 3 files changed, 61 insertions(+), 29 deletions(-) create mode 100644 tests/queries/0_stateless/02191_parse_date_time_best_effort_more_cases.reference create mode 100644 tests/queries/0_stateless/02191_parse_date_time_best_effort_more_cases.sql diff --git a/src/IO/parseDateTimeBestEffort.cpp b/src/IO/parseDateTimeBestEffort.cpp index 3b05d8c76b6..3c6f9b8f9f5 100644 --- a/src/IO/parseDateTimeBestEffort.cpp +++ b/src/IO/parseDateTimeBestEffort.cpp @@ -194,7 +194,7 @@ ReturnType parseDateTimeBestEffortImpl( } else if (num_digits == 6) { - /// This is YYYYMM + /// This is YYYYMM or hhmmss if (!year && !month) { readDecimalNumber<4>(year, digits); @@ -435,47 +435,59 @@ ReturnType parseDateTimeBestEffortImpl( else if (c == '+' || c == '-') { ++in.position(); - has_time_zone_offset = true; - if (c == '-') - time_zone_offset_negative = true; - num_digits = readDigits(digits, sizeof(digits), in); - if (num_digits == 4) + if (num_digits == 6 && !has_time && year && month && day_of_month) { - readDecimalNumber<2>(time_zone_offset_hour, digits); - readDecimalNumber<2>(time_zone_offset_minute, digits + 2); - } - else if (num_digits == 3) - { - readDecimalNumber<1>(time_zone_offset_hour, digits); - readDecimalNumber<2>(time_zone_offset_minute, digits + 1); - } - else if (num_digits == 2) - { - readDecimalNumber<2>(time_zone_offset_hour, digits); - } - else if (num_digits == 1) - { - readDecimalNumber<1>(time_zone_offset_hour, digits); + /// It looks like hhmmss + readDecimalNumber<2>(hour, digits); + readDecimalNumber<2>(minute, digits + 2); + readDecimalNumber<2>(second, digits + 4); + has_time = true; } else - return on_error("Cannot read DateTime: unexpected number of decimal digits for time zone offset: " + toString(num_digits), ErrorCodes::CANNOT_PARSE_DATETIME); - - if (num_digits < 3 && checkChar(':', in)) { - num_digits = readDigits(digits, sizeof(digits), in); + /// It looks like time zone offset + has_time_zone_offset = true; + if (c == '-') + time_zone_offset_negative = true; - if (num_digits == 2) + if (num_digits == 4) { - readDecimalNumber<2>(time_zone_offset_minute, digits); + readDecimalNumber<2>(time_zone_offset_hour, digits); + readDecimalNumber<2>(time_zone_offset_minute, digits + 2); + } + else if (num_digits == 3) + { + readDecimalNumber<1>(time_zone_offset_hour, digits); + readDecimalNumber<2>(time_zone_offset_minute, digits + 1); + } + else if (num_digits == 2) + { + readDecimalNumber<2>(time_zone_offset_hour, digits); } else if (num_digits == 1) { - readDecimalNumber<1>(time_zone_offset_minute, digits); + readDecimalNumber<1>(time_zone_offset_hour, digits); } else - return on_error("Cannot read DateTime: unexpected number of decimal digits for time zone offset in minutes: " + toString(num_digits), ErrorCodes::CANNOT_PARSE_DATETIME); + return on_error("Cannot read DateTime: unexpected number of decimal digits for time zone offset: " + toString(num_digits), ErrorCodes::CANNOT_PARSE_DATETIME); + + if (num_digits < 3 && checkChar(':', in)) + { + num_digits = readDigits(digits, sizeof(digits), in); + + if (num_digits == 2) + { + readDecimalNumber<2>(time_zone_offset_minute, digits); + } + else if (num_digits == 1) + { + readDecimalNumber<1>(time_zone_offset_minute, digits); + } + else + return on_error("Cannot read DateTime: unexpected number of decimal digits for time zone offset in minutes: " + toString(num_digits), ErrorCodes::CANNOT_PARSE_DATETIME); + } } } else diff --git a/tests/queries/0_stateless/02191_parse_date_time_best_effort_more_cases.reference b/tests/queries/0_stateless/02191_parse_date_time_best_effort_more_cases.reference new file mode 100644 index 00000000000..227e3b013b2 --- /dev/null +++ b/tests/queries/0_stateless/02191_parse_date_time_best_effort_more_cases.reference @@ -0,0 +1,10 @@ +2022-01-01 01:02:03 +2022-01-01 01:02:03 +2022-01-01 01:02:03 +2022-01-01 01:02:03 +2022-01-01 01:02:00 +2022-01-01 01:02:00 +2021-12-31 22:58:00 +2022-01-01 02:02:03 +2022-01-01 00:02:03 +2022-01-01 02:02:03 diff --git a/tests/queries/0_stateless/02191_parse_date_time_best_effort_more_cases.sql b/tests/queries/0_stateless/02191_parse_date_time_best_effort_more_cases.sql new file mode 100644 index 00000000000..d30834b90a3 --- /dev/null +++ b/tests/queries/0_stateless/02191_parse_date_time_best_effort_more_cases.sql @@ -0,0 +1,10 @@ +SELECT parseDateTimeBestEffort('20220101-010203', 'UTC'); +SELECT parseDateTimeBestEffort('20220101+010203', 'UTC'); +SELECT parseDateTimeBestEffort('20220101 010203', 'UTC'); +SELECT parseDateTimeBestEffort('20220101T010203', 'UTC'); +SELECT parseDateTimeBestEffort('20220101T01:02', 'UTC'); +SELECT parseDateTimeBestEffort('20220101-0102', 'UTC'); +SELECT parseDateTimeBestEffort('20220101+0102', 'UTC'); +SELECT parseDateTimeBestEffort('20220101-010203-01', 'UTC'); +SELECT parseDateTimeBestEffort('20220101-010203+0100', 'UTC'); +SELECT parseDateTimeBestEffort('20220101-010203-01:00', 'UTC'); From 7dbf0dede59320565a763ad6ac77a96eb8d5b753 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 1 Feb 2022 05:55:07 +0300 Subject: [PATCH 068/149] Change severity of the "Cancelled merging parts" message in logs --- .../MergeTree/MergeTreeBackgroundExecutor.cpp | 25 ++++++++++++++++--- .../MergeTree/MergeTreeBackgroundExecutor.h | 3 +-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp index 797caff8f69..51a7037e03f 100644 --- a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp @@ -3,11 +3,19 @@ #include #include +#include #include + namespace DB { +namespace ErrorCodes +{ + extern const int ABORTED; +} + + template void MergeTreeBackgroundExecutor::wait() { @@ -86,12 +94,18 @@ void MergeTreeBackgroundExecutor::routine(TaskRuntimeDataPtr item) ALLOW_ALLOCATIONS_IN_SCOPE; need_execute_again = item->task->executeStep(); } + catch (const Exception & e) + { + if (e.code() == ErrorCodes::ABORTED) /// Cancelled merging parts is not an error - log as info. + LOG_INFO(log, getCurrentExceptionMessage(false)); + else + tryLogCurrentException(__PRETTY_FUNCTION__); + } catch (...) { tryLogCurrentException(__PRETTY_FUNCTION__); } - if (need_execute_again) { std::lock_guard guard(mutex); @@ -118,7 +132,6 @@ void MergeTreeBackgroundExecutor::routine(TaskRuntimeDataPtr item) return; } - { std::lock_guard guard(mutex); erase_from_active(); @@ -132,12 +145,18 @@ void MergeTreeBackgroundExecutor::routine(TaskRuntimeDataPtr item) /// But it is rather safe, because we have try...catch block here, and another one in ThreadPool. item->task->onCompleted(); } + catch (const Exception & e) + { + if (e.code() == ErrorCodes::ABORTED) /// Cancelled merging parts is not an error - log as info. + LOG_INFO(log, getCurrentExceptionMessage(false)); + else + tryLogCurrentException(__PRETTY_FUNCTION__); + } catch (...) { tryLogCurrentException(__PRETTY_FUNCTION__); } - /// We have to call reset() under a lock, otherwise a race is possible. /// Imagine, that task is finally completed (last execution returned false), /// we removed the task from both queues, but still have pointer. diff --git a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h index f4635812e08..5cfa7b6ed7c 100644 --- a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h +++ b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h @@ -159,7 +159,6 @@ template class MergeTreeBackgroundExecutor final : public shared_ptr_helper> { public: - MergeTreeBackgroundExecutor( String name_, size_t threads_count_, @@ -194,7 +193,6 @@ public: void wait(); private: - String name; size_t threads_count{0}; size_t max_tasks_count{0}; @@ -210,6 +208,7 @@ private: std::condition_variable has_tasks; std::atomic_bool shutdown{false}; ThreadPool pool; + Poco::Logger * log = &Poco::Logger::get("MergeTreeBackgroundExecutor"); }; extern template class MergeTreeBackgroundExecutor; From 8e81f5084de8e25c9dd42ac19ecaabe3b767d9f2 Mon Sep 17 00:00:00 2001 From: tesw yew isal <278153+cnmade@users.noreply.github.com> Date: Tue, 1 Feb 2022 13:16:01 +0800 Subject: [PATCH 069/149] Translate zh/faq/operations/index: rename old files --- docs/zh/faq/operations/index.md | 1 - docs/zh/faq/operations/index.md.md | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 120000 docs/zh/faq/operations/index.md create mode 100644 docs/zh/faq/operations/index.md.md diff --git a/docs/zh/faq/operations/index.md b/docs/zh/faq/operations/index.md deleted file mode 120000 index fd141164fdc..00000000000 --- a/docs/zh/faq/operations/index.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/faq/operations/index.md \ No newline at end of file diff --git a/docs/zh/faq/operations/index.md.md b/docs/zh/faq/operations/index.md.md new file mode 100644 index 00000000000..fd141164fdc --- /dev/null +++ b/docs/zh/faq/operations/index.md.md @@ -0,0 +1 @@ +../../../en/faq/operations/index.md \ No newline at end of file From e736f77b98ab7f4db0650a5d895d53b557d91b70 Mon Sep 17 00:00:00 2001 From: tesw yew isal <278153+cnmade@users.noreply.github.com> Date: Tue, 1 Feb 2022 13:16:57 +0800 Subject: [PATCH 070/149] Translate zh/faq/operations/index: reimport file --- docs/zh/faq/operations/index.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 docs/zh/faq/operations/index.md diff --git a/docs/zh/faq/operations/index.md b/docs/zh/faq/operations/index.md new file mode 100644 index 00000000000..8e483febe14 --- /dev/null +++ b/docs/zh/faq/operations/index.md @@ -0,0 +1,19 @@ +--- +title: Question about operating ClickHouse servers and clusters +toc_hidden_folder: true +toc_priority: 3 +toc_title: Operations +--- + +# Question About Operating ClickHouse Servers and Clusters {#question-about-operating-clickhouse-servers-and-clusters} + +Questions: + +- [Which ClickHouse version to use in production?](../../faq/operations/production.md) +- [Is it possible to delete old records from a ClickHouse table?](../../faq/operations/delete-old-data.md) +- [Does ClickHouse support multi-region replication?](../../faq/operations/multi-region-replication.md) + +!!! info "Don’t see what you were looking for?" + Check out [other F.A.Q. categories](../../faq/index.md) or browse around main documentation articles found in the left sidebar. + +{## [Original article](https://clickhouse.com/docs/en/faq/production/) ##} From a96dc47854e3ee290f0990cab44c0f6b390871b4 Mon Sep 17 00:00:00 2001 From: tesw yew isal <278153+cnmade@users.noreply.github.com> Date: Tue, 1 Feb 2022 13:21:25 +0800 Subject: [PATCH 071/149] Translate zh/faq/operations/index: translate zh --- docs/zh/faq/operations/index.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/zh/faq/operations/index.md b/docs/zh/faq/operations/index.md index 8e483febe14..cdf4b9622ec 100644 --- a/docs/zh/faq/operations/index.md +++ b/docs/zh/faq/operations/index.md @@ -1,19 +1,20 @@ --- -title: Question about operating ClickHouse servers and clusters +title: 关于操作ClickHouse服务器和集群的问题 toc_hidden_folder: true toc_priority: 3 toc_title: Operations --- -# Question About Operating ClickHouse Servers and Clusters {#question-about-operating-clickhouse-servers-and-clusters} +# 关于操作ClickHouse服务器和集群的问题 {#question-about-operating-clickhouse-servers-and-clusters} -Questions: +问题: -- [Which ClickHouse version to use in production?](../../faq/operations/production.md) -- [Is it possible to delete old records from a ClickHouse table?](../../faq/operations/delete-old-data.md) -- [Does ClickHouse support multi-region replication?](../../faq/operations/multi-region-replication.md) +- [如果想在生产环境部署，需要用哪个版本的 ClickHouse 呢?](../../faq/operations/production.md) +- [是否可能从 ClickHouse 数据表中删除所有旧的数据记录?](../../faq/operations/delete-old-data.md) +- [ClickHouse支持多区域复制吗?](../../faq/operations/multi-region-replication.md) + -!!! info "Don’t see what you were looking for?" - Check out [other F.A.Q. categories](../../faq/index.md) or browse around main documentation articles found in the left sidebar. +!!! info "没看到你要找的东西吗?" + 查看[其他faq类别](../../faq/index.md)或浏览左边栏中的主要文档文章。 -{## [Original article](https://clickhouse.com/docs/en/faq/production/) ##} +{## [原文](https://clickhouse.com/docs/en/faq/production/) ##} From 78ed031bf01724985e9b2671e43c5fdea11b5be5 Mon Sep 17 00:00:00 2001 From: tesw yew isal <278153+cnmade@users.noreply.github.com> Date: Tue, 1 Feb 2022 13:22:15 +0800 Subject: [PATCH 072/149] Translate zh/faq/operations/index: remove old files --- docs/zh/faq/operations/index.md.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 docs/zh/faq/operations/index.md.md diff --git a/docs/zh/faq/operations/index.md.md b/docs/zh/faq/operations/index.md.md deleted file mode 100644 index fd141164fdc..00000000000 --- a/docs/zh/faq/operations/index.md.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/faq/operations/index.md \ No newline at end of file From f4c0b644209ca921fa1d05d6fe553bfd24b72463 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Sun, 30 Jan 2022 23:25:55 +0100 Subject: [PATCH 073/149] Clean up: insert_deduplication_token setting for INSERT statement + reduce number of allocations on replication merge tree path + bash test: move insert block settings into variable Issue: ClickHouse#7461 --- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp | 7 ++++--- ...2124_insert_deduplication_token_multiple_blocks.sh | 11 ++++++----- ...ert_deduplication_token_multiple_blocks_replica.sh | 11 ++++++----- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 69424c046a0..057d4ddf8df 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1644,7 +1644,7 @@ UInt32 IMergeTreeDataPart::getNumberOfRefereneces() const } -String IMergeTreeDataPart::getZeroLevelPartBlockID(std::string_view token) const +String IMergeTreeDataPart::getZeroLevelPartBlockID(const std::string_view token) const { if (info.level != 0) throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to get block id for non zero level part {}", name); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 6db50724787..b8f0fc21da9 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -2555,7 +2555,7 @@ bool MergeTreeData::renameTempPartAndReplace( /// deduplication. if (deduplication_log) { - String block_id = part->getZeroLevelPartBlockID(deduplication_token); + const String block_id = part->getZeroLevelPartBlockID(deduplication_token); auto res = deduplication_log->addPart(block_id, part_info); if (!res.second) { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index c14672fe382..dfa7d818617 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -140,6 +140,7 @@ void ReplicatedMergeTreeSink::consume(Chunk chunk) checkQuorumPrecondition(zookeeper); auto part_blocks = storage.writer.splitBlockIntoParts(block, max_parts_per_block, metadata_snapshot, context); + String block_dedup_token; for (auto & current_block : part_blocks) { @@ -161,12 +162,12 @@ void ReplicatedMergeTreeSink::consume(Chunk chunk) /// We add the hash from the data and partition identifier to deduplication ID. /// That is, do not insert the same data to the same partition twice. - String block_dedup_token = context->getSettingsRef().insert_deduplication_token; - if (!block_dedup_token.empty()) + const String& dedup_token = context->getSettingsRef().insert_deduplication_token; + if (!dedup_token.empty()) { /// multiple blocks can be inserted within the same insert query /// an ordinal number is added to dedup token to generate a distinctive block id for each block - block_dedup_token += fmt::format("_{}", chunk_dedup_seqnum); + block_dedup_token = fmt::format("{}_{}", dedup_token, chunk_dedup_seqnum); ++chunk_dedup_seqnum; } block_id = part->getZeroLevelPartBlockID(block_dedup_token); diff --git a/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks.sh b/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks.sh index b5f44794c60..04ccbda6235 100755 --- a/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks.sh +++ b/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks.sh @@ -5,31 +5,32 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) QUERY_COUNT_ORIGIN_BLOCKS="SELECT COUNT(*) FROM system.parts WHERE database = currentDatabase() AND table = 'block_dedup_token' AND min_block_number == max_block_number;" QUERY_SELECT_FROM_TABLE_ORDERED="SELECT * FROM block_dedup_token ORDER BY id;" +INSERT_BLOCK_SETTINGS="max_insert_block_size=1&min_insert_block_size_rows=0&min_insert_block_size_bytes=0" $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS block_dedup_token SYNC" $CLICKHOUSE_CLIENT --query="CREATE TABLE block_dedup_token (id Int32) ENGINE=MergeTree() ORDER BY id SETTINGS non_replicated_deduplication_window=0xFFFFFFFF;" $CLICKHOUSE_CLIENT --query="SELECT 'insert 2 blocks with dedup token, 1 row per block'" DEDUP_TOKEN='dedup1' -echo 'INSERT INTO block_dedup_token VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&max_insert_block_size=1&min_insert_block_size_rows=0&min_insert_block_size_bytes=0&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- +echo 'INSERT INTO block_dedup_token VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- $CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED" $CLICKHOUSE_CLIENT --query="SELECT 'insert deduplicated by token'" -echo 'INSERT INTO block_dedup_token VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&max_insert_block_size=1&min_insert_block_size_rows=0&min_insert_block_size_bytes=0&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- +echo 'INSERT INTO block_dedup_token VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- $CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED" $CLICKHOUSE_CLIENT --query="SELECT 'insert the same data by providing different dedup token'" DEDUP_TOKEN='dedup2' -echo 'INSERT INTO block_dedup_token VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&max_insert_block_size=1&min_insert_block_size_rows=0&min_insert_block_size_bytes=0&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- +echo 'INSERT INTO block_dedup_token VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- $CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED" $CLICKHOUSE_CLIENT --query="SELECT 'insert 4 blocks, 2 deduplicated, 2 inserted'" -echo 'INSERT INTO block_dedup_token VALUES (1), (2), (3), (4)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&max_insert_block_size=1&min_insert_block_size_rows=0&min_insert_block_size_bytes=0&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- +echo 'INSERT INTO block_dedup_token VALUES (1), (2), (3), (4)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- $CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED" $CLICKHOUSE_CLIENT --query="SELECT 'disable token based deduplication, insert the same data as with token'" DEDUP_TOKEN='' -echo 'INSERT INTO block_dedup_token VALUES (1), (2), (3), (4)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&max_insert_block_size=1&min_insert_block_size_rows=0&min_insert_block_size_bytes=0&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- +echo 'INSERT INTO block_dedup_token VALUES (1), (2), (3), (4)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- $CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED" $CLICKHOUSE_CLIENT --query="DROP TABLE block_dedup_token SYNC" diff --git a/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks_replica.sh b/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks_replica.sh index 928defd329f..1c776263f78 100755 --- a/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks_replica.sh +++ b/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks_replica.sh @@ -5,31 +5,32 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) QUERY_COUNT_ORIGIN_BLOCKS="SELECT COUNT(*) FROM system.parts WHERE database = currentDatabase() AND table = 'block_dedup_token_replica' AND min_block_number == max_block_number;" QUERY_SELECT_FROM_TABLE_ORDERED="SELECT * FROM block_dedup_token_replica ORDER BY id;" +INSERT_BLOCK_SETTINGS="max_insert_block_size=1&min_insert_block_size_rows=0&min_insert_block_size_bytes=0" $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS block_dedup_token_replica SYNC" $CLICKHOUSE_CLIENT --query="CREATE TABLE block_dedup_token_replica (id Int32) ENGINE=ReplicatedMergeTree('/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/{table}', '{replica}') ORDER BY id" $CLICKHOUSE_CLIENT --query="SELECT 'insert 2 blocks with dedup token, 1 row per block'" DEDUP_TOKEN='dedup1' -echo 'INSERT INTO block_dedup_token_replica VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&max_insert_block_size=1&min_insert_block_size_rows=0&min_insert_block_size_bytes=0&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- +echo 'INSERT INTO block_dedup_token_replica VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- $CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED" $CLICKHOUSE_CLIENT --query="SELECT 'insert deduplicated by token'" -echo 'INSERT INTO block_dedup_token_replica VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&max_insert_block_size=1&min_insert_block_size_rows=0&min_insert_block_size_bytes=0&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- +echo 'INSERT INTO block_dedup_token_replica VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- $CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED" $CLICKHOUSE_CLIENT --query="SELECT 'insert the same data by providing different dedup token'" DEDUP_TOKEN='dedup2' -echo 'INSERT INTO block_dedup_token_replica VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&max_insert_block_size=1&min_insert_block_size_rows=0&min_insert_block_size_bytes=0&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- +echo 'INSERT INTO block_dedup_token_replica VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- $CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED" $CLICKHOUSE_CLIENT --query="SELECT 'insert 4 blocks, 2 deduplicated, 2 inserted'" -echo 'INSERT INTO block_dedup_token_replica VALUES (1), (2), (3), (4)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&max_insert_block_size=1&min_insert_block_size_rows=0&min_insert_block_size_bytes=0&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- +echo 'INSERT INTO block_dedup_token_replica VALUES (1), (2), (3), (4)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- $CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED" $CLICKHOUSE_CLIENT --query="SELECT 'disable token based deduplication, insert the same data as with token'" DEDUP_TOKEN='' -echo 'INSERT INTO block_dedup_token_replica VALUES (1), (2), (3), (4)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&max_insert_block_size=1&min_insert_block_size_rows=0&min_insert_block_size_bytes=0&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- +echo 'INSERT INTO block_dedup_token_replica VALUES (1), (2), (3), (4)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- $CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED" $CLICKHOUSE_CLIENT --query="DROP TABLE block_dedup_token_replica SYNC" From 06477c2a7e5111478113b30c7348979d913bb65d Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 1 Feb 2022 03:36:07 +0300 Subject: [PATCH 074/149] Update ReplicatedMergeTreeSink.cpp --- src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index dfa7d818617..837765e03d6 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -162,7 +162,7 @@ void ReplicatedMergeTreeSink::consume(Chunk chunk) /// We add the hash from the data and partition identifier to deduplication ID. /// That is, do not insert the same data to the same partition twice. - const String& dedup_token = context->getSettingsRef().insert_deduplication_token; + const String & dedup_token = context->getSettingsRef().insert_deduplication_token; if (!dedup_token.empty()) { /// multiple blocks can be inserted within the same insert query From 2a6eb593be62ae073b06008f6d6df01dd668112b Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 1 Feb 2022 13:36:51 +0300 Subject: [PATCH 075/149] Revert "Revert "Add pool to WriteBufferFromS3"" --- src/Disks/DiskCacheWrapper.cpp | 59 +++-- src/Disks/DiskDecorator.cpp | 5 + src/Disks/DiskDecorator.h | 1 + src/Disks/DiskRestartProxy.cpp | 6 + src/Disks/DiskRestartProxy.h | 1 + src/Disks/IDisk.h | 26 +++ src/Disks/IDiskRemote.cpp | 21 ++ src/Disks/IDiskRemote.h | 3 + src/Disks/S3/DiskS3.cpp | 5 +- src/IO/WriteBuffer.h | 19 +- src/IO/WriteBufferFromFileDecorator.cpp | 4 + src/IO/WriteBufferFromFileDecorator.h | 11 + src/IO/WriteBufferFromS3.cpp | 215 ++++++++++++++++-- src/IO/WriteBufferFromS3.h | 37 ++- src/Storages/MergeTree/DataPartsExchange.cpp | 6 +- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 33 ++- src/Storages/MergeTree/IMergeTreeDataPart.h | 6 +- .../MergeTree/IMergeTreeDataPartWriter.h | 4 +- src/Storages/MergeTree/MergeTask.cpp | 13 +- src/Storages/MergeTree/MergeTask.h | 1 + .../MergeTree/MergeTreeDataPartInMemory.cpp | 4 +- .../MergeTreeDataPartWriterCompact.cpp | 28 ++- .../MergeTreeDataPartWriterCompact.h | 6 +- .../MergeTreeDataPartWriterInMemory.cpp | 2 +- .../MergeTreeDataPartWriterInMemory.h | 3 +- .../MergeTreeDataPartWriterOnDisk.cpp | 38 +++- .../MergeTree/MergeTreeDataPartWriterOnDisk.h | 10 +- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 34 ++- .../MergeTree/MergeTreeDataPartWriterWide.h | 7 +- .../MergeTree/MergeTreeDataWriter.cpp | 57 +++-- src/Storages/MergeTree/MergeTreeDataWriter.h | 30 ++- src/Storages/MergeTree/MergeTreePartition.cpp | 11 +- src/Storages/MergeTree/MergeTreePartition.h | 6 +- src/Storages/MergeTree/MergeTreeSink.cpp | 73 +++++- src/Storages/MergeTree/MergeTreeSink.h | 20 +- .../MergeTree/MergeTreeWriteAheadLog.cpp | 8 +- .../MergeTree/MergedBlockOutputStream.cpp | 126 +++++++--- .../MergeTree/MergedBlockOutputStream.h | 32 ++- .../MergedColumnOnlyOutputStream.cpp | 12 +- .../MergeTree/MergedColumnOnlyOutputStream.h | 5 +- src/Storages/MergeTree/MutateTask.cpp | 22 +- .../MergeTree/ReplicatedMergeTreeSink.cpp | 77 ++++++- .../MergeTree/ReplicatedMergeTreeSink.h | 9 + src/Storages/StorageReplicatedMergeTree.cpp | 2 +- .../s3_endpoint/endpoint.py | 45 ++-- 45 files changed, 928 insertions(+), 215 deletions(-) diff --git a/src/Disks/DiskCacheWrapper.cpp b/src/Disks/DiskCacheWrapper.cpp index 46ea46f85ef..da27eff0b54 100644 --- a/src/Disks/DiskCacheWrapper.cpp +++ b/src/Disks/DiskCacheWrapper.cpp @@ -8,15 +8,22 @@ namespace DB { /** - * Write buffer with possibility to set and invoke callback after 'finalize' call. + * This buffer writes to cache, but after finalize() copy written file from cache to disk. */ -class CompletionAwareWriteBuffer : public WriteBufferFromFileDecorator +class WritingToCacheWriteBuffer final : public WriteBufferFromFileDecorator { public: - CompletionAwareWriteBuffer(std::unique_ptr impl_, std::function completion_callback_) - : WriteBufferFromFileDecorator(std::move(impl_)), completion_callback(completion_callback_) { } + WritingToCacheWriteBuffer( + std::unique_ptr impl_, + std::function()> create_read_buffer_, + std::function()> create_write_buffer_) + : WriteBufferFromFileDecorator(std::move(impl_)) + , create_read_buffer(std::move(create_read_buffer_)) + , create_write_buffer(std::move(create_write_buffer_)) + { + } - virtual ~CompletionAwareWriteBuffer() override + virtual ~WritingToCacheWriteBuffer() override { try { @@ -28,15 +35,36 @@ public: } } + void preFinalize() override + { + impl->next(); + impl->preFinalize(); + impl->finalize(); + + read_buffer = create_read_buffer(); + write_buffer = create_write_buffer(); + copyData(*read_buffer, *write_buffer); + write_buffer->next(); + write_buffer->preFinalize(); + + is_prefinalized = true; + } + void finalizeImpl() override { - WriteBufferFromFileDecorator::finalizeImpl(); + if (!is_prefinalized) + preFinalize(); - completion_callback(); + write_buffer->finalize(); } private: - const std::function completion_callback; + std::function()> create_read_buffer; + std::function()> create_write_buffer; + std::unique_ptr read_buffer; + std::unique_ptr write_buffer; + + bool is_prefinalized = false; }; enum FileDownloadStatus @@ -165,21 +193,22 @@ DiskCacheWrapper::writeFile(const String & path, size_t buf_size, WriteMode mode if (!cache_file_predicate(path)) return DiskDecorator::writeFile(path, buf_size, mode); - LOG_TRACE(log, "Write file {} to cache", backQuote(path)); + LOG_TEST(log, "Write file {} to cache", backQuote(path)); auto dir_path = directoryPath(path); if (!cache_disk->exists(dir_path)) cache_disk->createDirectories(dir_path); - return std::make_unique( + return std::make_unique( cache_disk->writeFile(path, buf_size, mode), - [this, path, buf_size, mode]() + [this, path]() { /// Copy file from cache to actual disk when cached buffer is finalized. - auto src_buffer = cache_disk->readFile(path, ReadSettings(), /* read_hint= */ {}, /* file_size= */ {}); - auto dst_buffer = DiskDecorator::writeFile(path, buf_size, mode); - copyData(*src_buffer, *dst_buffer); - dst_buffer->finalize(); + return cache_disk->readFile(path, ReadSettings(), /* read_hint= */ {}, /* file_size= */ {}); + }, + [this, path, buf_size, mode]() + { + return DiskDecorator::writeFile(path, buf_size, mode); }); } diff --git a/src/Disks/DiskDecorator.cpp b/src/Disks/DiskDecorator.cpp index d4acb6fab0d..37911f16913 100644 --- a/src/Disks/DiskDecorator.cpp +++ b/src/Disks/DiskDecorator.cpp @@ -151,6 +151,11 @@ void DiskDecorator::removeSharedFile(const String & path, bool keep_s3) delegate->removeSharedFile(path, keep_s3); } +void DiskDecorator::removeSharedFiles(const RemoveBatchRequest & files, bool keep_in_remote_fs) +{ + delegate->removeSharedFiles(files, keep_in_remote_fs); +} + void DiskDecorator::removeSharedRecursive(const String & path, bool keep_s3) { delegate->removeSharedRecursive(path, keep_s3); diff --git a/src/Disks/DiskDecorator.h b/src/Disks/DiskDecorator.h index ff4f16fdf3d..0bdfffa8f01 100644 --- a/src/Disks/DiskDecorator.h +++ b/src/Disks/DiskDecorator.h @@ -52,6 +52,7 @@ public: void removeRecursive(const String & path) override; void removeSharedFile(const String & path, bool keep_s3) override; void removeSharedRecursive(const String & path, bool keep_s3) override; + void removeSharedFiles(const RemoveBatchRequest & files, bool keep_in_remote_fs) override; void setLastModified(const String & path, const Poco::Timestamp & timestamp) override; Poco::Timestamp getLastModified(const String & path) override; void setReadOnly(const String & path) override; diff --git a/src/Disks/DiskRestartProxy.cpp b/src/Disks/DiskRestartProxy.cpp index 9bd59513040..fe9dd8421b1 100644 --- a/src/Disks/DiskRestartProxy.cpp +++ b/src/Disks/DiskRestartProxy.cpp @@ -234,6 +234,12 @@ void DiskRestartProxy::removeSharedFile(const String & path, bool keep_s3) DiskDecorator::removeSharedFile(path, keep_s3); } +void DiskRestartProxy::removeSharedFiles(const RemoveBatchRequest & files, bool keep_in_remote_fs) +{ + ReadLock lock (mutex); + DiskDecorator::removeSharedFiles(files, keep_in_remote_fs); +} + void DiskRestartProxy::removeSharedRecursive(const String & path, bool keep_s3) { ReadLock lock (mutex); diff --git a/src/Disks/DiskRestartProxy.h b/src/Disks/DiskRestartProxy.h index 3644539e941..30f553f4fe0 100644 --- a/src/Disks/DiskRestartProxy.h +++ b/src/Disks/DiskRestartProxy.h @@ -54,6 +54,7 @@ public: void removeDirectory(const String & path) override; void removeRecursive(const String & path) override; void removeSharedFile(const String & path, bool keep_s3) override; + void removeSharedFiles(const RemoveBatchRequest & files, bool keep_in_remote_fs) override; void removeSharedRecursive(const String & path, bool keep_s3) override; void setLastModified(const String & path, const Poco::Timestamp & timestamp) override; Poco::Timestamp getLastModified(const String & path) override; diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index e3e95e008b5..223d2d48e30 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -197,6 +197,32 @@ public: /// Second bool param is a flag to remove (true) or keep (false) shared data on S3 virtual void removeSharedFileIfExists(const String & path, bool) { removeFileIfExists(path); } + struct RemoveRequest + { + String path; + bool if_exists = false; + + explicit RemoveRequest(String path_, bool if_exists_ = false) + : path(std::move(path_)), if_exists(std::move(if_exists_)) + { + } + }; + + using RemoveBatchRequest = std::vector; + + /// Batch request to remove multiple files. + /// May be much faster for blob storage. + virtual void removeSharedFiles(const RemoveBatchRequest & files, bool keep_in_remote_fs) + { + for (const auto & file : files) + { + if (file.if_exists) + removeSharedFileIfExists(file.path, keep_in_remote_fs); + else + removeSharedFile(file.path, keep_in_remote_fs); + } + } + /// Set last modified time to file or directory at `path`. virtual void setLastModified(const String & path, const Poco::Timestamp & timestamp) = 0; diff --git a/src/Disks/IDiskRemote.cpp b/src/Disks/IDiskRemote.cpp index 706f0f84f32..05aa4d3350b 100644 --- a/src/Disks/IDiskRemote.cpp +++ b/src/Disks/IDiskRemote.cpp @@ -361,6 +361,19 @@ void IDiskRemote::removeSharedFileIfExists(const String & path, bool keep_in_rem } } +void IDiskRemote::removeSharedFiles(const RemoveBatchRequest & files, bool keep_in_remote_fs) +{ + RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper(); + for (const auto & file : files) + { + bool skip = file.if_exists && !metadata_disk->exists(file.path); + if (!skip) + removeMeta(file.path, fs_paths_keeper); + } + + if (!keep_in_remote_fs) + removeFromRemoteFS(fs_paths_keeper); +} void IDiskRemote::removeSharedRecursive(const String & path, bool keep_in_remote_fs) { @@ -531,4 +544,12 @@ UInt32 IDiskRemote::getRefCount(const String & path) const return meta.ref_count; } +ThreadPool & IDiskRemote::getThreadPoolWriter() +{ + constexpr size_t pool_size = 100; + constexpr size_t queue_size = 1000000; + static ThreadPool writer(pool_size, pool_size, queue_size); + return writer; +} + } diff --git a/src/Disks/IDiskRemote.h b/src/Disks/IDiskRemote.h index c4f475f5b3e..8bb93cc345d 100644 --- a/src/Disks/IDiskRemote.h +++ b/src/Disks/IDiskRemote.h @@ -98,6 +98,8 @@ public: void removeSharedFileIfExists(const String & path, bool keep_in_remote_fs) override; + void removeSharedFiles(const RemoveBatchRequest & files, bool keep_in_remote_fs) override; + void removeSharedRecursive(const String & path, bool keep_in_remote_fs) override; void listFiles(const String & path, std::vector & file_names) override; @@ -135,6 +137,7 @@ public: virtual RemoteFSPathKeeperPtr createFSPathKeeper() const = 0; static AsynchronousReaderPtr getThreadPoolReader(); + static ThreadPool & getThreadPoolWriter(); virtual std::unique_ptr readMetaFile( const String & path, diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index ed960528abe..7c39148c38c 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -262,6 +262,8 @@ std::unique_ptr DiskS3::writeFile(const String & path, LOG_TRACE(log, "{} to file by path: {}. S3 path: {}", mode == WriteMode::Rewrite ? "Write" : "Append", backQuote(metadata_disk->getPath() + path), remote_fs_root_path + s3_path); + ThreadPool * writer = &getThreadPoolWriter(); + auto s3_buffer = std::make_unique( settings->client, bucket, @@ -269,7 +271,8 @@ std::unique_ptr DiskS3::writeFile(const String & path, settings->s3_min_upload_part_size, settings->s3_max_single_part_upload_size, std::move(object_metadata), - buf_size); + buf_size, + writer); return std::make_unique>(std::move(s3_buffer), std::move(metadata), s3_path); } diff --git a/src/IO/WriteBuffer.h b/src/IO/WriteBuffer.h index 9440ac0a855..bd2a628355e 100644 --- a/src/IO/WriteBuffer.h +++ b/src/IO/WriteBuffer.h @@ -104,10 +104,14 @@ public: ++pos; } - virtual void sync() - { - next(); - } + /// This method may be called before finalize() to tell there would not be any more data written. + /// Used does not have to call it, implementation should check it itself if needed. + /// + /// The idea is similar to prefetch. In case if all data is written, we can flush the buffer + /// and start sending data asynchronously. It may improve writing performance in case you have + /// multiple files to finalize. Mainly, for blob storage, finalization has high latency, + /// and calling preFinalize in a loop may parallelize it. + virtual void preFinalize() { next(); } /// Write the last data. void finalize() @@ -130,6 +134,13 @@ public: } } + /// Wait for data to be reliably written. Mainly, call fsync for fd. + /// May be called after finalize() if needed. + virtual void sync() + { + next(); + } + protected: virtual void finalizeImpl() { diff --git a/src/IO/WriteBufferFromFileDecorator.cpp b/src/IO/WriteBufferFromFileDecorator.cpp index 56d789ec23e..ac801534b4f 100644 --- a/src/IO/WriteBufferFromFileDecorator.cpp +++ b/src/IO/WriteBufferFromFileDecorator.cpp @@ -14,6 +14,10 @@ WriteBufferFromFileDecorator::WriteBufferFromFileDecorator(std::unique_ptrfinalize(); } diff --git a/src/IO/WriteBufferFromFileDecorator.h b/src/IO/WriteBufferFromFileDecorator.h index 18fdb8168f9..dde05276c28 100644 --- a/src/IO/WriteBufferFromFileDecorator.h +++ b/src/IO/WriteBufferFromFileDecorator.h @@ -17,6 +17,15 @@ public: std::string getFileName() const override; + void preFinalize() override + { + next(); + impl->preFinalize(); + is_prefinalized = true; + } + + const WriteBuffer & getImpl() const { return *impl; } + protected: void finalizeImpl() override; @@ -24,6 +33,8 @@ protected: private: void nextImpl() override; + + bool is_prefinalized = false; }; } diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 09193c4c396..6cb44fb5e52 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -11,6 +11,7 @@ # include # include # include +# include # include @@ -34,6 +35,20 @@ namespace ErrorCodes extern const int S3_ERROR; } +struct WriteBufferFromS3::UploadPartTask +{ + Aws::S3::Model::UploadPartRequest req; + bool is_finised = false; + std::string tag; + std::exception_ptr exception; +}; + +struct WriteBufferFromS3::PutObjectTask +{ + Aws::S3::Model::PutObjectRequest req; + bool is_finised = false; + std::exception_ptr exception; +}; WriteBufferFromS3::WriteBufferFromS3( std::shared_ptr client_ptr_, @@ -42,7 +57,8 @@ WriteBufferFromS3::WriteBufferFromS3( size_t minimum_upload_part_size_, size_t max_single_part_upload_size_, std::optional> object_metadata_, - size_t buffer_size_) + size_t buffer_size_, + ThreadPool * thread_pool_) : BufferWithOwnMemory(buffer_size_, nullptr, 0) , bucket(bucket_) , key(key_) @@ -50,6 +66,7 @@ WriteBufferFromS3::WriteBufferFromS3( , client_ptr(std::move(client_ptr_)) , minimum_upload_part_size(minimum_upload_part_size_) , max_single_part_upload_size(max_single_part_upload_size_) + , thread_pool(thread_pool_) { allocateBuffer(); } @@ -74,6 +91,8 @@ void WriteBufferFromS3::nextImpl() writePart(); allocateBuffer(); } + + waitForReadyBackGroundTasks(); } void WriteBufferFromS3::allocateBuffer() @@ -88,7 +107,7 @@ WriteBufferFromS3::~WriteBufferFromS3() finalize(); } -void WriteBufferFromS3::finalizeImpl() +void WriteBufferFromS3::preFinalize() { next(); @@ -100,8 +119,20 @@ void WriteBufferFromS3::finalizeImpl() { /// Write rest of the data as last part. writePart(); - completeMultipartUpload(); } + + is_prefinalized = true; +} + +void WriteBufferFromS3::finalizeImpl() +{ + if (!is_prefinalized) + preFinalize(); + + waitForAllBackGroundTasks(); + + if (!multipart_upload_id.empty()) + completeMultipartUpload(); } void WriteBufferFromS3::createMultipartUpload() @@ -144,22 +175,73 @@ void WriteBufferFromS3::writePart() LOG_WARNING(log, "Maximum part number in S3 protocol has reached (too many parts). Server may not accept this whole upload."); } - Aws::S3::Model::UploadPartRequest req; + if (thread_pool) + { + UploadPartTask * task = nullptr; + int part_number; + { + std::lock_guard lock(bg_tasks_mutex); + task = &upload_object_tasks.emplace_back(); + ++num_added_bg_tasks; + part_number = num_added_bg_tasks; + } + fillUploadRequest(task->req, part_number); + thread_pool->scheduleOrThrow([this, task, thread_group = CurrentThread::getGroup()]() + { + if (thread_group) + CurrentThread::attachTo(thread_group); + + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); + + try + { + processUploadRequest(*task); + } + catch (...) + { + task->exception = std::current_exception(); + } + + { + std::lock_guard lock(bg_tasks_mutex); + task->is_finised = true; + ++num_finished_bg_tasks; + } + + bg_tasks_condvar.notify_one(); + }); + } + else + { + UploadPartTask task; + fillUploadRequest(task.req, part_tags.size() + 1); + processUploadRequest(task); + part_tags.push_back(task.tag); + } +} + +void WriteBufferFromS3::fillUploadRequest(Aws::S3::Model::UploadPartRequest & req, int part_number) +{ req.SetBucket(bucket); req.SetKey(key); - req.SetPartNumber(part_tags.size() + 1); + req.SetPartNumber(part_number); req.SetUploadId(multipart_upload_id); - req.SetContentLength(size); + req.SetContentLength(temporary_buffer->tellp()); req.SetBody(temporary_buffer); +} - auto outcome = client_ptr->UploadPart(req); +void WriteBufferFromS3::processUploadRequest(UploadPartTask & task) +{ + auto outcome = client_ptr->UploadPart(task.req); if (outcome.IsSuccess()) { - auto etag = outcome.GetResult().GetETag(); - part_tags.push_back(etag); - LOG_DEBUG(log, "Writing part finished. Bucket: {}, Key: {}, Upload_id: {}, Etag: {}, Parts: {}", bucket, key, multipart_upload_id, etag, part_tags.size()); + task.tag = outcome.GetResult().GetETag(); + LOG_DEBUG(log, "Writing part finished. Bucket: {}, Key: {}, Upload_id: {}, Etag: {}, Parts: {}", bucket, key, multipart_upload_id, task.tag, part_tags.size()); } else throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); @@ -191,14 +273,19 @@ void WriteBufferFromS3::completeMultipartUpload() if (outcome.IsSuccess()) LOG_DEBUG(log, "Multipart upload has completed. Bucket: {}, Key: {}, Upload_id: {}, Parts: {}", bucket, key, multipart_upload_id, part_tags.size()); else - throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); + { + throw Exception(ErrorCodes::S3_ERROR, "{} Tags:{}", + outcome.GetError().GetMessage(), + fmt::join(part_tags.begin(), part_tags.end(), " ")); + } } void WriteBufferFromS3::makeSinglepartUpload() { auto size = temporary_buffer->tellp(); + bool with_pool = thread_pool != nullptr; - LOG_DEBUG(log, "Making single part upload. Bucket: {}, Key: {}, Size: {}", bucket, key, size); + LOG_DEBUG(log, "Making single part upload. Bucket: {}, Key: {}, Size: {}, WithPool: {}", bucket, key, size, with_pool); if (size < 0) throw Exception("Failed to make single part upload. Buffer in invalid state", ErrorCodes::S3_ERROR); @@ -209,22 +296,118 @@ void WriteBufferFromS3::makeSinglepartUpload() return; } - Aws::S3::Model::PutObjectRequest req; + if (thread_pool) + { + put_object_task = std::make_unique(); + fillPutRequest(put_object_task->req); + thread_pool->scheduleOrThrow([this, thread_group = CurrentThread::getGroup()]() + { + if (thread_group) + CurrentThread::attachTo(thread_group); + + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); + + try + { + processPutRequest(*put_object_task); + } + catch (...) + { + put_object_task->exception = std::current_exception(); + } + + { + std::lock_guard lock(bg_tasks_mutex); + put_object_task->is_finised = true; + } + + bg_tasks_condvar.notify_one(); + }); + } + else + { + PutObjectTask task; + fillPutRequest(task.req); + processPutRequest(task); + } +} + +void WriteBufferFromS3::fillPutRequest(Aws::S3::Model::PutObjectRequest & req) +{ req.SetBucket(bucket); req.SetKey(key); - req.SetContentLength(size); + req.SetContentLength(temporary_buffer->tellp()); req.SetBody(temporary_buffer); if (object_metadata.has_value()) req.SetMetadata(object_metadata.value()); +} - auto outcome = client_ptr->PutObject(req); +void WriteBufferFromS3::processPutRequest(PutObjectTask & task) +{ + auto outcome = client_ptr->PutObject(task.req); + bool with_pool = thread_pool != nullptr; if (outcome.IsSuccess()) - LOG_DEBUG(log, "Single part upload has completed. Bucket: {}, Key: {}, Object size: {}", bucket, key, req.GetContentLength()); + LOG_DEBUG(log, "Single part upload has completed. Bucket: {}, Key: {}, Object size: {}, WithPool: {}", bucket, key, task.req.GetContentLength(), with_pool); else throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); } +void WriteBufferFromS3::waitForReadyBackGroundTasks() +{ + if (thread_pool) + { + std::lock_guard lock(bg_tasks_mutex); + { + while (!upload_object_tasks.empty() && upload_object_tasks.front().is_finised) + { + auto & task = upload_object_tasks.front(); + auto exception = std::move(task.exception); + auto tag = std::move(task.tag); + upload_object_tasks.pop_front(); + + if (exception) + { + waitForAllBackGroundTasks(); + std::rethrow_exception(exception); + } + + part_tags.push_back(tag); + } + } + } +} + +void WriteBufferFromS3::waitForAllBackGroundTasks() +{ + if (thread_pool) + { + std::unique_lock lock(bg_tasks_mutex); + bg_tasks_condvar.wait(lock, [this]() { return num_added_bg_tasks == num_finished_bg_tasks; }); + + while (!upload_object_tasks.empty()) + { + auto & task = upload_object_tasks.front(); + if (task.exception) + std::rethrow_exception(std::move(task.exception)); + + part_tags.push_back(task.tag); + + upload_object_tasks.pop_front(); + } + + if (put_object_task) + { + bg_tasks_condvar.wait(lock, [this]() { return put_object_task->is_finised; }); + if (put_object_task->exception) + std::rethrow_exception(std::move(put_object_task->exception)); + } + } +} + } #endif diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index e4633282061..aaea718fe70 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -14,11 +14,19 @@ # include +# include + namespace Aws::S3 { class S3Client; } +namespace Aws::S3::Model +{ + class UploadPartRequest; + class PutObjectRequest; +} + namespace DB { @@ -29,7 +37,7 @@ namespace DB * Data is divided on chunks with size greater than 'minimum_upload_part_size'. Last chunk can be less than this threshold. * Each chunk is written as a part to S3. */ -class WriteBufferFromS3 : public BufferWithOwnMemory +class WriteBufferFromS3 final : public BufferWithOwnMemory { public: explicit WriteBufferFromS3( @@ -39,12 +47,15 @@ public: size_t minimum_upload_part_size_, size_t max_single_part_upload_size_, std::optional> object_metadata_ = std::nullopt, - size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE); + size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE, + ThreadPool * thread_pool_ = nullptr); ~WriteBufferFromS3() override; void nextImpl() override; + void preFinalize() override; + private: void allocateBuffer(); @@ -57,6 +68,17 @@ private: /// Receives response from the server after sending all data. void finalizeImpl() override; + struct UploadPartTask; + void fillUploadRequest(Aws::S3::Model::UploadPartRequest & req, int part_number); + void processUploadRequest(UploadPartTask & task); + + struct PutObjectTask; + void fillPutRequest(Aws::S3::Model::PutObjectRequest & req); + void processPutRequest(PutObjectTask & task); + + void waitForReadyBackGroundTasks(); + void waitForAllBackGroundTasks(); + String bucket; String key; std::optional> object_metadata; @@ -72,6 +94,17 @@ private: String multipart_upload_id; std::vector part_tags; + bool is_prefinalized = false; + + /// Following fields are for background uploads in thread pool (if specified). + ThreadPool * thread_pool; + std::unique_ptr put_object_task; + std::list upload_object_tasks; + size_t num_added_bg_tasks = 0; + size_t num_finished_bg_tasks = 0; + std::mutex bg_tasks_mutex; + std::condition_variable bg_tasks_condvar; + Poco::Logger * log = &Poco::Logger::get("WriteBufferFromS3"); }; diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 37167038f63..3d846c27fcc 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -300,7 +300,7 @@ MergeTreeData::DataPart::Checksums Service::sendPartFromDisk( throw Exception("Transferring part to replica was cancelled", ErrorCodes::ABORTED); if (hashing_out.count() != size) - throw Exception("Unexpected size of file " + path, ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + throw Exception(ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART, "Unexpected size of file {}, expected {} got {}", path, hashing_out.count(), size); writePODBinary(hashing_out.getHash(), out); @@ -595,7 +595,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToMemory( CompressionCodecFactory::instance().get("NONE", {})); part_out.write(block); - part_out.writeSuffixAndFinalizePart(new_projection_part); + part_out.finalizePart(new_projection_part, false); new_projection_part->checksums.checkEqual(checksums, /* have_uncompressed = */ true); new_data_part->addProjectionPart(projection_name, std::move(new_projection_part)); } @@ -619,7 +619,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToMemory( CompressionCodecFactory::instance().get("NONE", {})); part_out.write(block); - part_out.writeSuffixAndFinalizePart(new_data_part); + part_out.finalizePart(new_data_part, false); new_data_part->checksums.checkEqual(checksums, /* have_uncompressed = */ true); return new_data_part; diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 69424c046a0..73169b634cc 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -99,7 +99,7 @@ void IMergeTreeDataPart::MinMaxIndex::load(const MergeTreeData & data, const Dis initialized = true; } -void IMergeTreeDataPart::MinMaxIndex::store( +IMergeTreeDataPart::MinMaxIndex::WrittenFiles IMergeTreeDataPart::MinMaxIndex::store( const MergeTreeData & data, const DiskPtr & disk_, const String & part_path, Checksums & out_checksums) const { auto metadata_snapshot = data.getInMemoryMetadataPtr(); @@ -108,10 +108,10 @@ void IMergeTreeDataPart::MinMaxIndex::store( auto minmax_column_names = data.getMinMaxColumnsNames(partition_key); auto minmax_column_types = data.getMinMaxColumnsTypes(partition_key); - store(minmax_column_names, minmax_column_types, disk_, part_path, out_checksums); + return store(minmax_column_names, minmax_column_types, disk_, part_path, out_checksums); } -void IMergeTreeDataPart::MinMaxIndex::store( +IMergeTreeDataPart::MinMaxIndex::WrittenFiles IMergeTreeDataPart::MinMaxIndex::store( const Names & column_names, const DataTypes & data_types, const DiskPtr & disk_, @@ -122,6 +122,8 @@ void IMergeTreeDataPart::MinMaxIndex::store( throw Exception("Attempt to store uninitialized MinMax index for part " + part_path + ". This is a bug.", ErrorCodes::LOGICAL_ERROR); + WrittenFiles written_files; + for (size_t i = 0; i < column_names.size(); ++i) { String file_name = "minmax_" + escapeForFileName(column_names[i]) + ".idx"; @@ -134,8 +136,11 @@ void IMergeTreeDataPart::MinMaxIndex::store( out_hashing.next(); out_checksums.files[file_name].file_size = out_hashing.count(); out_checksums.files[file_name].file_hash = out_hashing.getHash(); - out->finalize(); + out->preFinalize(); + written_files.emplace_back(std::move(out)); } + + return written_files; } void IMergeTreeDataPart::MinMaxIndex::update(const Block & block, const Names & column_names) @@ -1280,6 +1285,7 @@ void IMergeTreeDataPart::remove() const try { /// Remove each expected file in directory, then remove directory itself. + IDisk::RemoveBatchRequest request; #if !defined(__clang__) # pragma GCC diagnostic push @@ -1288,18 +1294,19 @@ void IMergeTreeDataPart::remove() const for (const auto & [file, _] : checksums.files) { if (projection_directories.find(file) == projection_directories.end()) - disk->removeSharedFile(fs::path(to) / file, *keep_shared_data); + request.emplace_back(fs::path(to) / file); } #if !defined(__clang__) # pragma GCC diagnostic pop #endif for (const auto & file : {"checksums.txt", "columns.txt"}) - disk->removeSharedFile(fs::path(to) / file, *keep_shared_data); + request.emplace_back(fs::path(to) / file); - disk->removeSharedFileIfExists(fs::path(to) / DEFAULT_COMPRESSION_CODEC_FILE_NAME, *keep_shared_data); - disk->removeSharedFileIfExists(fs::path(to) / DELETE_ON_DESTROY_MARKER_FILE_NAME, *keep_shared_data); + request.emplace_back(fs::path(to) / DEFAULT_COMPRESSION_CODEC_FILE_NAME, true); + request.emplace_back(fs::path(to) / DELETE_ON_DESTROY_MARKER_FILE_NAME, true); + disk->removeSharedFiles(request, *keep_shared_data); disk->removeDirectory(to); } catch (...) @@ -1333,22 +1340,24 @@ void IMergeTreeDataPart::projectionRemove(const String & parent_to, bool keep_sh try { /// Remove each expected file in directory, then remove directory itself. + IDisk::RemoveBatchRequest request; #if !defined(__clang__) # pragma GCC diagnostic push # pragma GCC diagnostic ignored "-Wunused-variable" #endif for (const auto & [file, _] : checksums.files) - disk->removeSharedFile(fs::path(to) / file, keep_shared_data); + request.emplace_back(fs::path(to) / file); #if !defined(__clang__) # pragma GCC diagnostic pop #endif for (const auto & file : {"checksums.txt", "columns.txt"}) - disk->removeSharedFile(fs::path(to) / file, keep_shared_data); - disk->removeSharedFileIfExists(fs::path(to) / DEFAULT_COMPRESSION_CODEC_FILE_NAME, keep_shared_data); - disk->removeSharedFileIfExists(fs::path(to) / DELETE_ON_DESTROY_MARKER_FILE_NAME, keep_shared_data); + request.emplace_back(fs::path(to) / file); + request.emplace_back(fs::path(to) / DEFAULT_COMPRESSION_CODEC_FILE_NAME, true); + request.emplace_back(fs::path(to) / DELETE_ON_DESTROY_MARKER_FILE_NAME, true); + disk->removeSharedFiles(request, keep_shared_data); disk->removeSharedRecursive(to, keep_shared_data); } catch (...) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index c96cad4b039..b6e3b1c0e9d 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -300,9 +300,11 @@ public: { } + using WrittenFiles = std::vector>; + void load(const MergeTreeData & data, const DiskPtr & disk_, const String & part_path); - void store(const MergeTreeData & data, const DiskPtr & disk_, const String & part_path, Checksums & checksums) const; - void store(const Names & column_names, const DataTypes & data_types, const DiskPtr & disk_, const String & part_path, Checksums & checksums) const; + [[nodiscard]] WrittenFiles store(const MergeTreeData & data, const DiskPtr & disk_, const String & part_path, Checksums & checksums) const; + [[nodiscard]] WrittenFiles store(const Names & column_names, const DataTypes & data_types, const DiskPtr & disk_, const String & part_path, Checksums & checksums) const; void update(const Block & block, const Names & column_names); void merge(const MinMaxIndex & other); diff --git a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index d0d3f283478..34c53eda846 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -32,7 +32,9 @@ public: virtual void write(const Block & block, const IColumn::Permutation * permutation) = 0; - virtual void finish(IMergeTreeDataPart::Checksums & checksums, bool sync) = 0; + virtual void fillChecksums(IMergeTreeDataPart::Checksums & checksums) = 0; + + virtual void finish(bool sync) = 0; Columns releaseIndexColumns(); const MergeTreeIndexGranularity & getIndexGranularity() const { return index_granularity; } diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 7d4b731551c..5d5035d62ca 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -478,9 +478,11 @@ void MergeTask::VerticalMergeStage::finalizeVerticalMergeForOneColumn() const throw Exception("Cancelled merging parts", ErrorCodes::ABORTED); ctx->executor.reset(); - auto changed_checksums = ctx->column_to->writeSuffixAndGetChecksums(global_ctx->new_data_part, global_ctx->checksums_gathered_columns, ctx->need_sync); + auto changed_checksums = ctx->column_to->fillChecksums(global_ctx->new_data_part, global_ctx->checksums_gathered_columns); global_ctx->checksums_gathered_columns.add(std::move(changed_checksums)); + ctx->delayed_streams.emplace_back(std::move(ctx->column_to)); + if (global_ctx->rows_written != ctx->column_elems_written) { throw Exception("Written " + toString(ctx->column_elems_written) + " elements of column " + column_name + @@ -505,9 +507,8 @@ void MergeTask::VerticalMergeStage::finalizeVerticalMergeForOneColumn() const bool MergeTask::VerticalMergeStage::finalizeVerticalMergeForAllColumns() const { - /// No need to execute this part if it is horizontal merge. - if (global_ctx->chosen_merge_algorithm != MergeAlgorithm::Vertical) - return false; + for (auto & stream : ctx->delayed_streams) + stream->finish(ctx->need_sync); return false; } @@ -633,9 +634,9 @@ bool MergeTask::MergeProjectionsStage::finalizeProjectionsAndWholeMerge() const } if (global_ctx->chosen_merge_algorithm != MergeAlgorithm::Vertical) - global_ctx->to->writeSuffixAndFinalizePart(global_ctx->new_data_part, ctx->need_sync); + global_ctx->to->finalizePart(global_ctx->new_data_part, ctx->need_sync); else - global_ctx->to->writeSuffixAndFinalizePart( + global_ctx->to->finalizePart( global_ctx->new_data_part, ctx->need_sync, &global_ctx->storage_columns, &global_ctx->checksums_gathered_columns); global_ctx->promise.set_value(global_ctx->new_data_part); diff --git a/src/Storages/MergeTree/MergeTask.h b/src/Storages/MergeTree/MergeTask.h index d8e48c1794a..aa64c4c2265 100644 --- a/src/Storages/MergeTree/MergeTask.h +++ b/src/Storages/MergeTree/MergeTask.h @@ -268,6 +268,7 @@ private: Float64 progress_before = 0; std::unique_ptr column_to{nullptr}; + std::vector> delayed_streams; size_t column_elems_written{0}; QueryPipeline column_parts_pipeline; std::unique_ptr executor; diff --git a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp index 4ec53d88339..e4a174a7d29 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp @@ -125,12 +125,12 @@ void MergeTreeDataPartInMemory::flushToDisk(const String & base_path, const Stri projection_compression_codec); projection_out.write(projection_part->block); - projection_out.writeSuffixAndFinalizePart(projection_data_part); + projection_out.finalizePart(projection_data_part, false); new_data_part->addProjectionPart(projection_name, std::move(projection_data_part)); } } - out.writeSuffixAndFinalizePart(new_data_part); + out.finalizePart(new_data_part, false); } void MergeTreeDataPartInMemory::makeCloneInDetached(const String & prefix, const StorageMetadataPtr & metadata_snapshot) const diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index ce85bc75c80..d7b8f2c4165 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -218,7 +218,7 @@ void MergeTreeDataPartWriterCompact::writeDataBlock(const Block & block, const G } } -void MergeTreeDataPartWriterCompact::finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync) +void MergeTreeDataPartWriterCompact::fillDataChecksums(IMergeTreeDataPart::Checksums & checksums) { if (columns_buffer.size() != 0) { @@ -253,6 +253,12 @@ void MergeTreeDataPartWriterCompact::finishDataSerialization(IMergeTreeDataPart: marks.next(); addToChecksums(checksums); + plain_file->preFinalize(); + marks_file->preFinalize(); +} + +void MergeTreeDataPartWriterCompact::finishDataSerialization(bool sync) +{ plain_file->finalize(); marks_file->finalize(); if (sync) @@ -356,16 +362,28 @@ size_t MergeTreeDataPartWriterCompact::ColumnsBuffer::size() const return accumulated_columns.at(0)->size(); } -void MergeTreeDataPartWriterCompact::finish(IMergeTreeDataPart::Checksums & checksums, bool sync) +void MergeTreeDataPartWriterCompact::fillChecksums(IMergeTreeDataPart::Checksums & checksums) { // If we don't have anything to write, skip finalization. if (!columns_list.empty()) - finishDataSerialization(checksums, sync); + fillDataChecksums(checksums); if (settings.rewrite_primary_key) - finishPrimaryIndexSerialization(checksums, sync); + fillPrimaryIndexChecksums(checksums); - finishSkipIndicesSerialization(checksums, sync); + fillSkipIndicesChecksums(checksums); +} + +void MergeTreeDataPartWriterCompact::finish(bool sync) +{ + // If we don't have anything to write, skip finalization. + if (!columns_list.empty()) + finishDataSerialization(sync); + + if (settings.rewrite_primary_key) + finishPrimaryIndexSerialization(sync); + + finishSkipIndicesSerialization(sync); } } diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index 8b86a9701c9..cc33d8404c2 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -20,11 +20,13 @@ public: void write(const Block & block, const IColumn::Permutation * permutation) override; - void finish(IMergeTreeDataPart::Checksums & checksums, bool sync) override; + void fillChecksums(IMergeTreeDataPart::Checksums & checksums) override; + void finish(bool sync) override; private: /// Finish serialization of the data. Flush rows in buffer to disk, compute checksums. - void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync); + void fillDataChecksums(IMergeTreeDataPart::Checksums & checksums); + void finishDataSerialization(bool sync); void fillIndexGranularity(size_t index_granularity_for_block, size_t rows_in_block) override; diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterInMemory.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterInMemory.cpp index 5efcc63b2fc..0c715a7c27f 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterInMemory.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterInMemory.cpp @@ -76,7 +76,7 @@ void MergeTreeDataPartWriterInMemory::calculateAndSerializePrimaryIndex(const Bl } } -void MergeTreeDataPartWriterInMemory::finish(IMergeTreeDataPart::Checksums & checksums, bool /* sync */) +void MergeTreeDataPartWriterInMemory::fillChecksums(IMergeTreeDataPart::Checksums & checksums) { /// If part is empty we still need to initialize block by empty columns. if (!part_in_memory->block) diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterInMemory.h b/src/Storages/MergeTree/MergeTreeDataPartWriterInMemory.h index 495585ad4d2..233ca81a697 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterInMemory.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterInMemory.h @@ -18,7 +18,8 @@ public: /// You can write only one block. In-memory part can be written only at INSERT. void write(const Block & block, const IColumn::Permutation * permutation) override; - void finish(IMergeTreeDataPart::Checksums & checksums, bool sync) override; + void fillChecksums(IMergeTreeDataPart::Checksums & checksums) override; + void finish(bool /*sync*/) override {} private: void calculateAndSerializePrimaryIndex(const Block & primary_index_block); diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp index 2e299bb2447..8dca93f574f 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp @@ -2,6 +2,7 @@ #include #include +#include "IO/WriteBufferFromFileDecorator.h" namespace DB { @@ -10,13 +11,24 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -void MergeTreeDataPartWriterOnDisk::Stream::finalize() +void MergeTreeDataPartWriterOnDisk::Stream::preFinalize() { compressed.next(); /// 'compressed_buf' doesn't call next() on underlying buffer ('plain_hashing'). We should do it manually. plain_hashing.next(); marks.next(); + plain_file->preFinalize(); + marks_file->preFinalize(); + + is_prefinalized = true; +} + +void MergeTreeDataPartWriterOnDisk::Stream::finalize() +{ + if (!is_prefinalized) + preFinalize(); + plain_file->finalize(); marks_file->finalize(); } @@ -245,8 +257,7 @@ void MergeTreeDataPartWriterOnDisk::calculateAndSerializeSkipIndices(const Block } } -void MergeTreeDataPartWriterOnDisk::finishPrimaryIndexSerialization( - MergeTreeData::DataPart::Checksums & checksums, bool sync) +void MergeTreeDataPartWriterOnDisk::fillPrimaryIndexChecksums(MergeTreeData::DataPart::Checksums & checksums) { bool write_final_mark = (with_final_mark && data_written); if (write_final_mark && compute_granularity) @@ -269,6 +280,14 @@ void MergeTreeDataPartWriterOnDisk::finishPrimaryIndexSerialization( index_stream->next(); checksums.files["primary.idx"].file_size = index_stream->count(); checksums.files["primary.idx"].file_hash = index_stream->getHash(); + index_file_stream->preFinalize(); + } +} + +void MergeTreeDataPartWriterOnDisk::finishPrimaryIndexSerialization(bool sync) +{ + if (index_stream) + { index_file_stream->finalize(); if (sync) index_file_stream->sync(); @@ -276,8 +295,7 @@ void MergeTreeDataPartWriterOnDisk::finishPrimaryIndexSerialization( } } -void MergeTreeDataPartWriterOnDisk::finishSkipIndicesSerialization( - MergeTreeData::DataPart::Checksums & checksums, bool sync) +void MergeTreeDataPartWriterOnDisk::fillSkipIndicesChecksums(MergeTreeData::DataPart::Checksums & checksums) { for (size_t i = 0; i < skip_indices.size(); ++i) { @@ -288,8 +306,16 @@ void MergeTreeDataPartWriterOnDisk::finishSkipIndicesSerialization( for (auto & stream : skip_indices_streams) { - stream->finalize(); + stream->preFinalize(); stream->addToChecksums(checksums); + } +} + +void MergeTreeDataPartWriterOnDisk::finishSkipIndicesSerialization(bool sync) +{ + for (auto & stream : skip_indices_streams) + { + stream->finalize(); if (sync) stream->sync(); } diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h index fb46175c2aa..5af8cbc1650 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h @@ -71,6 +71,10 @@ public: std::unique_ptr marks_file; HashingWriteBuffer marks; + bool is_prefinalized = false; + + void preFinalize(); + void finalize(); void sync() const; @@ -107,9 +111,11 @@ protected: void calculateAndSerializeSkipIndices(const Block & skip_indexes_block, const Granules & granules_to_write); /// Finishes primary index serialization: write final primary index row (if required) and compute checksums - void finishPrimaryIndexSerialization(MergeTreeData::DataPart::Checksums & checksums, bool sync); + void fillPrimaryIndexChecksums(MergeTreeData::DataPart::Checksums & checksums); + void finishPrimaryIndexSerialization(bool sync); /// Finishes skip indices serialization: write all accumulated data to disk and compute checksums - void finishSkipIndicesSerialization(MergeTreeData::DataPart::Checksums & checksums, bool sync); + void fillSkipIndicesChecksums(MergeTreeData::DataPart::Checksums & checksums); + void finishSkipIndicesSerialization(bool sync); /// Get global number of the current which we are writing (or going to start to write) size_t getCurrentMark() const { return current_mark; } diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index b620bf8130e..62d53e58373 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB { @@ -514,7 +515,7 @@ void MergeTreeDataPartWriterWide::validateColumnOfFixedSize(const NameAndTypePai } -void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync) +void MergeTreeDataPartWriterWide::fillDataChecksums(IMergeTreeDataPart::Checksums & checksums) { const auto & global_settings = storage.getContext()->getSettingsRef(); ISerialization::SerializeBinaryBulkSettings serialize_settings; @@ -547,10 +548,19 @@ void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Ch writeFinalMark(*it, offset_columns, serialize_settings.path); } } + + for (auto & stream : column_streams) + { + stream.second->preFinalize(); + stream.second->addToChecksums(checksums); + } +} + +void MergeTreeDataPartWriterWide::finishDataSerialization(bool sync) +{ for (auto & stream : column_streams) { stream.second->finalize(); - stream.second->addToChecksums(checksums); if (sync) stream.second->sync(); } @@ -574,16 +584,28 @@ void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Ch } -void MergeTreeDataPartWriterWide::finish(IMergeTreeDataPart::Checksums & checksums, bool sync) +void MergeTreeDataPartWriterWide::fillChecksums(IMergeTreeDataPart::Checksums & checksums) { // If we don't have anything to write, skip finalization. if (!columns_list.empty()) - finishDataSerialization(checksums, sync); + fillDataChecksums(checksums); if (settings.rewrite_primary_key) - finishPrimaryIndexSerialization(checksums, sync); + fillPrimaryIndexChecksums(checksums); - finishSkipIndicesSerialization(checksums, sync); + fillSkipIndicesChecksums(checksums); +} + +void MergeTreeDataPartWriterWide::finish(bool sync) +{ + // If we don't have anything to write, skip finalization. + if (!columns_list.empty()) + finishDataSerialization(sync); + + if (settings.rewrite_primary_key) + finishPrimaryIndexSerialization(sync); + + finishSkipIndicesSerialization(sync); } void MergeTreeDataPartWriterWide::writeFinalMark( diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index 6303fbbac0d..b82fcd652ae 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -29,12 +29,15 @@ public: void write(const Block & block, const IColumn::Permutation * permutation) override; - void finish(IMergeTreeDataPart::Checksums & checksums, bool sync) final; + void fillChecksums(IMergeTreeDataPart::Checksums & checksums) final; + + void finish(bool sync) final; private: /// Finish serialization of data: write final mark if required and compute checksums /// Also validate written data in debug mode - void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync); + void fillDataChecksums(IMergeTreeDataPart::Checksums & checksums); + void finishDataSerialization(bool sync); /// Write data of one column. /// Return how many marks were written and diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 825fea38272..d16b5274a45 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -137,6 +137,12 @@ void updateTTL( } +void MergeTreeDataWriter::TemporaryPart::finalize() +{ + for (auto & stream : streams) + stream.finalizer.finish(); +} + BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts( const Block & block, size_t max_parts, const StorageMetadataPtr & metadata_snapshot, ContextPtr context) { @@ -270,9 +276,10 @@ Block MergeTreeDataWriter::mergeBlock( return block.cloneWithColumns(status.chunk.getColumns()); } -MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart( +MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPart( BlockWithPartition & block_with_partition, const StorageMetadataPtr & metadata_snapshot, ContextPtr context) { + TemporaryPart temp_part; Block & block = block_with_partition.block; static const String TMP_PREFIX = "tmp_insert_"; @@ -343,7 +350,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart( /// If optimize_on_insert is true, block may become empty after merge. /// There is no need to create empty part. if (expected_size == 0) - return nullptr; + return temp_part; DB::IMergeTreeDataPart::TTLInfos move_ttl_infos; const auto & move_ttl_entries = metadata_snapshot->getMoveTTLs(); @@ -419,30 +426,37 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart( auto compression_codec = data.getContext()->chooseCompressionCodec(0, 0); const auto & index_factory = MergeTreeIndexFactory::instance(); - MergedBlockOutputStream out(new_data_part, metadata_snapshot,columns, + auto out = std::make_unique(new_data_part, metadata_snapshot,columns, index_factory.getMany(metadata_snapshot->getSecondaryIndices()), compression_codec); - bool sync_on_insert = data_settings->fsync_after_insert; - - out.writeWithPermutation(block, perm_ptr); + out->writeWithPermutation(block, perm_ptr); for (const auto & projection : metadata_snapshot->getProjections()) { auto projection_block = projection.calculate(block, context); if (projection_block.rows()) - new_data_part->addProjectionPart( - projection.name, writeProjectionPart(data, log, projection_block, projection, new_data_part.get())); + { + auto proj_temp_part = writeProjectionPart(data, log, projection_block, projection, new_data_part.get()); + new_data_part->addProjectionPart(projection.name, std::move(proj_temp_part.part)); + for (auto & stream : proj_temp_part.streams) + temp_part.streams.emplace_back(std::move(stream)); + } } - out.writeSuffixAndFinalizePart(new_data_part, sync_on_insert); + auto finalizer = out->finalizePartAsync(new_data_part, data_settings->fsync_after_insert); + + temp_part.part = new_data_part; + temp_part.streams.emplace_back(TemporaryPart::Stream{.stream = std::move(out), .finalizer = std::move(finalizer)}); + + /// out.finish(new_data_part, std::move(written_files), sync_on_insert); ProfileEvents::increment(ProfileEvents::MergeTreeDataWriterRows, block.rows()); ProfileEvents::increment(ProfileEvents::MergeTreeDataWriterUncompressedBytes, block.bytes()); ProfileEvents::increment(ProfileEvents::MergeTreeDataWriterCompressedBytes, new_data_part->getBytesOnDisk()); - return new_data_part; + return temp_part; } -MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeProjectionPartImpl( +MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPartImpl( const String & part_name, MergeTreeDataPartType part_type, const String & relative_path, @@ -453,6 +467,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeProjectionPartImpl( Block block, const ProjectionDescription & projection) { + TemporaryPart temp_part; const StorageMetadataPtr & metadata_snapshot = projection.metadata; MergeTreePartInfo new_part_info("all", 0, 0, 0); auto new_data_part = data.createPart( @@ -525,24 +540,28 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeProjectionPartImpl( /// either default lz4 or compression method with zero thresholds on absolute and relative part size. auto compression_codec = data.getContext()->chooseCompressionCodec(0, 0); - MergedBlockOutputStream out( + auto out = std::make_unique( new_data_part, metadata_snapshot, columns, - {}, + MergeTreeIndices{}, compression_codec); - out.writeWithPermutation(block, perm_ptr); - out.writeSuffixAndFinalizePart(new_data_part); + out->writeWithPermutation(block, perm_ptr); + auto finalizer = out->finalizePartAsync(new_data_part, false); + temp_part.part = new_data_part; + temp_part.streams.emplace_back(TemporaryPart::Stream{.stream = std::move(out), .finalizer = std::move(finalizer)}); + + // out.finish(new_data_part, std::move(written_files), false); ProfileEvents::increment(ProfileEvents::MergeTreeDataProjectionWriterRows, block.rows()); ProfileEvents::increment(ProfileEvents::MergeTreeDataProjectionWriterUncompressedBytes, block.bytes()); ProfileEvents::increment(ProfileEvents::MergeTreeDataProjectionWriterCompressedBytes, new_data_part->getBytesOnDisk()); - return new_data_part; + return temp_part; } -MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeProjectionPart( +MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPart( MergeTreeData & data, Poco::Logger * log, Block block, const ProjectionDescription & projection, const IMergeTreeDataPart * parent_part) { String part_name = projection.name; @@ -574,7 +593,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeProjectionPart( /// This is used for projection materialization process which may contain multiple stages of /// projection part merges. -MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempProjectionPart( +MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempProjectionPart( MergeTreeData & data, Poco::Logger * log, Block block, @@ -609,7 +628,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempProjectionPart( projection); } -MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeInMemoryProjectionPart( +MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeInMemoryProjectionPart( const MergeTreeData & data, Poco::Logger * log, Block block, diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.h b/src/Storages/MergeTree/MergeTreeDataWriter.h index f16ec877113..ae46a94ccd7 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.h +++ b/src/Storages/MergeTree/MergeTreeDataWriter.h @@ -10,6 +10,7 @@ #include #include +#include namespace DB @@ -46,11 +47,28 @@ public: */ MergeTreeData::MutableDataPartPtr writeTempPart(BlockWithPartition & block, const StorageMetadataPtr & metadata_snapshot, bool optimize_on_insert); - MergeTreeData::MutableDataPartPtr - writeTempPart(BlockWithPartition & block, const StorageMetadataPtr & metadata_snapshot, ContextPtr context); + /// This structure contains not completely written temporary part. + /// Some writes may happen asynchronously, e.g. for blob storages. + /// You should call finalize() to wait until all data is written. + struct TemporaryPart + { + MergeTreeData::MutableDataPartPtr part; + + struct Stream + { + std::unique_ptr stream; + MergedBlockOutputStream::Finalizer finalizer; + }; + + std::vector streams; + + void finalize(); + }; + + TemporaryPart writeTempPart(BlockWithPartition & block, const StorageMetadataPtr & metadata_snapshot, ContextPtr context); /// For insertion. - static MergeTreeData::MutableDataPartPtr writeProjectionPart( + static TemporaryPart writeProjectionPart( MergeTreeData & data, Poco::Logger * log, Block block, @@ -58,7 +76,7 @@ public: const IMergeTreeDataPart * parent_part); /// For mutation: MATERIALIZE PROJECTION. - static MergeTreeData::MutableDataPartPtr writeTempProjectionPart( + static TemporaryPart writeTempProjectionPart( MergeTreeData & data, Poco::Logger * log, Block block, @@ -67,7 +85,7 @@ public: size_t block_num); /// For WriteAheadLog AddPart. - static MergeTreeData::MutableDataPartPtr writeInMemoryProjectionPart( + static TemporaryPart writeInMemoryProjectionPart( const MergeTreeData & data, Poco::Logger * log, Block block, @@ -82,7 +100,7 @@ public: const MergeTreeData::MergingParams & merging_params); private: - static MergeTreeData::MutableDataPartPtr writeProjectionPartImpl( + static TemporaryPart writeProjectionPartImpl( const String & part_name, MergeTreeDataPartType part_type, const String & relative_path, diff --git a/src/Storages/MergeTree/MergeTreePartition.cpp b/src/Storages/MergeTree/MergeTreePartition.cpp index 5b3c93aa553..706d72771e9 100644 --- a/src/Storages/MergeTree/MergeTreePartition.cpp +++ b/src/Storages/MergeTree/MergeTreePartition.cpp @@ -375,17 +375,17 @@ void MergeTreePartition::load(const MergeTreeData & storage, const DiskPtr & dis partition_key_sample.getByPosition(i).type->getDefaultSerialization()->deserializeBinary(value[i], *file); } -void MergeTreePartition::store(const MergeTreeData & storage, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const +std::unique_ptr MergeTreePartition::store(const MergeTreeData & storage, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const { auto metadata_snapshot = storage.getInMemoryMetadataPtr(); const auto & partition_key_sample = adjustPartitionKey(metadata_snapshot, storage.getContext()).sample_block; - store(partition_key_sample, disk, part_path, checksums); + return store(partition_key_sample, disk, part_path, checksums); } -void MergeTreePartition::store(const Block & partition_key_sample, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const +std::unique_ptr MergeTreePartition::store(const Block & partition_key_sample, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const { if (!partition_key_sample) - return; + return nullptr; auto out = disk->writeFile(part_path + "partition.dat"); HashingWriteBuffer out_hashing(*out); @@ -395,7 +395,8 @@ void MergeTreePartition::store(const Block & partition_key_sample, const DiskPtr out_hashing.next(); checksums.files["partition.dat"].file_size = out_hashing.count(); checksums.files["partition.dat"].file_hash = out_hashing.getHash(); - out->finalize(); + out->preFinalize(); + return out; } void MergeTreePartition::create(const StorageMetadataPtr & metadata_snapshot, Block block, size_t row, ContextPtr context) diff --git a/src/Storages/MergeTree/MergeTreePartition.h b/src/Storages/MergeTree/MergeTreePartition.h index d501d615621..f149fcbcb7e 100644 --- a/src/Storages/MergeTree/MergeTreePartition.h +++ b/src/Storages/MergeTree/MergeTreePartition.h @@ -38,8 +38,10 @@ public: void serializeText(const MergeTreeData & storage, WriteBuffer & out, const FormatSettings & format_settings) const; void load(const MergeTreeData & storage, const DiskPtr & disk, const String & part_path); - void store(const MergeTreeData & storage, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const; - void store(const Block & partition_key_sample, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const; + /// Store functions return write buffer with written but not finalized data. + /// User must call finish() for returned object. + [[nodiscard]] std::unique_ptr store(const MergeTreeData & storage, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const; + [[nodiscard]] std::unique_ptr store(const Block & partition_key_sample, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const; void assign(const MergeTreePartition & other) { value = other.value; } diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index 3029fc41bd3..ea252954b7e 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -7,6 +7,21 @@ namespace DB { +MergeTreeSink::~MergeTreeSink() = default; + +MergeTreeSink::MergeTreeSink( + StorageMergeTree & storage_, + StorageMetadataPtr metadata_snapshot_, + size_t max_parts_per_block_, + ContextPtr context_) + : SinkToStorage(metadata_snapshot_->getSampleBlock()) + , storage(storage_) + , metadata_snapshot(metadata_snapshot_) + , max_parts_per_block(max_parts_per_block_) + , context(context_) +{ +} + void MergeTreeSink::onStart() { /// Only check "too many parts" before write, @@ -14,22 +29,42 @@ void MergeTreeSink::onStart() storage.delayInsertOrThrowIfNeeded(); } +void MergeTreeSink::onFinish() +{ + finishDelayedChunk(); +} + +struct MergeTreeSink::DelayedChunk +{ + struct Partition + { + MergeTreeDataWriter::TemporaryPart temp_part; + UInt64 elapsed_ns; + String block_dedup_token; + }; + + std::vector partitions; +}; + void MergeTreeSink::consume(Chunk chunk) { auto block = getHeader().cloneWithColumns(chunk.detachColumns()); - String block_dedup_token; auto part_blocks = storage.writer.splitBlockIntoParts(block, max_parts_per_block, metadata_snapshot, context); + std::vector partitions; for (auto & current_block : part_blocks) { Stopwatch watch; + String block_dedup_token; - MergeTreeData::MutableDataPartPtr part = storage.writer.writeTempPart(current_block, metadata_snapshot, context); + auto temp_part = storage.writer.writeTempPart(current_block, metadata_snapshot, context); + + UInt64 elapsed_ns = watch.elapsed(); /// If optimize_on_insert setting is true, current_block could become empty after merge /// and we didn't create part. - if (!part) + if (!temp_part.part) continue; if (storage.getDeduplicationLog()) @@ -44,15 +79,41 @@ void MergeTreeSink::consume(Chunk chunk) } } - /// Part can be deduplicated, so increment counters and add to part log only if it's really added - if (storage.renameTempPartAndAdd(part, &storage.increment, nullptr, storage.getDeduplicationLog(), block_dedup_token)) + partitions.emplace_back(MergeTreeSink::DelayedChunk::Partition { - PartLog::addNewPart(storage.getContext(), part, watch.elapsed()); + .temp_part = std::move(temp_part), + .elapsed_ns = elapsed_ns, + .block_dedup_token = std::move(block_dedup_token) + }); + } + + finishDelayedChunk(); + delayed_chunk = std::make_unique(); + delayed_chunk->partitions = std::move(partitions); +} + +void MergeTreeSink::finishDelayedChunk() +{ + if (!delayed_chunk) + return; + + for (auto & partition : delayed_chunk->partitions) + { + partition.temp_part.finalize(); + + auto & part = partition.temp_part.part; + + /// Part can be deduplicated, so increment counters and add to part log only if it's really added + if (storage.renameTempPartAndAdd(part, &storage.increment, nullptr, storage.getDeduplicationLog(), partition.block_dedup_token)) + { + PartLog::addNewPart(storage.getContext(), part, partition.elapsed_ns); /// Initiate async merge - it will be done if it's good time for merge and if there are space in 'background_pool'. storage.background_operations_assignee.trigger(); } } + + delayed_chunk.reset(); } } diff --git a/src/Storages/MergeTree/MergeTreeSink.h b/src/Storages/MergeTree/MergeTreeSink.h index 96231fe668c..65a565d7f57 100644 --- a/src/Storages/MergeTree/MergeTreeSink.h +++ b/src/Storages/MergeTree/MergeTreeSink.h @@ -16,20 +16,16 @@ class MergeTreeSink : public SinkToStorage public: MergeTreeSink( StorageMergeTree & storage_, - const StorageMetadataPtr metadata_snapshot_, + StorageMetadataPtr metadata_snapshot_, size_t max_parts_per_block_, - ContextPtr context_) - : SinkToStorage(metadata_snapshot_->getSampleBlock()) - , storage(storage_) - , metadata_snapshot(metadata_snapshot_) - , max_parts_per_block(max_parts_per_block_) - , context(context_) - { - } + ContextPtr context_); + + ~MergeTreeSink() override; String getName() const override { return "MergeTreeSink"; } void consume(Chunk chunk) override; void onStart() override; + void onFinish() override; private: StorageMergeTree & storage; @@ -37,6 +33,12 @@ private: size_t max_parts_per_block; ContextPtr context; uint64_t chunk_dedup_seqnum = 0; /// input chunk ordinal number in case of dedup token + + /// We can delay processing for previous chunk and start writing a new one. + struct DelayedChunk; + std::unique_ptr delayed_chunk; + + void finishDelayedChunk(); }; } diff --git a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp index 694357ab0c2..9a41204c7e5 100644 --- a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp +++ b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp @@ -208,12 +208,12 @@ MergeTreeData::MutableDataPartsVector MergeTreeWriteAheadLog::restore(const Stor for (const auto & projection : metadata_snapshot->getProjections()) { auto projection_block = projection.calculate(block, context); + auto temp_part = MergeTreeDataWriter::writeInMemoryProjectionPart(storage, log, projection_block, projection, part.get()); + temp_part.finalize(); if (projection_block.rows()) - part->addProjectionPart( - projection.name, - MergeTreeDataWriter::writeInMemoryProjectionPart(storage, log, projection_block, projection, part.get())); + part->addProjectionPart(projection.name, std::move(temp_part.part)); } - part_out.writeSuffixAndFinalizePart(part); + part_out.finalizePart(part, false); min_block_number = std::min(min_block_number, part->info.min_block); max_block_number = std::max(max_block_number, part->info.max_block); diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 5274118df29..f8ed4d1806a 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -51,7 +51,67 @@ void MergedBlockOutputStream::writeWithPermutation(const Block & block, const IC writeImpl(block, permutation); } -void MergedBlockOutputStream::writeSuffixAndFinalizePart( +struct MergedBlockOutputStream::Finalizer::Impl +{ + IMergeTreeDataPartWriter & writer; + MergeTreeData::MutableDataPartPtr part; + std::vector> written_files; + bool sync; + + Impl(IMergeTreeDataPartWriter & writer_, MergeTreeData::MutableDataPartPtr part_, bool sync_) + : writer(writer_), part(std::move(part_)), sync(sync_) {} + + void finish(); +}; + +void MergedBlockOutputStream::Finalizer::finish() +{ + if (impl) + impl->finish(); + + impl.reset(); +} + +void MergedBlockOutputStream::Finalizer::Impl::finish() +{ + writer.finish(sync); + + for (auto & file : written_files) + { + file->finalize(); + if (sync) + file->sync(); + } + + part->storage.lockSharedData(*part); +} + +MergedBlockOutputStream::Finalizer::~Finalizer() +{ + try + { + finish(); + } + catch (...) + { + tryLogCurrentException("MergedBlockOutputStream"); + } +} + +MergedBlockOutputStream::Finalizer::Finalizer(Finalizer &&) = default; +MergedBlockOutputStream::Finalizer & MergedBlockOutputStream::Finalizer::operator=(Finalizer &&) = default; +MergedBlockOutputStream::Finalizer::Finalizer(std::unique_ptr impl_) : impl(std::move(impl_)) {} + +void MergedBlockOutputStream::finalizePart( + MergeTreeData::MutableDataPartPtr & new_part, + bool sync, + const NamesAndTypesList * total_columns_list, + MergeTreeData::DataPart::Checksums * additional_column_checksums) +{ + finalizePartAsync(new_part, sync, total_columns_list, additional_column_checksums).finish(); +} + +MergedBlockOutputStream::Finalizer MergedBlockOutputStream::finalizePartAsync( MergeTreeData::MutableDataPartPtr & new_part, bool sync, const NamesAndTypesList * total_columns_list, @@ -64,7 +124,9 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( checksums = std::move(*additional_column_checksums); /// Finish columns serialization. - writer->finish(checksums, sync); + writer->fillChecksums(checksums); + + LOG_TRACE(&Poco::Logger::get("MergedBlockOutputStream"), "filled checksums {}", new_part->getNameWithState()); for (const auto & [projection_name, projection_part] : new_part->getProjectionParts()) checksums.addFile( @@ -84,8 +146,9 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( new_part->setSerializationInfos(serialization_infos); } + auto finalizer = std::make_unique(*writer, new_part, sync); if (new_part->isStoredOnDisk()) - finalizePartOnDisk(new_part, checksums, sync); + finalizer->written_files = finalizePartOnDisk(new_part, checksums); new_part->rows_count = rows_count; new_part->modification_time = time(nullptr); @@ -97,15 +160,15 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( if (default_codec != nullptr) new_part->default_codec = default_codec; - new_part->storage.lockSharedData(*new_part); + + return Finalizer(std::move(finalizer)); } -void MergedBlockOutputStream::finalizePartOnDisk( +MergedBlockOutputStream::WrittenFiles MergedBlockOutputStream::finalizePartOnDisk( const MergeTreeData::DataPartPtr & new_part, - MergeTreeData::DataPart::Checksums & checksums, - bool sync) + MergeTreeData::DataPart::Checksums & checksums) { - + WrittenFiles written_files; if (new_part->isProjectionPart()) { if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING || isCompactPart(new_part)) @@ -116,6 +179,8 @@ void MergedBlockOutputStream::finalizePartOnDisk( count_out_hashing.next(); checksums.files["count.txt"].file_size = count_out_hashing.count(); checksums.files["count.txt"].file_hash = count_out_hashing.getHash(); + count_out->preFinalize(); + written_files.emplace_back(std::move(count_out)); } } else @@ -127,16 +192,21 @@ void MergedBlockOutputStream::finalizePartOnDisk( writeUUIDText(new_part->uuid, out_hashing); checksums.files[IMergeTreeDataPart::UUID_FILE_NAME].file_size = out_hashing.count(); checksums.files[IMergeTreeDataPart::UUID_FILE_NAME].file_hash = out_hashing.getHash(); - out->finalize(); - if (sync) - out->sync(); + out->preFinalize(); + written_files.emplace_back(std::move(out)); } if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) { - new_part->partition.store(storage, volume->getDisk(), part_path, checksums); + if (auto file = new_part->partition.store(storage, volume->getDisk(), part_path, checksums)) + written_files.emplace_back(std::move(file)); + if (new_part->minmax_idx->initialized) - new_part->minmax_idx->store(storage, volume->getDisk(), part_path, checksums); + { + auto files = new_part->minmax_idx->store(storage, volume->getDisk(), part_path, checksums); + for (auto & file : files) + written_files.emplace_back(std::move(file)); + } else if (rows_count) throw Exception("MinMax index was not initialized for new non-empty part " + new_part->name + ". It is a bug.", ErrorCodes::LOGICAL_ERROR); @@ -149,9 +219,8 @@ void MergedBlockOutputStream::finalizePartOnDisk( count_out_hashing.next(); checksums.files["count.txt"].file_size = count_out_hashing.count(); checksums.files["count.txt"].file_hash = count_out_hashing.getHash(); - count_out->finalize(); - if (sync) - count_out->sync(); + count_out->preFinalize(); + written_files.emplace_back(std::move(count_out)); } } @@ -163,9 +232,8 @@ void MergedBlockOutputStream::finalizePartOnDisk( new_part->ttl_infos.write(out_hashing); checksums.files["ttl.txt"].file_size = out_hashing.count(); checksums.files["ttl.txt"].file_hash = out_hashing.getHash(); - out->finalize(); - if (sync) - out->sync(); + out->preFinalize(); + written_files.emplace_back(std::move(out)); } if (!new_part->getSerializationInfos().empty()) @@ -175,25 +243,24 @@ void MergedBlockOutputStream::finalizePartOnDisk( new_part->getSerializationInfos().writeJSON(out_hashing); checksums.files[IMergeTreeDataPart::SERIALIZATION_FILE_NAME].file_size = out_hashing.count(); checksums.files[IMergeTreeDataPart::SERIALIZATION_FILE_NAME].file_hash = out_hashing.getHash(); - out->finalize(); - if (sync) - out->sync(); + out->preFinalize(); + written_files.emplace_back(std::move(out)); } { /// Write a file with a description of columns. auto out = volume->getDisk()->writeFile(fs::path(part_path) / "columns.txt", 4096); new_part->getColumns().writeText(*out); - out->finalize(); - if (sync) - out->sync(); + out->preFinalize(); + written_files.emplace_back(std::move(out)); } if (default_codec != nullptr) { auto out = volume->getDisk()->writeFile(part_path + IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME, 4096); DB::writeText(queryToString(default_codec->getFullCodecDesc()), *out); - out->finalize(); + out->preFinalize(); + written_files.emplace_back(std::move(out)); } else { @@ -205,10 +272,11 @@ void MergedBlockOutputStream::finalizePartOnDisk( /// Write file with checksums. auto out = volume->getDisk()->writeFile(fs::path(part_path) / "checksums.txt", 4096); checksums.write(*out); - out->finalize(); - if (sync) - out->sync(); + out->preFinalize(); + written_files.emplace_back(std::move(out)); } + + return written_files; } void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Permutation * permutation) diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.h b/src/Storages/MergeTree/MergedBlockOutputStream.h index 21e3c794239..b38395d56c2 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.h +++ b/src/Storages/MergeTree/MergedBlockOutputStream.h @@ -32,11 +32,33 @@ public: */ void writeWithPermutation(const Block & block, const IColumn::Permutation * permutation); + /// Finalizer is a structure which is returned from by finalizePart(). + /// Files from part may be written asynchronously, e.g. for blob storages. + /// You should call finish() to wait until all data is written. + struct Finalizer + { + struct Impl; + std::unique_ptr impl; + + explicit Finalizer(std::unique_ptr impl_); + ~Finalizer(); + Finalizer(Finalizer &&); + Finalizer & operator=(Finalizer &&); + + void finish(); + }; + /// Finalize writing part and fill inner structures /// If part is new and contains projections, they should be added before invoking this method. - void writeSuffixAndFinalizePart( + Finalizer finalizePartAsync( MergeTreeData::MutableDataPartPtr & new_part, - bool sync = false, + bool sync, + const NamesAndTypesList * total_columns_list = nullptr, + MergeTreeData::DataPart::Checksums * additional_column_checksums = nullptr); + + void finalizePart( + MergeTreeData::MutableDataPartPtr & new_part, + bool sync, const NamesAndTypesList * total_columns_list = nullptr, MergeTreeData::DataPart::Checksums * additional_column_checksums = nullptr); @@ -46,10 +68,10 @@ private: */ void writeImpl(const Block & block, const IColumn::Permutation * permutation); - void finalizePartOnDisk( + using WrittenFiles = std::vector>; + WrittenFiles finalizePartOnDisk( const MergeTreeData::DataPartPtr & new_part, - MergeTreeData::DataPart::Checksums & checksums, - bool sync); + MergeTreeData::DataPart::Checksums & checksums); NamesAndTypesList columns_list; IMergeTreeDataPart::MinMaxIndex minmax_idx; diff --git a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 4c43e93e809..2cc20186808 100644 --- a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -55,14 +55,13 @@ void MergedColumnOnlyOutputStream::write(const Block & block) } MergeTreeData::DataPart::Checksums -MergedColumnOnlyOutputStream::writeSuffixAndGetChecksums( +MergedColumnOnlyOutputStream::fillChecksums( MergeTreeData::MutableDataPartPtr & new_part, - MergeTreeData::DataPart::Checksums & all_checksums, - bool sync) + MergeTreeData::DataPart::Checksums & all_checksums) { /// Finish columns serialization. MergeTreeData::DataPart::Checksums checksums; - writer->finish(checksums, sync); + writer->fillChecksums(checksums); for (const auto & [projection_name, projection_part] : new_part->getProjectionParts()) checksums.addFile( @@ -85,4 +84,9 @@ MergedColumnOnlyOutputStream::writeSuffixAndGetChecksums( return checksums; } +void MergedColumnOnlyOutputStream::finish(bool sync) +{ + writer->finish(sync); +} + } diff --git a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h index 4b75bc52f72..7b587d01dab 100644 --- a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h +++ b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h @@ -25,8 +25,11 @@ public: Block getHeader() const { return header; } void write(const Block & block) override; + MergeTreeData::DataPart::Checksums - writeSuffixAndGetChecksums(MergeTreeData::MutableDataPartPtr & new_part, MergeTreeData::DataPart::Checksums & all_checksums, bool sync = false); + fillChecksums(MergeTreeData::MutableDataPartPtr & new_part, MergeTreeData::DataPart::Checksums & all_checksums); + + void finish(bool sync); private: Block header; diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 985098bb2a3..cc690287ef6 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -804,8 +804,12 @@ bool PartMergerWriter::mutateOriginalPartAndPrepareProjections() const auto & projection = *ctx->projections_to_build[i]; auto projection_block = projection_squashes[i].add(projection.calculate(cur_block, ctx->context)); if (projection_block) - projection_parts[projection.name].emplace_back(MergeTreeDataWriter::writeTempProjectionPart( - *ctx->data, ctx->log, projection_block, projection, ctx->new_data_part.get(), ++block_num)); + { + auto tmp_part = MergeTreeDataWriter::writeTempProjectionPart( + *ctx->data, ctx->log, projection_block, projection, ctx->new_data_part.get(), ++block_num); + tmp_part.finalize(); + projection_parts[projection.name].emplace_back(std::move(tmp_part.part)); + } } (*ctx->mutate_entry)->rows_written += cur_block.rows(); @@ -823,8 +827,10 @@ bool PartMergerWriter::mutateOriginalPartAndPrepareProjections() auto projection_block = projection_squash.add({}); if (projection_block) { - projection_parts[projection.name].emplace_back(MergeTreeDataWriter::writeTempProjectionPart( - *ctx->data, ctx->log, projection_block, projection, ctx->new_data_part.get(), ++block_num)); + auto temp_part = MergeTreeDataWriter::writeTempProjectionPart( + *ctx->data, ctx->log, projection_block, projection, ctx->new_data_part.get(), ++block_num); + temp_part.finalize(); + projection_parts[projection.name].emplace_back(std::move(temp_part.part)); } } @@ -976,7 +982,7 @@ private: ctx->mutating_executor.reset(); ctx->mutating_pipeline.reset(); - static_pointer_cast(ctx->out)->writeSuffixAndFinalizePart(ctx->new_data_part, ctx->need_sync); + static_pointer_cast(ctx->out)->finalizePart(ctx->new_data_part, ctx->need_sync); ctx->out.reset(); } @@ -1132,9 +1138,11 @@ private: ctx->mutating_pipeline.reset(); auto changed_checksums = - static_pointer_cast(ctx->out)->writeSuffixAndGetChecksums( - ctx->new_data_part, ctx->new_data_part->checksums, ctx->need_sync); + static_pointer_cast(ctx->out)->fillChecksums( + ctx->new_data_part, ctx->new_data_part->checksums); ctx->new_data_part->checksums.add(std::move(changed_checksums)); + + static_pointer_cast(ctx->out)->finish(ctx->need_sync); } for (const auto & [rename_from, rename_to] : ctx->files_to_rename) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index c14672fe382..15fe7a8ee77 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -32,6 +32,17 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +struct ReplicatedMergeTreeSink::DelayedChunk +{ + struct Partition + { + MergeTreeDataWriter::TemporaryPart temp_part; + UInt64 elapsed_ns; + String block_id; + }; + + std::vector partitions; +}; ReplicatedMergeTreeSink::ReplicatedMergeTreeSink( StorageReplicatedMergeTree & storage_, @@ -60,6 +71,8 @@ ReplicatedMergeTreeSink::ReplicatedMergeTreeSink( quorum = 0; } +ReplicatedMergeTreeSink::~ReplicatedMergeTreeSink() = default; + /// Allow to verify that the session in ZooKeeper is still alive. static void assertSessionIsNotExpired(zkutil::ZooKeeperPtr & zookeeper) @@ -126,8 +139,6 @@ void ReplicatedMergeTreeSink::consume(Chunk chunk) { auto block = getHeader().cloneWithColumns(chunk.detachColumns()); - last_block_is_duplicate = false; - auto zookeeper = storage.getZooKeeper(); assertSessionIsNotExpired(zookeeper); @@ -140,6 +151,7 @@ void ReplicatedMergeTreeSink::consume(Chunk chunk) checkQuorumPrecondition(zookeeper); auto part_blocks = storage.writer.splitBlockIntoParts(block, max_parts_per_block, metadata_snapshot, context); + std::vector partitions; for (auto & current_block : part_blocks) { @@ -147,11 +159,11 @@ void ReplicatedMergeTreeSink::consume(Chunk chunk) /// Write part to the filesystem under temporary name. Calculate a checksum. - MergeTreeData::MutableDataPartPtr part = storage.writer.writeTempPart(current_block, metadata_snapshot, context); + auto temp_part = storage.writer.writeTempPart(current_block, metadata_snapshot, context); /// If optimize_on_insert setting is true, current_block could become empty after merge /// and we didn't create part. - if (!part) + if (!temp_part.part) continue; String block_id; @@ -169,7 +181,7 @@ void ReplicatedMergeTreeSink::consume(Chunk chunk) block_dedup_token += fmt::format("_{}", chunk_dedup_seqnum); ++chunk_dedup_seqnum; } - block_id = part->getZeroLevelPartBlockID(block_dedup_token); + block_id = temp_part.part->getZeroLevelPartBlockID(block_dedup_token); LOG_DEBUG(log, "Wrote block with ID '{}', {} rows", block_id, current_block.block.rows()); } else @@ -177,27 +189,63 @@ void ReplicatedMergeTreeSink::consume(Chunk chunk) LOG_DEBUG(log, "Wrote block with {} rows", current_block.block.rows()); } + UInt64 elapsed_ns = watch.elapsed(); + + partitions.emplace_back(ReplicatedMergeTreeSink::DelayedChunk::Partition{ + .temp_part = std::move(temp_part), + .elapsed_ns = elapsed_ns, + .block_id = std::move(block_id) + }); + } + + finishDelayedChunk(zookeeper); + delayed_chunk = std::make_unique(); + delayed_chunk->partitions = std::move(partitions); + + /// If deduplicated data should not be inserted into MV, we need to set proper + /// value for `last_block_is_duplicate`, which is possible only after the part is committed. + /// Othervide we can delay commit. + /// TODO: we can also delay commit if there is no MVs. + if (!context->getSettingsRef().deduplicate_blocks_in_dependent_materialized_views) + finishDelayedChunk(zookeeper); +} + +void ReplicatedMergeTreeSink::finishDelayedChunk(zkutil::ZooKeeperPtr & zookeeper) +{ + if (!delayed_chunk) + return; + + last_block_is_duplicate = false; + + for (auto & partition : delayed_chunk->partitions) + { + partition.temp_part.finalize(); + + auto & part = partition.temp_part.part; + try { - commitPart(zookeeper, part, block_id); + commitPart(zookeeper, part, partition.block_id); + + last_block_is_duplicate = last_block_is_duplicate || part->is_duplicate; /// Set a special error code if the block is duplicate - int error = (deduplicate && last_block_is_duplicate) ? ErrorCodes::INSERT_WAS_DEDUPLICATED : 0; - PartLog::addNewPart(storage.getContext(), part, watch.elapsed(), ExecutionStatus(error)); + int error = (deduplicate && part->is_duplicate) ? ErrorCodes::INSERT_WAS_DEDUPLICATED : 0; + PartLog::addNewPart(storage.getContext(), part, partition.elapsed_ns, ExecutionStatus(error)); } catch (...) { - PartLog::addNewPart(storage.getContext(), part, watch.elapsed(), ExecutionStatus::fromCurrentException(__PRETTY_FUNCTION__)); + PartLog::addNewPart(storage.getContext(), part, partition.elapsed_ns, ExecutionStatus::fromCurrentException(__PRETTY_FUNCTION__)); throw; } } + + delayed_chunk.reset(); } void ReplicatedMergeTreeSink::writeExistingPart(MergeTreeData::MutableDataPartPtr & part) { - last_block_is_duplicate = false; - /// NOTE: No delay in this case. That's Ok. auto zookeeper = storage.getZooKeeper(); @@ -355,7 +403,6 @@ void ReplicatedMergeTreeSink::commitPart( if (storage.getActiveContainingPart(existing_part_name)) { part->is_duplicate = true; - last_block_is_duplicate = true; ProfileEvents::increment(ProfileEvents::DuplicatedInsertedBlocks); if (quorum) { @@ -530,6 +577,12 @@ void ReplicatedMergeTreeSink::onStart() storage.delayInsertOrThrowIfNeeded(&storage.partial_shutdown_event); } +void ReplicatedMergeTreeSink::onFinish() +{ + auto zookeeper = storage.getZooKeeper(); + assertSessionIsNotExpired(zookeeper); + finishDelayedChunk(zookeeper); +} void ReplicatedMergeTreeSink::waitForQuorum( zkutil::ZooKeeperPtr & zookeeper, diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h index 300791ff25b..41953e034fa 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h @@ -35,8 +35,11 @@ public: // needed to set the special LogEntryType::ATTACH_PART bool is_attach_ = false); + ~ReplicatedMergeTreeSink() override; + void onStart() override; void consume(Chunk chunk) override; + void onFinish() override; String getName() const override { return "ReplicatedMergeTreeSink"; } @@ -90,6 +93,12 @@ private: ContextPtr context; UInt64 chunk_dedup_seqnum = 0; /// input chunk ordinal number in case of dedup token + + /// We can delay processing for previous chunk and start writing a new one. + struct DelayedChunk; + std::unique_ptr delayed_chunk; + + void finishDelayedChunk(zkutil::ZooKeeperPtr & zookeeper); }; } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index a4b5e76c99d..de0931da667 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -7537,7 +7537,7 @@ bool StorageReplicatedMergeTree::createEmptyPartInsteadOfLost(zkutil::ZooKeeperP /// TODO(ab): What projections should we add to the empty part? How can we make sure that it /// won't block future merges? Perhaps we should also check part emptiness when selecting parts /// to merge. - out.writeSuffixAndFinalizePart(new_data_part, sync_on_insert); + out.finalizePart(new_data_part, sync_on_insert); try { diff --git a/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py b/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py index 3f219b6ba57..9f5c7b1c8ce 100644 --- a/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py +++ b/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py @@ -1,4 +1,5 @@ from bottle import request, route, run, response +from threading import Lock # Endpoint can be configured to throw 500 error on N-th request attempt. @@ -6,6 +7,7 @@ from bottle import request, route, run, response # Dict to the number of request should be failed. cache = {} +mutex = Lock() @route('/fail_request/<_request_number>') @@ -38,23 +40,34 @@ def delete(_bucket): @route('/<_bucket>/<_path:path>', ['GET', 'POST', 'PUT', 'DELETE']) def server(_bucket, _path): - if cache.get('request_number', None): - request_number = cache.pop('request_number') - 1 - if request_number > 0: - cache['request_number'] = request_number - else: - response.status = 500 - response.content_type = 'text/xml' - return 'ExpectedErrorExpected Errortxfbd566d03042474888193-00608d7537' - if cache.get('throttle_request_number', None): - request_number = cache.pop('throttle_request_number') - 1 - if request_number > 0: - cache['throttle_request_number'] = request_number - else: - response.status = 429 - response.content_type = 'text/xml' - return 'TooManyRequestsExceptionPlease reduce your request rate.txfbd566d03042474888193-00608d7538' + # It's delete query for failed part + if _path.endswith('delete'): + response.set_header("Location", "http://minio1:9001/" + _bucket + '/' + _path) + response.status = 307 + return 'Redirected' + + mutex.acquire() + try: + if cache.get('request_number', None): + request_number = cache.pop('request_number') - 1 + if request_number > 0: + cache['request_number'] = request_number + else: + response.status = 500 + response.content_type = 'text/xml' + return 'ExpectedErrorExpected Errortxfbd566d03042474888193-00608d7537' + + if cache.get('throttle_request_number', None): + request_number = cache.pop('throttle_request_number') - 1 + if request_number > 0: + cache['throttle_request_number'] = request_number + else: + response.status = 429 + response.content_type = 'text/xml' + return 'TooManyRequestsExceptionPlease reduce your request rate.txfbd566d03042474888193-00608d7538' + finally: + mutex.release() response.set_header("Location", "http://minio1:9001/" + _bucket + '/' + _path) response.status = 307 From e29c77f793e198f090ff66b28ffcf4641196f281 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Tue, 1 Feb 2022 16:19:26 +0800 Subject: [PATCH 076/149] Fix link order issue of WriteBufferFromS3 (cherry picked from commit f22b09f4fca2066ae606b525ec348a55816c3b49) --- src/Disks/S3/DiskS3.cpp | 18 ++++++++++++++++-- src/IO/WriteBufferFromS3.cpp | 37 ++++++++++-------------------------- src/IO/WriteBufferFromS3.h | 7 +++++-- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index 7c39148c38c..f6c25e7439f 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -9,6 +9,7 @@ #include +#include #include #include @@ -262,7 +263,20 @@ std::unique_ptr DiskS3::writeFile(const String & path, LOG_TRACE(log, "{} to file by path: {}. S3 path: {}", mode == WriteMode::Rewrite ? "Write" : "Append", backQuote(metadata_disk->getPath() + path), remote_fs_root_path + s3_path); - ThreadPool * writer = &getThreadPoolWriter(); + ScheduleFunc schedule = [](auto && callback) + { + getThreadPoolWriter().scheduleOrThrow([&callback, thread_group = CurrentThread::getGroup()]() + { + if (thread_group) + CurrentThread::attachTo(thread_group); + + SCOPE_EXIT_SAFE( + if (thread_group) + CurrentThread::detachQueryIfNotDetached(); + ); + callback(); + }); + }; auto s3_buffer = std::make_unique( settings->client, @@ -272,7 +286,7 @@ std::unique_ptr DiskS3::writeFile(const String & path, settings->s3_max_single_part_upload_size, std::move(object_metadata), buf_size, - writer); + std::move(schedule)); return std::make_unique>(std::move(s3_buffer), std::move(metadata), s3_path); } diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 6cb44fb5e52..66eedee9bd3 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -11,7 +11,6 @@ # include # include # include -# include # include @@ -58,7 +57,7 @@ WriteBufferFromS3::WriteBufferFromS3( size_t max_single_part_upload_size_, std::optional> object_metadata_, size_t buffer_size_, - ThreadPool * thread_pool_) + ScheduleFunc && schedule_) : BufferWithOwnMemory(buffer_size_, nullptr, 0) , bucket(bucket_) , key(key_) @@ -66,7 +65,7 @@ WriteBufferFromS3::WriteBufferFromS3( , client_ptr(std::move(client_ptr_)) , minimum_upload_part_size(minimum_upload_part_size_) , max_single_part_upload_size(max_single_part_upload_size_) - , thread_pool(thread_pool_) + , schedule(std::move(schedule_)) { allocateBuffer(); } @@ -175,7 +174,7 @@ void WriteBufferFromS3::writePart() LOG_WARNING(log, "Maximum part number in S3 protocol has reached (too many parts). Server may not accept this whole upload."); } - if (thread_pool) + if (schedule) { UploadPartTask * task = nullptr; int part_number; @@ -187,16 +186,8 @@ void WriteBufferFromS3::writePart() } fillUploadRequest(task->req, part_number); - thread_pool->scheduleOrThrow([this, task, thread_group = CurrentThread::getGroup()]() + schedule([this, task]() { - if (thread_group) - CurrentThread::attachTo(thread_group); - - SCOPE_EXIT_SAFE( - if (thread_group) - CurrentThread::detachQueryIfNotDetached(); - ); - try { processUploadRequest(*task); @@ -283,7 +274,7 @@ void WriteBufferFromS3::completeMultipartUpload() void WriteBufferFromS3::makeSinglepartUpload() { auto size = temporary_buffer->tellp(); - bool with_pool = thread_pool != nullptr; + bool with_pool = bool(schedule); LOG_DEBUG(log, "Making single part upload. Bucket: {}, Key: {}, Size: {}, WithPool: {}", bucket, key, size, with_pool); @@ -296,20 +287,12 @@ void WriteBufferFromS3::makeSinglepartUpload() return; } - if (thread_pool) + if (schedule) { put_object_task = std::make_unique(); fillPutRequest(put_object_task->req); - thread_pool->scheduleOrThrow([this, thread_group = CurrentThread::getGroup()]() + schedule([this]() { - if (thread_group) - CurrentThread::attachTo(thread_group); - - SCOPE_EXIT_SAFE( - if (thread_group) - CurrentThread::detachQueryIfNotDetached(); - ); - try { processPutRequest(*put_object_task); @@ -348,7 +331,7 @@ void WriteBufferFromS3::fillPutRequest(Aws::S3::Model::PutObjectRequest & req) void WriteBufferFromS3::processPutRequest(PutObjectTask & task) { auto outcome = client_ptr->PutObject(task.req); - bool with_pool = thread_pool != nullptr; + bool with_pool = bool(schedule); if (outcome.IsSuccess()) LOG_DEBUG(log, "Single part upload has completed. Bucket: {}, Key: {}, Object size: {}, WithPool: {}", bucket, key, task.req.GetContentLength(), with_pool); @@ -358,7 +341,7 @@ void WriteBufferFromS3::processPutRequest(PutObjectTask & task) void WriteBufferFromS3::waitForReadyBackGroundTasks() { - if (thread_pool) + if (schedule) { std::lock_guard lock(bg_tasks_mutex); { @@ -383,7 +366,7 @@ void WriteBufferFromS3::waitForReadyBackGroundTasks() void WriteBufferFromS3::waitForAllBackGroundTasks() { - if (thread_pool) + if (schedule) { std::unique_lock lock(bg_tasks_mutex); bg_tasks_condvar.wait(lock, [this]() { return num_added_bg_tasks == num_finished_bg_tasks; }); diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index aaea718fe70..f322dd4b567 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -30,6 +30,8 @@ namespace Aws::S3::Model namespace DB { +using ScheduleFunc = std::function)>; + /** * Buffer to write a data to a S3 object with specified bucket and key. * If data size written to the buffer is less than 'max_single_part_upload_size' write is performed using singlepart upload. @@ -48,7 +50,7 @@ public: size_t max_single_part_upload_size_, std::optional> object_metadata_ = std::nullopt, size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE, - ThreadPool * thread_pool_ = nullptr); + ScheduleFunc && schedule_ = {}); ~WriteBufferFromS3() override; @@ -97,7 +99,8 @@ private: bool is_prefinalized = false; /// Following fields are for background uploads in thread pool (if specified). - ThreadPool * thread_pool; + /// We use std::function to avoid dependency of Interpreters + ScheduleFunc schedule; std::unique_ptr put_object_task; std::list upload_object_tasks; size_t num_added_bg_tasks = 0; From a6cc61bd14eb04d2e133f71b8b609f876057c5d9 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 1 Feb 2022 14:17:46 +0300 Subject: [PATCH 077/149] Revert "Revert "Additionally check remote_fs_execute_merges_on_single_replica_time_threshold inside ReplicatedMergeTreeQueue"" --- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 2a8c4d32578..94a779a699d 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1197,15 +1197,30 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( return false; } - if (merge_strategy_picker.shouldMergeOnSingleReplica(entry)) + bool should_execute_on_single_replica = merge_strategy_picker.shouldMergeOnSingleReplica(entry); + if (!should_execute_on_single_replica) { + /// Separate check. If we use only s3, check remote_fs_execute_merges_on_single_replica_time_threshold as well. + auto disks = storage.getDisks(); + bool only_s3_storage = true; + for (const auto & disk : disks) + if (disk->getType() != DB::DiskType::S3) + only_s3_storage = false; + + if (!disks.empty() && only_s3_storage) + should_execute_on_single_replica = merge_strategy_picker.shouldMergeOnSingleReplicaShared(entry); + } + + if (should_execute_on_single_replica) + { + auto replica_to_execute_merge = merge_strategy_picker.pickReplicaToExecuteMerge(entry); if (replica_to_execute_merge && !merge_strategy_picker.isMergeFinishedByReplica(replica_to_execute_merge.value(), entry)) { - String reason = "Not executing merge for the part " + entry.new_part_name - + ", waiting for " + replica_to_execute_merge.value() + " to execute merge."; - out_postpone_reason = reason; + const char * format_str = "Not executing merge for the part {}, waiting for {} to execute merge."; + LOG_DEBUG(log, format_str, entry.new_part_name, replica_to_execute_merge.value()); + out_postpone_reason = out_postpone_reason = fmt::format(format_str, format_str, entry.new_part_name, replica_to_execute_merge.value()); return false; } } From e4697a8d416f6229f856d43272c40a22c0949948 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Mon, 31 Jan 2022 20:51:20 +0300 Subject: [PATCH 078/149] Update fmtlib (7.0.0 -> 8.1.1) Signed-off-by: Azat Khuzhin --- contrib/fmtlib | 2 +- contrib/fmtlib-cmake/CMakeLists.txt | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/contrib/fmtlib b/contrib/fmtlib index c108ee1d590..b6f4ceaed0a 160000 --- a/contrib/fmtlib +++ b/contrib/fmtlib @@ -1 +1 @@ -Subproject commit c108ee1d590089ccf642fc85652b845924067af2 +Subproject commit b6f4ceaed0a0a24ccf575fab6c56dd50ccf6f1a9 diff --git a/contrib/fmtlib-cmake/CMakeLists.txt b/contrib/fmtlib-cmake/CMakeLists.txt index d8cb721b9ba..fecec5f3e43 100644 --- a/contrib/fmtlib-cmake/CMakeLists.txt +++ b/contrib/fmtlib-cmake/CMakeLists.txt @@ -1,7 +1,10 @@ set (SRCS + # NOTE: do not build module for now: + # ../fmtlib/src/fmt.cc ../fmtlib/src/format.cc ../fmtlib/src/os.cc + ../fmtlib/include/fmt/args.h ../fmtlib/include/fmt/chrono.h ../fmtlib/include/fmt/color.h ../fmtlib/include/fmt/compile.h @@ -11,9 +14,9 @@ set (SRCS ../fmtlib/include/fmt/locale.h ../fmtlib/include/fmt/os.h ../fmtlib/include/fmt/ostream.h - ../fmtlib/include/fmt/posix.h ../fmtlib/include/fmt/printf.h ../fmtlib/include/fmt/ranges.h + ../fmtlib/include/fmt/xchar.h ) add_library(_fmt ${SRCS}) From a09bc1d72edefbedca4845dd298dfc5f9b63ee4d Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 1 Feb 2022 12:12:21 +0300 Subject: [PATCH 079/149] Add fmt::runtime() in Exception ctor Signed-off-by: Azat Khuzhin --- src/Common/Exception.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Common/Exception.h b/src/Common/Exception.h index 3aa06f8c988..b6bc31a5821 100644 --- a/src/Common/Exception.h +++ b/src/Common/Exception.h @@ -37,7 +37,7 @@ public: // Format message with fmt::format, like the logging functions. template Exception(int code, const std::string & fmt, Args&&... args) - : Exception(fmt::format(fmt, std::forward(args)...), code) + : Exception(fmt::format(fmt::runtime(fmt), std::forward(args)...), code) {} struct CreateFromPocoTag {}; @@ -55,7 +55,7 @@ public: template void addMessage(const std::string& format, Args&&... args) { - extendedMessage(fmt::format(format, std::forward(args)...)); + extendedMessage(fmt::format(fmt::runtime(format), std::forward(args)...)); } void addMessage(const std::string& message) @@ -119,7 +119,7 @@ public: // Format message with fmt::format, like the logging functions. template ParsingException(int code, const std::string & fmt, Args&&... args) - : Exception(fmt::format(fmt, std::forward(args)...), code) + : Exception(fmt::format(fmt::runtime(fmt), std::forward(args)...), code) {} From bedf208cbd3243982833eb5468827a84d579ec95 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 1 Feb 2022 12:10:27 +0300 Subject: [PATCH 080/149] Use fmt::runtime() for LOG_* for non constexpr Here is oneliner: $ gg 'LOG_\(DEBUG\|TRACE\|INFO\|TEST\|WARNING\|ERROR\|FATAL\)([^,]*, [a-zA-Z]' -- :*.cpp :*.h | cut -d: -f1 | sort -u | xargs -r sed -E -i 's#(LOG_[A-Z]*)\(([^,]*), ([A-Za-z][^,)]*)#\1(\2, fmt::runtime(\3)#' Note, that I tried to do this with coccinelle (tool for semantic patchin), but it cannot parse C++: $ cat fmt.cocci @@ expression log; expression var; @@ -LOG_DEBUG(log, var) +LOG_DEBUG(log, fmt::runtime(var)) I've also tried to use some macros/templates magic to do this implicitly in logger_useful.h, but I failed to do so, and apparently it is not possible for now. Signed-off-by: Azat Khuzhin v2: manual fixes Signed-off-by: Azat Khuzhin --- base/daemon/BaseDaemon.cpp | 6 ++--- programs/keeper/Keeper.cpp | 2 +- programs/library-bridge/Handlers.cpp | 6 ++--- programs/odbc-bridge/ColumnInfoHandler.cpp | 2 +- .../odbc-bridge/IdentifierQuoteHandler.cpp | 2 +- programs/odbc-bridge/MainHandler.cpp | 4 ++-- programs/odbc-bridge/SchemaAllowedHandler.cpp | 2 +- src/Client/ClientBase.cpp | 2 +- src/Client/Connection.cpp | 2 +- src/Client/ConnectionEstablisher.cpp | 6 ++--- src/Common/DNSResolver.cpp | 2 +- src/Coordination/LoggerWrapper.h | 2 +- src/Databases/DatabaseAtomic.cpp | 8 +++---- src/Databases/DatabaseOnDisk.cpp | 2 +- src/Databases/SQLite/SQLiteUtils.cpp | 2 +- src/Dictionaries/CassandraHelpers.cpp | 10 ++++---- src/Dictionaries/MySQLDictionarySource.cpp | 6 ++--- .../PostgreSQLDictionarySource.cpp | 4 ++-- src/Dictionaries/XDBCDictionarySource.cpp | 4 ++-- src/Disks/DiskSelector.cpp | 2 +- src/Functions/logTrace.cpp | 2 +- src/Interpreters/executeDDLQueryOnCluster.cpp | 2 +- src/Parsers/DumpASTNode.h | 2 +- src/Server/GRPCServer.cpp | 2 +- src/Server/HTTPHandler.cpp | 5 +++- src/Server/InterserverIOHTTPHandler.cpp | 6 ++--- src/Server/PostgreSQLHandler.cpp | 2 +- src/Server/TCPHandler.cpp | 2 +- src/Storages/MergeTree/DataPartsExchange.cpp | 2 +- .../MergeTree/MergeFromLogEntryTask.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- .../ReplicatedMergeMutateTaskBase.cpp | 6 ++--- .../ReplicatedMergeTreePartCheckThread.cpp | 4 ++-- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 24 +++++++++---------- src/Storages/StorageMergeTree.cpp | 4 ++-- src/Storages/StorageReplicatedMergeTree.cpp | 18 +++++++------- 36 files changed, 82 insertions(+), 79 deletions(-) diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index f3026d7c87a..0c3acc4a3df 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -317,7 +317,7 @@ private: else error_message = "Sanitizer trap."; - LOG_FATAL(log, error_message); + LOG_FATAL(log, fmt::runtime(error_message)); if (stack_trace.getSize()) { @@ -330,11 +330,11 @@ private: for (size_t i = stack_trace.getOffset(); i < stack_trace.getSize(); ++i) bare_stacktrace << ' ' << stack_trace.getFramePointers()[i]; - LOG_FATAL(log, bare_stacktrace.str()); + LOG_FATAL(log, fmt::runtime(bare_stacktrace.str())); } /// Write symbolized stack trace line by line for better grep-ability. - stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL(log, s); }); + stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL(log, fmt::runtime(s)); }); #if defined(OS_LINUX) /// Write information about binary checksum. It can be difficult to calculate, so do it only after printing stack trace. diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index 636ce129d63..88df4d5b3e7 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -324,7 +324,7 @@ int Keeper::main(const std::vector & /*args*/) } else { - LOG_WARNING(log, message); + LOG_WARNING(log, fmt::runtime(message)); } } diff --git a/programs/library-bridge/Handlers.cpp b/programs/library-bridge/Handlers.cpp index bf9ace679ba..d16ac18ab56 100644 --- a/programs/library-bridge/Handlers.cpp +++ b/programs/library-bridge/Handlers.cpp @@ -37,7 +37,7 @@ namespace if (!response.sent()) *response.send() << message << std::endl; - LOG_WARNING(&Poco::Logger::get("LibraryBridge"), message); + LOG_WARNING(&Poco::Logger::get("LibraryBridge"), fmt::runtime(message)); } std::shared_ptr parseColumns(std::string && column_string) @@ -178,7 +178,7 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe catch (const Exception & ex) { processError(response, "Invalid 'sample_block' parameter in request body '" + ex.message() + "'"); - LOG_WARNING(log, ex.getStackTraceString()); + LOG_WARNING(log, fmt::runtime(ex.getStackTraceString())); return; } @@ -278,7 +278,7 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe catch (const Exception & ex) { processError(response, "Invalid 'requested_block' parameter in request body '" + ex.message() + "'"); - LOG_WARNING(log, ex.getStackTraceString()); + LOG_WARNING(log, fmt::runtime(ex.getStackTraceString())); return; } diff --git a/programs/odbc-bridge/ColumnInfoHandler.cpp b/programs/odbc-bridge/ColumnInfoHandler.cpp index 8ceeddcd7ab..4d9a6b7a692 100644 --- a/programs/odbc-bridge/ColumnInfoHandler.cpp +++ b/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -77,7 +77,7 @@ void ODBCColumnsInfoHandler::handleRequest(HTTPServerRequest & request, HTTPServ response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); if (!response.sent()) *response.send() << message << std::endl; - LOG_WARNING(log, message); + LOG_WARNING(log, fmt::runtime(message)); }; if (!params.has("table")) diff --git a/programs/odbc-bridge/IdentifierQuoteHandler.cpp b/programs/odbc-bridge/IdentifierQuoteHandler.cpp index c7cad68f19e..7f809da4b10 100644 --- a/programs/odbc-bridge/IdentifierQuoteHandler.cpp +++ b/programs/odbc-bridge/IdentifierQuoteHandler.cpp @@ -29,7 +29,7 @@ void IdentifierQuoteHandler::handleRequest(HTTPServerRequest & request, HTTPServ response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); if (!response.sent()) *response.send() << message << std::endl; - LOG_WARNING(log, message); + LOG_WARNING(log, fmt::runtime(message)); }; if (!params.has("connection_string")) diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index 1252d1ae70a..02bdabe8ffa 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -46,7 +46,7 @@ void ODBCHandler::processError(HTTPServerResponse & response, const std::string response.setStatusAndReason(HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); if (!response.sent()) *response.send() << message << std::endl; - LOG_WARNING(log, message); + LOG_WARNING(log, fmt::runtime(message)); } @@ -102,7 +102,7 @@ void ODBCHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse catch (const Exception & ex) { processError(response, "Invalid 'sample_block' parameter in request body '" + ex.message() + "'"); - LOG_ERROR(log, ex.getStackTraceString()); + LOG_ERROR(log, fmt::runtime(ex.getStackTraceString())); return; } diff --git a/programs/odbc-bridge/SchemaAllowedHandler.cpp b/programs/odbc-bridge/SchemaAllowedHandler.cpp index 7b526bd8041..0c58af2f7c1 100644 --- a/programs/odbc-bridge/SchemaAllowedHandler.cpp +++ b/programs/odbc-bridge/SchemaAllowedHandler.cpp @@ -37,7 +37,7 @@ void SchemaAllowedHandler::handleRequest(HTTPServerRequest & request, HTTPServer response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); if (!response.sent()) *response.send() << message << std::endl; - LOG_WARNING(log, message); + LOG_WARNING(log, fmt::runtime(message)); }; if (!params.has("connection_string")) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 006307cb433..27deace416d 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -1261,7 +1261,7 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin for (const auto & query_id_format : query_id_formats) { writeString(query_id_format.first, std_out); - writeString(fmt::format(query_id_format.second, fmt::arg("query_id", global_context->getCurrentQueryId())), std_out); + writeString(fmt::format(fmt::runtime(query_id_format.second), fmt::arg("query_id", global_context->getCurrentQueryId())), std_out); writeChar('\n', std_out); std_out.next(); } diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 505a6514812..ad2fc76f090 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -405,7 +405,7 @@ bool Connection::ping() } catch (const Poco::Exception & e) { - LOG_TRACE(log_wrapper.get(), e.displayText()); + LOG_TRACE(log_wrapper.get(), fmt::runtime(e.displayText())); return false; } diff --git a/src/Client/ConnectionEstablisher.cpp b/src/Client/ConnectionEstablisher.cpp index 4d27c9efc69..3385834e386 100644 --- a/src/Client/ConnectionEstablisher.cpp +++ b/src/Client/ConnectionEstablisher.cpp @@ -58,9 +58,9 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std:: auto table_status_it = status_response.table_states_by_id.find(*table_to_check); if (table_status_it == status_response.table_states_by_id.end()) { - const char * message_pattern = "There is no table {}.{} on server: {}"; - fail_message = fmt::format(message_pattern, backQuote(table_to_check->database), backQuote(table_to_check->table), result.entry->getDescription()); - LOG_WARNING(log, fail_message); + fail_message = fmt::format("There is no table {}.{} on server: {}", + backQuote(table_to_check->database), backQuote(table_to_check->table), result.entry->getDescription()); + LOG_WARNING(log, fmt::runtime(fail_message)); ProfileEvents::increment(ProfileEvents::DistributedConnectionMissingTable); return; } diff --git a/src/Common/DNSResolver.cpp b/src/Common/DNSResolver.cpp index 36d0c13b153..13da3efd57a 100644 --- a/src/Common/DNSResolver.cpp +++ b/src/Common/DNSResolver.cpp @@ -272,7 +272,7 @@ bool DNSResolver::updateCacheImpl(UpdateF && update_func, ElemsT && elems, const } if (!lost_elems.empty()) - LOG_INFO(log, log_msg, lost_elems); + LOG_INFO(log, fmt::runtime(log_msg), lost_elems); return updated; } diff --git a/src/Coordination/LoggerWrapper.h b/src/Coordination/LoggerWrapper.h index 002fa870241..a2493763633 100644 --- a/src/Coordination/LoggerWrapper.h +++ b/src/Coordination/LoggerWrapper.h @@ -39,7 +39,7 @@ public: const std::string & msg) override { LogsLevel db_level = static_cast(level_); - LOG_IMPL(log, db_level, LEVELS.at(db_level), msg); + LOG_IMPL(log, db_level, LEVELS.at(db_level), fmt::runtime(msg)); } void set_level(int level_) override diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index cb0c1cdae95..721bf79199b 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -80,7 +80,7 @@ void DatabaseAtomic::drop(ContextPtr) } catch (...) { - LOG_WARNING(log, getCurrentExceptionMessage(true)); + LOG_WARNING(log, fmt::runtime(getCurrentExceptionMessage(true))); } fs::remove_all(getMetadataPath()); } @@ -469,7 +469,7 @@ void DatabaseAtomic::tryCreateSymlink(const String & table_name, const String & } catch (...) { - LOG_WARNING(log, getCurrentExceptionMessage(true)); + LOG_WARNING(log, fmt::runtime(getCurrentExceptionMessage(true))); } } @@ -482,7 +482,7 @@ void DatabaseAtomic::tryRemoveSymlink(const String & table_name) } catch (...) { - LOG_WARNING(log, getCurrentExceptionMessage(true)); + LOG_WARNING(log, fmt::runtime(getCurrentExceptionMessage(true))); } } @@ -527,7 +527,7 @@ void DatabaseAtomic::renameDatabase(ContextPtr query_context, const String & new } catch (...) { - LOG_WARNING(log, getCurrentExceptionMessage(true)); + LOG_WARNING(log, fmt::runtime(getCurrentExceptionMessage(true))); } auto new_name_escaped = escapeForFileName(new_name); diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 165bad950f5..29591a5f88f 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -316,7 +316,7 @@ void DatabaseOnDisk::dropTable(ContextPtr local_context, const String & table_na } catch (...) { - LOG_WARNING(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__)); + LOG_WARNING(log, fmt::runtime(getCurrentExceptionMessage(__PRETTY_FUNCTION__))); attachTable(local_context, table_name, table, table_data_path_relative); if (renamed) fs::rename(table_metadata_path_drop, table_metadata_path); diff --git a/src/Databases/SQLite/SQLiteUtils.cpp b/src/Databases/SQLite/SQLiteUtils.cpp index 954576d9c05..5b38caeabee 100644 --- a/src/Databases/SQLite/SQLiteUtils.cpp +++ b/src/Databases/SQLite/SQLiteUtils.cpp @@ -20,7 +20,7 @@ void processSQLiteError(const String & message, bool throw_on_error) if (throw_on_error) throw Exception(ErrorCodes::PATH_ACCESS_DENIED, message); else - LOG_ERROR(&Poco::Logger::get("SQLiteEngine"), message); + LOG_ERROR(&Poco::Logger::get("SQLiteEngine"), fmt::runtime(message)); } diff --git a/src/Dictionaries/CassandraHelpers.cpp b/src/Dictionaries/CassandraHelpers.cpp index a33ab288a34..235e29b5bd8 100644 --- a/src/Dictionaries/CassandraHelpers.cpp +++ b/src/Dictionaries/CassandraHelpers.cpp @@ -58,15 +58,15 @@ void cassandraLogCallback(const CassLogMessage * message, void * data) { Poco::Logger * logger = static_cast(data); if (message->severity == CASS_LOG_CRITICAL || message->severity == CASS_LOG_ERROR) - LOG_ERROR(logger, message->message); + LOG_ERROR(logger, fmt::runtime(message->message)); else if (message->severity == CASS_LOG_WARN) - LOG_WARNING(logger, message->message); + LOG_WARNING(logger, fmt::runtime(message->message)); else if (message->severity == CASS_LOG_INFO) - LOG_INFO(logger, message->message); + LOG_INFO(logger, fmt::runtime(message->message)); else if (message->severity == CASS_LOG_DEBUG) - LOG_DEBUG(logger, message->message); + LOG_DEBUG(logger, fmt::runtime(message->message)); else if (message->severity == CASS_LOG_TRACE) - LOG_TRACE(logger, message->message); + LOG_TRACE(logger, fmt::runtime(message->message)); } } diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index a291fcea47f..29d70f3a7c4 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -193,7 +193,7 @@ Pipe MySQLDictionarySource::loadAll() auto connection = pool->get(); last_modification = getLastModification(connection, false); - LOG_TRACE(log, load_all_query); + LOG_TRACE(log, fmt::runtime(load_all_query)); return loadFromQuery(load_all_query); } @@ -203,7 +203,7 @@ Pipe MySQLDictionarySource::loadUpdatedAll() last_modification = getLastModification(connection, false); std::string load_update_query = getUpdateFieldAndDate(); - LOG_TRACE(log, load_update_query); + LOG_TRACE(log, fmt::runtime(load_update_query)); return loadFromQuery(load_update_query); } @@ -289,7 +289,7 @@ LocalDateTime MySQLDictionarySource::getLastModification(mysqlxx::Pool::Entry & { auto query = connection->query("SHOW TABLE STATUS LIKE " + quoteForLike(configuration.table)); - LOG_TRACE(log, query.str()); + LOG_TRACE(log, fmt::runtime(query.str())); auto result = query.use(); diff --git a/src/Dictionaries/PostgreSQLDictionarySource.cpp b/src/Dictionaries/PostgreSQLDictionarySource.cpp index 9af3ea06838..6fdf486fdbf 100644 --- a/src/Dictionaries/PostgreSQLDictionarySource.cpp +++ b/src/Dictionaries/PostgreSQLDictionarySource.cpp @@ -80,7 +80,7 @@ PostgreSQLDictionarySource::PostgreSQLDictionarySource(const PostgreSQLDictionar Pipe PostgreSQLDictionarySource::loadAll() { - LOG_TRACE(log, load_all_query); + LOG_TRACE(log, fmt::runtime(load_all_query)); return loadBase(load_all_query); } @@ -88,7 +88,7 @@ Pipe PostgreSQLDictionarySource::loadAll() Pipe PostgreSQLDictionarySource::loadUpdatedAll() { auto load_update_query = getUpdateFieldAndDate(); - LOG_TRACE(log, load_update_query); + LOG_TRACE(log, fmt::runtime(load_update_query)); return loadBase(load_update_query); } diff --git a/src/Dictionaries/XDBCDictionarySource.cpp b/src/Dictionaries/XDBCDictionarySource.cpp index e95094cac47..f08abcdc516 100644 --- a/src/Dictionaries/XDBCDictionarySource.cpp +++ b/src/Dictionaries/XDBCDictionarySource.cpp @@ -121,7 +121,7 @@ std::string XDBCDictionarySource::getUpdateFieldAndDate() Pipe XDBCDictionarySource::loadAll() { - LOG_TRACE(log, load_all_query); + LOG_TRACE(log, fmt::runtime(load_all_query)); return loadFromQuery(bridge_url, sample_block, load_all_query); } @@ -130,7 +130,7 @@ Pipe XDBCDictionarySource::loadUpdatedAll() { std::string load_query_update = getUpdateFieldAndDate(); - LOG_TRACE(log, load_query_update); + LOG_TRACE(log, fmt::runtime(load_query_update)); return loadFromQuery(bridge_url, sample_block, load_query_update); } diff --git a/src/Disks/DiskSelector.cpp b/src/Disks/DiskSelector.cpp index df77006addc..4c80b128b4b 100644 --- a/src/Disks/DiskSelector.cpp +++ b/src/Disks/DiskSelector.cpp @@ -101,7 +101,7 @@ DiskSelectorPtr DiskSelector::updateFromConfig( } writeString(" disappeared from configuration, this change will be applied after restart of ClickHouse", warning); - LOG_WARNING(&Poco::Logger::get("DiskSelector"), warning.str()); + LOG_WARNING(&Poco::Logger::get("DiskSelector"), fmt::runtime(warning.str())); } return result; diff --git a/src/Functions/logTrace.cpp b/src/Functions/logTrace.cpp index acf2a2041ec..05315f4dff6 100644 --- a/src/Functions/logTrace.cpp +++ b/src/Functions/logTrace.cpp @@ -48,7 +48,7 @@ namespace "First argument for function " + getName() + " must be Constant string", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); static auto * log = &Poco::Logger::get("FunctionLogTrace"); - LOG_TRACE(log, message); + LOG_TRACE(log, fmt::runtime(message)); return DataTypeUInt8().createColumnConst(input_rows_count, 0); } diff --git a/src/Interpreters/executeDDLQueryOnCluster.cpp b/src/Interpreters/executeDDLQueryOnCluster.cpp index 629c15cdae0..ce00676b2ed 100644 --- a/src/Interpreters/executeDDLQueryOnCluster.cpp +++ b/src/Interpreters/executeDDLQueryOnCluster.cpp @@ -325,7 +325,7 @@ Chunk DDLQueryStatusSource::generate() return {}; } - LOG_INFO(log, msg_format, node_path, timeout_seconds, num_unfinished_hosts, num_active_hosts); + LOG_INFO(log, fmt::runtime(msg_format), node_path, timeout_seconds, num_unfinished_hosts, num_active_hosts); NameSet unfinished_hosts = waiting_hosts; for (const auto & host_id : finished_hosts) diff --git a/src/Parsers/DumpASTNode.h b/src/Parsers/DumpASTNode.h index e1071f02dd3..e8efeb4b59c 100644 --- a/src/Parsers/DumpASTNode.h +++ b/src/Parsers/DumpASTNode.h @@ -102,7 +102,7 @@ public: ~DebugASTLog() { if constexpr (_enable) - LOG_DEBUG(log, buf.str()); + LOG_DEBUG(log, fmt::runtime(buf.str())); } WriteBuffer * stream() { return (_enable ? &buf : nullptr); } diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index 85b1d345c6e..0faafb92ad5 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -1262,7 +1262,7 @@ namespace { io.onException(); - LOG_ERROR(log, getExceptionMessage(exception, true)); + LOG_ERROR(log, fmt::runtime(getExceptionMessage(exception, true))); if (responder && !responder_finished) { diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 5253e66be92..a42df54aed7 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -915,7 +915,10 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse } processQuery(request, params, response, used_output, query_scope); - LOG_DEBUG(log, (request_credentials ? "Authentication in progress..." : "Done processing query")); + if (request_credentials) + LOG_DEBUG(log, "Authentication in progress..."); + else + LOG_DEBUG(log, "Done processing query"); } catch (...) { diff --git a/src/Server/InterserverIOHTTPHandler.cpp b/src/Server/InterserverIOHTTPHandler.cpp index 082f7cc2e33..9506c5c133f 100644 --- a/src/Server/InterserverIOHTTPHandler.cpp +++ b/src/Server/InterserverIOHTTPHandler.cpp @@ -138,9 +138,9 @@ void InterserverIOHTTPHandler::handleRequest(HTTPServerRequest & request, HTTPSe write_response(message); if (is_real_error) - LOG_ERROR(log, message); + LOG_ERROR(log, fmt::runtime(message)); else - LOG_INFO(log, message); + LOG_INFO(log, fmt::runtime(message)); } catch (...) { @@ -148,7 +148,7 @@ void InterserverIOHTTPHandler::handleRequest(HTTPServerRequest & request, HTTPSe std::string message = getCurrentExceptionMessage(false); write_response(message); - LOG_ERROR(log, message); + LOG_ERROR(log, fmt::runtime(message)); } } diff --git a/src/Server/PostgreSQLHandler.cpp b/src/Server/PostgreSQLHandler.cpp index 9808b538280..d6b834fceb7 100644 --- a/src/Server/PostgreSQLHandler.cpp +++ b/src/Server/PostgreSQLHandler.cpp @@ -105,7 +105,7 @@ void PostgreSQLHandler::run() "0A000", "Command is not supported"), true); - LOG_ERROR(log, Poco::format("Command is not supported. Command code %d", static_cast(message_type))); + LOG_ERROR(log, "Command is not supported. Command code {:d}", static_cast(message_type)); message_transport->dropMessage(); } } diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 6fa2b25d181..49595a9c658 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -466,7 +466,7 @@ void TCPHandler::runImpl() } const auto & e = *exception; - LOG_ERROR(log, getExceptionMessage(e, true)); + LOG_ERROR(log, fmt::runtime(getExceptionMessage(e, true))); sendException(*exception, send_exception_with_stack_trace); } } diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 37167038f63..11a2e8b0b94 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -525,7 +525,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( { if (e.code() != ErrorCodes::S3_ERROR && e.code() != ErrorCodes::ZERO_COPY_REPLICATION_ERROR) throw; - LOG_WARNING(log, e.message() + " Will retry fetching part without zero-copy."); + LOG_WARNING(log, fmt::runtime(e.message() + " Will retry fetching part without zero-copy.")); /// Try again but without zero-copy return fetchPart(metadata_snapshot, context, part_name, replica_path, host, port, timeouts, user, password, interserver_scheme, throttler, to_detached, tmp_prefix_, nullptr, false, disk); diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp index d92eaf85f3d..b0b8aad2841 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp @@ -84,7 +84,7 @@ std::pair MergeFromLogEntryT /// 3. We have two intersecting parts, both cover source_part_name. It's logical error. /// TODO Why 1 and 2 can happen? Do we need more assertions here or somewhere else? constexpr const char * message = "Part {} is covered by {} but should be merged into {}. This shouldn't happen often."; - LOG_WARNING(log, message, source_part_name, source_part_or_covering->name, entry.new_part_name); + LOG_WARNING(log, fmt::runtime(message), source_part_name, source_part_or_covering->name, entry.new_part_name); if (!source_part_or_covering->info.contains(MergeTreePartInfo::fromPartName(entry.new_part_name, storage.format_version))) throw Exception(ErrorCodes::LOGICAL_ERROR, message, source_part_name, source_part_or_covering->name, entry.new_part_name); return {false, {}}; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 6db50724787..4a41358b1d1 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -5918,7 +5918,7 @@ ReservationPtr MergeTreeData::balancedReservation( writeCString("\nbalancer: \n", log_str); for (const auto & [disk_name, per_disk_parts] : disk_parts_for_logging) writeString(fmt::format(" {}: [{}]\n", disk_name, fmt::join(per_disk_parts, ", ")), log_str); - LOG_DEBUG(log, log_str.str()); + LOG_DEBUG(log, fmt::runtime(log_str.str())); if (ttl_infos) reserved_space = tryReserveSpacePreferringTTLRules( diff --git a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp index db5ca15ce8a..880e729e534 100644 --- a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp @@ -45,17 +45,17 @@ bool ReplicatedMergeMutateTaskBase::executeStep() if (e.code() == ErrorCodes::NO_REPLICA_HAS_PART) { /// If no one has the right part, probably not all replicas work; We will not write to log with Error level. - LOG_INFO(log, e.displayText()); + LOG_INFO(log, fmt::runtime(e.displayText())); } else if (e.code() == ErrorCodes::ABORTED) { /// Interrupted merge or downloading a part is not an error. - LOG_INFO(log, e.message()); + LOG_INFO(log, fmt::runtime(e.message())); } else if (e.code() == ErrorCodes::PART_IS_TEMPORARILY_LOCKED) { /// Part cannot be added temporarily - LOG_INFO(log, e.displayText()); + LOG_INFO(log, fmt::runtime(e.displayText())); storage.cleanup_thread.wakeup(); } else diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index bc3dd093f36..af877bdbdf0 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -359,7 +359,7 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na tryLogCurrentException(log, __PRETTY_FUNCTION__); String message = "Part " + part_name + " looks broken. Removing it and will try to fetch."; - LOG_ERROR(log, message); + LOG_ERROR(log, fmt::runtime(message)); /// Delete part locally. storage.forgetPartAndMoveToDetached(part, "broken"); @@ -378,7 +378,7 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na ProfileEvents::increment(ProfileEvents::ReplicatedPartChecksFailed); String message = "Unexpected part " + part_name + " in filesystem. Removing."; - LOG_ERROR(log, message); + LOG_ERROR(log, fmt::runtime(message)); storage.forgetPartAndMoveToDetached(part, "unexpected"); return {part_name, false, message}; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 2a8c4d32578..05b0f8821a9 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1062,7 +1062,7 @@ bool ReplicatedMergeTreeQueue::isNotCoveredByFuturePartsImpl(const LogEntry & en const LogEntry & another_entry = *entry_for_same_part_it->second; const char * format_str = "Not executing log entry {} of type {} for part {} " "because another log entry {} of type {} for the same part ({}) is being processed. This shouldn't happen often."; - LOG_INFO(log, format_str, entry.znode_name, entry.type, entry.new_part_name, + LOG_INFO(log, fmt::runtime(format_str), entry.znode_name, entry.type, entry.new_part_name, another_entry.znode_name, another_entry.type, another_entry.new_part_name); out_reason = fmt::format(format_str, entry.znode_name, entry.type, entry.new_part_name, another_entry.znode_name, another_entry.type, another_entry.new_part_name); @@ -1088,7 +1088,7 @@ bool ReplicatedMergeTreeQueue::isNotCoveredByFuturePartsImpl(const LogEntry & en { const char * format_str = "Not executing log entry {} for part {} " "because it is covered by part {} that is currently executing."; - LOG_TRACE(log, format_str, entry.znode_name, new_part_name, future_part_elem.first); + LOG_TRACE(log, fmt::runtime(format_str), entry.znode_name, new_part_name, future_part_elem.first); out_reason = fmt::format(format_str, entry.znode_name, new_part_name, future_part_elem.first); return false; } @@ -1173,7 +1173,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( { const char * format_str = "Not executing log entry {} of type {} for part {} " "because part {} is not ready yet (log entry for that part is being processed)."; - LOG_TRACE(log, format_str, entry.znode_name, entry.typeToString(), entry.new_part_name, name); + LOG_TRACE(log, fmt::runtime(format_str), entry.znode_name, entry.typeToString(), entry.new_part_name, name); /// Copy-paste of above because we need structured logging (instead of already formatted message). out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.typeToString(), entry.new_part_name, name); return false; @@ -1192,7 +1192,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( if (merger_mutator.merges_blocker.isCancelled()) { const char * format_str = "Not executing log entry {} of type {} for part {} because merges and mutations are cancelled now."; - LOG_DEBUG(log, format_str, entry.znode_name, entry.typeToString(), entry.new_part_name); + LOG_DEBUG(log, fmt::runtime(format_str), entry.znode_name, entry.typeToString(), entry.new_part_name); out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.typeToString(), entry.new_part_name); return false; } @@ -1229,7 +1229,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( if (merger_mutator.ttl_merges_blocker.isCancelled()) { const char * format_str = "Not executing log entry {} for part {} because merges with TTL are cancelled now."; - LOG_DEBUG(log, format_str, + LOG_DEBUG(log, fmt::runtime(format_str), entry.znode_name, entry.new_part_name); out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.new_part_name); return false; @@ -1239,7 +1239,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( { const char * format_str = "Not executing log entry {} for part {}" " because {} merges with TTL already executing, maximum {}."; - LOG_DEBUG(log, format_str, + LOG_DEBUG(log, fmt::runtime(format_str), entry.znode_name, entry.new_part_name, total_merges_with_ttl, data_settings->max_number_of_merges_with_ttl_in_pool); @@ -1258,7 +1258,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( entry.znode_name, entry.typeToString(), entry.new_part_name, ReadableSize(sum_parts_size_in_bytes), ReadableSize(max_source_parts_size)); - LOG_DEBUG(log, out_postpone_reason); + LOG_DEBUG(log, fmt::runtime(out_postpone_reason)); return false; } @@ -1272,7 +1272,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( { int head_alter = alter_sequence.getHeadAlterVersion(state_lock); const char * format_str = "Cannot execute alter metadata {} with version {} because another alter {} must be executed before"; - LOG_TRACE(log, format_str, entry.znode_name, entry.alter_version, head_alter); + LOG_TRACE(log, fmt::runtime(format_str), entry.znode_name, entry.alter_version, head_alter); out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.alter_version, head_alter); return false; } @@ -1287,13 +1287,13 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( if (head_alter == entry.alter_version) { const char * format_str = "Cannot execute alter data {} with version {} because metadata still not altered"; - LOG_TRACE(log, format_str, entry.znode_name, entry.alter_version); + LOG_TRACE(log, fmt::runtime(format_str), entry.znode_name, entry.alter_version); out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.alter_version); } else { const char * format_str = "Cannot execute alter data {} with version {} because another alter {} must be executed before"; - LOG_TRACE(log, format_str, entry.znode_name, entry.alter_version, head_alter); + LOG_TRACE(log, fmt::runtime(format_str), entry.znode_name, entry.alter_version, head_alter); out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.alter_version, head_alter); } @@ -1312,7 +1312,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( const char * format_str = "Not executing log entry {} of type {} for part {} " "because another DROP_RANGE or REPLACE_RANGE entry are currently executing."; - LOG_TRACE(log, format_str, entry.znode_name, entry.typeToString(), entry.new_part_name); + LOG_TRACE(log, fmt::runtime(format_str), entry.znode_name, entry.typeToString(), entry.new_part_name); out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.typeToString(), entry.new_part_name); return false; } @@ -1340,7 +1340,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( { const char * format_str = "Not executing log entry {} of type {} for part {} " "because it probably depends on {} (REPLACE_RANGE)."; - LOG_TRACE(log, format_str, entry.znode_name, entry.typeToString(), entry.new_part_name, replace_entry->znode_name); + LOG_TRACE(log, fmt::runtime(format_str), entry.znode_name, entry.typeToString(), entry.new_part_name, replace_entry->znode_name); out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.typeToString(), entry.new_part_name, replace_entry->znode_name); return false; } diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 43cd42606a4..6cd1e2d66af 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1230,7 +1230,7 @@ bool StorageMergeTree::optimize( constexpr const char * message = "Cannot OPTIMIZE table: {}"; if (disable_reason.empty()) disable_reason = "unknown reason"; - LOG_INFO(log, message, disable_reason); + LOG_INFO(log, fmt::runtime(message), disable_reason); if (local_context->getSettingsRef().optimize_throw_if_noop) throw Exception(ErrorCodes::CANNOT_ASSIGN_OPTIMIZE, message, disable_reason); @@ -1256,7 +1256,7 @@ bool StorageMergeTree::optimize( constexpr const char * message = "Cannot OPTIMIZE table: {}"; if (disable_reason.empty()) disable_reason = "unknown reason"; - LOG_INFO(log, message, disable_reason); + LOG_INFO(log, fmt::runtime(message), disable_reason); if (local_context->getSettingsRef().optimize_throw_if_noop) throw Exception(ErrorCodes::CANNOT_ASSIGN_OPTIMIZE, message, disable_reason); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index a4b5e76c99d..9c907074531 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -1211,7 +1211,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) if (unexpected_parts_nonnew_rows > 0) { - LOG_WARNING(log, sanity_report_fmt, getStorageID().getNameForLogs(), + LOG_WARNING(log, fmt::runtime(sanity_report_fmt), getStorageID().getNameForLogs(), formatReadableQuantity(unexpected_parts_rows), formatReadableQuantity(total_rows_on_filesystem), unexpected_parts.size(), unexpected_parts_rows, unexpected_parts_nonnew, unexpected_parts_nonnew_rows, parts_to_fetch.size(), parts_to_fetch_blocks); @@ -2861,17 +2861,17 @@ bool StorageReplicatedMergeTree::processQueueEntry(ReplicatedMergeTreeQueue::Sel if (e.code() == ErrorCodes::NO_REPLICA_HAS_PART) { /// If no one has the right part, probably not all replicas work; We will not write to log with Error level. - LOG_INFO(log, e.displayText()); + LOG_INFO(log, fmt::runtime(e.displayText())); } else if (e.code() == ErrorCodes::ABORTED) { /// Interrupted merge or downloading a part is not an error. - LOG_INFO(log, e.message()); + LOG_INFO(log, fmt::runtime(e.message())); } else if (e.code() == ErrorCodes::PART_IS_TEMPORARILY_LOCKED) { /// Part cannot be added temporarily - LOG_INFO(log, e.displayText()); + LOG_INFO(log, fmt::runtime(e.displayText())); cleanup_thread.wakeup(); } else @@ -4383,7 +4383,7 @@ bool StorageReplicatedMergeTree::optimize( if (!partition_id.empty()) disable_reason += fmt::format(" (in partition {})", partition_id); String message = fmt::format(message_fmt, disable_reason); - LOG_INFO(log, message); + LOG_INFO(log, fmt::runtime(message)); return handle_noop(message); } @@ -4397,7 +4397,7 @@ bool StorageReplicatedMergeTree::optimize( if (create_result == CreateMergeEntryResult::MissingPart) { String message = "Can't create merge queue node in ZooKeeper, because some parts are missing"; - LOG_TRACE(log, message); + LOG_TRACE(log, fmt::runtime(message)); return handle_noop(message); } @@ -4410,7 +4410,7 @@ bool StorageReplicatedMergeTree::optimize( assert(try_no == max_retries); String message = fmt::format("Can't create merge queue node in ZooKeeper, because log was updated in every of {} tries", try_no); - LOG_TRACE(log, message); + LOG_TRACE(log, fmt::runtime(message)); return handle_noop(message); }; @@ -5569,7 +5569,7 @@ void StorageReplicatedMergeTree::fetchPartition( && e.code() != ErrorCodes::CANNOT_READ_ALL_DATA) throw; - LOG_INFO(log, e.displayText()); + LOG_INFO(log, fmt::runtime(e.displayText())); } return; } @@ -5706,7 +5706,7 @@ void StorageReplicatedMergeTree::fetchPartition( && e.code() != ErrorCodes::CANNOT_READ_ALL_DATA) throw; - LOG_INFO(log, e.displayText()); + LOG_INFO(log, fmt::runtime(e.displayText())); } if (!fetched) From 220ed206c1e022ad5a106205ba102bc076baff94 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 1 Feb 2022 12:23:30 +0300 Subject: [PATCH 081/149] logger_useful: implicitly convert fmt::basic_runtime for std::string ctor Signed-off-by: Azat Khuzhin --- base/base/logger_useful.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/base/base/logger_useful.h b/base/base/logger_useful.h index 1237c6bd47c..ad7d6583f5e 100644 --- a/base/base/logger_useful.h +++ b/base/base/logger_useful.h @@ -12,6 +12,8 @@ namespace { template constexpr size_t numArgs(Ts &&...) { return sizeof...(Ts); } template constexpr auto firstArg(T && x, Ts &&...) { return std::forward(x); } + /// For implicit conversion of fmt::basic_runtime<> to char* for std::string ctor + template constexpr auto firstArg(fmt::basic_runtime && data, Ts &&...) { return data.str.data(); } } From cf93732de9c8555ca534f3674284ffe85b1953ce Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 1 Feb 2022 12:42:59 +0300 Subject: [PATCH 082/149] DatabaseSQLite: fix fmt error in DatabaseSQLite::checkSQLiteTable() Signed-off-by: Azat Khuzhin --- src/Databases/SQLite/DatabaseSQLite.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Databases/SQLite/DatabaseSQLite.cpp b/src/Databases/SQLite/DatabaseSQLite.cpp index f4dab8a91a8..fb742587285 100644 --- a/src/Databases/SQLite/DatabaseSQLite.cpp +++ b/src/Databases/SQLite/DatabaseSQLite.cpp @@ -94,7 +94,7 @@ bool DatabaseSQLite::checkSQLiteTable(const String & table_name) const if (!sqlite_db) sqlite_db = openSQLiteDB(database_path, getContext(), /* throw_on_error */true); - const String query = fmt::format("SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}';", table_name); + const String query = fmt::format("SELECT name FROM sqlite_master WHERE type='table' AND name='{}';", table_name); auto callback_get_data = [](void * res, int, char **, char **) -> int { From 743096a8836abceb6499e4f2d19674df2d0f35fe Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 1 Feb 2022 13:00:23 +0300 Subject: [PATCH 083/149] Use proper fmt:: like Exception ctor in DataPartsExchange Signed-off-by: Azat Khuzhin --- src/Storages/MergeTree/DataPartsExchange.cpp | 42 +++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 11a2e8b0b94..c553637222d 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -222,7 +222,7 @@ void Service::sendPartFromMemory( auto projection_sample_block = metadata_snapshot->projections.get(name).sample_block; auto part_in_memory = asInMemoryPart(projection); if (!part_in_memory) - throw Exception("Projection " + name + " of part " + part->name + " is not stored in memory", ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Projection {} of part {} is not stored in memory", name, part->name); writeStringBinary(name, out); projection->checksums.write(out); @@ -232,7 +232,7 @@ void Service::sendPartFromMemory( auto part_in_memory = asInMemoryPart(part); if (!part_in_memory) - throw Exception("Part " + part->name + " is not stored in memory", ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Part {} is not stored in memory", part->name); NativeWriter block_out(out, 0, metadata_snapshot->getSampleBlock()); part->checksums.write(out); @@ -300,7 +300,7 @@ MergeTreeData::DataPart::Checksums Service::sendPartFromDisk( throw Exception("Transferring part to replica was cancelled", ErrorCodes::ABORTED); if (hashing_out.count() != size) - throw Exception("Unexpected size of file " + path, ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + throw Exception(ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART, "Unexpected size of file {}", path); writePODBinary(hashing_out.getHash(), out); @@ -323,7 +323,7 @@ void Service::sendPartFromDiskRemoteMeta(const MergeTreeData::DataPartPtr & part auto disk = part->volume->getDisk(); if (!disk->supportZeroCopyReplication()) - throw Exception(fmt::format("disk {} doesn't support zero-copy replication", disk->getName()), ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "disk {} doesn't support zero-copy replication", disk->getName()); part->storage.lockSharedData(*part); @@ -340,9 +340,9 @@ void Service::sendPartFromDiskRemoteMeta(const MergeTreeData::DataPartPtr & part fs::path metadata(metadata_file); if (!fs::exists(metadata)) - throw Exception("Remote metadata '" + file_name + "' is not exists", ErrorCodes::CORRUPTED_DATA); + throw Exception(ErrorCodes::CORRUPTED_DATA, "Remote metadata '{}' is not exists", file_name); if (!fs::is_regular_file(metadata)) - throw Exception("Remote metadata '" + file_name + "' is not a file", ErrorCodes::CORRUPTED_DATA); + throw Exception(ErrorCodes::CORRUPTED_DATA, "Remote metadata '{}' is not a file", file_name); UInt64 file_size = fs::file_size(metadata); writeStringBinary(it.first, out); @@ -355,7 +355,7 @@ void Service::sendPartFromDiskRemoteMeta(const MergeTreeData::DataPartPtr & part throw Exception("Transferring part to replica was cancelled", ErrorCodes::ABORTED); if (hashing_out.count() != file_size) - throw Exception("Unexpected size of file " + metadata_file, ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + throw Exception(ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART, "Unexpected size of file {}", metadata_file); writePODBinary(hashing_out.getHash(), out); } @@ -370,7 +370,7 @@ MergeTreeData::DataPartPtr Service::findPart(const String & name) if (part) return part; - throw Exception("No part " + name + " in table", ErrorCodes::NO_SUCH_DATA_PART); + throw Exception(ErrorCodes::NO_SUCH_DATA_PART, "No part {} in table", name); } MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( @@ -511,9 +511,9 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( if (!try_zero_copy) throw Exception("Got unexpected 'remote_fs_metadata' cookie", ErrorCodes::LOGICAL_ERROR); if (std::find(capability.begin(), capability.end(), remote_fs_metadata) == capability.end()) - throw Exception(fmt::format("Got 'remote_fs_metadata' cookie {}, expect one from {}", remote_fs_metadata, fmt::join(capability, ", ")), ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Got 'remote_fs_metadata' cookie {}, expect one from {}", remote_fs_metadata, fmt::join(capability, ", ")); if (server_protocol_version < REPLICATION_PROTOCOL_VERSION_WITH_PARTS_ZERO_COPY) - throw Exception(fmt::format("Got 'remote_fs_metadata' cookie with old protocol version {}", server_protocol_version), ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Got 'remote_fs_metadata' cookie with old protocol version {}", server_protocol_version); if (part_type == "InMemory") throw Exception("Got 'remote_fs_metadata' cookie for in-memory part", ErrorCodes::INCORRECT_PART_TYPE); @@ -649,9 +649,10 @@ void Fetcher::downloadBaseOrProjectionPartToDisk( /// Otherwise malicious ClickHouse replica may force us to write to arbitrary path. String absolute_file_path = fs::weakly_canonical(fs::path(part_download_path) / file_name); if (!startsWith(absolute_file_path, fs::weakly_canonical(part_download_path).string())) - throw Exception("File path (" + absolute_file_path + ") doesn't appear to be inside part path (" + part_download_path + ")." - " This may happen if we are trying to download part from malicious replica or logical error.", - ErrorCodes::INSECURE_PATH); + throw Exception(ErrorCodes::INSECURE_PATH, + "File path ({}) doesn't appear to be inside part path ({}). " + "This may happen if we are trying to download part from malicious replica or logical error.", + absolute_file_path, part_download_path); auto file_out = disk->writeFile(fs::path(part_download_path) / file_name); HashingWriteBuffer hashing_out(*file_out); @@ -670,8 +671,10 @@ void Fetcher::downloadBaseOrProjectionPartToDisk( readPODBinary(expected_hash, in); if (expected_hash != hashing_out.getHash()) - throw Exception("Checksum mismatch for file " + fullPath(disk, (fs::path(part_download_path) / file_name).string()) + " transferred from " + replica_path, - ErrorCodes::CHECKSUM_DOESNT_MATCH); + throw Exception(ErrorCodes::CHECKSUM_DOESNT_MATCH, + "Checksum mismatch for file {} transferred from {}", + fullPath(disk, (fs::path(part_download_path) / file_name).string()), + replica_path); if (file_name != "checksums.txt" && file_name != "columns.txt" && @@ -762,7 +765,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDiskRemoteMeta( if (!disk->supportZeroCopyReplication() || !disk->checkUniqueId(part_id)) { - throw Exception(fmt::format("Part {} unique id {} doesn't exist on {}.", part_name, part_id, disk->getName()), ErrorCodes::ZERO_COPY_REPLICATION_ERROR); + throw Exception(ErrorCodes::ZERO_COPY_REPLICATION_ERROR, "Part {} unique id {} doesn't exist on {}.", part_name, part_id, disk->getName()); } LOG_DEBUG(log, "Downloading Part {} unique id {} metadata onto disk {}.", part_name, part_id, disk->getName()); @@ -774,7 +777,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDiskRemoteMeta( String part_download_path = fs::path(data.getRelativeDataPath()) / part_relative_path / ""; if (disk->exists(part_download_path)) - throw Exception("Directory " + fullPath(disk, part_download_path) + " already exists.", ErrorCodes::DIRECTORY_ALREADY_EXISTS); + throw Exception(ErrorCodes::DIRECTORY_ALREADY_EXISTS, "Directory {} already exists.", fullPath(disk, part_download_path)); CurrentMetrics::Increment metric_increment{CurrentMetrics::ReplicatedFetch}; @@ -817,8 +820,9 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDiskRemoteMeta( if (expected_hash != hashing_out.getHash()) { - throw Exception("Checksum mismatch for file " + metadata_file + " transferred from " + replica_path, - ErrorCodes::CHECKSUM_DOESNT_MATCH); + throw Exception(ErrorCodes::CHECKSUM_DOESNT_MATCH, + "Checksum mismatch for file {} transferred from {}", + metadata_file, replica_path); } } } From 3b3635c6d5bd3d4bc6bfab2fd0e2a2c22c77d4d8 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 1 Feb 2022 12:30:58 +0300 Subject: [PATCH 084/149] Fix formatting error in logging messages Signed-off-by: Azat Khuzhin --- programs/library-bridge/Handlers.cpp | 2 +- src/Common/ZooKeeper/ZooKeeper.cpp | 2 +- src/IO/S3Common.cpp | 2 +- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 6 +++++- src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp | 3 ++- 5 files changed, 10 insertions(+), 5 deletions(-) diff --git a/programs/library-bridge/Handlers.cpp b/programs/library-bridge/Handlers.cpp index d16ac18ab56..58f9bd0a936 100644 --- a/programs/library-bridge/Handlers.cpp +++ b/programs/library-bridge/Handlers.cpp @@ -123,7 +123,7 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe } else { - LOG_TRACE(log, "Cannot clone from dictionary with id: {}, will call libNew instead"); + LOG_TRACE(log, "Cannot clone from dictionary with id: {}, will call libNew instead", from_dictionary_id); lib_new = true; } } diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index c8753c8edaf..b1574341c40 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -1145,7 +1145,7 @@ std::string normalizeZooKeeperPath(std::string zookeeper_path, bool check_starts if (check_starts_with_slash) throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "ZooKeeper path must starts with '/', got '{}'", zookeeper_path); if (log) - LOG_WARNING(log, "ZooKeeper path ('{}') does not start with '/'. It will not be supported in future releases"); + LOG_WARNING(log, "ZooKeeper path ('{}') does not start with '/'. It will not be supported in future releases", zookeeper_path); zookeeper_path = "/" + zookeeper_path; } diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index 432dc443300..59a4dab837b 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -317,7 +317,7 @@ public: , load_frequency_ms(Aws::Auth::REFRESH_THRESHOLD) , logger(&Poco::Logger::get("AWSInstanceProfileCredentialsProvider")) { - LOG_INFO(logger, "Creating Instance with injected EC2MetadataClient and refresh rate {}."); + LOG_INFO(logger, "Creating Instance with injected EC2MetadataClient and refresh rate."); } Aws::Auth::AWSCredentials GetAWSCredentials() override diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 05b0f8821a9..4435343aa67 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1616,7 +1616,11 @@ MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( auto end = in_partition->second.lower_bound(desired_mutation_version); if (end == in_partition->second.end() || end->first != desired_mutation_version) - LOG_WARNING(log, "Mutation with version {} not found in partition ID {} (trying to mutate part {}", desired_mutation_version, part->info.partition_id, part->name + ")"); + LOG_WARNING(log, + "Mutation with version {} not found in partition ID {} (trying to mutate part {})", + desired_mutation_version, + part->info.partition_id, + part->name); else ++end; diff --git a/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp b/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp index f02653d9167..db040584536 100644 --- a/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp +++ b/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp @@ -450,7 +450,8 @@ void MaterializedPostgreSQLConsumer::processReplicationMessage(const char * repl if (replica_identity != 'd' && replica_identity != 'i') { LOG_WARNING(log, - "Table has replica identity {} - not supported. A table must have a primary key or a replica identity index"); + "Table has replica identity {} - not supported. A table must have a primary key or a replica identity index", + replica_identity); markTableAsSkipped(relation_id, table_name); return; } From 5be76bc969fff7cb629100f4ff88a40226e9f0e3 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 1 Feb 2022 12:52:02 +0300 Subject: [PATCH 085/149] Use proper fmt-like logging Signed-off-by: Azat Khuzhin --- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 2 +- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- src/Storages/MergeTree/MergeTreePartsMover.cpp | 2 +- src/Storages/MergeTree/MutateFromLogEntryTask.cpp | 8 +++++--- src/Storages/StorageReplicatedMergeTree.cpp | 6 +++--- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 69424c046a0..6d8b348870b 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1428,7 +1428,7 @@ void IMergeTreeDataPart::makeCloneOnDisk(const DiskPtr & disk, const String & di if (disk->exists(fs::path(path_to_clone) / relative_path)) { - LOG_WARNING(storage.log, "Path " + fullPath(disk, path_to_clone + relative_path) + " already exists. Will remove it and clone again."); + LOG_WARNING(storage.log, "Path {} already exists. Will remove it and clone again.", fullPath(disk, path_to_clone + relative_path)); disk->removeRecursive(fs::path(path_to_clone) / relative_path / ""); } disk->createDirectories(path_to_clone); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 4a41358b1d1..abf418c0921 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1677,7 +1677,7 @@ size_t MergeTreeData::clearOldWriteAheadLogs() auto min_max_block_number = MergeTreeWriteAheadLog::tryParseMinMaxBlockNumber(it->name()); if (min_max_block_number && is_range_on_disk(min_max_block_number->first, min_max_block_number->second)) { - LOG_DEBUG(log, "Removing from filesystem the outdated WAL file " + it->name()); + LOG_DEBUG(log, "Removing from filesystem the outdated WAL file {}", it->name()); disk_ptr->removeFile(relative_data_path + it->name()); ++cleared_count; } diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index 8391e96ffba..9cc3ffe6e9e 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -211,7 +211,7 @@ MergeTreeData::DataPartPtr MergeTreePartsMover::clonePart(const MergeTreeMoveEnt String relative_path = part->relative_path; if (disk->exists(path_to_clone + relative_path)) { - LOG_WARNING(log, "Path " + fullPath(disk, path_to_clone + relative_path) + " already exists. Will remove it and clone again."); + LOG_WARNING(log, "Path {} already exists. Will remove it and clone again.", fullPath(disk, path_to_clone + relative_path)); disk->removeRecursive(fs::path(path_to_clone) / relative_path / ""); } disk->createDirectories(path_to_clone); diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp index 713f6a68612..16dd578a7eb 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp @@ -28,9 +28,11 @@ std::pair MutateFromLogEntry if (source_part->name != source_part_name) { - LOG_WARNING(log, "Part " + source_part_name + " is covered by " + source_part->name - + " but should be mutated to " + entry.new_part_name + ". " - + "Possibly the mutation of this part is not needed and will be skipped. This shouldn't happen often."); + LOG_WARNING(log, + "Part {} is covered by {} but should be mutated to {}. " + "Possibly the mutation of this part is not needed and will be skipped. " + "This shouldn't happen often.", + source_part_name, source_part->name, entry.new_part_name); return {false, {}}; } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 9c907074531..47b2edf9d78 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -2203,7 +2203,7 @@ void StorageReplicatedMergeTree::executeClonePartFromShard(const LogEntry & entr if (replica.empty()) throw Exception(ErrorCodes::NO_REPLICA_HAS_PART, "Not found active replica on shard {} to clone part {}", entry.source_shard, entry.new_part_name); - LOG_INFO(log, "Will clone part from shard " + entry.source_shard + " and replica " + replica); + LOG_INFO(log, "Will clone part from shard {} and replica {}", entry.source_shard, replica); MutableDataPartPtr part; @@ -6685,7 +6685,7 @@ void StorageReplicatedMergeTree::movePartitionToShard( zkutil::KeeperMultiException::check(rc, ops, responses); String task_znode_path = dynamic_cast(*responses.back()).path_created; - LOG_DEBUG(log, "Created task for part movement between shards at " + task_znode_path); + LOG_DEBUG(log, "Created task for part movement between shards at {}", task_znode_path); /// TODO(nv): Nice to have support for `replication_alter_partitions_sync`. /// For now use the system.part_moves_between_shards table for status. @@ -6824,7 +6824,7 @@ bool StorageReplicatedMergeTree::waitForShrinkingQueueSize(size_t queue_size, UI bool StorageReplicatedMergeTree::dropPartImpl( zkutil::ZooKeeperPtr & zookeeper, String part_name, LogEntry & entry, bool detach, bool throw_if_noop) { - LOG_TRACE(log, "Will try to insert a log entry to DROP_RANGE for part: " + part_name); + LOG_TRACE(log, "Will try to insert a log entry to DROP_RANGE for part {}", part_name); auto part_info = MergeTreePartInfo::fromPartName(part_name, format_version); From 5dfafd68a7cc6fabb0f0cf17b9fcc8c2a6617049 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 1 Feb 2022 13:13:44 +0300 Subject: [PATCH 086/149] ReplicatedMergeTreeQueue: Fix fmt:: and reduce copy-paste of logging and out reason Signed-off-by: Azat Khuzhin --- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 96 ++++++++++--------- 1 file changed, 49 insertions(+), 47 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 4435343aa67..373a97aa915 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1060,12 +1060,12 @@ bool ReplicatedMergeTreeQueue::isNotCoveredByFuturePartsImpl(const LogEntry & en if (entry_for_same_part_it != future_parts.end()) { const LogEntry & another_entry = *entry_for_same_part_it->second; - const char * format_str = "Not executing log entry {} of type {} for part {} " - "because another log entry {} of type {} for the same part ({}) is being processed. This shouldn't happen often."; - LOG_INFO(log, fmt::runtime(format_str), entry.znode_name, entry.type, entry.new_part_name, - another_entry.znode_name, another_entry.type, another_entry.new_part_name); - out_reason = fmt::format(format_str, entry.znode_name, entry.type, entry.new_part_name, - another_entry.znode_name, another_entry.type, another_entry.new_part_name); + out_reason = fmt::format( + "Not executing log entry {} of type {} for part {} " + "because another log entry {} of type {} for the same part ({}) is being processed. This shouldn't happen often.", + entry.znode_name, entry.type, entry.new_part_name, + another_entry.znode_name, another_entry.type, another_entry.new_part_name); + LOG_INFO(log, fmt::runtime(out_reason)); return false; /** When the corresponding action is completed, then `isNotCoveredByFuturePart` next time, will succeed, @@ -1086,10 +1086,11 @@ bool ReplicatedMergeTreeQueue::isNotCoveredByFuturePartsImpl(const LogEntry & en if (future_part.contains(result_part)) { - const char * format_str = "Not executing log entry {} for part {} " - "because it is covered by part {} that is currently executing."; - LOG_TRACE(log, fmt::runtime(format_str), entry.znode_name, new_part_name, future_part_elem.first); - out_reason = fmt::format(format_str, entry.znode_name, new_part_name, future_part_elem.first); + out_reason = fmt::format( + "Not executing log entry {} for part {} " + "because it is covered by part {} that is currently executing.", + entry.znode_name, new_part_name, future_part_elem.first); + LOG_TRACE(log, fmt::runtime(out_reason)); return false; } } @@ -1171,11 +1172,11 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( { if (future_parts.count(name)) { - const char * format_str = "Not executing log entry {} of type {} for part {} " - "because part {} is not ready yet (log entry for that part is being processed)."; - LOG_TRACE(log, fmt::runtime(format_str), entry.znode_name, entry.typeToString(), entry.new_part_name, name); - /// Copy-paste of above because we need structured logging (instead of already formatted message). - out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.typeToString(), entry.new_part_name, name); + out_postpone_reason = fmt::format( + "Not executing log entry {} of type {} for part {} " + "because part {} is not ready yet (log entry for that part is being processed).", + entry.znode_name, entry.typeToString(), entry.new_part_name, name); + LOG_TRACE(log, fmt::runtime(out_postpone_reason)); return false; } @@ -1191,9 +1192,10 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( if (merger_mutator.merges_blocker.isCancelled()) { - const char * format_str = "Not executing log entry {} of type {} for part {} because merges and mutations are cancelled now."; - LOG_DEBUG(log, fmt::runtime(format_str), entry.znode_name, entry.typeToString(), entry.new_part_name); - out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.typeToString(), entry.new_part_name); + out_postpone_reason = fmt::format( + "Not executing log entry {} of type {} for part {} because merges and mutations are cancelled now.", + entry.znode_name, entry.typeToString(), entry.new_part_name); + LOG_DEBUG(log, fmt::runtime(out_postpone_reason)); return false; } @@ -1228,24 +1230,20 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( { if (merger_mutator.ttl_merges_blocker.isCancelled()) { - const char * format_str = "Not executing log entry {} for part {} because merges with TTL are cancelled now."; - LOG_DEBUG(log, fmt::runtime(format_str), - entry.znode_name, entry.new_part_name); - out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.new_part_name); + out_postpone_reason = fmt::format( + "Not executing log entry {} for part {} because merges with TTL are cancelled now.", + entry.znode_name, entry.new_part_name); + LOG_DEBUG(log, fmt::runtime(out_postpone_reason)); return false; } size_t total_merges_with_ttl = data.getTotalMergesWithTTLInMergeList(); if (total_merges_with_ttl >= data_settings->max_number_of_merges_with_ttl_in_pool) { - const char * format_str = "Not executing log entry {} for part {}" - " because {} merges with TTL already executing, maximum {}."; - LOG_DEBUG(log, fmt::runtime(format_str), - entry.znode_name, entry.new_part_name, total_merges_with_ttl, - data_settings->max_number_of_merges_with_ttl_in_pool); - - out_postpone_reason = fmt::format(format_str, + out_postpone_reason = fmt::format( + "Not executing log entry {} for part {} because {} merges with TTL already executing, maximum {}.", entry.znode_name, entry.new_part_name, total_merges_with_ttl, data_settings->max_number_of_merges_with_ttl_in_pool); + LOG_DEBUG(log, fmt::runtime(out_postpone_reason)); return false; } } @@ -1271,9 +1269,10 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( if (!alter_sequence.canExecuteMetaAlter(entry.alter_version, state_lock)) { int head_alter = alter_sequence.getHeadAlterVersion(state_lock); - const char * format_str = "Cannot execute alter metadata {} with version {} because another alter {} must be executed before"; - LOG_TRACE(log, fmt::runtime(format_str), entry.znode_name, entry.alter_version, head_alter); - out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.alter_version, head_alter); + out_postpone_reason = fmt::format( + "Cannot execute alter metadata {} with version {} because another alter {} must be executed before", + entry.znode_name, entry.alter_version, head_alter); + LOG_TRACE(log, fmt::runtime(out_postpone_reason)); return false; } } @@ -1286,15 +1285,17 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( int head_alter = alter_sequence.getHeadAlterVersion(state_lock); if (head_alter == entry.alter_version) { - const char * format_str = "Cannot execute alter data {} with version {} because metadata still not altered"; - LOG_TRACE(log, fmt::runtime(format_str), entry.znode_name, entry.alter_version); - out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.alter_version); + out_postpone_reason = fmt::format( + "Cannot execute alter data {} with version {} because metadata still not altered", + entry.znode_name, entry.alter_version); + LOG_TRACE(log, fmt::runtime(out_postpone_reason)); } else { - const char * format_str = "Cannot execute alter data {} with version {} because another alter {} must be executed before"; - LOG_TRACE(log, fmt::runtime(format_str), entry.znode_name, entry.alter_version, head_alter); - out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.alter_version, head_alter); + out_postpone_reason = fmt::format( + "Cannot execute alter data {} with version {} because another alter {} must be executed before", + entry.znode_name, entry.alter_version, head_alter); + LOG_TRACE(log, fmt::runtime(out_postpone_reason)); } return false; @@ -1309,11 +1310,11 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( /// See also removePartProducingOpsInRange(...) and ReplicatedMergeTreeQueue::CurrentlyExecuting. if (currently_executing_drop_or_replace_range) { - - const char * format_str = "Not executing log entry {} of type {} for part {} " - "because another DROP_RANGE or REPLACE_RANGE entry are currently executing."; - LOG_TRACE(log, fmt::runtime(format_str), entry.znode_name, entry.typeToString(), entry.new_part_name); - out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.typeToString(), entry.new_part_name); + out_postpone_reason = fmt::format( + "Not executing log entry {} of type {} for part {} " + "because another DROP_RANGE or REPLACE_RANGE entry are currently executing.", + entry.znode_name, entry.typeToString(), entry.new_part_name); + LOG_TRACE(log, fmt::runtime(out_postpone_reason)); return false; } @@ -1338,10 +1339,11 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( auto new_part_info = MergeTreePartInfo::fromPartName(new_part_name, format_version); if (!new_part_info.isDisjoint(drop_part_info)) { - const char * format_str = "Not executing log entry {} of type {} for part {} " - "because it probably depends on {} (REPLACE_RANGE)."; - LOG_TRACE(log, fmt::runtime(format_str), entry.znode_name, entry.typeToString(), entry.new_part_name, replace_entry->znode_name); - out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.typeToString(), entry.new_part_name, replace_entry->znode_name); + out_postpone_reason = fmt::format( + "Not executing log entry {} of type {} for part {} " + "because it probably depends on {} (REPLACE_RANGE).", + entry.znode_name, entry.typeToString(), entry.new_part_name, replace_entry->znode_name); + LOG_TRACE(log, fmt::runtime(out_postpone_reason)); return false; } } From de80014eb5d1a82a336229cb303e614b0745d1c6 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 1 Feb 2022 13:17:43 +0300 Subject: [PATCH 087/149] Use fmt::format over Poco::format in PostgreSQLHandler Signed-off-by: Azat Khuzhin --- src/Server/PostgreSQLHandler.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Server/PostgreSQLHandler.cpp b/src/Server/PostgreSQLHandler.cpp index d6b834fceb7..04e43ed63aa 100644 --- a/src/Server/PostgreSQLHandler.cpp +++ b/src/Server/PostgreSQLHandler.cpp @@ -222,7 +222,7 @@ void PostgreSQLHandler::cancelRequest() std::unique_ptr msg = message_transport->receiveWithPayloadSize(8); - String query = Poco::format("KILL QUERY WHERE query_id = 'postgres:%d:%d'", msg->process_id, msg->secret_key); + String query = fmt::format("KILL QUERY WHERE query_id = 'postgres:{:d}:{:d}'", msg->process_id, msg->secret_key); ReadBufferFromString replacement(query); auto query_context = session->makeQueryContext(); @@ -287,7 +287,7 @@ void PostgreSQLHandler::processQuery() { secret_key = dis(gen); auto query_context = session->makeQueryContext(); - query_context->setCurrentQueryId(Poco::format("postgres:%d:%d", connection_id, secret_key)); + query_context->setCurrentQueryId(fmt::format("postgres:{:d}:{:d}", connection_id, secret_key)); CurrentThread::QueryScope query_scope{query_context}; ReadBufferFromString read_buf(spl_query); From 2ef2479ddcb2eff064aa46a82d6cb0c08693c191 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Tue, 1 Feb 2022 18:38:52 +0700 Subject: [PATCH 088/149] Add const to make clang-tidy happy --- src/Server/GRPCServer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index 85b1d345c6e..cfbe86d1adc 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -754,7 +754,7 @@ namespace // Parse the OpenTelemetry traceparent header. ClientInfo client_info = session->getClientInfo(); - auto & client_metadata = responder->grpc_context.client_metadata(); + const auto & client_metadata = responder->grpc_context.client_metadata(); auto traceparent = client_metadata.find("traceparent"); if (traceparent != client_metadata.end()) { From d0d71180dd7b89d5c3eab11c59a8355803cbe7bc Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Tue, 1 Feb 2022 14:44:19 +0300 Subject: [PATCH 089/149] Add fmt::runtime in gtest_log Signed-off-by: Azat Khuzhin --- src/Common/tests/gtest_log.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/tests/gtest_log.cpp b/src/Common/tests/gtest_log.cpp index 5addb5acf5d..b25f1cf117a 100644 --- a/src/Common/tests/gtest_log.cpp +++ b/src/Common/tests/gtest_log.cpp @@ -17,7 +17,7 @@ TEST(Logger, Log) Poco::Logger * log = &Poco::Logger::get("Log"); /// This test checks that we don't pass this string to fmtlib, because it is the only argument. - EXPECT_NO_THROW(LOG_INFO(log, "Hello {} World")); + EXPECT_NO_THROW(LOG_INFO(log, fmt::runtime("Hello {} World"))); } TEST(Logger, TestLog) From c28255850a3e3b0c32a930d7fb6b3f6e2e6a49bc Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 1 Feb 2022 15:06:49 +0300 Subject: [PATCH 090/149] fix metric Query --- src/Interpreters/ProcessList.cpp | 6 ++++++ src/Interpreters/ProcessList.h | 4 ++++ .../0_stateless/02190_current_metrics_query.reference | 1 + tests/queries/0_stateless/02190_current_metrics_query.sql | 2 ++ 4 files changed, 13 insertions(+) create mode 100644 tests/queries/0_stateless/02190_current_metrics_query.reference create mode 100644 tests/queries/0_stateless/02190_current_metrics_query.sql diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 37b2992d657..3c91f00ebf4 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -17,6 +17,11 @@ #include +namespace CurrentMetrics +{ + extern const Metric Query; +} + namespace DB { @@ -313,6 +318,7 @@ QueryStatus::QueryStatus( , client_info(client_info_) , priority_handle(std::move(priority_handle_)) , query_kind(query_kind_) + , num_queries_increment(CurrentMetrics::Query) { auto settings = getContext()->getSettings(); limits.max_execution_time = settings.max_execution_time; diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index 545e5b07345..2ba0b0814ee 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -121,6 +121,10 @@ protected: IAST::QueryKind query_kind; + /// This field is unused in this class, but it + /// increments/decrements metric in constructor/destructor. + CurrentMetrics::Increment num_queries_increment; + public: QueryStatus( diff --git a/tests/queries/0_stateless/02190_current_metrics_query.reference b/tests/queries/0_stateless/02190_current_metrics_query.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02190_current_metrics_query.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02190_current_metrics_query.sql b/tests/queries/0_stateless/02190_current_metrics_query.sql new file mode 100644 index 00000000000..e8b22e92a99 --- /dev/null +++ b/tests/queries/0_stateless/02190_current_metrics_query.sql @@ -0,0 +1,2 @@ +-- This query itself is also accounted in metric. +SELECT value > 0 FROM system.metrics WHERE metric = 'Query'; From 1f81e43f1edf269249490d16fd9d6a90ca10d14d Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Tue, 1 Feb 2022 19:55:24 +0700 Subject: [PATCH 091/149] Fix checking grants for SHOW GRANTS. --- .../Access/InterpreterShowGrantsQuery.cpp | 43 ++++++++++++++++++- .../integration/test_grant_and_revoke/test.py | 9 ++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/Access/InterpreterShowGrantsQuery.cpp b/src/Interpreters/Access/InterpreterShowGrantsQuery.cpp index cd98d8d4575..e11843f52e1 100644 --- a/src/Interpreters/Access/InterpreterShowGrantsQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowGrantsQuery.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -135,15 +136,53 @@ QueryPipeline InterpreterShowGrantsQuery::executeImpl() std::vector InterpreterShowGrantsQuery::getEntities() const { - const auto & show_query = query_ptr->as(); + const auto & access = getContext()->getAccess(); const auto & access_control = getContext()->getAccessControl(); + + const auto & show_query = query_ptr->as(); auto ids = RolesOrUsersSet{*show_query.for_roles, access_control, getContext()->getUserID()}.getMatchingIDs(access_control); + bool throw_if_access_denied = !show_query.for_roles->all; + std::optional show_users_check_result; + std::optional show_roles_check_result; + + auto has_grant_show_users = [&]() + { + if (show_users_check_result) + return *show_users_check_result; + if (throw_if_access_denied) + { + access->checkAccess(AccessType::SHOW_USERS); + show_users_check_result = true; + return true; + } + show_users_check_result = access->isGranted(AccessType::SHOW_USERS); + return *show_users_check_result; + }; + + auto has_grant_show_roles = [&]() + { + if (show_roles_check_result) + return *show_roles_check_result; + if (throw_if_access_denied) + { + access->checkAccess(AccessType::SHOW_ROLES); + show_roles_check_result = true; + return true; + } + show_roles_check_result = access->isGranted(AccessType::SHOW_ROLES); + return *show_roles_check_result; + }; + std::vector entities; for (const auto & id : ids) { auto entity = access_control.tryRead(id); - if (entity) + if (!entity) + continue; + if ((id == access->getUserID()) + || (entity->isTypeOf() && has_grant_show_users()) + || (entity->isTypeOf() && has_grant_show_roles())) entities.push_back(entity); } diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py index b905e4df219..89e07fecb0a 100644 --- a/tests/integration/test_grant_and_revoke/test.py +++ b/tests/integration/test_grant_and_revoke/test.py @@ -250,6 +250,15 @@ def test_introspection(): assert instance.query("SHOW GRANTS", user='A') == TSV(["GRANT SELECT ON test.table TO A"]) assert instance.query("SHOW GRANTS", user='B') == TSV(["GRANT CREATE ON *.* TO B WITH GRANT OPTION"]) + assert instance.query("SHOW GRANTS FOR ALL", user='A') == TSV(["GRANT SELECT ON test.table TO A"]) + assert instance.query("SHOW GRANTS FOR ALL", user='B') == TSV(["GRANT CREATE ON *.* TO B WITH GRANT OPTION"]) + assert instance.query("SHOW GRANTS FOR ALL") == TSV(["GRANT SELECT ON test.table TO A", + "GRANT CREATE ON *.* TO B WITH GRANT OPTION", + "GRANT ALL ON *.* TO default WITH GRANT OPTION"]) + + expected_error = "necessary to have grant SHOW USERS" + assert expected_error in instance.query_and_get_error("SHOW GRANTS FOR B", user='A') + expected_access1 = "CREATE USER A\n" \ "CREATE USER B\n" \ "CREATE USER default IDENTIFIED WITH plaintext_password SETTINGS PROFILE default" From a4aa1f37cec54535c0f2f7f5d162aea2a0aae44a Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Tue, 1 Feb 2022 17:44:23 +0300 Subject: [PATCH 092/149] remove unused code --- src/Interpreters/TreeCNFConverter.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/Interpreters/TreeCNFConverter.h b/src/Interpreters/TreeCNFConverter.h index 0ec5368ba71..7ea5fa54680 100644 --- a/src/Interpreters/TreeCNFConverter.h +++ b/src/Interpreters/TreeCNFConverter.h @@ -89,15 +89,6 @@ public: return *this; } - template - const CNFQuery & iterateAtoms(F func) const - { - for (const auto & group : statements) - for (const auto & atom : group) - func(atom); - return *this; - } - CNFQuery & appendGroup(AndGroup&& and_group) { for (auto && or_group : and_group) From dd81eff3015d47dd774a015433486649500738af Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 1 Feb 2022 15:28:22 +0000 Subject: [PATCH 093/149] Fix tests. --- src/Server/GRPCServer.cpp | 2 +- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index 85b1d345c6e..cfbe86d1adc 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -754,7 +754,7 @@ namespace // Parse the OpenTelemetry traceparent header. ClientInfo client_info = session->getClientInfo(); - auto & client_metadata = responder->grpc_context.client_metadata(); + const auto & client_metadata = responder->grpc_context.client_metadata(); auto traceparent = client_metadata.find("traceparent"); if (traceparent != client_metadata.end()) { diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 94a779a699d..c71ead74be4 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1220,7 +1220,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( { const char * format_str = "Not executing merge for the part {}, waiting for {} to execute merge."; LOG_DEBUG(log, format_str, entry.new_part_name, replica_to_execute_merge.value()); - out_postpone_reason = out_postpone_reason = fmt::format(format_str, format_str, entry.new_part_name, replica_to_execute_merge.value()); + out_postpone_reason = fmt::format(format_str, entry.new_part_name, replica_to_execute_merge.value()); return false; } } From 0cac4b4a6e1c5c3b6791c5fc43599ad354526fce Mon Sep 17 00:00:00 2001 From: Aaron Katz <95644870+aaronstephenkatz@users.noreply.github.com> Date: Tue, 1 Feb 2022 09:40:17 -0800 Subject: [PATCH 094/149] adding # as a recognized start of single line comment --- src/Parsers/Lexer.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/Parsers/Lexer.cpp b/src/Parsers/Lexer.cpp index 654174c18f7..fd5d0b556f0 100644 --- a/src/Parsers/Lexer.cpp +++ b/src/Parsers/Lexer.cpp @@ -280,6 +280,18 @@ Token Lexer::nextTokenImpl() } return Token(TokenType::Slash, token_begin, pos); } + case '#': /// start of single line comment, MySQL style + { /// PostgreSQL has some operators using '#' character. + /// For less ambiguity, we will recognize a comment only if # is followed by whitespace. + /// or #! as a special case for "shebang". + /// #hello - not a comment + /// # hello - a comment + /// #!/usr/bin/clickhouse-local --queries-file - a comment + ++pos; + if (pos < end && (*pos == ' ' || *pos == '!')) + return comment_until_end_of_line(); + return Token(TokenType::Error, token_begin, pos); + } case '%': return Token(TokenType::Percent, token_begin, ++pos); case '=': /// =, == From ed90a0fc1ccff841b2ff6404207b37250a3be598 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 1 Feb 2022 18:57:56 +0000 Subject: [PATCH 095/149] Add tests. --- .../queries/0_stateless/02192_comment.reference | 7 +++++++ tests/queries/0_stateless/02192_comment.sql | 16 ++++++++++++++++ .../0_stateless/02192_comment_failure.reference | 5 +++++ .../queries/0_stateless/02192_comment_failure.sh | 11 +++++++++++ 4 files changed, 39 insertions(+) create mode 100644 tests/queries/0_stateless/02192_comment.reference create mode 100644 tests/queries/0_stateless/02192_comment.sql create mode 100644 tests/queries/0_stateless/02192_comment_failure.reference create mode 100755 tests/queries/0_stateless/02192_comment_failure.sh diff --git a/tests/queries/0_stateless/02192_comment.reference b/tests/queries/0_stateless/02192_comment.reference new file mode 100644 index 00000000000..700b3808344 --- /dev/null +++ b/tests/queries/0_stateless/02192_comment.reference @@ -0,0 +1,7 @@ +1 +2 +3 +1 +1 +1 +\n# hello 1 diff --git a/tests/queries/0_stateless/02192_comment.sql b/tests/queries/0_stateless/02192_comment.sql new file mode 100644 index 00000000000..ff56caa77ee --- /dev/null +++ b/tests/queries/0_stateless/02192_comment.sql @@ -0,0 +1,16 @@ +# comment +#! comment2 +select 1; +# comment3 +#! comment4 +select 2; # another one comemnt +# +#! +select 3; + +select 1; #! +SELECT # hello +1; +SELECT /* # hello */ 1; +SELECT ' +# hello', 1; diff --git a/tests/queries/0_stateless/02192_comment_failure.reference b/tests/queries/0_stateless/02192_comment_failure.reference new file mode 100644 index 00000000000..21da4d2be9e --- /dev/null +++ b/tests/queries/0_stateless/02192_comment_failure.reference @@ -0,0 +1,5 @@ +OK +OK +OK +OK +OK diff --git a/tests/queries/0_stateless/02192_comment_failure.sh b/tests/queries/0_stateless/02192_comment_failure.sh new file mode 100755 index 00000000000..78ff474ae84 --- /dev/null +++ b/tests/queries/0_stateless/02192_comment_failure.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query="#" 2>&1 | grep -F -q 'Syntax error' && echo 'OK' || echo 'FAIL' +${CLICKHOUSE_CLIENT} --query="#not a comemnt" 2>&1 | grep -F -q 'Syntax error' && echo 'OK' || echo 'FAIL' +${CLICKHOUSE_CLIENT} --query="select 1 #not a comemnt" 2>&1 | grep -F -q 'Syntax error' && echo 'OK' || echo 'FAIL' +${CLICKHOUSE_CLIENT} --query="select 1 #" 2>&1 | grep -F -q 'Syntax error' && echo 'OK' || echo 'FAIL' +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "select 42 #" 2>&1 | grep -F -q 'Syntax error' && echo 'OK' || echo 'FAIL' From 392d60eb0df294dc3982b58c17ae770927f872f8 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 1 Feb 2022 19:09:29 +0000 Subject: [PATCH 096/149] Update documentation. --- docs/en/sql-reference/syntax.md | 6 +++--- docs/ru/sql-reference/syntax.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/syntax.md b/docs/en/sql-reference/syntax.md index 207b2b82cd2..19efef3dc6a 100644 --- a/docs/en/sql-reference/syntax.md +++ b/docs/en/sql-reference/syntax.md @@ -30,7 +30,7 @@ There may be any number of space symbols between syntactical constructions (incl ClickHouse supports either SQL-style and C-style comments: -- SQL-style comments start with `--` and continue to the end of the line, a space after `--` can be omitted. +- SQL-style comments start with `--`, `#!` or `# ` and continue to the end of the line, a space after `--` and `#!` can be omitted. - C-style are from `/*` to `*/`and can be multiline, spaces are not required either. ## Keywords {#syntax-keywords} @@ -106,9 +106,9 @@ In queries, you can check `NULL` using the [IS NULL](../sql-reference/operators/ ### Heredoc {#heredeoc} -A [heredoc](https://en.wikipedia.org/wiki/Here_document) is a way to define a string (often multiline), while maintaining the original formatting. A heredoc is defined as a custom string literal, placed between two `$` symbols, for example `$heredoc$`. A value between two heredocs is processed "as-is". +A [heredoc](https://en.wikipedia.org/wiki/Here_document) is a way to define a string (often multiline), while maintaining the original formatting. A heredoc is defined as a custom string literal, placed between two `$` symbols, for example `$heredoc$`. A value between two heredocs is processed "as-is". -You can use a heredoc to embed snippets of SQL, HTML, or XML code, etc. +You can use a heredoc to embed snippets of SQL, HTML, or XML code, etc. **Example** diff --git a/docs/ru/sql-reference/syntax.md b/docs/ru/sql-reference/syntax.md index 6705b1068fe..7e9260915a8 100644 --- a/docs/ru/sql-reference/syntax.md +++ b/docs/ru/sql-reference/syntax.md @@ -28,7 +28,7 @@ INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') ## Комментарии {#comments} Поддерживаются комментарии в SQL-стиле и C-стиле. -Комментарии в SQL-стиле: от `--` до конца строки. Пробел после `--` может не ставиться. +Комментарии в SQL-стиле: от `--`, `#!` или `# ` до конца строки. Пробел после `--` и `#!` может не ставиться. Комментарии в C-стиле: от `/*` до `*/`. Такие комментарии могут быть многострочными. Пробелы тоже не обязательны. ## Ключевые слова {#syntax-keywords} @@ -104,9 +104,9 @@ INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') ### Heredoc {#heredeoc} -Синтаксис [heredoc](https://ru.wikipedia.org/wiki/Heredoc-синтаксис) — это способ определения строк с сохранением исходного формата (часто с переносом строки). `Heredoc` задается как произвольный строковый литерал между двумя символами `$`, например `$heredoc$`. Значение между двумя `heredoc` обрабатывается "как есть". +Синтаксис [heredoc](https://ru.wikipedia.org/wiki/Heredoc-синтаксис) — это способ определения строк с сохранением исходного формата (часто с переносом строки). `Heredoc` задается как произвольный строковый литерал между двумя символами `$`, например `$heredoc$`. Значение между двумя `heredoc` обрабатывается "как есть". -Синтаксис `heredoc` часто используют для вставки кусков кода SQL, HTML, XML и т.п. +Синтаксис `heredoc` часто используют для вставки кусков кода SQL, HTML, XML и т.п. **Пример** From 9ea4f8d9558de06871a616e566c87a73f18663d9 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 1 Feb 2022 19:12:39 +0000 Subject: [PATCH 097/149] Fix test name. --- ...92_comment_failure.reference => 02192_comment_error.reference} | 0 .../{02192_comment_failure.sh => 02192_comment_error.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{02192_comment_failure.reference => 02192_comment_error.reference} (100%) rename tests/queries/0_stateless/{02192_comment_failure.sh => 02192_comment_error.sh} (100%) diff --git a/tests/queries/0_stateless/02192_comment_failure.reference b/tests/queries/0_stateless/02192_comment_error.reference similarity index 100% rename from tests/queries/0_stateless/02192_comment_failure.reference rename to tests/queries/0_stateless/02192_comment_error.reference diff --git a/tests/queries/0_stateless/02192_comment_failure.sh b/tests/queries/0_stateless/02192_comment_error.sh similarity index 100% rename from tests/queries/0_stateless/02192_comment_failure.sh rename to tests/queries/0_stateless/02192_comment_error.sh From efd15a92defb9a1970458ef3db02f6b0247e7242 Mon Sep 17 00:00:00 2001 From: tesw yew isal <278153+cnmade@users.noreply.github.com> Date: Wed, 2 Feb 2022 08:47:49 +0800 Subject: [PATCH 098/149] Translate zh/faq/integration/index: rename old file --- docs/zh/faq/integration/index.md | 1 - docs/zh/faq/integration/index.md.md | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 120000 docs/zh/faq/integration/index.md create mode 100644 docs/zh/faq/integration/index.md.md diff --git a/docs/zh/faq/integration/index.md b/docs/zh/faq/integration/index.md deleted file mode 120000 index 8323d6218a3..00000000000 --- a/docs/zh/faq/integration/index.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/faq/integration/index.md \ No newline at end of file diff --git a/docs/zh/faq/integration/index.md.md b/docs/zh/faq/integration/index.md.md new file mode 100644 index 00000000000..8323d6218a3 --- /dev/null +++ b/docs/zh/faq/integration/index.md.md @@ -0,0 +1 @@ +../../../en/faq/integration/index.md \ No newline at end of file From 5496d4d8409958fe2ea3ae3ce0aa9fee0aaf849c Mon Sep 17 00:00:00 2001 From: tesw yew isal <278153+cnmade@users.noreply.github.com> Date: Wed, 2 Feb 2022 08:48:41 +0800 Subject: [PATCH 099/149] Translate zh/faq/integration/index: reimport file --- docs/zh/faq/integration/index.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 docs/zh/faq/integration/index.md diff --git a/docs/zh/faq/integration/index.md b/docs/zh/faq/integration/index.md new file mode 100644 index 00000000000..644eef0c7b7 --- /dev/null +++ b/docs/zh/faq/integration/index.md @@ -0,0 +1,19 @@ +--- +title: Questions about integrating ClickHouse and other systems +toc_hidden_folder: true +toc_priority: 4 +toc_title: Integration +--- + +# Questions About Integrating ClickHouse and Other Systems {#question-about-integrating-clickhouse-and-other-systems} + +Questions: + +- [How do I export data from ClickHouse to a file?](../../faq/integration/file-export.md) +- [How to import JSON into ClickHouse?](../../faq/integration/json-import.md) +- [What if I have a problem with encodings when connecting to Oracle via ODBC?](../../faq/integration/oracle-odbc.md) + +!!! info "Don’t see what you were looking for?" + Check out [other F.A.Q. categories](../../faq/index.md) or browse around main documentation articles found in the left sidebar. + +{## [Original article](https://clickhouse.com/docs/en/faq/integration/) ##} From f2a8ba403dee7ec56fcfc4ba95d2457da1ad26bf Mon Sep 17 00:00:00 2001 From: tesw yew isal <278153+cnmade@users.noreply.github.com> Date: Wed, 2 Feb 2022 08:52:12 +0800 Subject: [PATCH 100/149] Translate zh/faq/integration/index: translate to zh --- docs/zh/faq/integration/index.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/zh/faq/integration/index.md b/docs/zh/faq/integration/index.md index 644eef0c7b7..2bfd728ec8c 100644 --- a/docs/zh/faq/integration/index.md +++ b/docs/zh/faq/integration/index.md @@ -1,19 +1,21 @@ --- -title: Questions about integrating ClickHouse and other systems +title: 关于集成ClickHouse和其他系统的问题 toc_hidden_folder: true toc_priority: 4 toc_title: Integration --- -# Questions About Integrating ClickHouse and Other Systems {#question-about-integrating-clickhouse-and-other-systems} +# 关于集成ClickHouse和其他系统的问题 {#question-about-integrating-clickhouse-and-other-systems} -Questions: +问题: -- [How do I export data from ClickHouse to a file?](../../faq/integration/file-export.md) -- [How to import JSON into ClickHouse?](../../faq/integration/json-import.md) -- [What if I have a problem with encodings when connecting to Oracle via ODBC?](../../faq/integration/oracle-odbc.md) +- [如何从 ClickHouse 导出数据到一个文件?](../../faq/integration/file-export.md) +- [如何导入JSON到ClickHouse?](../../faq/integration/json-import.md) +- [如果我用ODBC链接Oracle数据库出现编码问题该怎么办?](../../faq/integration/oracle-odbc.md) -!!! info "Don’t see what you were looking for?" - Check out [other F.A.Q. categories](../../faq/index.md) or browse around main documentation articles found in the left sidebar. -{## [Original article](https://clickhouse.com/docs/en/faq/integration/) ##} + +!!! info "没看到你要找的东西吗?" + 查看[其他faq类别](../../faq/index.md)或浏览左边栏中的主要文档文章。 + +{## [原文](https://clickhouse.com/docs/en/faq/integration/) ##} \ No newline at end of file From 057194c773d08276d909ed5ea87d3663eb6f0338 Mon Sep 17 00:00:00 2001 From: tesw yew isal <278153+cnmade@users.noreply.github.com> Date: Wed, 2 Feb 2022 08:52:49 +0800 Subject: [PATCH 101/149] Translate zh/faq/integration/index: remove old file --- docs/zh/faq/integration/index.md.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 docs/zh/faq/integration/index.md.md diff --git a/docs/zh/faq/integration/index.md.md b/docs/zh/faq/integration/index.md.md deleted file mode 100644 index 8323d6218a3..00000000000 --- a/docs/zh/faq/integration/index.md.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/faq/integration/index.md \ No newline at end of file From 15993cb13bd5052488bf73189c0619838d0a0c21 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 2 Feb 2022 11:37:10 +0300 Subject: [PATCH 102/149] Add missing fmt::runtime() in MergeTreeBackgroundExecutor Signed-off-by: Azat Khuzhin --- src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp index 51a7037e03f..499a8fbbaa6 100644 --- a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp @@ -97,7 +97,7 @@ void MergeTreeBackgroundExecutor::routine(TaskRuntimeDataPtr item) catch (const Exception & e) { if (e.code() == ErrorCodes::ABORTED) /// Cancelled merging parts is not an error - log as info. - LOG_INFO(log, getCurrentExceptionMessage(false)); + LOG_INFO(log, fmt::runtime(getCurrentExceptionMessage(false))); else tryLogCurrentException(__PRETTY_FUNCTION__); } @@ -148,7 +148,7 @@ void MergeTreeBackgroundExecutor::routine(TaskRuntimeDataPtr item) catch (const Exception & e) { if (e.code() == ErrorCodes::ABORTED) /// Cancelled merging parts is not an error - log as info. - LOG_INFO(log, getCurrentExceptionMessage(false)); + LOG_INFO(log, fmt::runtime(getCurrentExceptionMessage(false))); else tryLogCurrentException(__PRETTY_FUNCTION__); } From 9f8cf1f0e506a1145dff290cf6f497d7790190b8 Mon Sep 17 00:00:00 2001 From: CoolT2 Date: Wed, 2 Feb 2022 13:29:05 +0300 Subject: [PATCH 103/149] Add quick start alert to RU installation guide Add quick start note as in EN version --- docs/ru/development/build-osx.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/ru/development/build-osx.md b/docs/ru/development/build-osx.md index a1192b509df..d63128836c4 100644 --- a/docs/ru/development/build-osx.md +++ b/docs/ru/development/build-osx.md @@ -2,8 +2,13 @@ toc_priority: 65 toc_title: Сборка на Mac OS X --- + # Как собрать ClickHouse на Mac OS X {#how-to-build-clickhouse-on-mac-os-x} +!!! info "Вам не нужно собирать ClickHouse самостоятельно" + Вы можете установить готовый ClickHouse, как описано в [Быстрый старте](https://clickhouse.com/#quick-start). + Следуйте инструкциям по установке `macOS (Intel)` или `macOS (Apple Silicon)`. + Сборка должна запускаться с x86_64 (Intel) на macOS версии 10.15 (Catalina) и выше в последней версии компилятора Xcode's native AppleClang, Homebrew's vanilla Clang или в GCC-компиляторах. ## Установка Homebrew {#install-homebrew} From 1d1985159050db39a292cea32037223f11076514 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 2 Feb 2022 10:29:10 +0300 Subject: [PATCH 104/149] Disable data skipping indexes by default for queries with FINAL This patch adds use_skip_indexes_if_final setting, that is OFF by default. Since skipping data for queries with FINAL may produce incorrect result. Signed-off-by: Azat Khuzhin --- src/Core/Settings.h | 1 + .../QueryPlan/ReadFromMergeTree.cpp | 7 ++++++- .../02200_use_skip_indexes.reference | 4 ++++ .../0_stateless/02200_use_skip_indexes.sql | 14 ++++++++++++++ .../02201_use_skip_indexes_if_final.reference | 6 ++++++ .../02201_use_skip_indexes_if_final.sql | 16 ++++++++++++++++ .../02202_use_skip_indexes_if_final.reference | 6 ++++++ .../02202_use_skip_indexes_if_final.sql | 19 +++++++++++++++++++ 8 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/02200_use_skip_indexes.reference create mode 100644 tests/queries/0_stateless/02200_use_skip_indexes.sql create mode 100644 tests/queries/0_stateless/02201_use_skip_indexes_if_final.reference create mode 100644 tests/queries/0_stateless/02201_use_skip_indexes_if_final.sql create mode 100644 tests/queries/0_stateless/02202_use_skip_indexes_if_final.reference create mode 100644 tests/queries/0_stateless/02202_use_skip_indexes_if_final.sql diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 54307b25efa..48dd637a943 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -170,6 +170,7 @@ class IColumn; M(Bool, force_index_by_date, false, "Throw an exception if there is a partition key in a table, and it is not used.", 0) \ M(Bool, force_primary_key, false, "Throw an exception if there is primary key in a table, and it is not used.", 0) \ M(Bool, use_skip_indexes, true, "Use data skipping indexes during query execution.", 0) \ + M(Bool, use_skip_indexes_if_final, false, "If query has FINAL, then skipping data based on indexes may produce incorrect result, hence disabled by default.", 0) \ M(String, force_data_skipping_indices, "", "Comma separated list of strings or literals with the name of the data skipping indices that should be used during query execution, otherwise an exception will be thrown.", 0) \ \ M(Float, max_streams_to_max_threads_ratio, 1, "Allows you to use more sources than the number of threads - to more evenly distribute work across threads. It is assumed that this is a temporary solution, since it will be possible in the future to make the number of sources equal to the number of threads, but for each source to dynamically select available work for itself.", 0) \ diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index c8c36f6c595..21505610e09 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -912,6 +912,11 @@ MergeTreeDataSelectAnalysisResultPtr ReadFromMergeTree::selectRangesToRead( parts_before_pk = parts.size(); auto reader_settings = getMergeTreeReaderSettings(context); + + bool use_skip_indexes = context->getSettings().use_skip_indexes; + if (select.final() && !context->getSettings().use_skip_indexes_if_final) + use_skip_indexes = false; + result.parts_with_ranges = MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipIndexes( std::move(parts), metadata_snapshot, @@ -922,7 +927,7 @@ MergeTreeDataSelectAnalysisResultPtr ReadFromMergeTree::selectRangesToRead( log, num_streams, result.index_stats, - context->getSettings().use_skip_indexes); + use_skip_indexes); } catch (...) { diff --git a/tests/queries/0_stateless/02200_use_skip_indexes.reference b/tests/queries/0_stateless/02200_use_skip_indexes.reference new file mode 100644 index 00000000000..f11db7af711 --- /dev/null +++ b/tests/queries/0_stateless/02200_use_skip_indexes.reference @@ -0,0 +1,4 @@ +-- { echoOn } +SELECT * FROM data_02200 WHERE value = 1 SETTINGS use_skip_indexes=1, max_rows_to_read=1; +1 1 +SELECT * FROM data_02200 WHERE value = 1 SETTINGS use_skip_indexes=0, max_rows_to_read=1; -- { serverError TOO_MANY_ROWS } diff --git a/tests/queries/0_stateless/02200_use_skip_indexes.sql b/tests/queries/0_stateless/02200_use_skip_indexes.sql new file mode 100644 index 00000000000..64003285abb --- /dev/null +++ b/tests/queries/0_stateless/02200_use_skip_indexes.sql @@ -0,0 +1,14 @@ +CREATE TABLE data_02200 ( + key Int, + value Int, + INDEX idx value TYPE minmax GRANULARITY 1 +) +Engine=MergeTree() +ORDER BY key +PARTITION BY key; + +INSERT INTO data_02200 SELECT number, number FROM numbers(10); + +-- { echoOn } +SELECT * FROM data_02200 WHERE value = 1 SETTINGS use_skip_indexes=1, max_rows_to_read=1; +SELECT * FROM data_02200 WHERE value = 1 SETTINGS use_skip_indexes=0, max_rows_to_read=1; -- { serverError TOO_MANY_ROWS } diff --git a/tests/queries/0_stateless/02201_use_skip_indexes_if_final.reference b/tests/queries/0_stateless/02201_use_skip_indexes_if_final.reference new file mode 100644 index 00000000000..423a9fa46d6 --- /dev/null +++ b/tests/queries/0_stateless/02201_use_skip_indexes_if_final.reference @@ -0,0 +1,6 @@ +-- { echoOn } +SELECT * FROM data_02201 FINAL WHERE value = 1 SETTINGS use_skip_indexes=0, use_skip_indexes_if_final=0, max_rows_to_read=1; -- { serverError TOO_MANY_ROWS } +SELECT * FROM data_02201 FINAL WHERE value = 1 SETTINGS use_skip_indexes=1, use_skip_indexes_if_final=0, max_rows_to_read=1; -- { serverError TOO_MANY_ROWS } +SELECT * FROM data_02201 FINAL WHERE value = 1 SETTINGS use_skip_indexes=0, use_skip_indexes_if_final=1, max_rows_to_read=1; -- { serverError TOO_MANY_ROWS } +SELECT * FROM data_02201 FINAL WHERE value = 1 SETTINGS use_skip_indexes=1, use_skip_indexes_if_final=1, max_rows_to_read=1; +1 1 diff --git a/tests/queries/0_stateless/02201_use_skip_indexes_if_final.sql b/tests/queries/0_stateless/02201_use_skip_indexes_if_final.sql new file mode 100644 index 00000000000..2afc4941c9e --- /dev/null +++ b/tests/queries/0_stateless/02201_use_skip_indexes_if_final.sql @@ -0,0 +1,16 @@ +CREATE TABLE data_02201 ( + key Int, + value Int, + INDEX idx value TYPE minmax GRANULARITY 1 +) +Engine=AggregatingMergeTree() +ORDER BY key +PARTITION BY key; + +INSERT INTO data_02201 SELECT number, number FROM numbers(10); + +-- { echoOn } +SELECT * FROM data_02201 FINAL WHERE value = 1 SETTINGS use_skip_indexes=0, use_skip_indexes_if_final=0, max_rows_to_read=1; -- { serverError TOO_MANY_ROWS } +SELECT * FROM data_02201 FINAL WHERE value = 1 SETTINGS use_skip_indexes=1, use_skip_indexes_if_final=0, max_rows_to_read=1; -- { serverError TOO_MANY_ROWS } +SELECT * FROM data_02201 FINAL WHERE value = 1 SETTINGS use_skip_indexes=0, use_skip_indexes_if_final=1, max_rows_to_read=1; -- { serverError TOO_MANY_ROWS } +SELECT * FROM data_02201 FINAL WHERE value = 1 SETTINGS use_skip_indexes=1, use_skip_indexes_if_final=1, max_rows_to_read=1; diff --git a/tests/queries/0_stateless/02202_use_skip_indexes_if_final.reference b/tests/queries/0_stateless/02202_use_skip_indexes_if_final.reference new file mode 100644 index 00000000000..7d543cfcaf6 --- /dev/null +++ b/tests/queries/0_stateless/02202_use_skip_indexes_if_final.reference @@ -0,0 +1,6 @@ +-- { echoOn } +SELECT * FROM data_02201 FINAL WHERE value_max = 1 ORDER BY key, value_max SETTINGS use_skip_indexes=1, use_skip_indexes_if_final=0; +0 1 +SELECT * FROM data_02201 FINAL WHERE value_max = 1 ORDER BY key, value_max SETTINGS use_skip_indexes=1, use_skip_indexes_if_final=1; +0 1 +1 1 diff --git a/tests/queries/0_stateless/02202_use_skip_indexes_if_final.sql b/tests/queries/0_stateless/02202_use_skip_indexes_if_final.sql new file mode 100644 index 00000000000..cce785eb17d --- /dev/null +++ b/tests/queries/0_stateless/02202_use_skip_indexes_if_final.sql @@ -0,0 +1,19 @@ +-- This tests will show the difference in data with use_skip_indexes_if_final and w/o + +CREATE TABLE data_02201 ( + key Int, + value_max SimpleAggregateFunction(max, Int), + INDEX idx value_max TYPE minmax GRANULARITY 1 +) +Engine=AggregatingMergeTree() +ORDER BY key +PARTITION BY key; + +SYSTEM STOP MERGES data_02201; + +INSERT INTO data_02201 SELECT number, number FROM numbers(10); +INSERT INTO data_02201 SELECT number, number+1 FROM numbers(10); + +-- { echoOn } +SELECT * FROM data_02201 FINAL WHERE value_max = 1 ORDER BY key, value_max SETTINGS use_skip_indexes=1, use_skip_indexes_if_final=0; +SELECT * FROM data_02201 FINAL WHERE value_max = 1 ORDER BY key, value_max SETTINGS use_skip_indexes=1, use_skip_indexes_if_final=1; From 88ae61173907381e28fd8361deb2439e57d82b8b Mon Sep 17 00:00:00 2001 From: CoolT2 Date: Wed, 2 Feb 2022 13:51:29 +0300 Subject: [PATCH 105/149] Apply suggestions from code review Co-authored-by: Nikolay Degterinsky <43110995+evillique@users.noreply.github.com> --- docs/ru/development/build-osx.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ru/development/build-osx.md b/docs/ru/development/build-osx.md index d63128836c4..48d92501f06 100644 --- a/docs/ru/development/build-osx.md +++ b/docs/ru/development/build-osx.md @@ -6,8 +6,8 @@ toc_title: Сборка на Mac OS X # Как собрать ClickHouse на Mac OS X {#how-to-build-clickhouse-on-mac-os-x} !!! info "Вам не нужно собирать ClickHouse самостоятельно" - Вы можете установить готовый ClickHouse, как описано в [Быстрый старте](https://clickhouse.com/#quick-start). - Следуйте инструкциям по установке `macOS (Intel)` или `macOS (Apple Silicon)`. + Вы можете установить предварительно собранный ClickHouse, как описано в [Быстром старте](https://clickhouse.com/#quick-start). + Следуйте инструкциям по установке для `macOS (Intel)` или `macOS (Apple Silicon)`. Сборка должна запускаться с x86_64 (Intel) на macOS версии 10.15 (Catalina) и выше в последней версии компилятора Xcode's native AppleClang, Homebrew's vanilla Clang или в GCC-компиляторах. From 720407c88baba174336bdec763e45bea51672ad1 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Tue, 1 Feb 2022 22:06:53 +0100 Subject: [PATCH 106/149] Update list-versions.sh, update version_date.tsv list-versions.sh used to use a commit date instead of a tagging date with `--format="%ai"`, now it uses the tag date itself. --- utils/list-versions/list-versions.sh | 4 +- utils/list-versions/version_date.tsv | 242 ++++++++++++++++----------- 2 files changed, 143 insertions(+), 103 deletions(-) diff --git a/utils/list-versions/list-versions.sh b/utils/list-versions/list-versions.sh index 01364c78f83..a93552025e4 100755 --- a/utils/list-versions/list-versions.sh +++ b/utils/list-versions/list-versions.sh @@ -1,3 +1,5 @@ #!/bin/bash -git tag --list | grep -P 'v.+-(stable|lts)' | sort -V | xargs git show --format='%ai' | awk '/^v/ { version = $1 } /^[0-9]+/ { if (version) { date = $1 } } { if (version && date) { print version "\t" date; version = ""; date = ""; } }' | tac +# refname:strip=2: default tag name when format is not set +# creatordate is always defined for all tags +git tag --list 'v*-lts' 'v*-stable' --format='%(refname:strip=2) %(creatordate:format:%F)' | sort -rV diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 539aa6f1b19..cff412d9fd8 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,56 +1,87 @@ -v21.9.4.35-stable 2021-09-22 -v21.9.3.30-stable 2021-09-16 +v22.1.3.7-stable 2022-01-23 +v22.1.2.2-stable 2022-01-19 +v21.12.4.1-stable 2022-01-23 +v21.12.3.32-stable 2021-12-27 +v21.12.2.17-stable 2021-12-16 +v21.11.11.1-stable 2022-01-23 +v21.11.10.1-stable 2022-01-12 +v21.11.9.1-stable 2021-12-27 +v21.11.8.4-stable 2021-12-22 +v21.11.7.9-stable 2021-12-15 +v21.11.6.7-stable 2021-12-10 +v21.11.5.33-stable 2021-12-02 +v21.11.4.14-stable 2021-11-17 +v21.11.3.6-stable 2021-11-11 +v21.11.2.2-stable 2021-11-09 +v21.10.6.2-stable 2022-01-24 +v21.10.5.3-stable 2021-12-10 +v21.10.4.26-stable 2021-12-02 +v21.10.3.9-stable 2021-11-17 +v21.10.2.15-stable 2021-10-18 +v21.9.6.24-stable 2021-12-02 +v21.9.5.16-stable 2021-10-19 +v21.9.4.35-stable 2021-09-24 +v21.9.3.30-stable 2021-09-17 v21.9.2.17-stable 2021-09-09 -v21.8.8.29-lts 2021-09-28 -v21.8.7.22-lts 2021-09-22 -v21.8.6.15-lts 2021-09-16 +v21.8.14.5-lts 2022-01-26 +v21.8.13.6-lts 2021-12-27 +v21.8.12.29-lts 2021-12-02 +v21.8.11.4-lts 2021-11-17 +v21.8.10.19-lts 2021-10-21 +v21.8.9.13-lts 2021-10-19 +v21.8.8.29-lts 2021-09-29 +v21.8.7.22-lts 2021-09-24 +v21.8.6.15-lts 2021-09-17 v21.8.5.7-lts 2021-09-02 -v21.8.4.51-lts 2021-08-17 +v21.8.4.51-lts 2021-08-18 v21.8.3.44-lts 2021-08-12 -v21.7.11.3-stable 2021-09-23 -v21.7.10.4-stable 2021-09-16 +v21.7.11.3-stable 2021-09-24 +v21.7.10.4-stable 2021-09-18 v21.7.9.7-stable 2021-09-02 -v21.7.8.58-stable 2021-08-17 -v21.7.7.47-stable 2021-08-09 +v21.7.8.58-stable 2021-08-18 +v21.7.7.47-stable 2021-08-10 v21.7.6.39-stable 2021-08-06 -v21.7.5.29-stable 2021-07-28 -v21.7.4.18-stable 2021-07-17 -v21.7.3.14-stable 2021-07-13 +v21.7.5.29-stable 2021-07-29 +v21.7.4.18-stable 2021-07-19 +v21.7.3.14-stable 2021-07-14 v21.7.2.7-stable 2021-07-09 -v21.6.9.7-stable 2021-09-02 -v21.6.8.62-stable 2021-07-13 -v21.6.7.57-stable 2021-07-09 -v21.6.6.51-stable 2021-07-02 +v21.6.9.7-stable 2021-09-03 +v21.6.8.62-stable 2021-07-16 +v21.6.7.57-stable 2021-07-10 +v21.6.6.51-stable 2021-07-04 v21.6.5.37-stable 2021-06-19 v21.6.4.26-stable 2021-06-11 -v21.6.3.14-stable 2021-06-04 +v21.6.3.14-stable 2021-06-05 v21.5.9.4-stable 2021-07-10 -v21.5.8.21-stable 2021-07-02 -v21.5.7.9-stable 2021-06-22 -v21.5.6.6-stable 2021-05-29 +v21.5.8.21-stable 2021-07-04 +v21.5.7.9-stable 2021-06-23 +v21.5.6.6-stable 2021-05-30 v21.5.5.12-stable 2021-05-20 -v21.4.7.3-stable 2021-05-19 -v21.4.6.55-stable 2021-04-30 +v21.4.7.3-stable 2021-05-20 +v21.4.6.55-stable 2021-05-01 v21.4.5.46-stable 2021-04-24 v21.4.4.30-stable 2021-04-16 v21.4.3.21-stable 2021-04-12 -v21.3.17.2-lts 2021-09-16 -v21.3.16.5-lts 2021-09-03 +v21.3.20.1-lts 2022-01-26 +v21.3.19.1-lts 2021-12-10 +v21.3.18.4-lts 2021-10-21 +v21.3.17.2-lts 2021-09-17 +v21.3.16.5-lts 2021-09-04 v21.3.15.4-stable 2021-07-10 -v21.3.14.1-lts 2021-07-01 -v21.3.13.9-lts 2021-06-22 -v21.3.12.2-lts 2021-05-25 -v21.3.11.5-lts 2021-05-14 -v21.3.10.1-lts 2021-05-09 -v21.3.9.83-lts 2021-04-28 +v21.3.14.1-lts 2021-07-04 +v21.3.13.9-lts 2021-06-23 +v21.3.12.2-lts 2021-05-26 +v21.3.11.5-lts 2021-05-16 +v21.3.10.1-lts 2021-05-10 +v21.3.9.83-lts 2021-05-01 v21.3.8.76-lts 2021-04-24 -v21.3.7.62-stable 2021-04-16 +v21.3.7.62-stable 2021-04-17 v21.3.6.55-lts 2021-04-12 v21.3.5.42-lts 2021-04-07 v21.3.4.25-lts 2021-03-28 v21.3.3.14-lts 2021-03-19 v21.3.2.5-lts 2021-03-12 -v21.2.10.48-stable 2021-04-16 +v21.2.10.48-stable 2021-04-17 v21.2.9.41-stable 2021-04-12 v21.2.8.31-stable 2021-04-07 v21.2.7.11-stable 2021-03-28 @@ -93,8 +124,8 @@ v20.9.5.5-stable 2020-11-13 v20.9.4.76-stable 2020-10-29 v20.9.3.45-stable 2020-10-09 v20.9.2.20-stable 2020-09-22 -v20.8.19.4-stable 2021-07-10 -v20.8.18.32-lts 2021-04-16 +v20.8.19.4-stable 2021-07-11 +v20.8.18.32-lts 2021-04-17 v20.8.17.25-lts 2021-04-08 v20.8.16.20-lts 2021-04-06 v20.8.15.11-lts 2021-04-01 @@ -239,97 +270,104 @@ v19.9.2.4-stable 2019-06-24 v19.8.3.8-stable 2019-06-11 v19.7.5.29-stable 2019-07-05 v19.7.5.27-stable 2019-06-09 -v19.7.3.9-stable 2019-05-05 +v19.7.3.9-stable 2019-05-27 v19.6.3.18-stable 2019-06-15 -v19.6.2.11-stable 2019-04-30 -v19.5.4.22-stable 2019-04-30 -v19.5.3.8-stable 2019-04-17 +v19.6.2.11-stable 2019-05-14 +v19.5.4.22-stable 2019-05-13 +v19.5.3.8-stable 2019-04-18 v19.5.2.6-stable 2019-04-15 -v19.4.5.35-stable 2019-05-05 -v19.4.4.33-stable 2019-04-16 -v19.4.3.11-stable 2019-04-01 -v19.4.2.7-stable 2019-03-29 +v19.4.5.35-stable 2019-05-13 +v19.4.4.33-stable 2019-04-17 +v19.4.3.11-stable 2019-04-02 +v19.4.2.7-stable 2019-03-30 v19.4.1.3-stable 2019-03-19 v19.4.0.49-stable 2019-03-09 -v19.3.9.12-stable 2019-04-01 -v19.3.8.6-stable 2019-03-04 -v19.3.7-stable 2019-03-11 +v19.3.9.12-stable 2019-04-02 +v19.3.8.6-stable 2019-03-19 +v19.3.7-stable 2019-03-12 v19.3.6-stable 2019-03-02 -v19.3.5-stable 2019-02-20 -v19.3.4-stable 2019-02-15 +v19.3.5-stable 2019-02-21 +v19.3.4-stable 2019-02-16 v19.3.3-stable 2019-02-13 -v19.1.16.79-stable 2019-03-22 -v19.1.15.73-stable 2019-03-04 +v19.1.16.79-stable 2019-04-02 +v19.1.15.73-stable 2019-03-19 v19.1.14-stable 2019-03-14 v19.1.13-stable 2019-03-12 v19.1.10-stable 2019-03-03 -v19.1.9-stable 2019-02-20 -v19.1.8-stable 2019-02-15 -v19.1.7-stable 2019-02-13 +v19.1.9-stable 2019-02-21 +v19.1.8-stable 2019-02-16 +v19.1.7-stable 2019-02-15 v19.1.6-stable 2019-01-24 -v19.1.5-stable 2019-01-22 -v18.16.1-stable 2018-12-20 -v18.16.0-stable 2018-12-14 +v19.1.5-stable 2019-01-23 +v18.16.1-stable 2018-12-21 +v18.16.0-stable 2018-12-15 v18.14.19-stable 2018-12-19 v18.14.18-stable 2018-12-04 -v18.14.17-stable 2018-11-29 +v18.14.17-stable 2018-11-30 v18.14.15-stable 2018-11-21 -v18.14.14-stable 2018-11-20 -v18.14.13-stable 2018-11-07 -v18.14.12-stable 2018-11-01 -v18.14.11-stable 2018-10-26 -v18.14.10-stable 2018-10-23 +v18.14.14-stable 2018-11-21 +v18.14.13-stable 2018-11-08 +v18.14.12-stable 2018-11-02 +v18.14.11-stable 2018-10-29 +v18.14.10-stable 2018-10-24 v18.14.9-stable 2018-10-16 v18.14.8-stable 2018-10-13 v18.12.17-stable 2018-09-16 v18.12.14-stable 2018-09-13 -v18.12.13-stable 2018-09-10 -v1.1.54390-stable 2018-07-06 -v1.1.54388-stable 2018-06-27 +v18.12.13-stable 2018-09-11 +v18.10.3-stable 2018-08-13 +v18.6.0-stable 2018-08-01 +v18.5.1-stable 2018-07-31 +v18.4.0-stable 2018-07-28 +v18.1.0-stable 2018-07-20 +v1.1.54394-stable 2018-07-12 +v1.1.54390-stable 2018-07-09 +v1.1.54388-stable 2018-06-28 v1.1.54385-stable 2018-06-01 -v1.1.54383-stable 2018-05-18 -v1.1.54381-stable 2018-04-26 -v1.1.54380-stable 2018-04-20 -v1.1.54378-stable 2018-04-13 +v1.1.54383-stable 2018-05-22 +v1.1.54381-stable 2018-05-14 +v1.1.54380-stable 2018-04-21 +v1.1.54378-stable 2018-04-16 v1.1.54370-stable 2018-03-16 -v1.1.54362-stable 2018-03-10 +v1.1.54362-stable 2018-03-11 v1.1.54358-stable 2018-03-08 -v1.1.54343-stable 2018-01-24 +v1.1.54343-stable 2018-02-08 v1.1.54342-stable 2018-01-22 -v1.1.54337-stable 2018-01-17 -v1.1.54336-stable 2018-01-16 -v1.1.54335-stable 2018-01-16 -v1.1.54327-stable 2017-12-20 -v1.1.54318-stable 2017-11-30 +v1.1.54337-stable 2018-01-18 +v1.1.54336-stable 2018-01-17 +v1.1.54335-stable 2018-01-17 +v1.1.54327-stable 2017-12-21 +v1.1.54318-stable 2017-12-01 v1.1.54310-stable 2017-11-01 -v1.1.54304-stable 2017-10-19 -v1.1.54292-stable 2017-09-20 -v1.1.54289-stable 2017-09-13 -v1.1.54284-stable 2017-08-29 +v1.1.54304-stable 2017-10-24 +v1.1.54292-stable 2017-09-27 +v1.1.54289-stable 2017-09-14 +v1.1.54284-stable 2017-08-30 v1.1.54282-stable 2017-08-23 v1.1.54276-stable 2017-08-16 -v1.1.54245-stable 2017-07-04 -v1.1.54242-stable 2017-06-26 -v1.1.54236-stable 2017-05-23 -v1.1.54231-stable 2017-04-29 -v1.1.54198-stable 2017-03-28 -v1.1.54190-stable 2017-03-21 +v1.1.54245-stable 2017-07-05 +v1.1.54242-stable 2017-06-27 +v1.1.54236-stable 2017-05-25 +v1.1.54231-stable 2017-05-15 +v1.1.54198-stable 2017-03-29 +v1.1.54190-stable 2017-03-22 v1.1.54188-stable 2017-03-17 -v1.1.54181-stable 2017-03-09 -v1.1.54165-stable 2017-02-14 -v1.1.54159-stable 2017-02-03 -v1.1.54144-stable 2017-01-31 +v1.1.54181-stable 2017-03-11 +v1.1.54165-stable 2017-02-15 +v1.1.54159-stable 2017-02-07 +v1.1.54144-stable 2017-02-01 v1.1.54135-stable 2017-01-20 -v1.1.54134-stable 2017-01-14 -v1.1.54133-stable 2017-01-12 -v1.1.54127-stable 2016-12-22 -v1.1.54112-stable 2016-12-08 -v1.1.54083-stable 2016-11-28 +v1.1.54134-stable 2017-01-16 +v1.1.54133-stable 2017-01-13 +v1.1.54127-stable 2016-12-23 +v1.1.54112-stable 2016-12-13 +v1.1.54083-stable 2016-12-01 v1.1.54080-stable 2016-11-25 -v1.1.54074-stable 2016-11-23 -v1.1.54046-stable 2016-10-27 -v1.1.54030-stable 2016-10-24 -v1.1.54023-stable 2016-10-07 -v1.1.54022-stable 2016-09-26 -v1.1.54020-stable 2016-09-19 -v1.1.54019-stable 2016-09-07 +v1.1.54074-stable 2016-11-24 +v1.1.54046-stable 2016-11-08 +v1.1.54030-stable 2016-10-25 +v1.1.54023-stable 2016-10-10 +v1.1.54022-stable 2016-09-28 +v1.1.54020-stable 2016-09-21 +v1.1.54019-stable 2016-09-14 +v1.1.54011-stable 2016-08-18 From 7bfd526f465b23975f547cf86810b73266581006 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 2 Feb 2022 11:49:32 +0100 Subject: [PATCH 107/149] Add workflow for stable and lts tags --- .github/workflows/tags_stable.yml | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/tags_stable.yml diff --git a/.github/workflows/tags_stable.yml b/.github/workflows/tags_stable.yml new file mode 100644 index 00000000000..0ac40a1f06d --- /dev/null +++ b/.github/workflows/tags_stable.yml @@ -0,0 +1,33 @@ +name: TagsStableWorkflow +# - Gets artifacts from S3 +# - Sends it to JFROG Artifactory +# - Adds them to the release assets + +on: # yamllint disable-line rule:truthy + push: + tags: + - 'v*-stable' + - 'v*-lts' + + +jobs: + UpdateVersions: + runs-on: [self-hosted, style-checker] + steps: + - name: Get tag name + run: echo "GITHUB_TAG=${GITHUB_REF#refs/tags/}" >> "$GITHUB_ENV" + - name: Check out repository code + uses: actions/checkout@v2 + with: + ref: master + - name: Generate versions + run: | + git fetch --tags + ./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv + - name: Create Pull Request + uses: peter-evans/create-pull-request@v3 + with: + commit-message: Update version_date.tsv on ${{ env.GITHUB_TAG }} + branch: auto/${{ env.GITHUB_TAG }} + delete-branch: true + title: Update version_date.tsv on ${{ env.GITHUB_TAG }} From d2860a6d87541a7a2ac32f3c64fe454ab6722c79 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 2 Feb 2022 14:44:26 +0100 Subject: [PATCH 108/149] Add body for created pull request --- .github/workflows/tags_stable.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tags_stable.yml b/.github/workflows/tags_stable.yml index 0ac40a1f06d..423fc64c4fc 100644 --- a/.github/workflows/tags_stable.yml +++ b/.github/workflows/tags_stable.yml @@ -27,7 +27,12 @@ jobs: - name: Create Pull Request uses: peter-evans/create-pull-request@v3 with: - commit-message: Update version_date.tsv on ${{ env.GITHUB_TAG }} + commit-message: Update version_date.tsv after ${{ env.GITHUB_TAG }} branch: auto/${{ env.GITHUB_TAG }} delete-branch: true - title: Update version_date.tsv on ${{ env.GITHUB_TAG }} + title: Update version_date.tsv after ${{ env.GITHUB_TAG }} + body: | + Update version_date.tsv after ${{ env.GITHUB_TAG }} + + Changelog category (leave one): + - Not for changelog (changelog entry is not required) From 5472aef0848b3d451db7340d6a7c8df71ef0b379 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 2 Feb 2022 16:34:00 +0300 Subject: [PATCH 109/149] Fix current_user/current_address for interserver mode Before this patch current_user/current_address will be preserved from the previous query. Signed-off-by: Azat Khuzhin --- src/Server/TCPHandler.cpp | 6 +++++ .../test.py | 24 +++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 49595a9c658..668017f8ef8 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1392,6 +1392,12 @@ void TCPHandler::receiveQuery() if (is_interserver_mode) { ClientInfo original_session_client_info = session->getClientInfo(); + + /// Cleanup fields that should not be reused from previous query. + original_session_client_info.current_user.clear(); + original_session_client_info.current_query_id.clear(); + original_session_client_info.current_address = {}; + session = std::make_unique(server.context(), ClientInfo::Interface::TCP_INTERSERVER); session->getClientInfo() = original_session_client_info; } diff --git a/tests/integration/test_distributed_inter_server_secret/test.py b/tests/integration/test_distributed_inter_server_secret/test.py index 73d338ba870..2601163d790 100644 --- a/tests/integration/test_distributed_inter_server_secret/test.py +++ b/tests/integration/test_distributed_inter_server_secret/test.py @@ -87,6 +87,17 @@ def get_query_user_info(node, query_pattern): type = 'QueryFinish' """.format(query_pattern)).strip().split('\t') +# @return -- [user, initial_user] +def get_query_user_info_by_id(node, query_id): + node.query("SYSTEM FLUSH LOGS") + return node.query(""" + SELECT user, initial_user + FROM system.query_log + WHERE + query_id = '{}' AND + type = 'QueryFinish' + """.format(query_id)).strip().split('\t') + # @return -- settings def get_query_setting_on_shard(node, query_pattern, setting): node.query("SYSTEM FLUSH LOGS") @@ -183,6 +194,7 @@ def test_secure_insert_buffer_async(): # previous connection that was instantiated with "ro" user (using # interserver secret) assert not n1.contains_in_log('{' + query_id + '} Connection (n2:9000): Connecting.') + assert get_query_user_info_by_id(n1, query_id) == ['default', 'default'] # And before the bug was fixed this query will fail with the following error: # @@ -191,6 +203,18 @@ def test_secure_insert_buffer_async(): n1.query('OPTIMIZE TABLE dist_secure_buffer') n1.query('SYSTEM FLUSH DISTRIBUTED ON CLUSTER secure dist_secure_from_buffer') + # Check user from which the INSERT on the remote node will be executed + # + # Incorrect example: + # + # {2c55669f-71ad-48fe-98fa-7b475b80718e} executeQuery: (from 172.16.1.1:44636, user: ro) INSERT INTO default.data_from_buffer (key) VALUES + # + # Correct example: + # + # {2c55669f-71ad-48fe-98fa-7b475b80718e} executeQuery: (from 0.0.0.0:0, user: ) INSERT INTO default.data_from_buffer (key) VALUES + # + assert n2.contains_in_log('executeQuery: (from 0.0.0.0:0, user: ) INSERT INTO default.data_from_buffer (key) VALUES') + assert int(n1.query('SELECT count() FROM dist_secure_from_buffer')) == 2 n1.query('TRUNCATE TABLE data_from_buffer ON CLUSTER secure') From 744cb06a9148e6d8017abd6989fbd18530fbb90c Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 2 Feb 2022 16:04:19 +0100 Subject: [PATCH 110/149] Replace custom formats by `short` alias --- utils/list-versions/list-versions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/list-versions/list-versions.sh b/utils/list-versions/list-versions.sh index a93552025e4..8d4f286124e 100755 --- a/utils/list-versions/list-versions.sh +++ b/utils/list-versions/list-versions.sh @@ -2,4 +2,4 @@ # refname:strip=2: default tag name when format is not set # creatordate is always defined for all tags -git tag --list 'v*-lts' 'v*-stable' --format='%(refname:strip=2) %(creatordate:format:%F)' | sort -rV +git tag --list 'v*-lts' 'v*-stable' --format='%(refname:short) %(creatordate:short)' | sort -rV From 30557aebfb5cdb0a114d09286bcb52692e629f9f Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Wed, 2 Feb 2022 21:29:47 +0700 Subject: [PATCH 111/149] Add helper class to cache the result of checking access. --- src/Access/CachedAccessChecking.cpp | 44 +++++++++++++++++++ src/Access/CachedAccessChecking.h | 29 ++++++++++++ .../Access/InterpreterShowGrantsQuery.cpp | 39 +++------------- 3 files changed, 79 insertions(+), 33 deletions(-) create mode 100644 src/Access/CachedAccessChecking.cpp create mode 100644 src/Access/CachedAccessChecking.h diff --git a/src/Access/CachedAccessChecking.cpp b/src/Access/CachedAccessChecking.cpp new file mode 100644 index 00000000000..aa8ef6073d3 --- /dev/null +++ b/src/Access/CachedAccessChecking.cpp @@ -0,0 +1,44 @@ +#include +#include + + +namespace DB +{ +CachedAccessChecking::CachedAccessChecking(const std::shared_ptr & access_, AccessFlags access_flags_) + : CachedAccessChecking(access_, AccessRightsElement{access_flags_}) +{ +} + +CachedAccessChecking::CachedAccessChecking(const std::shared_ptr & access_, const AccessRightsElement & element_) + : access(access_), element(element_) +{ +} + +CachedAccessChecking::~CachedAccessChecking() = default; + +bool CachedAccessChecking::checkAccess(bool throw_if_denied) +{ + if (checked) + return result; + if (throw_if_denied) + { + try + { + access->checkAccess(element); + result = true; + } + catch (...) + { + result = false; + throw; + } + } + else + { + result = access->isGranted(element); + } + checked = true; + return result; +} + +} diff --git a/src/Access/CachedAccessChecking.h b/src/Access/CachedAccessChecking.h new file mode 100644 index 00000000000..e87c28dd823 --- /dev/null +++ b/src/Access/CachedAccessChecking.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class ContextAccess; + +/// Checks if the current user has a specified access type granted, +/// and if it's checked another time later, it will just return the first result. +class CachedAccessChecking +{ +public: + CachedAccessChecking(const std::shared_ptr & access_, AccessFlags access_flags_); + CachedAccessChecking(const std::shared_ptr & access_, const AccessRightsElement & element_); + ~CachedAccessChecking(); + + bool checkAccess(bool throw_if_denied = true); + +private: + const std::shared_ptr access; + const AccessRightsElement element; + bool checked = false; + bool result = false; +}; + +} diff --git a/src/Interpreters/Access/InterpreterShowGrantsQuery.cpp b/src/Interpreters/Access/InterpreterShowGrantsQuery.cpp index e11843f52e1..17d9f321b56 100644 --- a/src/Interpreters/Access/InterpreterShowGrantsQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowGrantsQuery.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -142,37 +143,9 @@ std::vector InterpreterShowGrantsQuery::getEntities() const const auto & show_query = query_ptr->as(); auto ids = RolesOrUsersSet{*show_query.for_roles, access_control, getContext()->getUserID()}.getMatchingIDs(access_control); + CachedAccessChecking show_users(access, AccessType::SHOW_USERS); + CachedAccessChecking show_roles(access, AccessType::SHOW_ROLES); bool throw_if_access_denied = !show_query.for_roles->all; - std::optional show_users_check_result; - std::optional show_roles_check_result; - - auto has_grant_show_users = [&]() - { - if (show_users_check_result) - return *show_users_check_result; - if (throw_if_access_denied) - { - access->checkAccess(AccessType::SHOW_USERS); - show_users_check_result = true; - return true; - } - show_users_check_result = access->isGranted(AccessType::SHOW_USERS); - return *show_users_check_result; - }; - - auto has_grant_show_roles = [&]() - { - if (show_roles_check_result) - return *show_roles_check_result; - if (throw_if_access_denied) - { - access->checkAccess(AccessType::SHOW_ROLES); - show_roles_check_result = true; - return true; - } - show_roles_check_result = access->isGranted(AccessType::SHOW_ROLES); - return *show_roles_check_result; - }; std::vector entities; for (const auto & id : ids) @@ -180,9 +153,9 @@ std::vector InterpreterShowGrantsQuery::getEntities() const auto entity = access_control.tryRead(id); if (!entity) continue; - if ((id == access->getUserID()) - || (entity->isTypeOf() && has_grant_show_users()) - || (entity->isTypeOf() && has_grant_show_roles())) + if ((id == access->getUserID() /* Any user can see his own grants */) + || (entity->isTypeOf() && show_users.checkAccess(throw_if_access_denied)) + || (entity->isTypeOf() && show_roles.checkAccess(throw_if_access_denied))) entities.push_back(entity); } From 1117e6095d39e447cc4e0dab237e5855b3aec00f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 2 Feb 2022 18:34:05 +0300 Subject: [PATCH 112/149] Update src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp Co-authored-by: Azat Khuzhin --- src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index c71ead74be4..a488dd52f1f 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1218,9 +1218,10 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( if (replica_to_execute_merge && !merge_strategy_picker.isMergeFinishedByReplica(replica_to_execute_merge.value(), entry)) { - const char * format_str = "Not executing merge for the part {}, waiting for {} to execute merge."; - LOG_DEBUG(log, format_str, entry.new_part_name, replica_to_execute_merge.value()); - out_postpone_reason = fmt::format(format_str, entry.new_part_name, replica_to_execute_merge.value()); + out_postpone_reason = fmt::format( + "Not executing merge for the part {}, waiting for {} to execute merge.", + entry.new_part_name, replica_to_execute_merge.value()); + LOG_DEBUG(log, fmt::runtime(out_postpone_reason)); return false; } } From 2003a96b58ae29442d80190c3052ed3ee3d0f443 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 2 Feb 2022 17:48:41 +0000 Subject: [PATCH 113/149] Probably fix something --- src/Disks/S3/DiskS3.cpp | 4 ++-- src/IO/WriteBufferFromS3.cpp | 2 +- src/IO/WriteBufferFromS3.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index f6c25e7439f..2638365c7ad 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -263,9 +263,9 @@ std::unique_ptr DiskS3::writeFile(const String & path, LOG_TRACE(log, "{} to file by path: {}. S3 path: {}", mode == WriteMode::Rewrite ? "Write" : "Append", backQuote(metadata_disk->getPath() + path), remote_fs_root_path + s3_path); - ScheduleFunc schedule = [](auto && callback) + ScheduleFunc schedule = [pool = &getThreadPoolWriter()](auto callback) { - getThreadPoolWriter().scheduleOrThrow([&callback, thread_group = CurrentThread::getGroup()]() + pool->scheduleOrThrow([callback = std::move(callback), thread_group = CurrentThread::getGroup()]() { if (thread_group) CurrentThread::attachTo(thread_group); diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 66eedee9bd3..4445d821a31 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -57,7 +57,7 @@ WriteBufferFromS3::WriteBufferFromS3( size_t max_single_part_upload_size_, std::optional> object_metadata_, size_t buffer_size_, - ScheduleFunc && schedule_) + ScheduleFunc schedule_) : BufferWithOwnMemory(buffer_size_, nullptr, 0) , bucket(bucket_) , key(key_) diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index f322dd4b567..1eb8a771944 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -50,7 +50,7 @@ public: size_t max_single_part_upload_size_, std::optional> object_metadata_ = std::nullopt, size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE, - ScheduleFunc && schedule_ = {}); + ScheduleFunc schedule_ = {}); ~WriteBufferFromS3() override; From 5bb1b3ce775b63db15dc957d2e24af07491eb32e Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 2 Feb 2022 20:53:04 +0300 Subject: [PATCH 114/149] support async inserts in clickhouse-client for queries with inlined data --- programs/client/Client.cpp | 2 +- src/Client/ClientBase.cpp | 34 ++++++++++++++----- src/Client/ClientBase.h | 2 ++ src/Common/CurrentMetrics.cpp | 1 + src/Common/ProfileEvents.cpp | 1 + src/Interpreters/AsynchronousInsertQueue.cpp | 15 ++++++++ .../02193_async_insert_tcp_client_1.reference | 5 +++ .../02193_async_insert_tcp_client_1.sql | 27 +++++++++++++++ .../02193_async_insert_tcp_client_2.reference | 4 +++ .../02193_async_insert_tcp_client_2.sh | 21 ++++++++++++ 10 files changed, 102 insertions(+), 10 deletions(-) create mode 100644 tests/queries/0_stateless/02193_async_insert_tcp_client_1.reference create mode 100644 tests/queries/0_stateless/02193_async_insert_tcp_client_1.sql create mode 100644 tests/queries/0_stateless/02193_async_insert_tcp_client_2.reference create mode 100755 tests/queries/0_stateless/02193_async_insert_tcp_client_2.sh diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index a5e4517824d..1a5c7d3e492 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -286,7 +286,7 @@ bool Client::executeMultiQuery(const String & all_queries_text) // , where the inline data is delimited by semicolon and not by a // newline. auto * insert_ast = parsed_query->as(); - if (insert_ast && insert_ast->data) + if (insert_ast && isSyncInsertWithData(*insert_ast, global_context)) { this_query_end = insert_ast->end; adjustQueryEnd(this_query_end, all_queries_end, global_context->getSettingsRef().max_parser_depth); diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 27deace416d..3141ec9d35e 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -573,6 +573,18 @@ void ClientBase::updateSuggest(const ASTCreateQuery & ast_create) suggest->addWords(std::move(new_words)); } +bool ClientBase::isSyncInsertWithData(const ASTInsertQuery & insert_query, const ContextPtr & context) +{ + if (!insert_query.data) + return false; + + auto settings = context->getSettings(); + if (insert_query.settings_ast) + settings.applyChanges(insert_query.settings_ast->as()->changes); + + return !settings.async_insert; +} + void ClientBase::processTextAsSingleQuery(const String & full_query) { /// Some parts of a query (result output and formatting) are executed @@ -597,10 +609,12 @@ void ClientBase::processTextAsSingleQuery(const String & full_query) updateSuggest(*create); } - // An INSERT query may have the data that follow query text. Remove the + /// An INSERT query may have the data that follow query text. Remove the /// Send part of query without data, because data will be sent separately. - auto * insert = parsed_query->as(); - if (insert && insert->data) + /// For asynchronous inserts we don't extract data, because it's needed + /// to be done on server side in that case. + const auto * insert = parsed_query->as(); + if (insert && isSyncInsertWithData(*insert, global_context)) query_to_execute = full_query.substr(0, insert->data - full_query.data()); else query_to_execute = full_query; @@ -1303,8 +1317,10 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin if (insert && insert->select) insert->tryFindInputFunction(input_function); + bool is_async_insert = global_context->getSettings().async_insert && insert && insert->hasInlinedData(); + /// INSERT query for which data transfer is needed (not an INSERT SELECT or input()) is processed separately. - if (insert && (!insert->select || input_function) && !insert->watch) + if (insert && (!insert->select || input_function) && !insert->watch && !is_async_insert) { if (input_function && insert->format.empty()) throw Exception("FORMAT must be specified for function input()", ErrorCodes::INVALID_USAGE_OF_INPUT); @@ -1434,17 +1450,17 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText( // row input formats (e.g. TSV) can't tell when the input stops, // unlike VALUES. auto * insert_ast = parsed_query->as(); + const char * query_to_execute_end = this_query_end; + if (insert_ast && insert_ast->data) { this_query_end = find_first_symbols<'\n'>(insert_ast->data, all_queries_end); insert_ast->end = this_query_end; - query_to_execute = all_queries_text.substr(this_query_begin - all_queries_text.data(), insert_ast->data - this_query_begin); - } - else - { - query_to_execute = all_queries_text.substr(this_query_begin - all_queries_text.data(), this_query_end - this_query_begin); + query_to_execute_end = isSyncInsertWithData(*insert_ast, global_context) ? insert_ast->data : this_query_end; } + query_to_execute = all_queries_text.substr(this_query_begin - all_queries_text.data(), query_to_execute_end - this_query_begin); + // Try to include the trailing comment with test hints. It is just // a guess for now, because we don't yet know where the query ends // if it is an INSERT query with inline data. We will do it again diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index 89e0770182b..5d753cfd702 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -139,6 +139,8 @@ private: void updateSuggest(const ASTCreateQuery & ast_create); protected: + static bool isSyncInsertWithData(const ASTInsertQuery & insert_query, const ContextPtr & context); + bool is_interactive = false; /// Use either interactive line editing interface or batch mode. bool is_multiquery = false; bool delayed_interactive = false; diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 896168253cf..a741f1f1bfc 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -80,6 +80,7 @@ M(SyncDrainedConnections, "Number of connections drained synchronously.") \ M(ActiveSyncDrainedConnections, "Number of active connections drained synchronously.") \ M(AsynchronousReadWait, "Number of threads waiting for asynchronous read.") \ + M(PendingAsyncInsert, "Number of asynchronous inserts that are waiting for flush.") \ namespace CurrentMetrics { diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index ea6c782ebb4..aa507b1ce59 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -8,6 +8,7 @@ M(Query, "Number of queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.") \ M(SelectQuery, "Same as Query, but only for SELECT queries.") \ M(InsertQuery, "Same as Query, but only for INSERT queries.") \ + M(AsyncInsertQuery, "Same as InsertQuery, but only for asynchronous INSERT queries.") \ M(FailedQuery, "Number of failed queries.") \ M(FailedSelectQuery, "Same as FailedQuery, but only for SELECT queries.") \ M(FailedInsertQuery, "Same as FailedQuery, but only for INSERT queries.") \ diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index 0e2605fa2e2..c3ec9fa42f1 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -24,6 +24,16 @@ #include +namespace CurrentMetrics +{ + extern const Metric PendingAsyncInsert; +} + +namespace ProfileEvents +{ + extern const Event AsyncInsertQuery; +} + namespace DB { @@ -223,6 +233,9 @@ void AsynchronousInsertQueue::pushImpl(InsertData::EntryPtr entry, QueueIterator if (data->size > max_data_size) scheduleDataProcessingJob(it->first, std::move(data), getContext()); + + CurrentMetrics::add(CurrentMetrics::PendingAsyncInsert); + ProfileEvents::increment(ProfileEvents::AsyncInsertQuery); } void AsynchronousInsertQueue::waitForProcessingQuery(const String & query_id, const Milliseconds & timeout) @@ -437,6 +450,8 @@ try for (const auto & entry : data->entries) if (!entry->isFinished()) entry->finish(); + + CurrentMetrics::sub(CurrentMetrics::PendingAsyncInsert, data->entries.size()); } catch (const Exception & e) { diff --git a/tests/queries/0_stateless/02193_async_insert_tcp_client_1.reference b/tests/queries/0_stateless/02193_async_insert_tcp_client_1.reference new file mode 100644 index 00000000000..333b3b69fb6 --- /dev/null +++ b/tests/queries/0_stateless/02193_async_insert_tcp_client_1.reference @@ -0,0 +1,5 @@ +1 aaa +2 bbb +3 ccc +4 ddd +4 4 diff --git a/tests/queries/0_stateless/02193_async_insert_tcp_client_1.sql b/tests/queries/0_stateless/02193_async_insert_tcp_client_1.sql new file mode 100644 index 00000000000..795a27883e6 --- /dev/null +++ b/tests/queries/0_stateless/02193_async_insert_tcp_client_1.sql @@ -0,0 +1,27 @@ +DROP TABLE IF EXISTS t_async_insert_02193_1; + +CREATE TABLE t_async_insert_02193_1 (id UInt32, s String) ENGINE = Memory; + +INSERT INTO t_async_insert_02193_1 FORMAT CSV SETTINGS async_insert = 1 +1,aaa +; + +INSERT INTO t_async_insert_02193_1 FORMAT Values SETTINGS async_insert = 1 (2, 'bbb'); + +SET async_insert = 1; + +INSERT INTO t_async_insert_02193_1 VALUES (3, 'ccc'); +INSERT INTO t_async_insert_02193_1 FORMAT JSONEachRow {"id": 4, "s": "ddd"}; + +SELECT * FROM t_async_insert_02193_1 ORDER BY id; + +SYSTEM FLUSH LOGS; + +SELECT count(), sum(ProfileEvents['AsyncInsertQuery']) FROM system.query_log +WHERE + event_date >= yesterday() AND + type = 'QueryFinish' AND + current_database = currentDatabase() AND + query ILIKE 'INSERT INTO t_async_insert_02193_1%'; + +DROP TABLE IF EXISTS t_async_insert_02193_1; diff --git a/tests/queries/0_stateless/02193_async_insert_tcp_client_2.reference b/tests/queries/0_stateless/02193_async_insert_tcp_client_2.reference new file mode 100644 index 00000000000..398d1450c6c --- /dev/null +++ b/tests/queries/0_stateless/02193_async_insert_tcp_client_2.reference @@ -0,0 +1,4 @@ +1 aaa +2 bbb +3 ccc +4 ddd diff --git a/tests/queries/0_stateless/02193_async_insert_tcp_client_2.sh b/tests/queries/0_stateless/02193_async_insert_tcp_client_2.sh new file mode 100755 index 00000000000..e99d0fa69b8 --- /dev/null +++ b/tests/queries/0_stateless/02193_async_insert_tcp_client_2.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Tags: long + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t_async_insert_02193_2" + +${CLICKHOUSE_CLIENT} -q "CREATE TABLE t_async_insert_02193_2 (id UInt32, s String) ENGINE = Memory" + +${CLICKHOUSE_CLIENT} -q "INSERT INTO t_async_insert_02193_2 FORMAT CSV SETTINGS async_insert = 1 1,aaa" +${CLICKHOUSE_CLIENT} -q "INSERT INTO t_async_insert_02193_2 FORMAT Values SETTINGS async_insert = 1 (2, 'bbb')" + +${CLICKHOUSE_CLIENT} -q "INSERT INTO t_async_insert_02193_2 VALUES (3, 'ccc')" +${CLICKHOUSE_CLIENT} -q 'INSERT INTO t_async_insert_02193_2 FORMAT JSONEachRow {"id": 4, "s": "ddd"}' + +${CLICKHOUSE_CLIENT} -q "SELECT * FROM t_async_insert_02193_2 ORDER BY id" +${CLICKHOUSE_CLIENT} -q "TRUNCATE TABLE t_async_insert_02193_2" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t_async_insert_02193_2" From f327e6723242baa5ee42131d83f010b2c1743e65 Mon Sep 17 00:00:00 2001 From: FArthur-cmd <613623@mail.ru> Date: Wed, 2 Feb 2022 20:01:25 +0000 Subject: [PATCH 115/149] correct config --- programs/server/config.xml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/programs/server/config.xml b/programs/server/config.xml index f306195993f..d88773a3fc4 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -237,7 +237,6 @@ - /etc/clickhouse-server/server.crt /etc/clickhouse-server/server.key - + /etc/clickhouse-server/dhparam.pem none true true From 9b0a8d577043824852ac7720da1b436608b6332c Mon Sep 17 00:00:00 2001 From: Cody Baker Date: Wed, 2 Feb 2022 13:24:43 -0700 Subject: [PATCH 116/149] Update and add new team members (#34268) --- website/images/photos/artur-filatenkov.jpg | Bin 0 -> 12541 bytes website/images/photos/bastian-spanneberg.jpg | Bin 0 -> 9108 bytes website/images/photos/manas-alekar.jpg | Bin 0 -> 25471 bytes website/images/photos/marcelo-rodriguez.jpg | Bin 0 -> 27061 bytes website/images/photos/martin-choluj.jpg | Bin 0 -> 20690 bytes website/images/photos/mikhail-fursov.jpg | Bin 0 -> 14426 bytes website/templates/company/team.html | 80 ++++++++++++++++++- 7 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 website/images/photos/artur-filatenkov.jpg create mode 100644 website/images/photos/bastian-spanneberg.jpg create mode 100644 website/images/photos/manas-alekar.jpg create mode 100644 website/images/photos/marcelo-rodriguez.jpg create mode 100644 website/images/photos/martin-choluj.jpg create mode 100644 website/images/photos/mikhail-fursov.jpg diff --git a/website/images/photos/artur-filatenkov.jpg b/website/images/photos/artur-filatenkov.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4fdc7758268197d0f2479406f62863bd507ccacf GIT binary patch literal 12541 zcmb7~byQnX)2~C(mg4SK+`YI%ad&rzV#TdE1Omm27K(e2;!=VIcQ5Yl4wv@*-tXRj z?_{l=oXqc;>~*qc?b&no%iPN<09{U6RvG{U0|3Chz5p+a00{s*+^fO<9TDK+5s*-j zkPs1((2?K2MZrSH#==C$#KguUA;88Z#>K=Wq#`6HC8MCAz$W-W`+=N>gq(u>Zx9%G z1Oy~RBs3%>G;$nF9PHqrwpHHxzXrlkmB~0b^ zg_Gf>*FlA%X@>MjV3Ba5pHF@7Q0M<@;1N;cTAg$9^>=2()}kme-*UwItxtK7wmzq} z3&i}djBWTXKPP}u$mM5KdFayDR?#JJTD}{esfOJBij^&_4#IZ~E+pRZ?pX+ScL~Cg4!tVr>P~EX{YqyaI77U0XsJw-}r) z`W6B}G;{y?5fwk%(eVL}A6o<2V44Cc*+F2xK2NFh0kr^F>d<8pQ`}0#8lrRb`_ERm z`=88!wxi#o|3DOKV=ULp)$NnB6MujG9K#hXj#h_H=7kj+JBf+k!L0^6Q9Vv_f4s62 z27OO|sfHYo*{$Rh8T?~mV@8z|&D%vlMIwD2&V$QkG^y{bZTrAC4Cy|4Op4;{b4>x9JKOryNxiVfMHh+rDmi(Mh*TWh z1j43WfAeG?zbPk&8!}JN^M4?A*#;?TocHMX_^nU#v%!SnutVdx6T{R=x~BX@)FOUd zao8`qIG;Y94bmz>J7Bx;)zz)lKawd2&fEX$;%N5HhgWOJ@7H6#kqcS&t7YWy7bVB?cwnTSNxF%VyhmE?^}LV<+h}wyn4U~yxG-%IK6X!rckRs zzrV-Um3SeRvp^y!yE%uyclgjFBc)ewg9NAam+o*|>|_;F(WjnT4d9@wZK~kTcz2aH zS-HQk=-(i)-Lba;CxcmnD;lnQcr2*yJKHr!ceexCwLACde%VJLZ%27a|z(vHYnfL zJ>Y)1bqA#g3aK`Z(}DQhGO^5)12NgOubCEZkSz)+8!QS<2|A=BVVUj za(b2|xnXMT&O+vv|Ii`O?jvmTk0l>9GF*MadCuk|z!^@qIje&TJJ=6Yre!P8xb}~1 z`D{W{T9tTsi8wtQHtzgcT25|>jE(XCp@=-^{)fuU=>hgl0m{nU`^A|(%5ul9gkjv3 z7h4`p$LX{h3A%4MNPA}Gvf~yn1BDn({&BEMYnRo2{%i+Hk)?)q3@N-h26kY)QEG4O z$<~{I9g9j=X9qb(a%pcrYUO-zZZZSXVzYa>!dU!E}^9C^d#Y6 z6osg2_dmU9Oy}PF?U2gvDXnE`Dv|Q=>`i;~(3({EZ-Ob;m zNjLD^ZTs9kMEu&(coqM%1{YS5N?NiX!VEk7jf1*6-Y3!7e-1Z2M}>M76_d^C(Td(z zT5H`KGur-_a2xhJ*}S=|6fWbOq>oxigVr0_Tv);7DVY;hs8b~J(&|5u(eCed{*NB$ z*Ec;X95{ZM9jMqg2c5HXWoU+eXSV1Z>h4zgS<(tM=LGs!f$m|TqW>YLO1-N5)=9o+ z9?zaXgS^k~&xxwvnYV{Cz8G+fHvtFkp;LEFgZwhs4-R;-`ZEhrtAHo1vynwaJYc`j(Z>VF?jeii-uK z>-qh2&|Uj-DxK~JtnH)iwRuFV6l*D^SoMq1nY&x3c5iCEHaSaajr(-MEQN%*#vQkd zlb%0^-8>W)^$fO>h1`bEDvX2dFMl+*3mse;xCwq^_UmTkj zy(a_<+&A<;9M3Opwb6o}lrt*bH>5RX+qJlPlGO)375u(C5P7X>WYGaIuy0n0cxm>0klPU8rX=t|vm2%bmwap}FkB56_`i1s2mSw^YGySHAfQD+ikq47r^)oOsebs{Xq+-;iYA)KZTl_oW`vRif+Rl$+gINS+jEN zy7S(CBy3Q4H_dnAKi~2>&l<|Aiq=K&l&2CM=ygOQ9Y)W^&(Wrb%UFer5f|^O_y}|> zOljAKaXds}08<%g7^``u6bZl>);}0S)!Atq?mR$C!d)3Rl9{)lv5n8Es$Tn4_j71< zWEo!f(*aa6VG}}%4#U`v7lCdp-`sy!VMgwmwc%iVevtKlOoWmPhjQH3?t_!69ILL# zgL~>wh++!k9tj^=2ErG%;9dNY$X4T!9eb=5scf)NpY8PAq6Rh=QI*?}yVXh6TH;Ty z(>t0|2=W%Al^TT4-=#~YFSj%2b)J%Ty!LI{7kA_@Z38xuLyOx0GhcRS0?lexu!L%c8>={@eexIfE( zf6b{3{0NfABip*t-Hyx7mfiA&bJ$c~ZI%V$vPaAE7*sgagx$OgPzLvNR38R^8WgN* z&5!A-cW1^wA{d}uh003I!8ivL7bec9vfk?H zVFeahQYUu0R5b?VIdT3_IZ6g!8RG4Unssk$et9=z|BF5%!|UQoDbp;#UW^2tB-%0W z3|BoqGgQ*|mct>oxX&5uPqa4kN;Fy*&UL zAw_MxoxzEJ-LB~aZ^=BvJVJ(%8vobYZm znaqK*$NFZ5FqMW&!uNzPe4k{O(<>iRcVNTr|Q-0Q@1;$m5+k_64!P*v)Tqu_p%PvQu{RT+4M6WRn*WP>c>=q+WG>i zl_jWqmbe1p=G-d|=bPpgWm$aj=URC&6W_UFZh{c3{eI3iKj8@=^V%)gWhd*<)R|V*g0;+9FH4t&H5Ll2t=j@tIM5-!gry8kD%qSQ3`q+DN~*CE~2r&`n#)h znS~BF@9DUSD&BlyCsq5AAU3{~pkdmxZ3)2h9fJoqsYB&a)3`cMv<~5L5w3*7%yMbf z4+7h$ii#wQvwl%MESspgw-JXc(w^63sZEn)s?0aYZBZ>{Khlc!h|xZTd6(pri5%Vf zfRqmi-&5kVmSi2RLS36|^IhDT(=J~Cmhkk`_>+)DK|@n71^02DkOd4! zTk;)3jX~QjdN=9rK4n8^`wX+~kcz-9f3o{~kOITwP64;SR(^WE^^LPKk;e#1+xKn5 z866fU<>O3yW``1q_hJ3#TP)*N)F8}6B`q=1yjV0(!k{1 zTg);d$N?tkL;5Wb=vW)t(>pV*j{F=ycnpqBGsAEk8h&^b_blkPvqP*Wc2Yc1ZK+Mi zv#IjbNWri5l_@Ld(l53cNc48dN={DXu+mk1L!ktU_7ItFhPc*2;F~5_N|U^*OA*>E zsJrGgl)c36Y6G6V?)PmePGX`nQB!Qqgva?aXX(lA=1^#S!GWk zaOgIlCnc@C#A5keL(%SI#xb}pS(?e=7<-hY1(JhPY(2JM3}d{`XZ9eHQFRu~I~QHF z=PRdcM8sgA#89z-<&5_y?Sg5LzqA<2QuA&hu82u03>KcL?^{OkP%|Tm~Eoyvp zv@Rsgu4rEgN;XNpS)Xd6ca!T49mYiq&YIwUS??z@8@E%4lMuv>CVh!G_oTWUsY4r@%g zpC#2JR%r-C++}Huuh(jme3;giV;gcHMD}9B&$hEUkgv2Mm5Ez+YIfGt(eE`%xWByt zraLH}C@P5S6P6nMzLWRCyR>Uv!eCdVs-OBCB;a73aPzLJuq5Q|MU*uiRKjK>oH>E$ zjP}p|=gRj^`DpLDDT~4i=?OJujiWr^nT~eYVdsbI*fi!+4VaTIqMSakY^JBMBKR@n zXX}@Xpnfln%|}W2CZBQ3vAf0PlqV%j@7!W@8yb92!;5a=5@~ z1JpEdI%+;Z31^_<9NeS!O?RP=GYv5a@R0Vkn752wVR6O@GYS55_l~6y&o|NNSkA6i z7~`;^Vm99Tkp}Hkzb`U|{nt>bpR;v8Y)e2qpJ~rDF{KP6lBqv!wda{uWF@$} zKs%~zy30G9h0@h0)W=j?6=`*>OH#SGLP#X;s^>m}qJzk2${0!sFbu;$!7d~<9xTc}Ewex22SLw{HC{fNzP@UsA_^H%rJ6A=NMC3wjp=7;mqGM6& z#$UNH>)AOq(M+3-J5q$rbbVkRZCI9-&9W3 z4P*O)k$a5K zw9lJA4;6$(IxO(xGprXgO|&MQ2fW>hN3=22(3EAg#Pwz6#=BD!tVb8CCr4@$8!=Q<=J+H&XZ7gyf-1V28;xNs-dz$95O;MJCu6)`e2HwTl6 zy_I`@ryzmOUfq@aXvl76;M(z(&z|`TCD7-o{<$Z8r3V}5_%2)CizALn!cEaKqE8)1 zfa}5obS}BOOjPzc;s za$9~U*Dv=hJ&mocHs0jqeu?F|j7mg{=Z?*X-j~|1^pl%_UDovw9GbZ=!E-~SL&{9b z9^u`oFX)QR!TFiO*QZ0F2C>_RTAvLkPI^D$aUsAy&jaswGz$!4$FP7E84huiJiW7m zr)YYLmaZ-v*u_F2OOX+op4KgqZhxO=UKLkXvPo!mypa9{Knk%Txn^)v5WWu7O0h9{ ze*win<7DKx|J1izT7Ap5&k&zOfA!5`X+bDea%5}@jS6q;FxMY@eA}7Sjz0nj9=K4I z8%dk-$g-AV;M0{?>F$x&P$$*7aZ`~=ZZ(d!QcF>xJvBtGuoF%*Jx{0(#ufK_T^vfa zFJln9bsx7IIe%3XN!>DO?~$stdTEfz2jUtUZ+Rr>Pr)nB4XHtyw#yq2l9-1^(x8*bido>@a+ed1IJ8%@K&^gE`dLf0mSX1r+ z4h2Q!pc?9NSv^7Z0RsaS2{r#FY%Vj_^egxAdFfekx=TaG(*0B0;2b+?D3sTke=WE9s)z5q zE9wUJbG{$cQgVChrhoEOi3es_CLxP6-}wa~!&G{c?^DmG2$5t-8_8E-h43?ovb^0j zS5~6x`dCZ5`GZ6-ktEDLy0)?$+VhkDVel&=RC_bC@_qW)H*@esW=ZLwSG)E$G^D`m z2)-F8r;U64!SCp{oOO>fTONV&(;40w#dNz|2u^nm`KW|0@$>upU)}v%mXT(+UvO}J z7UDti56Y)Hzb0@boAQ7E*oVC>DhXWEGIP@$y3o`hJ{Q6C%_}|y)8ky7N)J_v?!_Hu zO<(>Z{gj2gA{92Bta z2XFVb>19kxzf?w?d}+5w-cv%0nG+Al3tSt_f5ee;Nc^NGPa$X78hack&U~C@$j5kx zQaMmJuXKEL#|+)e(S#d?>tf@?sr&M!9n@x;918~|cT&a8J?Tm27VT%QN_Y(6QQY?_ zT%ym6(bm-vv-HqZ8Ha5t*J`QwCL2Q!$G_>;VC_G!eA8ZeztYz~x}KI3*G?b(5Z_>? zP+7}Z=>FlTjBcOg*W+Y!Gb>f$;X#)bY2XK?lhvlXx7X>ddHY}Um3b+BYySu<4yTYA zY*^81Ory{houk)qC#uIz!Y9Xrv$bG!>OJNmS`<9ygf8slAlv1}#Ap2k+>%6*BzXqw z7TZln`7RFYbHl*bh7<$(>w~mcdwmY~`VRZrkb(hVP`tLHOkIM}MH7D1bg_-AtdXCd z{cpz!{RJ@m$I8m`1+eNwE5ogQIw&F#MvR5SfxVyF5@vT=Dggne}?JT3}vlEwLVQg zkguaF9qY5sRNq2egOBXG!FsFEZKUq%>^UWcWp2rwJWr>J`j%O}`9wSI0H{$= zbE8DGC0J6+y#*7ANnfLpxSZ)UR$6Z7MP`e~sx{IAVN9}dd=>5lbL022`pr1c118a$ zzXCVSH+4>F0@dQTojT76Za-!FSL~$NfAu>NHA{sbE@`R9ZwoWP``& z>E&~mJjM5JCG;!w8Ga!lOuHMujH{&fsJ7(sY|?puZvf?4TX$5PT%M7*QMe zp|rx4;R2O)YT7MRwQes?zJ}fJtIK9|l@E=>ZD6ncY&XGGagJSzA_>cj4*3MQ>+rZiFtYX=6>UF^-*1X*yK(n5w5+%?ENZ&X+;)$?WRtC%0 zAq$rCPPHAOK0K~#?0~s@%lgC~)EM4*6J61>!4D;6G*bY%f7u606W(V|i=v>sMwQ`{H4B7BD{vL@# za|wnu;r4}Y`S0sdFBlo8suCknAiuLmZ!)@WO44dC!{~uS0l>{-%x6l(2)~I+!0@ay z+qbJBTo5*>UVc*6>Wquv!4dU|5L`}f8?0I88!xHg$3-n%MT3x$P z>hCf{UVVCo`0_nFV0=9?Y)9jSUTUL)*nt^%h6+mb2!OBaYHEU$U4rV)hY~PhdV&%o zg6LHd%C6N|H4u_A>#t9zUEhK#*`KM>hLWDBp5AkS(MA)l(ExZ4Wmw5xC@V?tIW0H2 z+HYyzo)*c#Q6c1ZCgaQ98;?Z`nq)|%e>KJExjh*AiT}~XD&j1OunK<=qKuz zEHNXqMm>F@R!tQ~9W35fZ2jyB7Dd&Yl4`57RC(%R8T3{3sqS+l-big<`AHYr&cUG2V=+`(POJ5cTpC4j4uzjUzFSuvaPuX{{>*su0?GE=Gv=2LKbYKFYf;s1+=XRFsA1rR)F}|-;n3v@ z@7BT)cPdaQ2yJUl#RS&%To}BJCgM zOUh^8VJsg0UYS3gTwZCkI$;*uDy{V1>)jGqYVB51WXE*c&?zIK2S(l}Z%ZbqL$clt zd+o?i15QXv>`)|CEoJ6tI3@D8;tvD-@0R#uxyW6L4-u?un~@s^QC9jv^o8bv9Ob;E z<(B7{@@Ivi49JJ&7>NR%N^x^_w4^3FZSjkpmG?^cBbF}2xOBkf&g&GI^Ug;!n9rfY zzwdJ^nd&~d{oV_1Gk`?E%S)SkrXxx$5$d4M*_;w%M!--@j`>D1?s*a-_tXSoMskGy zpy?KD2~t^cp3tdOTnL_I;F?yq_y4KwmCamk8$GI}Z%o%o8d~tS2T|;+K?`r^7itcU z-<77ByXad#y0AAfRvg1J+Dakn($S zgJ#+iqaT8z5X|r49C9<}A4gp~K9fj#Aetgk4sw!;23ebK6J7-39csi@2puLFn~j)x z)^`NzN&3Iv!1;rpH-e}RBo*q^C}R8S9KG>t)72{+$A`emPW@OK4a^`1zUQL}xQ(Qi zPrU*t40EFArqy%Kb7Xw-P;sH7)Sg6YyI^vY%1MD``s6+SU{$gAz~tW-AWdY!(*5gdeo3jD)r!mBWB1_;p|E=;y6grL zgm@uC0o($`v8T@DUOTBV4P)r?QpTLHgKII?2wa zz+&QW_BQuccttjN2rs!2y_Z|ezHJbE$G~sz8`J+*t_Nji{d5*9jgmf1-3f~-Fn#Bq z0mr5$eDoTfHsomD3?|tfGvsNyXx{4GU~9H%Rv*8~H!R`&b^FEFiqMbS)|rvn*{|7eP0-93w>SG$ltxPU@s$CsvyFAoDE$=8R6T(#7`~EW9bIFMtHe{q@m2?~8CR5@Z z%w@Pf$y`6tiWdMwy1JA^FT&jvghaR-_C7`GnW`2AUgjqdjtudlW2wq_ba$97qbLlm zK0WZUl6m7x-G1F`8Ei)DWvhBi{8(=7bb~O_53F3{mg*_j!Sa`s1$X#{z*ctU9a2Dt zP#Mb-QbOkmx#6AQzil6OS`}t?xir;NbBf9r%T@t(E#J!b>ux1?ODoH}D63#^Kv=wj z5ZXNvV(^e6Z$FK{0LG5$$TD95zq~2Wq+b9my26pAAN!o0e{Mw(Z0I}(OuEGRT(T65 zFq}Bl4%rM>NTqi#>9dk_;@%KcLrQ$Z@1z=Gyw~%#x5f0z^nJA>jgAX~Ifk$bHExp} zf~<-(F*rZC1i7MtPqb!+X zkBAoB_N08zma&s&T~H{fR;LDWtzkbZ?)%14`6Lg17q|5r90v<>6S0{#Wk}oR(NzWp z)ORNpt3yRl*!st#wdHprh6;Eo=RtXY*2;H})mm8ceY4lg>Vq>-w1QBY@aO{_Wyk zPgTAiBX|P?2lMw#!C!-UgN}g-OHL`u#-RpB!7h%4O(kYx`fnHa1_nmtwLxRzjImEJ zG!jU8vPmz$NuTc5a;y`lwrJK<9LBPdJgByH;6pW~lY&P&xmC}vQ?Jg%AO@nCZqV32$dd0-?_~6=VkmrrfmWIpq}s00xM|RokdU695Wj8p z*Q^scsSjUG9d3}^UC3X}pSWv^r z2Qt;8=GT9Oxu%A!N)KY@ue@o5d?F806<4FmKgVS!xMJ+`Surz+%*P*`O94S#?1ysa z>H@@pX1rSdl3po`Zc&gXpHf?8sq z61d#ptsZc1D2FfWT1SGmk)Z2`7+fYUYJF8IxsQFWr+>4D?QizL!N9)S8^r%+5A19D zP{NUmVzF_EiK}6&n!;0jV3#m4a|!;JL$LqJq0e+p??N8Jq!HpD70n+|*&k)asUp_7 zcf?a5rua8jL*_v?F%#qB#E)MDKOh?pxee$MM2=DQa zy84H|lPt{kew|$!sQ=FH-EEACoSbU)-WazlL><@e$dL?_QD>e)vNO-d$NHy@osU`Y z9%*yo^|7MV{#XlFLZZe0T%ejAl6%#*RfijZkp z2qB}WA>e@gq{f=yil45wW3Vc=6N%rTyOv+4^!u@Tne=P#%l!`)?IMCtnHz5mWtJGu zQz3hdCw=>JdZ2_jaSro1$bD80*3}0_JHI?`E9mD1YNb@8Z)*3qB9o-cg6oQIb6SPa zseBG>YgCTNVmV{{)gP0+#08YDQ5mlZPD>)!_Mx_(4;7eyopkR>+y50PtDBUHS+9Jm zws?ISSg1KZBGUCvAEFh4)nLYl3Z2K|AxP+V8g`_EJPDd&Rbg%krscpg4>eNGT~lX&?~#rr$(&IhX01o* zQqVit2~)MyYeNcpsl5;6j7KA+tBRPz%AYC0?VrF(9FagfYc<32(!LpHFG!ZIhF_~5 zT2#Upi~jK5-UnUXv$NfHy-00`E+jeee5%Xy1O|=|z~~`M>vXTZeX&DElOj24mhY5% zYZUG|z$ld$K>Tj+hqX6O0xme|ngMqR=_K>9vY2;X^TGvapf9X(cem*r zCNfITMw8&f#0G_jc@Fa$Y%-Q-87c%st0P`W@|i>WHHX#ham;kQuT1^Bk>jhaa+yPm z;d2Y4<+o35gL7rfIZZjba3J_(`1p8u_*BFsgk-c-bhI>7G}QFWT&(nroQ%{oY=Z2ZJiJhTC>^V?m=K>B z7ax@GKa+rPAP{_9d`f(LN8y{Pz*;u`uKHSFBnuGMxxNOb}RkhV9AQ4!S+VrsJFXYzL(2|q-$mqVI za;>UaLhKrYuZnVZ87&hj&9a)L4(F3+ZxFAV+r_@U{FVO?$eDrFNfs}Vr+XpbW=1oX z*>9G`@_CU_TFGwet5r^KgtGTOghqhvk;QBN>! zA5T|*G9^oy)GA=uosSHlzf$xY<{UgZEZ=Zz+HLHTl|S1*_TA3&Qd%uHo{f!Y#W3Lz zvsL)?^;%S;l`1W^On%>T$LiftK1X+|=KiFYk^%I^qRp~)hTE3JEc5>0{Y|j;@*X-s z=Hu&B6lK#9;4~I(Kl-h)++?sj?~R#SPt9(9^3-4d*sr2V@8v0~vIMf{$6I|1G<{ag zZL2?Pmk0dy^_@SE$g*O!f;}u;zjv*`ukmrGVGdbJDKL0Gzjxbm%%kc~!mZMqcO~`n z%)0IeH{U{Yy#Vqq$ZXm^%WDDv7SjlGrDa~$=RFhOQ%IvU*=??$l3KGjwJWROV-MHW zi!9@vYTd0@db63Dw(7P?cz)N~+BpU@&<*-a6Yii^$DO1M>z8?L#9d5~5 zE}CkxE^g%kVK$Q~s#O{`Do`96)!u$lc*@UzlF)qD<*VJA^)op~lo{_=CYX5&A&h>$ z9RBs_7KdmvtMe9JK!y7T0JE|dM^Y<@f7;c&EdS#@`@!kBpyE!(JYjx!b~!0~Zj_DU zOb}E4P>sbT{dn*1Ywhs;<;EwDQ{M)o?0P_@F;>MrRwYk&!w()+$LDypOVm3beZ3M; zo?KjywQHeymXc$52_^v#nxGS>pZt-XKHAweb2&4AqO+m|;|rH#OHMzOic7aTo~jm@ z*4*%3b8cTQX#aU5{Wn2+qiF;vPL#v4rA8<2YIT!b2JlXO_Frs68yD~MoQ#@^HM6DA zaiIF`dF#cNrH0P-yHbx=j}uDo{^X~0=O}$r1pux>Ro!H_xYuL$6{AwV{?h)QLQ6tC ztPTR73BET&`Lm4Q6=_->$5FAbk5@>aZU#T@&RGcf6Y#_f6|Tz7oXD&%?I-o-(gc06 z;iKhsY{bMl9Pnu@#);s2(HsSbV}&Ls*`$G0YMQ^dRws_O|MKMbr(Mu{Ne5;uG6a|v zvo&YBj{fIu24ky7fLq63*YVi`3Kkn`tD7<>Qx0TEZ4Of< z#w`8da7&oj=k}w0U9B_Pd^+c(^O){{T2lgoyWS~V8r z(^^vJtofYwKO3euk-oLmt7iPxs=i$Be=o6mc~4w1nZ9c)IDiL$!*chvx*kWnh30e@ z?62*W8)w6Kz>V3R<>9={$qUI<9~d%wd%9^FINZU!pDp zJUr2csJ0O~6#{9kYFo#06cC@&I#k%8aqEZS!uF4uK2 zMTbK7&rH?BX*BjDNj_7d&0SR}T15Hb?PXVa9ba^={v4o3uTV+vz$#!^gcQEqU`CsJ z75XZ{NK2oOKowA>fC1oS7L$Z0yZ*tl z92dV`Mj_zxunR0xI!i=~aVzCj{%&{5=G~u+*Es+dWp`M?25fOA(dsFF)AI?-(zk((-Ozi%y`(}6L}V`+CLVXLRoofplc0*;lm6*a zpcGY<8s2Xu+-v8(60fX55DZFC{(kZEX^5#9LKH|sytTzndO?Fp%AF_Vd2%sbaC&&T z1_0qUii*#;sNC+phvM^yT*}|IeFk zw4=-buLYTxl+VT5;n}aTLW)VT>HX#XZrLM)>0(NX_W;+-yde?V(&tG)9fM40MgPz5(YX?t?s;tbzBJGL^;A+1 zL33Fx_4B56`i^%R`PL~7QH#$O*D#-cetazRH=(+D`)QXz^6TlV+P3?OnWv1;qAMm| zimKuANg&$5V0kX=5E z^T8NF9_$iG7T9X7Jr3Avt>#%w>sZ?|yvV+ESapp&{8ScL9q`}ot>VC++Tn{Zp0TEy zYMwEo(nB&*&fkXhf2z473;BFvUEKA ziB747sP9+?4uh0y1}+sD6sY6Xo0kH>LZ*W{#j{3o zGE%O6##_8`Y++9tSohbQL|p4%GCQ=Jz|*hOm7+G=n*}Fu% zn?ZV4w`2#JY?HGlt8atO9CHy_yt@vrdgNVEmG*qLk&L#<8~Vbp2nreHg>l!rz4fyF zc5Oey=48jB;r$NKps>&p)Xf&09degUc(sfblY@zn93i*0o)qgTAHH7Up1K}ev7vf& zBAR&z(TM%d4>4mRUtxN@%=khgf+&20QfF`wej0nsi}D?^b(95kNWBG`Q^zwV zX|9n%1_6*zv}Q}i^_-22Mx91c@pyIL zZ$)bnc^ARPN=L?moH~7ZK`y+ebji}ty{8fP-H|;i`rFGyMM*?sux2_f#NRmoa|W$b z?uxN_+5F`$W`}5vzg05bhp)YA+jFWKiJ#z)RQ84W=Y?rx310-5m}rKxF6FDSe;OQa ziiV`h;FS*RZAGiqN-jfgWEQ!j|FX@*tlYKl+nc7+z(zPYBsjTUBF1P_N)j6ToS~9Mjo12O z%AZO49MU30rF(-Z6J+!lY5`6)OL|YdM)r??-={v~#F3M7WCCi*mrMcz``P`WaJ%l9 zYNh#8V5i_CCr;?MoDA}4Crwz(y6K3X%$P4(QEJagPoQ8T~4CxQ$+t&bhvTvIdq3BW6J zj50R!_9CX8*M%B{A`qGQZJxg-T-P11bc^O3nRsZPziu}+8IYVcHYr!LP7>0=ETQ(| zF4MYY&Tt;aCpH}I8+A#Pc^eU2i@zxJZABjEb3V9uSP#?3p!%!c?l&S0U)(aW9*X7g z1+HS?IkYg+vCh0`{;iKhTkkIvUkrgUv7NGok73C%Zo~bC-@a(mUJ9?!{3we*=TT9r zNw4Opr6UJ!{Dr)52BUsCvmOl=LF6!wL`wpfbu=%w4D@08~aezcP4zjF4peoxFoL+V|N-ne*i zDb^s#h^*gvT)7jjybC+;w6V#0_32<#?jVU{L@=d~B=J!b%)FN^)kC{Sd)P~jE|c8W zyN13(>HHU$*Loj~3n5Xpl5vyCtH!ZBn%X7GX#DSdd=(aZF-0D(jEeQz$8ifnZ!br1 z7uQ2#4QovdbrzyL$we|R6~zrhbfBm6FKd|nPfS&|E69B)ZuzXGco)s>?OqOdB)%-NO#9=R6^mj6a7tJDLntv}JmG~Qq#sJTS%|d4!`BWjo>4v1;Un}Y zrV^LiHmi91vRRgzs*PMdvHrNQ zDlu?)jC_RHiSNF)m}2#e|F@s!l)F8LhzyF3do5>e2WodX-m;euv>e|%-kQWm__HWd z7cSMv$Wh3h;=vwqxSpHyCt-~t6cH7}eSt6MEftdw;&1U7G+b7GVwdmR4)n+LYjI$Dq_&u5EA}Ooo-ztntfDx&m)xnD zKf%B$uSsiit2Mo7lXTGI0e=&6ZOjC+39cb@@=hqeDMb9xgpT!z>JL^ll5M+`YhUX{ zo!(FKGP<8KBFSBi1%mu1e4mq`2vVM9ZVqR|3jKb_FzPUJ>!IO9ic4i43<7>B>Ng45 zmh&(1Q>}VK_Z^Z5_s8tl$GqylmCDz%w65{t*#UU`UTXcBk}eu_gx3`05`H+59M<;E zX9)SLDF+{Xbp)2|knAF{QltEy}=k@2{#6rsW8C1$9 zH_nzCS1=UKgi|_$d<-tr!=jsbjmagfz6R2;rsuykpA8nJL~B&^^x}HbWjADvFhJ(JW+VlT<$`>moh0}au9eoYRRP!M@31lUA-j!&};nTiDGZ{2L1+a zS(#4DN$|%~r_{&AM927y@>>g1N_|7?A|UK{a1c`^TX0(Ge&_V{hJL66iUe(p@3~P^ zEwOzQs^%EnHmWP_&(HfR$BsRL6M^NVM?YzHaShTIMMbxK;P;Anu?kitr0n{HvS&&= z!I|Nqv-!RT9URcHXiCccj+d*JY?nO#i@j;BcZUEv=+? z*+;-`!u{bOMgt+ePp_1u;#jPd%=DbA&_A_F6;dkZl^~1W*qr#bq{Sy9__VxMH4^UH z5vAB!PKnvqZ}|3{(YE%9$1^S#E8TvIVs{yTMuEvWEQ+Jpd?xR6h3R~VDq=MvG09P(n=%fhhZLf3IWLri1&7(LA7Ds zA5zC*2NOPGpM5T@ct`vtS}!{G6v0dN8x;$wreSG}Uh|eKbH0In`QY?layB9<YH{lBvB^tkoyb2~^bU@e5&LNGx~Mz`XuwtVHfZjvo;Qj}dXj zl4FSnW^YUch_ILL>&i|6sJPX#8$&IxOc!i48lQSsf|*K3SA}zk;5COsvLuX62M=#C zRrB7qhnsK>ihU9gZIA$#0Q})YReweqp#dVYw^|)NnmB3SHr0R9I*`&)#Mw!iYui*G zq%RDGZ#Tc{c$eK^NiqPn6I8{;_<=eM$@i7tUDcO=o@dN8@C4GYsSyVnBSmF97UdCihO36G%Z%PVXpiNX{U+|=#tUBw&3uC=31Y?#{cOx&C=Vk~!N z@p^8xM_nxD2BJ4ZJNA9fdbGvYOh_fwJA5c;1{#EQGH>u*DCqkF4Hzz7G%J&b^t%~P zZ0z#>=I)V9F1y7G6U`yf1RwFpa#*PRnp%nZ?dp)pJWgU!=%`5jq3FX6-UrS#M?=bc zEHh|dM%(7w8}qWow=`Zbc#yz?U9w5^N{c(&vN|<)A(F10i#RGPZX8<{RTO_db!bH45R~gD=RW4e*F!9`3)o$C=UmZ5aiVHI@2B$or#-5!WQe+Gqky{;< z9n}UF|EC5B>XAUfH?(M@^WM^G{$n}Wzugd;!Km~C=qiD(ZV z_TpUQD+Bj@!nI}8Y+j^>QYk*KNrCpf-Qc(*1+NGESq^qt(R3>JG`@CDh|4VLZ(KYt zw^D;tmmeQjwBi_bT;D4qXf@*JSwXI)x<3|%*1QY=!*4M?nitQa~g0t^HduYw}Ql2@O#^8Mr9ukO2JKVR;su9D;R%s($Alg3}XiN-N< z#cL`Pr>!5aMxnEb9aBUF7CxfICb%LAKff1KxvHc>PA&Zi`^br~Ei>QfNo=+bI#*ix z#2zLWpsZ?#oj6Q2pU+o|cvI|3u?2<%Q`KTPq2Z66B?K0QY$j;loA)(0L|zE%1@cI0 zJ)>UlEjw0uKG-zE(`B@F6J{Mz{Ug>-?&;T}XI82p>a9aKfwUtPN$m+*F%ee5IYO@^ z{*G4CIRA=-EyxDgHI6u2y|uXu#Re!o~4I+NjZq&Xe>m4IGpDU>F1hlZ```!7t)Ra=zQ5D6zAJ=xtt~J*|A+6rA2@O`zW~I!+_}aa=t4HevB_=%35ZP8Zu!MMr`wpj(_4tBq9V6QD1RUc}a1l)BSLB_E2l_x+@YE{G~;T)>Ru z{jv=ubsU)=`%9>7KDcm-7$aMv6nR1nRDcFPVbmPdkh5{SCa5z>KchrYqmDd%=J^S; zMCmeg3u?xTTiJajHS$UA@S6PM+as|P6QZD%CP92mN~FBdWhq9u+Rb_EAigQpVv7Oi zRja>cvpVSGkCt#P6$>3}T+dzW4`BJ3bt=Z}B|#$jcVwFb*Coe8XN9zKUtrJYx+p6i zC(mWeQ8Zau7kgB(2qm@RDHuA`+PE#+JigGaT|j7EI!J*@xkzf@C~Z8-Qi%QpBb5JzzvS5M?Cr*t<#WZ!E{uAV|p_Y%+K>xW*4H4r|`U-Ds-K;f0*#6F)j za-cwCl5{Bt;p-fR3jI_O`>mF{j`#;)jPmMn{5W~ltdUe8YUsC$9Fb+Oh=g%ZipDzX zmC8H7I7O2`YiNFzliR4}dL*{c@^KmIP@!puFmC>cgVP3|^Y+WVMoEIpA~hqKR?3Pb zB^Y`*N&FSTweS|_ZUP|ZDOTkkjY5&-PI}Ix#&6=FRi4*h{sFyvY1#8~wj0KtlChXj zSoAjm*?mfso|k^~2V@I&A#tB=_`9ae#Tgxh+SMPq6TYWu#|*+j4EA3FRV;aE6cC>5 zk~13FK`u)w>wK7RP#RyBk1QgBpbULazjM|gM;xmgYG2J-5AszG2&Tu6VJ#g>H#JWG z1Ij;WKP(PzJk*gz5vTEiHHBecrxuYPvTJH@HgsW+!{)pU7v#l}mr41c*R`L8d!K9nYd7hV#$oweeHYB^6Pb$|PG`qn&mHuP1+n7zH#S|>DB$ZKyt z?6di*O{B;#vIY}g|19IDEIW%)C+$CS=<3UO%_wZI^4as{RgUZA1wU6iQJ>jYTC#MI zQGECLI^1Kv{tv)k5B*BLX&qHS*V-JF$;6n(hB$OI5gHe-5Q3tI_o>; zaM&5~UyB!wH2iWM&)53f2+jU?!Nc}}&WU||T|cjSU)k1Vbn>b^<#*`Xy-dOS-+AWb z?$^IHhci06zYAzs%c$Tw&6Pi?mHUdw7BU#+LMky{@S&yxagZ7;C=hAk&;tDFT0c#yiryDHH5(xE!B}9k&8>ckkj~$~&c1Y;fU7ChxQNAw zBH%?4{G;vd-Et9kExnZTSi0CmL=n3D-*f3nUs#T%v4UQ}45Fzq0c3b9%k<1K%BbL(Lm#C5KVX}rCt%}^XH+Sc zxxL)0#37tRzDG9VB~oo;Z-|vXKb(Mi)+#NQm@S1C68KJVws0Bs?wE58Q5Yh;c%r~c+E^PH4<_oZ=%bUrI2(NhL4 zY1z234bzRp2-{J>p<|e$B8x9j3?lsY@Co&YzbYFd;(#Pb#1HE8n>;uljd7C24v6{4 zdU(g%p+PN{iWj4AoPU}rX0f7FQU`Bvt3(jLGJ#3Gz0KQ7d!(07F}96g)tGQ|Re%Yn zD1kI-JC=$|`1I^SV2q>y2m|YXm)#!*bXZ^(?g&A-|En+zSztSTYKvv2fd7>E|3|D3)&Kt`)&~j<{M8>k{RhbH{P6a( zi=W)wirez_RSxWwu%v0Y4AOqoRoCBjddAf(DMey;dbLF-z7midlzMx6>-@nh$ThUP zb|nz7vkUqYB%5>F65HNRw>lM_14uG#lF_fFKQeXuXigNJkL}Uk{vw*!$vi!()igf8czP*yypD6A+Sm(q7X2 zD)a$X9>zX?8o1qXuRFcPv$9rSUsV+iF!A$=2}9yVJ==%5IJd~^WO*Wu7t(Vm5;x^_z@agwKK?Q2m>1&i=`CqzPt4wDa_{%w zz8^igNOUqO|KHqaHnc6fM2BzfnO>EBuRf4Bs5U1$%#t;Qf2+tOlq*(pJ3Wn0@NED6 z%9ofeX^pKjo*3(1AdqprtgA9$Om2`(d4Psl#VXc0L<3yPQ0U4#M?`QAJ}~&g~xsS+S%f$0Ds z5kd~$gAOikd-cA9SVy>n@PXH7E_8rU* z0K9F%Fj={{IPkKt*f}#Bo7tO~Gn?9jSUij!SlF0ZSpfV(9uCH)Kyw#z6LU*zI{}J6 zZ5mfk#FXp`O^?$%D6y*OxaRCZYi2Ng!TuWYwT-4sloSd7Pjmeaim6e>EhuO^7#Msot zgpr(`m7R@+m4k(igNc=u_X`g%2M7886pD9oPG%OoDq<4 z-pP`MjfaPag_WI!ot^0&gUQ*`&c)b+$LNh# z-qioM4HR(>;FKMm;e7p1%dw2z`xi!yQrA|x4r*Y#?ES<4(2Q>=Fax6PNwGX z=#>8uJMfA+nH#&=tR_Rd!J4*#f(om}47)Y|SJz`qFO<$0y;oL!9V zOwFam1SsBFFk4%j@v`x7im`v?=HV9Q{>H{8E-E6y#v>xi$tA)e#=*fV!TxVrF?&;2 zkhz`9ziG|>ODpof(*9!%AcuF!V&+cPZsukZPWB-3fAyKy`hRZ=_y5Z8e`wAA_qKff zUujw18^iL?$o?NA{cq5_`TSG;kKw+*{Kxdo?cVL(>D{p3&H-ruGFO0E=lc%?^|k`| z4uFS+g@uEKhl7Lv0RR3VBf`TYBBLN7AtNE7V4(b~V4$F(W1ypAl!~fsPTPFY$5lR?J7zPRx0F4O+g9-K43-|#5Ktsbo zLH+mthlPWN0YJecAVLA&;qT?YpP-;&VBz2qFaXd{Ffh=tFzCV8I(ZvIi#o_=b&8*QG)^W%31Q%ZUmX@ghhxu|T!k)cG?W<~vhpXlN+de;$~B z1VCXze}wtO1}mbB1!qjbjt%d)^^G#{#}v-nG63Zr0~!+s6Ceb*KHOK*6=wE|T+3u1 z&jC3reEX61`OY3j#zR6S)!WA2wCZR4A$3Z|^Y;@?L<6SnIWP@`o*X=QF;&66G} zk&(~_#z!PT{38Lci`68FKfIbw^qj9 zd&u)}^!U;s)+jb~Q|VTp2R$Ied_{cHkn;v`$|o!#Sh@5C-6E?%!tOhgCC^b%$=AZ? zA=NcC+qBOeozI3t)k1AwBevG7Pgq1^8Zrpl=yfF^25oyp<6pR^;1wYcU#RcZDPp8w z=bBtjdtbRKZ#QgrX7pWB&1W6>NBR4ab_e!c`Dmhv&WxYqY-Z%ICafbZ>e||STPi+L z!hpWVl|47rm9>4m?h~tnT?LxToO+l&P^@Tn=ulxJ4~x2T8kgk61{_Hs({hBaS@-T~ zR!W19Hd~k@sN$DgpRCmx8IF7<_T$yLBvX52%8({i%)n=@{vUgL08E)uYUQiTMLV|*h4|TxiGb4QzF*I_f`KJa9i3?a>wE42( zBB;bFlww82VLu$_VJ?km=~5G?#!a^UF+7+rK+fe_H#jQbm)aN;3PNFgK6dqe5Z z%DqQ=`BHQYRZQEIW1$%5vgE)o&1sj{HiL?tF+bg>ng<&9p_R*xqsm9w8|J&;n4&`{ z$s!*uI)9~i)}l{Udao~V?)r^X%NIXB#I?QwBA+2`eq3B{fd2b;L2hcop63gQ?O6gPg@8sb8%m zvs8e*(;j9sbR4?`Fkf3HZz>|nbucvb(PKC!>m?g*AUf(=8#?*@ygC7uRfmYDq}W#k z4b#Ygga(1fc~qx*ekU!gVg?CR$6O92Jkj-na%BC}Mp)1vX_?(Dqq+8B5Bfo>n7H&~ zFC}fzsM%7(MLwfku{z-P$cEL%iNs*k^#Y>tv&vL>+}#Z#P$p%?T4)Z40W$oO!*%}s z2d}%dlrJnhhRu|`nTf0Uq!M2dv!{X7 zp#hH~kDVJ1rjIF~yaVjlqgHH6)xO&vkL=arWJw^N^Lb38u*IW>tms0`&Piv5=QJi- z+;i?gew&BQjX1(uL19MWF)9SS0pz{O@NW3d%ug=`H#A3m{yg9r)Q?v%3!M4-ug#iR zR--tUeEk~9Twu{!05f+s7k61>>6hCMVRPinlN|DYL2cs`x{+`5em94~5(EFpE=5sH zGIlm^0=vjluezqph55OjjVF-VYsc=)wttWJQ=31HZ|2DxVB;k_uek-KveP{0P70d> zE%GM>N3*sDy{*G;CG!Pg)32jq`{v>#mv|f=D~-LDyl(y_EW|X;0?k+ne#+O!+RW%% zx6Ep6gSiR=?V#6&y&|DxpfotyuxvgsG|wh3FT50KK(xy{qcGaUyv0LP%(;U6{*I#Y z_-yA5AR&6*UzRTKWO6UTTmcvaJqaRTiNK9$y^h8sr;4?8oRR7I^@>W!9!67z@h z&Jl$>Y$ejh#HMLCRs${b-MundWLK8R->&}GbrSCORH4rEdq-M0k-yO7AiFx5N z-*nd=>Ih||xtJ*OuSFzD^7(E?=CK*Dg8FwN#JEDk9R8FTQX%r%=9yk~r}*l^+nw#5 zhQ!tZY*Uv+w)()ooOtF^+X8)0!IYu>TZ?^U5~Gw)F>ip&xM4v7AZpVS_d?D2jx1l| zxGW!08(Rz29o0NY$*7|0T)JscmD8M?EQA6?eh9}Z$js~eqi6*Q(_vir32+Q7tk_xm zeP;3N!QnYBha+!Z|Mp4o4In1YO%}eM|N0c*jzc8Uu}8JKP%(G)DkS!rl)B+Q;uAOj zI&c8;XP&7je*^4f7iOH>R=~hoV=%&BwB_mZ=3P|#ZLGMRo%9rLzX5vMX$*76_0t}h zhepP8sUl1D?YX$!T_Gmu18}5|3WT!cQS$e9fSu1Jb=hNYbKlMjq~pII-MEv$udZw1d**u_ zMj87j!nOv%rY`yMD?7qgRjgyt-YOgmI1%${l-*mg70adc+uecUN6~ontn>NQ%+~PT zZuyJDoz~OgtPJop7A-Re60^HJfxTYjfP}KjN2P*4!{_vh+-VYNDjTgkAzrs#X_{gz`3Pt- z6>+A+>ndf6=j2#>GLqC@|0gjb-v*rfl#j`?(cni9{8c(w5^02oNv8(Z9b@33_p_DC#FPY+Jsp|)A(H>M z5l|Do4?kV1Iv4-RbynQoa$pMjG3_ko<)2IJ=@MxaU^8h+06Kc=QpzvaCS=Y-n(`;1 zRfe4KW@lC&j3|kP$7HvM$dytbk7~a)!P@5qJK`}Aggmcc9ovlau;oa+B)-pCHnF)% zPqjzvyUp(G4x zx;H(YzbNtEh_@y3!~G8eT~O;^^eEvlvT^+ea)G#CaHV1%=_s43$y*EG%soVnD9nA`X zT!Gc7JCzo4V5#m@!r%sa=Iy@6`%F2aF7WNVF%N{05EsQj`*n=Cdtr8YXHy&&bsvMs zT#|~recrVd^9{hhs)`~WX@*Ml0pt1~Cj#JqwlxP|#rUZ~SH|KPIq2z;xU=*JK>cd5 zgJGtVI0!6JsYfd>EtO$Q^k%6Sm^Fy%+haw@#41VuEdnp1Xx42(;lhwEhO!kNnH3^D`Up{0)ZF#=882`zx1uw|{Nh(~M)#l`#-cKd;fmc!c28N?k=4{M%KQ%j{ z1KmoK*HGUF(%snu4MnvXMxp2xKIkurCe9N$4-0logu}O*QMeRd@#1CVv|4#?NjGL> z_A$PrMAjpxm8?NY5*b~9FvEc+II|z-h)SI80?9At6k&A=ui= ztv7{Z=9E*_VjCbyC(NU2ER4qw&MyDg;o zzTGb*J$qBO>#R%{U9fiS$q?!PL&IE1D2LfJ%JM^`uI$UK6R}o;zIaIZ)Idm?*{+?O zcWkD|)Zj&UzW`c0_5k-GVEyQo5MGgOzW zy!MK0@e|e^NY|)iJ;TnfG*KLkgJd9%b_$o)se_QS$>?|EzxT?PCa=Jsc97eIXSzL_ zui1t~sj%%+`{Tr^3$eqdj8r+cO?)$_5P~qVrk0j*Tv7Dq=KWBj5CS#>J!i)S4?^Dt ztalCCJ)AQ3vzqKO-p^DYc5>714D8dMtPs&O=?%D1P`B3iL{jr9C-xwkFfHfGql?Fu zh#BCPnSD0C-T zGxG_RthE44BFwSWc8xGQGIP>tkx!*b-AQS?p3zI+@liiXNlz7FJ_w7f-&X5a1Al3U zUv0Q}|7A&7iwc70!VvmqrMrFbmrxuSoyFY0ndkE&>$$(>@!UY-CS!FqNgxtoUl`2jIBGrvz<;8G$2Kw5W zrddZeVP%MaO35M#Y$iWOypQxy&>ui}N;*fue<3h85XB5BiZvoV+5B)OJGD3eB)a-K zqhL>#xYOj|dHht|FKDYR>MQ!3c8T5C7%dplHQqBZpqDMJA+-qYR?8=SRtze+n%Ij; z*Aa%;M8=Cq?TS~4>}4vkf6t-oF4sD=*`lNV8Hr0J(Tu|*oRScZ9mjy3v``GiCM+`x zJLjdG=-AOB)Iw|szJFkguY4~0srC)Ao^JZ=veJFyaHB@C%BN@IDM~1H=!@Bi3)#2W zNJvoR2VpCAyipmd>1}!mQtTh1!*$jRQtRD-A57K{tC(uaDp6W z7u!P4{!Vp}ig1TZ-PwIh-GV4gq`c7$!jWIUNGa{wJF0w!#+-S4Vs-cS2B(MjcN1N! zZFxUf3^C^*6E_n-{>#48bYkYYZ2%m^kl##7gKiJ=T)M(-+>H0l@WG9X91e>mpxo2bplGK zbXQ+qjuIP{I}}R37aZ=nkP`yjVNqQvt5_%TbnmAT8;rx`t#{#lNKOfo&ka(`hm0f- zWzKMCPo58{xrEY2Io|*aIu1Xj>!8_$lrF$}rqZ6o@=Y_tt`Jbg`#!0UFG}~x{g=nl z{9GaK-#Ri}*U1ng^c7*=TTAYQ-oNRd7j@c>d)gSWO{)IA8r~=Aj<>JWF536$54G{e z!x7Zcv8j$;M<*+(^>wvsY>zQ^!7ZiWuEBMQuiv;nE}ThZ?P>><+|}WNt-qUpUo?nD zDIm>-S7D|zGAlEM(b1N`rN^378QKB|B-%?z;zU9Hkxdwr6<8RPJln9d_>0oELLk(T z%g(c$cCR;k))9w1BMsbDbhvuR&h>f7aLTaA7?C zPYk6lsgR13J2zFtDivZ5)P@s54S2=Dtc!^VijB6$~v(xOpdZ>kb;@m)5k^$AfhVp1Q zEP_`Y@c0I}70vKdy?p}^5U(o;N(^wH{5n?boH=Z_l{CA$MqDj}ek27$7U+^iMj~7g zl-GYfyHzBA;OLg5xvX+sK&f6*8!Ij^qJmQzaj9j;Q5QGykON`UvDwYS_8mJJ{VnL4 z=*@!>gd6M~R3*N@Lav}YwK>mFWaKBht5{s`yv=L8WdAx!K(Tvug(Nn?AKRXZIxf>R ziZtMB1}d3<^5DNZ=R$%y7yqnQTSq88>-xt&hn~}A(MK<|f6WVXOIzn~wx5|Ju6%ng z2Z$RD%3slS|7l$Ti%@oz+mPJZF;fL)Qeh|}#fT`{R*$#numE6P+u!gOg59stwV zVVVvzYgeN8C;Xs|C;3s$Pm2`$9>)qUX16>tg9Oa<2X%`Az#}k!3n@R43V>^I1yBj_ zO)djN&J;guP*e1ZVqRV3r*CL>r;B?fLL_a>CipP0_3p4cZC4#C4d!({!;Cc<1bztW z@rjjtzeh?M<`e6d#)hzhy1b%p{@Bg^%B|;wn_^jUG-y3W-%_NL{DL~DsqBy{VO(Qa za&06S`W~5+%RomT6C0J%46ItdHI^;+KYasik+c^sMPccuJI`0iaZ0hz_82J<*Z?qY zBCh15_@+XfN)ooW8b%Rr#^3ic(_kWR zpaB0je68MzZCgQ?zys8%H-H9CXoY`6plUtX&BG(If+Qs3ph z#_KPrlxkBF0YOD>M6ON!0m)k5M3)CH^H~g7eY^Kx;1EJ%q2q1gb7&b@Yp(lnw+H&m zY3K>!4IP#dOcmj*Y-3eP8G;>zPe62I%I+a!6Fa!{{qsi-;+jZVi@m!)l6BaLo*{=@ z!CAxYc(4zyf-pL)o@CJhuliLxf6K}f7_&h~e@|poTNADpQEDq<+q|CJG+J#BPUnWq zQrOSzX-$m=1SKIXJ@6+-s$DrR1H96I;6mCNzZEz`5K0yB{wb#|6u149?(w@S{qG=rwMwPXAjl#8}5Da!+K8jas-qq4Q zkWiD&q5AqBFW)h7Ef8^I;;)>a`f$z(7NpznAWJvdaZkQS(iGuCHQi-fjF|=~?zV%8 zf!fWzyPwpE@Nv#$j;HO~QC@l%b(TqNY}FP!+2{!uATr5g@VB@EE);>^FHDwwmP5E8 zjM@6kDg)DdG1q5a>`Zi{``;0Z5iMxOraRG!f79=pH{R0?-$}I)aS$RLc|S8~7ZNSW zI6(#&j`I#I*VZ5X7#IbquMftWiz z6)#Jxm`d~zzQmR>l!U||`q)>w9y$Ge;_PEe)bA5Tom{0)D(U}ot6t~gH@?IEfTm`& z)4x%kj86$`MbH3$P*o#y7V!A=ecNd3UJs2L*~^2KikoCfGX15>>*zRrZUQn!3};2j zNQ(l7O2rO#>0zWr|D@fWPyG}Y)?ZgQ!2pWiUF1rgC5z#w6Jsr{q^41_Zvww^oT=$E zI`dUX(n+W;g`aGRtsvckBC;4AVSSb;miedQsp=lo{#vvCo!GFWp-H^mr2XLvv(=pVv4y zMVWdgjI!DslTrC$emldWO75YWxySl`xfEIz#hSZ*Ehx)Mdb*@xzBL*@KMfzol$>id z?D9c_tCjPLnKpy!qh1{$%RnPIXQ?5q`RY8(2)D`G_31sRUa0kDN`t`{wasICx1vG- zBg%Yqh6*lLH8tZ|PT(^H#~9KMpQ!fT92QgVNrZw->&qu@T%*5r++{CTbhZyy>Ydm+@9VpmH5ebhzaciwWd0A z)^?^kZ-Dg3-7p_qK_gZ<43Ll}1h^KEcuq81q9*V9!roj^it2kQq6;-X(D#{Bdt?M? z%0|T&%lGNx75fc>@-|lW*rE5_alG@2F|vGYt+n@Y+R2m`fGKI%7q+YI97bG-MFg%S z^7z!}?Hm{Qhyt{o9#lS)CMXpqsFkGyTM9R9Ss(PC-Bz_bn{;8k-O9G7wJ3=`A^rMv z<@zvthE74jsI5nunb%Y-vh$%)eyA1&J`aSYzQZK>sLhr~L@h8m;`p_eFW6&_Zd0!V zi`OXw17yTjI4~D$@iA z0FhVxVl$?o&7Sz8C|#tfOb<$TWZacs8=zi?Vz2-tjlrd&&vdV{vKND2KCo@^iJseK z29*>7b*wA717}HAYPJ~|Ql~}veNu3dl}+`nSK;)b{_IH0n?=PS(u;@Z*GuMNN>d%u z7toQa7BFz0f@Z+hoaQo?BM$2pA6_U{(w{>>oaVOnq_)N61CD-al!mp+t_pG{yDd`b zFoZ%eH=IJT^!AdbPvJ0c0QKmpgt?tr^z^coDz95&6Q{2{;bQEeG{*`tcWQe586(s? zTaF<4zK9teXCPK%e7OOeV>+v{^A86m`&!p3&%HDh^bqK?MJW~6_DmlZnR593J-R4b zLR5&Psd{Ps`n{)}Pv4c+`dtf|73UJE-JAj++!`gYWQD}*<`sMTHf}MNq%Kyo*{SeT z>l0&B^+d$X@cv`0P|8X~_|4x@+~Wy@w4GR?Asyn(H-N4|EZlg}@9_3$ zaboh_jNLB6LHRbd*RMY1G>L}CHa-Yk{w*{&Y=r^4nNH+3#Bq%&)5J^LjsqQPha6Oz z0#z5SRKSm8j@vd?-Gf)9xJrx&vZP>cEj;4WVkln;gt8``TsD=ti>oiSe7d_*Fa|f= z>SI5j_4?FSN2@*^^BrlQ(U(lny>x|YwX6=|!x4No9}sWWkk>4AN(qU%X(zrTNj2G} z^j&6ELBA}kD@nsS{Vi8u$HgPkn((8Vk2JL1LgZkxa+)UXNB~r&(ez=(ka2XduP5%Q zst#MP?v#PgX+~e~)m@0brh=?A1Ys}Jo}*G|I0x^nz&dH=7n(E6sup{DO;Y+BU{)IW z78MRuql?<6hGrOJB-+LA_K>CS#~ZPN*OiF2#OBoWV^)&7i40`v3gqVMonYi@O)*~n zRoB=?xXr`ZDB!K(u94U(C#_D7H@*F9yYh#il(BjFwnqGwN*;T#<1BiLK^>na_wa6d zl6SNEA+rCc)%oRqB3sQUp<{M2CC`25!a-ZXznY<%#s-^V3ELK-JWh=z-_|c_ml4mL zH94Yh*8?!khuWofbG8BccTBU-w%&6llpLDK=&kJapW`s2LqFdT&`e@_L-`ZtfK$BA zjG8^e6%-4zh#Vh=ZMBYm-bapr7(m`6cm(GkOw&y*>a&iBdyedp?+JTAg5xpXUrxt8 zO9sh5sYSj4lqq$Tjt2|X4Zl>dE>s;~1@P~6tdtm6XdP*mHrOBSF2SX`bu8U3n0d}B zl7uAjK-1nON#yCtk}P?t|rd% z>u3Yop-=53awc-26ZtZ?2J7eew*L82<#6i?D(;`bVbd|kH;1YdHa|ouvW4KI6A$bq z8A+>J3238VRm)a1%zbevRcf#Q2_uW_yE(7PT2i*tK(A%5NmIDdG@ODDS}+YljK{FW zQLdOwbr4{mgqTX@%Emos|KK&R zdyVZ6IFzqb7WbWN-DXjP3RVz%`y;f}6eVZV%*Kg!qew+vC zp*O#Dcbwx2TVC6J68UMZ5VX@})36|@ywuJQzsXj&H^P&>ov|l$#1LePcdq84(YP~1 zS5^KpHX!kwcU;FUOS}o45anfCFuzi)w!~+p_bj0o{*e}6?#z^-PM;68;Q?=_VO>zr zWt*bQB}PENSG8g_>u5{aJ%;nRj$e?5O<&&`b)lTKSqN!-K9QFZf3cZL|N3a=wDQEm z@23t1BeWG1;mZv@6h%`-Ga2oqkGLAe^DLbFKswou7*m9KZ-1iF47r<#8I2PCp1Dc* zJjR|jo;9SJNi%r}R394yVq`+>y{uKGJ0&0?tq7XSXdo$Ysa4o}DT|Er9AvMl-sm!(^9yl}%1OGxWtMmF zINC{?>Y)7XMQXLq7WAbd)qd16@l<})hnN0r*I_U&(?Gy?;2m>aL)5hk){xlxrz*xxv{o*ybK-ozl+IywDlsLU=tUSq(>H>f~~Y zL>qMWchzljvbpQ)&|k(;^3lb(Ha9RN@(35&-vFBLr{RCJdm~xhYinsnytciZFuWHq z*Y&h1kLYMYE08hkn(z(TYr$K1#!E8pY>Hoo7q6~0JqrR;Lr@2?$a3q2M>I5$N8Lr#zY-o1n_pw%!w@Q9z<8%X`5Mb3QH2sI{cs4> z_94w3{dzhv>6rTy$dRgSA6rCt`r?+czL55mQ&$;me5WMp$=_2P<1J{&PY0O^sm)0= z++n~(iu+W(-o8s};mMQF!qalA*$NV?ID*fJy+AJEQ=5jsiz!4tdbA|MhB%bQo zIyEzJ`8f=Wjcww?mu)GI7E0wyNX6*O_tddG@kfJGpeypIV_e8tC5m27H~|#e9mVZw zVFmJCWo!AXA((VD^WKDtTb<^Nn|G=$O;Y#c;`Pb$j|TlZxHmuz5 z%!*$K0@@K!V{zhbzY`bs3^hvm3f(NW=42_4}%#APin;888jISsAx*q$fA1z;kqny96nMRxL9DxYlvv0&wG_9j? zOV*aJapY1*>XM;M&{b%8v%isZ*4e`d2i9Xb+toI8m#Z!ATd0hk(4-5gUDOD)DR(9Y zTQ2tKmjo2)1>3E~hNfHRc2?-ytHYYW(nu^L*jPZmL3y9UfN{f2pPVx&{(hsRL@bU8 zEAi3;7CZW^nNJ@Ql~;il4D;*QVp9$m^^XrMCua zUGKi{a?sKLSn#eC20-^q%BV@(IgtsWIKOj`?y|llziq;q7>fo)Ld_EUZR??Xln! z$2Do!7V^xeN(Jk-Fi-_Yq=X=Y%Bqxg;sqmzu1ILo06b$aw;H$qOMf`!G0$kX4aXBb||{zpgP19yQk#KM-Cm zuu$B(sra||YQ@t8)UAu37rgui=?f%1{;qC&$&HCfIJU9H&jCDlpyxQA>9=Nf-f?l_ zHqFyevQk0$#}McRupc0O$1(*b`n;z~?0YKc@nDE8n7Dv>FdG+DBakITlS~jli>q0t z$^8zhUqfV?MdE45tO?_#ZUKGZ#oCeNl$ZOFLhLC~q(_z`2;BbY>QTffKlutDYZ@is zfA&zxQjnYcfCsIevz;kn%`M>Fq{)o>tgc=yRN7o=A12Wzb7*S5s>j)cmNp~v z8JFv`^vCz(OnH`xWsOZ7yHh@&J_B&|5`OUj0=3{>4h%@T;tP5jwADor~$j#@-h9{Z#TVrr1QHuEjP z@nR{RIK!u|pEuF_x!kuQ+=+SQYqXWJFz|jEvJp|uyR+wKw{*4~dZU%&iZi_{CtuRt z>c#<|DsK4G;Dr{kPB+WtOpeW9Ij{$eh1oI zd)t>y>0ggK=lKu1F< zNA1O>OEhk3A*GSnL_m+zZ}zH~ClVLVP0;q@;|sM*%HMBWO_=8yj&W&|nI72O>WEU0 zF3*i%w*y{&;MeoB`{+IooImsP52RuHz$lN?;}*d%TB`t^N2KvqS8YLsIjS549obl# zHNi7I_iIYNVrQ@)-X9(7;!1CCiy~rw&Ojo(oUSz2rL0;K#qPO|oodA~yaVr^=!o!v z6fNG>L3?wA0mw1xHeDms=GNy^9K~w)mRGHAxnf&d@c6dCvnUB1-`WH zztz9IroKw6#T=u3R|Cxv9&MPrKcly%alry&*6>O5WAhhqAN7G#Cudor2X1ONl(V{H z_U%PSZuXxj>b*j~4cfvr7~5|%=;MQA>;H<$V6$mVMkVF{G1VFDvDx)K46>pG6|v_m z9~FbEb4zJ#HBYnkY+KCre!i$ji1rsMyUjTMJinOPf#AaK)NWcte2Aw*D$EzCK<8Ye zRp7?8t=vo0ZHN_LyfqBb*WbX*j()n@wb1npC;L?Fi=favZG7cBzgC4rYydWkPZ;oG zwMDlNFNB?n|0v%+%qstZpQFUuYbDu3IUrPN5TJF;&^tE=qDi~?nA(~rM`YLJwQfQ~ zpM{Z%w7pQujVMIK9yG`dR`p;dMj&LJuFPV~(j$7N{Xoxp@1gqj5o>vg-SP?Abwi`32I zW}{E)5TC$#TM^B$LZbk<$R}BBrpy9OQ@L4HLHx14K8{ECn!)=$36POg-D`~Kqinn@ zr13-+fv$<@oY@{*m(ZvV(LHA>Vpkd474*l?6r@m)kYqLbIKObOE$-AuVms5BT3B zYzDtuVy=`k*wX3fevRV0!T}Z9_R`)xd>*~$<@5*ss~`5~*Nm}fYrKYA`fNjXQV$Wb zJ9HHN?55;7Uas?vQ?Wy@8^0a@QqxI&;Kh!4CORhTPIIr!8@T*6l}BnNieCN#XG#)< z$BQNuoyR3c)%$+qM`f%ygx}M0qmh%gZ}1EA*8ax6lEe=m!Qtz61UqNJ8nQOUa=Qec zU!T+62{jAT1m@LM!~*%7N*M&e-Qf4D2AQ)n(fLOfwZIvwV9fXP;a}9L;P=ov<#8%V zA+-h%EjpshmRci<{Af-dkA8tnoRAlCiVc<#AEqCQw^7bnefO>GYm8f6HHv2e-kdk6F6|IlWU^MDT!` zB?HCryrpZh>XsAFxbtL8WAB4G6^v!2q`UVsejuju% z_$Htq&CLZ05!R^9lww5G%<>3zQ^ymC|zXYNh!TzCY1_C!09s*mBtobRrKq|DWb zbq{_;<3wBy{pPyjjeLJ3o1=F)>qmy!2j6CjVAoaC09-CP(*e%ZEOuV3Vwd~Md9%Id z0Pd&vbY4kVrewVpSFTV;i0IB)jBWfAOJ6`A1=nwPjS`ThcC~OP(c|xx$K!5ppwhY> zS&(T5CViQFMgCM_+331hzkAY+?U6hch7eqRkmh*ZGhxFs7ZT1dkn#_MR?-INVlP+f zT{bqhD9=Dt=h_4>h@IY(^(VQyAnvo}s>$1m5rF~>+>f*ipfmC79A8LUP&x}9%sDwl zR4VSwj)B3<$jXfU39O<}*(Y*L;>=TCwA=EzoFbK8vUv6AfJ!&*kV?+xmA^;216(sh z=*^$B?kJ8C|H90k3CKUqKe8=01&`Y@p4uiA6a^%Jkls_|{<86cVv4;jIGmi2f=Ov; ztn9*sB~tooY~vKxe0TF!`TvlhHXmxsN?pA$9s21U)0>>s#+dQYzM zG+{0e+)cLgEuIBDUyD4y{~YjFYBE9wUQTK#7ngoj4VixmsHH{shtc{|Q7E7N2KZgK z!j?>*NKs#N9Xy-p16l?jT@19F54@(QS~O1LK2@#101KJ*h>RTZACOQ`EtU8e7Ea5+ z&dRbOBPMl7PiH_oT-=N1VzVmnC;_hISA0;fG@~7tK-`XMXzMbypk%zTj-8;#*bO~Q z-0O@is_UYe-)1+G*zqOSWIZcccPgHn4U5Aj>gFI zD4=SuC8%u20nwf#QaiSCZ1&~4x~~_VA&w9yx_a_G9hIw;o(m=_;7_~;OR=^>=3=nI zS$ITgTB(uT;z?ey~i*JADF}Ap@J+g^j^c=GNvEv?I zrd0A6WAl>vgVsR=uo^FYQTU-~eOHAIx;quK@b{lrgYP~r#T{vi1I@+w?2oHT7L;?} zD-k*1AhNYhgE|M@we4%Q8l=wRAEA4jJ-O@V&w2B9=^3r7I7Y6ic0LJns8GeD6&Ib| zx|Yp*3S@Us1`~-gOPTJ$$;9UI-D^Ca4p`Bw;7@;-uZy6hR6q1Hnmza8z3U6L^5p@< zq!q*{Dm3Bb>SN(UH`~XML%#^t5LU~^51@&!LQT~uGFg;N8IAB$4Sb0AAMx_E?nN_1 zlWyCAs#BQyC1M|6njTfqG2K~S`4h`w!HN`{3?GIW3m*WUJhJhWSWX^;%MKxVjO1Zm z{)|otgOIF6;&ulXU(1DSQ0Dj3#SlWTFQj+!+v-i_fuW_~3y7PaYfPEw0JpD&DyJ7! zrPslF%--f{HYsN$Cx{XQppAZUK57S&_%NWF0%dXbJoF2>E#yzuI}>L5yDL|ADUnWC z1;r+5A}Ho`=Dm4dwZb&SRb(;CUUR{Xm2SWoX;>hrOIl^)^I0+T@QBrM-2+N`-4?Ve zmcX3Ba#n6y4X-ux=t}QIZ`Y`l0klPQnkA&GLzrntpJ-JP?rc$fc+kOYbhiNCk!Je) zYOMew*cG_JmN|TM&}C0Dq{!@&olTWXsuk_n-#==mI;%mJd%#qkc{wn9PDh=h93Rxb zl6-rwX>Vw@%;UCzno)u_e3=YblK!I^p4vln4gR2UUZ|@6N!_nC(0KN)iZzY(`wgyu zE6GU;IOse3Z{0aNKuh5#oMyPH{1ag*GU}cD{Pgd>!dB=yfZvSsk?Md%Dn4629~-v? z3RV6-xn_{j^vd8u9mKM#`%03EXYu&Q&Pq!wmM?;;?!F3g7F3>3h;4LqeLpV_xJ(1$ zZPJo0UD&(5AEGWdJ*-TSrqp=Sd<_EWJxLbV6#WP+3_k}Q>N+^Sh&_Zn1AZ=UCeW0? ztX~#arg8xc4t9*E6DD~QVUP^^`Enp`H|KmuReBP0litp88cJQ!_AKdodMQu*K!Fp5 zW*{XZ_L|12loH33cV;&Dx&xUSPn6(S+?YQZluWs;Q1n*e@QAuLbbwt~)vPOa^}UPh z8AE;WY`1V7`9Mau(9NF1;{#XCz&FL%EphhLI&c!%-qxEH#as+D%~IYa{z zecOe7&%^$z%MvkNk?nj=>f_^M)th>Q?^?>Nx3UWFc2kPn`l>(ZO%^sDsy&7m69v;- z(*$~~6|_&l;i`en-HQC2HO=(9Nm$a5L~O4MA>G~-Xct~*|J&OHO=~upG#KEa(%^cT(2(z!^dO#lh~CA!&WByNz+|4tb02q zzS!+Y`)(QJ<<}3CbBu7=y+o03vOsLWj+d-^EY@ZuPq<>ohdJ94STbBN5=#jsJjXzO zBjHgle3+m2IR5}fCgs4FB!H1)4c>I=r(svFq#xU!<#{{Ba1^FC*`j*Pz!xFIRhRsNtU`Eir2Wb{4t6#=Et-nPFmEu0>R-!ikpT z>tkAKRtGUS1nvgatWDC9qN3k!o0yQJpf$kSuZn4rRwO=VZ72J;Uj^SkU0($&MGRy~d1OQMC5^{z*R1!AbhsJn2`2SREK$dx zLt@3cT#p;n;ihs?$tF_cyz3!Ii>i}*HJIvtRO3C?V{&|kHVlqQ2ox{_uqMiF)vsF3 zC1cLOc|5c7>9J0ZikuqfCs<7DiYdBJ@f6KL~{CfS5^KJJ=UVoNi zz)t1W?Pk!mzr<4fabK2$pPkCy3o)|l!=?4=XyFsipYhYKM5x5sOK#o|Mkl|3HI~t) zl=e)eMD-CsB-zl59Xi^gM)zzH`;8-)4KgDISOZ{2q1XNuuCa#qCEQ;j3Sy~!2_+>x z+jz3;@U^t1zP?7AcvG@JRau{DX4vHRJ%OpfZRQQ-BgVjV9|1-V>`j?A>#p;6C56xB z8V|y@)-l?s$B9~1kXVZXS6gXTm!o@MKPMXv1`izijQz2nBdGNZ>m{Ru>Fk}Zmx>2cgucI_?k?S|U%@FxhY;Zir=_B)>15<1$t zdbgO7&$gqR9H8+dt*d;XG|{=}gx{rwYp&2KXxwO2kAPvbQ)U2 zzh4^8$6mXP#AxEk4JsFlM{Q4-okEL$6H>Db+@UB@Bu5mF@aDqC zz=BZ&fS`wdg_vH-HynR0yC?d$2F)8A-A;gm({k3BXJ|OxV9AyNc3=!-GW62u$@gnU)4^)Y z$TC<;G!F7B78(FR?`nv;t`wO|0VbbAGZ07wU+$_&enUjX*^^eakJ;B>6IT&eV8bQ1 zY(AO#z9D`zCt=4TP?KZH36UkYmO;=BLsvfd&REslZyl37!68%XmAac;Tg5=`D;~=v zIBSqoiN_fja`ZxCgrmgpBxRc5Oxw!=*&%X-R!Jy zlFVr`W#!ux$Y8~=MUf4FQGHd9jc=yjipX=9$mM>NV{s_c$r%YW%yl~0n&`c|Z7~_m zWb$kRP6Bc;I? z*G(!l@k+q5V`CU3jwCRK(kT~Plda8OPDLgqjFgU1D}j5FU~SY>JR^^i@UVD})?F=1 zpk_u$ofWOIm$>d|kR+Um+qIM*7U*ceSU*jYr6f>PYNY;Bbhs2tfC2EGWDw%WWJGPn z*lI<*D&-5em2IIx(*FP*J6?d`J>7*Ae5>Ydv{I>J!E6NG*Y2f;{dBt;Mv@q5ndxx7cz8p`N4j?yO z5*XEc7ds;iFZAB^J5!a>;rn|jl>}vA$6GmND%(NO5&=DHExb3?(8cVjITnfI%yhwy z!y%u@8cW$gB!GTGl-TS$pW9iyYDX3*fo91dMGW|a%r*LgoyAw;z*S*0XypCL!}20` z@RD-5V%5=Iwpz<_|vN%6g_ zNqi#3c{mF$0AvjpA%YrH4Xx`!4x~3t(cb?675Py-i&Um)YbrLv;F706SamfQRdNX= ztu9I=o5Z9NB~mUx?nv)X0@8e>f#z_|SYOK{9-`ekeN}Om)iiXO{{U6G?x3Qj_K-S- zK00^t(u?S8QpwHrNiiw8(pc2q{ue(Q&PJA@Hsr@4&G(JUssK@aP5aeCE67<)>TS8r zjqmn~xXjA|otta15y-5?m#zAKG`94bWr{I`X*|*fh_OP~yL9jHwQ+B5d^0`cGWa-f z<;2Hd`tM8Dal6<@_nBHuKQMduy=UrVLlF#Ly`q+mf%RU}nG;skql2DVP>d2?LwJy&2E+Vw*%<^Jq zjrW+w^+G{m;yYH_Z#HALwm7EJQW#oJpFZ~$O=*ODfR-OlL8x7-S3zJjrGi5rEJO-O z1E}aJoK5nvN-*?>;i|VnEOi6sC@`v)l0ZE1=woh%#8X8Vi8Kit`iXTOJ;CcjS};mY z;*7}~sbOxQQ2;TJ%8lhFC$t9Te00`pkvtK#2t3CB=uKqH6k;~%XK0F+D z$p$9iWn~4--2-dTDK@wtDq8Cxw$kCr&Z8q1RT)`GJxeKO3_&}*$5EjNy{PwatYQ>p zVuIUb>j7tOpI4aWTf^wX_Ps*&e>u6ATFtSc)s#Ma!PiwoG79lo0=t+=tt zj9I*F7;suMBuoU0A-$SqU`nVR6+zqP>S=H|=XYDkyF4h?lb68Zq2;|iDR$%Mn5KLj#ZrLH z51FCJ*KVK@yFv}COd{MC=VHD`8I_w7?3)b}A=C>ABP1g*JuauMsmPtF7JLp-_Mb3i zb|ijQ)2%dGkM`k)D_kt77@^Y5b8rvCtt$^&%t;zpw=vqfC?~15g}=hHv8<~yn{os} zw)Ro4nU9J14+a#l28HxBj})YUU}p4;OD%Pgb<(!(TTGuX2EdQ!Cbm0?E8 z-h`1KQ4NeD`_{)$MRIP!8u3!a{_zZkA442W8ePK4=>Btmm0{bhy4-tIvn&4qaDVMN zd1c7<`B=q@z~SI!H+xpaWfNpPZ^soWoet z7Dgt-FQ*4_-&^-Lr#7`V4#3XI#KK5z`fK0&hTfxG{UV;X>{thb%5j((^W#AyNj6r{ z`BHRCpQhgr8h2j?>HSEtOG+F5nlv7M&gZx_r(V-vDB5MrJYxF;57&hT@PRrd`o1VhmEljHR z0O;Btv?m6bxl8DgEtW;|0zN$|Y0HIqPx_b$ipBe1rNuZ_g^){*L{hnFEpmQz{g773 zg_yy+QK%=eHEPf7&+sZ9@m%lhkF?qJiiGm5qGHGBe0lBuC z)kjZW*@bu}C`6kCp>`BjJy^H-M@1B?qF11MYu~sWKNxt7Y!t;ASvZpAXP1%e7Ee0> zNZNJKgQY18Y`y;giQ;o}xFM0eW9MWs#m35gu>^#Jx#LsSwzkpZQ+3aioBckBX8V2| zE-H92oAn%uV}jHDS0RD#)ra9x#kb_km^?Qn3{9DbC|s<3Qw~(5mQgF&fn(C<`zY66 z1-AZFDVj;8Xo6w!@evWpklaNy$h#iek~*EDqisB{Z{f##@iMS*qQ}q1XtHL@^qO_r zp~P{3k74yGCjD!DXNDJMvE$@0{k6yBVx&LXe4NR0!Q6p~^rGuZ88z{j2r8}DuHBxV(lxr{yaBd5U7g&BmL92X2-cSx?S zp-@`l;CuxtYcSml56c?#9Z9{Sg1cpI@oD37B!RG8$(q4p{~_oh}JI0}?BzzvJ`8 zk2hVhRN-)r*b`v#`5vA$N7Rj>JJrJrUwG;AHNoH4zLzpCZ7QBm4-!lWF)$FwA~uM^ z0sjE%?r&LXz0Z@SOZxl>M0pD?IugK>qCUMoRjrd^bi73P88|{$#|*JH=@Af?l<95P zzLm|{t)3N$y5$C7_d3a(?j9sdAO1E2tQwO*Yv`zy8-;vRNcVV-xAS;MhJZ#DW{ z+qJ3PbrR9|9Ih<+c?pzDO&WS|B!st|olW~%qir!fD^$-HmW4w=zK+nxDcD#8+ULDY z+A7CG4oG|unJgp4fk&jwK;5Uto^#})EP9$kF3DVGA*GBptTi10How_OR+kqJT(P|VoZ83}q42o)(vupTL>-FTeog8(6{qj>`7;5Da`G2{0fb&gF6p1%r>y`a0@cv!iiP9g{XIGQ4VRV{mVMKsW5m<5ua{Zvyx{amaGmxa+yp2WCaEZz_|f?=Nr7Nmjn2 z_OCa_!|lKN+{DSsk0%u}lIBK)OJ&DavA2P53+qjozim&KGfR}kpoZpY@+V;JrC2m`-j4eWM#?wia2>VcFK(A9H#wR?l)gf0=V9gKbQK?%A8Fv zjek+?HO87lE+S0xW=RM>N5?>sVxaKc+I-D2p2gwu`F`5{yIkz3{{T&$o0uL5$h3@# z2~Q}tx|?p$4QSSVPirm1J*@4Ifds1}{-bbWfp-x(EX>YMfFhfP@us=KN%(wC^|Fgv>ipyO)4#EsF<;5 znq3eExD3R7!{bLIn%e64ekkL{uOJr!!2s?}?^x}&T{>xqaFYZj&qNm|<*hwN-ke7x z$jG;sOF1UqI^MlcqsYgW@Oiw{g=WWc(kj@3E&|^7_|h13xLFvSm|UJXEEx>n^)Uh! zNh9k}Is@PmlmATbssK(4_`JeSvB z%)dI0IWf4H+&)G-WIK?q;j&vx`E*rFYf-l;;ZlR;y}=A}d1Gm0+)QvtZi)trVdGb) zPD<^9#+Q)fCRwv2J6k}uHUM}nZN^VY&Ru}T!}Pzj3bPYuz3pzgS72K7e+US5wHR;wK;7nn>}Llv~scSP{@z z16od-Q&veH6hs`H&mgj}RJv=@#{M;5gZT))Cmrfa>ypS=SdyTCK0x-PZ;zSmrJ#7; zE-0jUpyVKyHaA$!<32s=HTc(GsM_Sp9BE;*&l*^Z18~8<8-|rTX<_Q4B0M=VJwili zBRiD^to;YViKVKuApngI;Ni!^x6Z0{R)3C^bNtT-42vF1xyn)*VvGVuhU40c(LU?e zVbAio**xAM|VcD~p+x$LA#2?i15DRV!i%(XJ0$gJVos z^j~ND_Zl48W6NcS6Cklwfj}}!bloPmB(9qDu(q_2_fKSVSa5Kn7@a*y*yOyjDNrI~ zwLsr{TphZa1B{E8gCq9_eA#6EW-0aK5O+ zWri5RLc1#+>@^^7U_jj8WiYil89m9rFtd6Wbcd|ooMJ<=IF$JrwA_Chch&^(Wx@307 z*H1NRBRsr2ndRij$;p)^c5JP|$v^x|+xFP@-S^V87otm$<8}6)qu@2Ew+fJM1{juB zu&^KLq3wM;_o&-!FI_TyjfanqZL0Fl-cS@?#OYyb$CG)hpyD_~jb@(4Z>RyPp*0t5 zx-{#-qYDxRg}Ry|sVBKJ{{Tsa=_vwSukFXN(&t;A!?E~L1u&N~ObOvkN80l`%7AIK1E{}+S}UsJ`QL4N%_q5V*ODkn z$5@8eU-Gfg8sqst9`8S`<5$;l>{l!9PiN%87~?pD$EsZ96DN_Ar^@xkuj{wgdE1p~ z%$qr~IS9kTmrwo(feN8VLONCJ@%MuuWHH95oTMz6b(Fc~~1U z)M!2?uSg=+%a(6jA*3r318&+5oqATqTx>q#T)E`Ohh(sc52#q(u1<~=4~>V7Mn)Ip z;$>pYUM#{fJEQrwc%i>vpEdN{tDpeVJQ@0h2GU3${yyJz z0S25FxhDGPKaB{2G=75_G(V*E(O!d}|GMbZ^#5oNU;Vtg9Sy zl{=RAwZXVI9tN+RVS4MP<9+$gL&wdF$TsMa9psdoE3*|4N3}-XiMK)Hb5$!lsh_fW z>G{4|OqNA};G(p-3GQ8quQo*EmBY7vv zy6si!Z$=L#9_9D6a6yR1F_j>fB69-XILxkg7m%Mb0K)d~RTD^L_W~p_M?7%5?Gwn( zS-m%v_kB0k!&M#>%Ph|Q#^uRm!cm2XD#{O}s*=pldq;D1@PnAvDDrroxJ)z1> zM3bt;8x1#|P5eNtcJGb#`1@aG`2O6(fGl%JB%1dz#d6m8p2D-ewzi!x?+1y9M#v#p zpe=H`Zewo|(z3gGZLeH2gAE`gpdaN>00ZSynz^nQC?INXO;VL+T z^p@O4U7Jm2YtK0@N=*FM%9}-DwiG=PI}e%fUY}l^W*$do_ieISl3GRBqWf-0_Jd6|`32J0rxnvE@byGkjJa*7mOgHWaT)3mG}KpF`CggPrQ?m+8m z0OZY(oy_)bm%a3-ka5HR0NYuTF!vrr(ZLV~0T<>!m=HcfxxP-D?LVip)|uu>?1-R9 z_!%;nc_St{00U_j+z(pfpBKB{*tGc*=g8ASqZTqT;=02eV^MCWPa3Uy_Ls~&mzie- zA5F#jn;O<>u{!u=5lo3Ul_Vm^#2qv>*M!lsU{13ji~Foi%8sMMttn0`lOa|{iSDwg zw$eJ)$Z3g`%8~B+tZWZzat8UXQ8@eqB(G$*eJ^k0_SM^nPIrmG*3e&=k|I7!mfz80H#f-?{)*m{I{&*nDpzL)7o=B56;3f|RBfdb6$ zGeEmk^&npRcOT+u{(3*wjX$&JV#Vb2;Eb1J5xHVVa&;%Q=>Gs^KkBkCW?(72kNwbf zBg{w6r~QqWJ~pe9gTTUYu%LEhatPkU`~|6A%T+q09gmGTmvX?wAhITkE&gI|X=QRR z{RV8zOtNfK)->z(p0zxcz+^mjBq|ap!g%Or}H1`Kgzu>uC6orc|HBy@9t|$>0?0IH)v2Y>>ZU`leiU3c{D zPE^?G$U4Nr=nnNh1o(XGBc~tS9*GP?l?LiAOLpl~AXz#-Oks{QNJ;+yQ(-nIOIDIq z5>NjC_1xqhqO(Wow(E-%->1T+ZH<*mXZ1b7%*T@3{{Y%?8I@xq!bueBE&T%e(QZoz zP{r#_>+L_LLIe@m`S$d79s}}^wwwY+=X;etCikEy%a&C>m{z&)@HAQjC+aZ}%Z%n6 z<_s}qDn`(BlOs{mAV?lBZv#>Y@G$c-Cz42VvfqwYDUTjQnVH;Z^9FrZ-{tQ}Jr9S% zXfYMYC1vtmuehrBxbbS+N#Q$J8~e4HlPlnGc={ze{A{OHKQh`ojJtNS+q&uespMiI76_01D7KT{#{U3$qa}!!>1Dalj~ysHHgSTR zo8Q9J%EWdx{*(7m5cFR6u_XK|kd%Z&8$5?4`Vdwvzu9Eg!3bW`_Um_-s(cVumQ*y})V9ywZV z;xbWwkVd9}A33C!azG?4lxfo4sZ&E51FWT6U!+*{qy&VM`cK_JTN+9IC+?symbu&r Y9j{Cv%tF&rB1gWafF>yo`p{4R*^DJB{{R30 literal 0 HcmV?d00001 diff --git a/website/images/photos/marcelo-rodriguez.jpg b/website/images/photos/marcelo-rodriguez.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a25be5309dad0eddc0bf2fbda38b2d47ec60f3d7 GIT binary patch literal 27061 zcmb4~Wo+G0yQNQ_5Z(A3=dbKtjWS0bu`C z@xcHP|BC-l6#(|H3HiGV{zk5F;-Qk6rSbQkWYN_3M3*n`vubUX|4@TR=*gKZ02<$^rgd zsrB$&dpX5gNZDAy+kD(xNqq(#tI2t7iN^MV)xfE@E!g`Nm1vNl$3_&mGxraY8EEB{ z@aALcw?a!;Mw{L>NXg>U+D1pWdU&!FRh(>%3b$fv`WB~sA3dGsk_uBc94arpD2%L- zgltl!PnMfMr!3T>r|WjY4?+w@qub?rVb;^24)86N*4 z6URlHAda@)ruSq}swXjQ*gWkn1RoOHSM?Mmrou8#8%%MIgVr}s6Ej41BdwU~m4eMu zON7u5{FXt0{{x3bO+Na0R7=Tlk&hO-;(L9a4#8Zu0Rxt;3|OReCcW5xi*e1l3<_xt zBW4Bq4N)q)Uy@D^mVTXTL+_P*NmdNgMNqU zD#q2-P(N=`4cABj6uCVkrp`gvs-hlibDDIPQ7$tW0tn`bI3~$oy)}gA`7ypA_VH;l z?{X=X*8n7^BDYF5txk$A?ikxI)5F_eUstF4l>4*E3i#cZN-0_Uo1eUelFIxczigx$ zhOCwZTs#=^uk!7;_s2;S_LuuDzW}++nPvCx_-Bn-(#kl;^kULjUg{csOQN^8m9ghh z?PwOII$^8FAY5J9r#TI;S!Ff#{^*z9Rj+JGZ)3$;-l-$2vK;g4P4`O#oJSi)efBnR zgjnud?{xPhA&SyP-Xz0!yB<7ttCU)@KOVcD)8~X8Yi%Q^#>ClvC3@SZDEJP7ze-VB z0^X_2jW#2Y(g11a+`iUHBlq6zqY(Q6=po5l|lEp zW1mDarDvZPWS+DvEea0x*kA-=pJZ;$&cu7$wE7s4-?sL}^NoK~TwJU zj!U8C)<~($*_HH^5uS63*W;ZtvZJgX3Ovhs99RgB#uxYfF5Lp&jA+aqD3QLPWG7^6 zH0M!fOZbUovt+Ud{WPq;H2KgJ%kSUXB0VgZa_&^y31Fn$r=a>mUdo5XpX{-X3gw&-iV= zIRck*?BlsGVeB}^C!D_`|J>C($Jn@JwWjBg7;4x3;dYHLh+bnj>-_a|M0=!vzD}Fh0>^ zXoVkjN_}|0?#^~%aBa%1T?$59-`tv9(pL87AL{q^j^%Q9zi9gH zhyB?Fg2f|8Xq;7`9#N$;Ub9c8gzZ{nEFE`!eI|=a?KzU(Inec*-yWW}w6W*LSeTj1 z?&?s~-}0ZUc|zf}zjtK(8{)flmfR-i*XM`0m+my#lLod5k?NQ*Z$`i!^Pp;8dov<0 z#`of{bg*5d?V}viL|-9wBv^2NW))$?T6>dzRQ~dz@$Caxrk{-iv}4>Z5O<+&fc10_ zP~5D4FL-=2CXmKKO_oUz-`?4?a zZset!#RYVqN3^#6jY9i=SuDAny}S|}Z|~W5Rl1~j>uz0s7Wsxiy?{S>a%;^=AO_cs zXFK$#shYbRgB-IB&CHdqyCE1Yp2*$qVCf&%tYNi52(l`%ZpTU-&)9;W6`J0a5hA^a zQF>@63mZG?H2hvYM-7=9HmpNMof-5Sm3FCX@VVOro)En2rSZ=-? zY4-4~|Kt+ixDfYwu`J=tTG9NA|I`{=&ts9hAV4`~M=^(@!`iH)-2;gr4@dnAV3U0# zwaHmjm}+e&aK=X;p4}z|#n!`ROf zpe0LJ(i4wQ9`rq()21Y|sp9A+@#@D^hoI+}B5N41*~_>0@Ui}a^&}zhBgG1ZaGONPaWbUId&DT>tJ`fK&)vHZ*zbLb(VYwt`BjHy;=lo;ipT|99 zU%%}M)sxATz%C5EGz%<}?w|I)kd)v<5G(rd{PNDAzfp-5ZVE^Rk*8N@A{qA6Yqk& zr04VmF_C#%zg|MadWjrTi^8cbp9VG!&bohSYJCk-0q15L@jXE3<3_ndQhvRIWu=jV^tBh!g;(bR<`i=4k<>fLV3bH|24!Po` zYRjp9$^59Ck&~zR$M>MQq7l9Bt%q&#oK2->9AZDwy@Dd0!Ii753qqlwXvO2?Dz~5V zm%L|#Df@gfWp|YPuqfbQxfEr%?1?ldV}yLI$|16QB(>)&ezt#k_#?aJD=F>E6@6?; z8YzN{=A0@_G$zU&=oWh-`7X=D2wySbU}&!sqb0tlh`QoK(_%7~7YX2vGzbPDPpi;0 zZ~JFpR=M;ER<+X{$|MMA?r~vT) z=>M9Hf2WWDC^U3b3`{IiXjW0|e~tt8pAi8A2j4DmZ}Q->{O)S)Hbl#r zqR`P7sON*ATVCy?&owpkb36z4je0LI^~@=bMx$9yRuK0p3`WhCtD+{Qhi)OMNn?$V zN<>#*Vu{ZwhY&rl|E+L>xG|w4Fpm`_tf@Lj8ynmY0mjPA#2Ddzf~18&f007GM*g#z z18Anncmdsq)!@;VV`dqlBgpM@p`9<2t#z@ptJ9U`eu>{1W~NA3fifZFDnt7zVuSCt#QpH>_S$5DIrn zehXgRz^0`7+2In$HqiR%(1B4f5V*Rv^0x&&&%uvfXQD%9wN0<4ECYUD4KW=mfZ5zr zdbRDZr)=`CqL~w+M5RL1oG5^!WTB9s+0l(K#U95E3MIUn8olO-= z`8d#IlOjq1ROjzj7q?c^WWy*6e%o=_ zb-kPRc-w~u)g&gNY!S73@VbjF3!wv;8kCW9yDf910%Mq!h5nYr>eHD23Rz`L=kL}? zdx{mf3 YcAKjDP`Zr!n<*M&<8`wkg!PBpYb|okNu-W`Aq}V1>-63!an~Gfvgi~= z^D!%qn`{Erpt>+BRrvLN}3md?7C)T`O{ zwIZFy6$U`|99@2bsqXK9!DFdw=F@Y=7ejfn#-7QHV>39B6RF{T-uyb0cyJ!;?ag=$ zbBc7(Sh&%1>y~)}$1*&7Xv`rlZ*`E(%HLSV6woj&N`mY+`026k7CWl<6IF7VL+Y>{ z?TnB(RhJv=V8zkcNgE6-g+;zkFgAxm3-W;WnlRgZ)?6mf*r9Y~#Z2&LHB@n_n_p&2 znjV#U7_>iCxpb-eA-=_e?yU5@v#p~7Hs^CItq|ynFwk=AGXKaeUbOQFY1L)Oh>^58 zVdb7;BbfPbCrJLKY7q9j$20tlJalKZsJYST(h)x+9!x)yhx_$ZJ`y%zMZ+#!xI+gA zx^qWctCN&=D~R%COo*j+N5>{V;&%PlbJ^^zNadO$mbpGaR)KHP?mKc1+iB=G!~t{I z+QT__r1n>ItcTiV%5~LKwGlC*Y!RFb*hrb}jluMa*6bc-D8>aA6DEdHu6&JlKlSD+ zQ#^H8SArbPT9Y%Jn!SJILoX-=h&7qrkIJAy!KBb*)qbzuz#Eg?(p-0djZqF=pi2B)GM_s!2iyfjx^fJ?}?fGy!I&UDE$wLu*BO2fNsG`!5F>tB(S zW+C499y6j23g_Jhhx@e(&|}Jy9Y@5R4fkwXegM&v zlj=0;R1`Mc_k*fQI@RqJvcHe`}m{F+SY1x$&2{wO(7_yQ1HGzu1IRJ=!H zR$$N>$mc>FedL&>t>m; zR18VI^H9pjS|h3sjxTPQ9)?O?$UnQEmC&{mh2fnu zQ64Xxr)@TH@o8}{ESH`5J`8$>MOh9Bow1Vr>xcG!c!GeBJ6Au&Z|^@RsB9o6k?G6v znvC|I&mOZ%qn$nZd4Ew^TUzrmhr4b($#T~+7czRv*2ZZoJMmf*O;SDQlr)KApvp%~ zLGe6+G|D_^w*&-!0T54s!6vhrkdkhrmcd=lnmEwiej@kEa@;&u7SZIqN58?(%Q}iJ zNKXFsBoS1=zZL`u2>}HI4hj7acKyS7;1Fo&WNc!P802cG>}C|=>TW}{NsX}Pp#@G&i-Z4eR}doLsU|H$&`sU4pf`6UpcVOn0mU*m+z9{O_`6aMf34|t z6A@PWi##oHFVG10MP*?$;|zI*KA5cI09(W47h<25QdFx-51{j9h2Iq7(2eLyk~HAR zq0TRh+JhpOb2y>TfII*y#!|fahBjj4q&=s%s?^tw7l~b(?FQG)A8OXlhc_DWJ!Mg` zU!MQc7a`h$5HbnfB&P;vsk=>PMVy0+qQh-N_!=>j`^<^9sCDqUG1rID79a4|;E)Y< zuwBt)ivL@hEFxj<3U=1iSweHC1+H1{gm9?mw4-n+RGMQbtHd#E6Xv$)Q-^UUF-wtinT5W+C)yGe*^O0C?4_P6?<&Ec2t#z6J21It(d9knQJTtx+uz28` zxePox*CaXV>Ri&IFmhych1(YLAQ!IDnG`2Zx=fM<`i6_CH660Fpb7gJovV_@2`4G} zUNw2^b-53T{5D=9dHU-io?_M2@~O#UJHn!WA^;VJ{FzqwaB*>iI*^uBS0w4$S=nA| zOTq5-BsN>i8#cFF-<^3tfXUG|(NhPm8spejS55RNkw*3J#oBVmgr}(mVm^LGR+8ra zWDxyObSh!$i(|AC6wYgQFikpYi8D4#e%QfRy;JeXp(vW!zV6Z)Jz(_t0z|xST|Xuk zp(6wOhunE~WM@P>&%3&%bRC`7yaeR_7W9s5`=lo0a!9}Y5M_}B9eFFUW1Q;dUsG2F zfPhYbq{$I-BBIK)^QxOa#h#HV$g)NdS(dMq!v-1lQ6s|V0|-3~2Rk143?fpK*1;Bk z!Fo;FToiZyRE|(4rr0yUDHAdFAk;on^XkwbdDDHUAkH|J?D@fMGwU9y9(DAq8FzFw zpilc|?Ya8v5R6<}_gtRQHO(c^*FQO}6{m=^#DC-t>%pBlpNRveY^&5t2qzfl{VYQa>H|v)7BnRyH z9g(~AfRqX?Np zw9WC|z;Mw{_@xfMrv>=Y#1H%Q$i<5nH&I{VD(^6k@X9A&uSn2U3 zLvNp-lIB9{oOA_mUF?hL9PRsOsfXn`voVBBQ#pkS3CrTYu~J6e)e4ltN-V;O9~CI7 zUEIHK&E?&;9jFbF2HZnm-=ySw%1(b$S%jo){Keqaz2o@(tQ!YBLuBmq;iv9ifBnl) zd4R{#Dm;zr%gZLvg^-kc7uCc}6Z1Ip_%LbQowh?VAm0XhXXc0?o#30cq2;Y`@1Ktx z;Bq>3Re!Yj0%R&%95+tMQmA^19j(~Nc&_in@fP=8V^I1ls?ie8sm4l3ELkD6Q^4-FegfBAt*diJC)H4`~E%ry#K>#A~FJR@s@S@^4F_;M^q2HDR? zIq2G3ne72YH7C`qwG|aAr)z{PbEtxXIJD?hPFPu%pA^rm=QF9MdFH#eU7oc!(7iTV z<5w>Ji53y!;V#F}YPy(7Q!&23TP(w_(}{R${RLoh9O&pIJh})b((nycCOw$?(?GBd zHTM^Trpa>TZlKvvifg@T;Sur1Lq%ci{kEATB3x4qxoZxG4)=O+p zvJe1@8=NAslOw4wlg}));v}GYCrps>8mc~oG3mQD-tiED(M+fC8oePd1jHj}1QNPa z0NI#Q1ZWld(y(Ym)?oTY1b0D(aqPZO?=NAvV$m|&^3I5t=Frp(R*dST>r8@qfa2|M zu-^5tw`BeYj8hbv|G+P5QVj zIonT^^(LD0Y>LhM{&f8GHlq3D43FkFHna1}7P!@3zR_l6k|xU)=E? z5R#M@c{?8D!y)W~aWGOlP#ZQ9>Fnw_b?ii5tNg z;ToEhv*^d68RJO#HCSE!4KJv-YN|svxw9Qr`gN3W7Cg*@t`)_l^f?hOmh&%weZJk) z=Vn*n4PPjLLqlWk-O!P)x0+4FMvDAgUX-u zRB7CK(7G@R`$$%#W#11N_im)Uyd8x!B1w`O@Wco!O!%y9>L@jg^l&<&gv6zyv^Ydu zrFF3{Ku!(;qhb>P%4c9l?T6x?mRC>nC8o%VusYGohQ#$q9vzB^6rqM|Z?Oaz{DbDa zAQMhwD$|NyMKSJAEd>Z!d@P5YM`efVA(NzFA%(p0VdmC0%bqNZ4ix(+84wYw<%lJI zp1eDT!(SF zk8W9HUl@ge5$V5KN%jY&`I?smhXPZkG2XEb$7xfBSx}%|MaqXWaT(yWscV#Cl?q&3C6re*6^RqxP!BnlYZLxJ|Br2DkszGF)ITTi^1$K;3q zckm$*j;8NLm8@W0N(A|4Yz?*^@r7P$w)ljajJU|$-CUz=3!D@8$*gMPLiv%my7#PB zv~zJsvK{w1iEW0Ch{Kj?aKIrq+yU0K?dqLOD(Y;|le!z?iM+)4S_U&Qjvv!I zH1fsJ@{4=S?*OvVv9QLq;r=uAbw=g8pM%NZlN@?Juy82mE%&YRSqOx>l&9vmJ8LM+ zr1<;NQykU}Vc}pX3Hio<4{)<+$8+Qw>eGtT-5&?Mq)6~>(jZiP9E@+4*eiwa_!aom%1yM$I&5)(L2L4=R)*mwL^J$ubmUQWd2HC?1? zQZGRo0m*|KYrmlL(QolNnBgp+#hsB32bm^QY|VW{GV0~^98b#BiDx@#l9p(W5xhE+ zax)bYQO~~85Q!}t1mq;AFR?!zBR9VQJJ6wp{BVmMlSnWv2|^XEjWscnd}?U~y;In( zx5HvM_cr##xIh-}AI<#QJxMx#mpK&8oN2*)bszFgYPYd?^=Sq<5-Mnn*T@gfIfVJA zhLV1b;+SF$9xe+o`J(Eh&l!w$4URX{q)srajB$=cFttl9-`SxQ ztmg~IlSFy1T2AtIj3YHOJyr9MLY93OM>+4NCj=-cIW=1*ZH^$w!De{pY9UM#U^wL6 zPxEsLQn4IRLcM=bFVAu!gM_-TyWbY7pE@9^g?wwq4PXj;;r z0fP(m70O3(ioaY3N>Y{}>bX6{O6E#i(xT7f(5CH{`7EMQXS|IdGE(aPxt5zoea(2C z5W)q6n*)bhA=CvMF+O{5{qV*}MuYEP0FKNNj`Q%lypn#5q1*&aq+GkI&4e2=u9}Ws_@kAPtAq&1V}vT6IY)eT%;%Jtu3ZNrEe`k# z5lNT0N2I|Haj9*}eehhCHb*hWRXg-fMDR1Kg~T;emw4Z8i)%)Y0r$5*=>JpKq~ zX1*)P!@?WZSl>5JhbZm5-T4IDvX-jc9-C%!!NGxrq%pjVD1BId#OXC{-Te|wqivE9 znZhZQULWKc)Hd22390`u$jdY>il|q2ezny03 zd6O0l4@k%V6QYg_Kmz|kGr;Iw&<&RgPnx!)fVSS|tl`%ukcfp8=VBGVprDPhRXP-K!U%FQR^UbXXT ztf?S~g7dv8Ub&oWg9yb7>Cm+uVhEg-r6rILnB|FsJ{gn`nRGT4iyc>}U1%T&q510W zTC40WPja#83XIlNv=DIKjs1nmgq9jW2?N94p)ut*@S*#oK-2U`D%=;~$@~0=$Lm{N zy_G?!AdyM-Oq^pf;5=ympBJVV4Utnmm{pvB{=LCtYDWY%iQiJ<7U*e|L3@9cZHxpC zy@`Z8S$J&<88{*p+NJ3}cqTVJ;{Rt>6~n5&9S51u_~cxJh9Yy*+!lnm^ct-pEun)& zNKwIkc$@F54z@D39&c}eTHMM)V%J_0ON;f=V-eutn>Jbzj6?#{_W;1%vgL z0Qnog+T5XAc%iS;u(e#L=OjW8zhvg5sfi=gzzIfEaQ*g0`l0;x)Wv9Nd;fx;Vxv?& z2g9x&;A_r}qehg;YiYa9oDs-)T$rM5k|;5Pq0^aaJkvO^AvV#8cpv^ARpYC7UE}R< zpZ#Z4niwQ=Oh6qv6P(A%Oc7VvT|xT=(5@kd^#nJ%7q~r(p?B4#3J6iZbbZ!zxv$@& zN@cg~b`x0si)AeNkwqdb&WQG*e*ArZ6B+f*Cnb3xwx%d`gq!)Qeoq9k^+T~2(e+V( z|6rix@4R3#n=S>!U6(IM5_L!C2YpF%w0}I@Tz-2!v)4Js!AOF&_sv1p!MwB{1ayCO zZvrAaa^3pLXRpz>yXg;N-ndb;1K-mwLbA<9n>2Ui*=eqBiXvp?us_rGfCThu zUZLZL(X3)>pa(l&w&N>2bq@8&eJdSsa86>VBDa$lTUXETBQFb$=Q+-4)3R4s*fxoLlLxpXq>xv2W_(@!DfM`tEoFjJV z-LF@VF6dO}9@Gs!4lF*{X0VRLnd;jRb2thqS<%!3al#l?OW2YqDdk!4zhjh#+bSXuQ*-NXKFb-esSRry#{v-3%%5mPah1$7)hmZn!h3<7gDa zT|%}+V+V`w-&7^6MqTFrOS+u+Ggv}rcrD??EC>als7^JY4QbWq|8zAWhFKCW`2x^l zjuqsClmELkmfgE+lTMH+_qX-w1@Ds-PSvz0@39AY+P+$cogk2l`z#fX1(r?%XNou6 zn~uj*(N4E)yV-RQ%f0lS4pV3vG7@Vgc|K)fhqGndUWTL<-N&Vv*)Am1n)`O4-=;e3 zbwCVcY&ypnZSN_zd>Gcm!OcWYY8>ZSBzFxIdB`qXG9L4=SJGJD_8WsjeXN2QON*qi=oqn%CBG>S#UeDKFMvM2VyDlI&dP*OQa3 zNV-t$Z&>QA`)l40w>ZdSxnpHXzx=7cQo)X+5nUBEL;AB3zgjZU?57AhYJo8qygR~k zq8#+lH*c+OR7_euqwuj);^n}i4DcBz0?kBu@fwwIjzQ32$4fy#%xHtBFx0a#&-uK? z{aJBew^c;Inox(Sqb$P6X2-@CK;sJBo$CuQ1k`ZI^X)gEQ2bdn&PkGbl#xRtXf9+f zF+}_0a-dkFSmGe5Z|_x?d`Vk_yg_9HA6h_Ls?GlkP)-|ne)moaj4)n1f{2`L1W8qS zcC(wkTSV%TiqGl^hOu6jV?B_sc)IE;O>xO-TdA1i(8-~G6!Uwygcj4~bYX+vpSm0V zF*QIK9QlLV@Umx?X8J?Lk@OSH1r9a1!Rn3Ce{5R%jCuZ@$ypD10{RtmKrkODm?**I zapywrl9f7XGb!ydVJ+UdwFc_9vLNaZ$}h#+9btu+3;izb-Lwj8Rqh4S5JXYJQuqU} zktA#6@%>3;kWLE;=@BxYz&-sXoT3i5AB1rw_wh!A+L*2I`#jZilxl|3UYUBv5WmeqnMg;I=V?;`#mU^| zCrbU;DUKC}oZ_Ot03PJIOmrXkoae@MXt=&p%!G;7HW-}KWbP!_ikoNm$T>?pU{8!<7s}5p!7cZxZjNdH zU>2e6T$O|)iWlsAQR~gK($cBc4Mqu4;&fdGkqQ?^^Anc&EnywVLlsaeiJce^Piw!V zHzXI=4J)auP_ig$(*_07^+y?LP=!81;`M=W^_O(vFE%x8!iTWRTxQ9oZpj%wVSCJ$ zLd1UjVBoVHXp&jA2H%lWt0MA_3CUqGp8Bi|qnR5RUS~f@{X+ImAiDVmgTijjKYpH-7X2@#IYJsV%TQ+MStcxKwBva6Z z1}Fca5vYPPL6c0S&^|xHea~e>3DJ`ToFq!DsghLB1Z@&;43?&{C~F_>XJ5!C{efT5 zC*+0KpH7)(rFfCFW`xRA;c#dA>srW6e8I830Qpx^O;b*wC~W2&n9rKME~ymlct8UY zj<=K;5HErRO8z9P*urS5D6-Q z-B#OA1UEBcUm)XYofRHSO9dCuMT5o@`^XKB=VRWBIX2*N>W3jsR~ANA!2W>wK+o%| zD;#0#v8-g@3QM1ogeyQ1}TK_1Yc zwalju(1|l9X}m%@HHO+vtw*3{>}w%sp&MpAu`kTSKO&pVq#VP!VGcljyIv@cExi^b zjc;EZ8W8uv?`Mb-v7J-b0S(c4{E*dbSQYdouW*vE==-J&%vRi>u1#RT^rgp3@3exK zVvUjw)kyzQkgJq<>J3{kpx~yk(|#RXr7xUlDkAs zLYJbd5$WxFM8ubdUPeXClt5Oeh^)ZJV|+I-%%fT z&t5N*HLC#$y|EJ@vw%eU0+XIp$_-rvSB>I%ubmM)`oTzDaI&X8%E}D5nZ|d0CrawuFJ*ZkR$3R=H z`AnQ+(gO1Uta@Yx87j!A1!^F&+Jfz!WXl00^^Oejk>zgKWHY5>E!$D(g)q}_6s$%l zqf|XCyiAj-oO4PP444QYqIk0w)2^^1`rctd5k*;ErONG>s>!<~fp`?DGwYvIl@b&k zjR)LZX7ZmDAh$n@xi;cx$JcFA?bJ-nWMnYt$P08x)um=4H?YD)WRb_|yzCO!qbt<2 zmOsp^InX7DoMfAYwiQhU|Aty(q&nYlzettJMQr_im0WftM7l|DqE16`o7~^$Qf)t+3D*M3_@AnF`PhUKEwlbNUw+(%NnE41_kduM_UzQ+^z3gf_c{9VV_MBu}P}qVn{c#;@M5+A*vS*dqm^a0ae6c@w2zyGV^KtU}<=tth_7x)XIX?Q9!} zP%1RTaSVA2maH@n@@aanOnM~;$ashjGfF)#J0i#UVoKTF^TaGOh4!0V8o%Hi6*WOI zN8}{Q@ne}HJc6&29gaWq4mP@~0%FQOV#OT=h5i*7IkTc{jYgrbEN1xQw~c`p=p z6;e@*W|qcm{j>M`G}peX4e$9dmd4fi1RpYXDpvmT?*2x_nXqKNKGw}aPlO-ki370F zjhRIv9{qUXh27p}{dn3N^nT;50!(?M+Ufj6x+v}KpD~lMLisk$$WkN?c}112V;kv3p@0JbySPb84DQR# zF^Oe={N0^KzaYiooLCM!g773>wi{C-=b7By@Ra`zJ8!`D*cqXC zGvkKBGM=5YmOM>8JeN&DHgaFP_(U!jIcm>RYf+;1^n2a*^K0n52<*POUC`KREC##o z%nMBx#vW)f_~ERW$43xTnyP85P!K)K^E$?S550KOY!KK*xRqLkpQt`NrMST!jQ)m> z57&ahOKUEATnCCvjtG=3D&`kE4z*9M!uV0bknUD!HMMV?rj6QR?>#rKUaZgl858H8 zwUP11{R&x3|z)0UQDl&Y96-rq?ze51lQtbd_e z8WyYWRS6p*2BREZxQnqRAnRY0A$RCVAeV?_c67&wr|iN^ zncI)@5W-MEFP}0n8{6l0UngX@!bOGBG#oEa-&Z$~j16#*)87jnp)WV5=nZY<{03-!X1{nTG#n@dm`gnN-OIGL2Gw0LDn1vRK?Wvk)%UERX~mz=Rqd{M)E z?GAlPvO22GyR5kHx&^O*{|CZOP$%^T4!djN`%r! zn`pQG#~b_IX;(IinUsZvu{42{ByWKAlbu`Tkfb?hSU7&?KzFE*aUO7{1yat|>sGR}X*D8f^6gsnMxz;j#{K8kX93=R7dot*(or35Xa7swGV{S&m;Kxv1p&0QIWj=Y+B za1jY6f%yO{cE)4^`2~<(q)rQWEC#6-vF6+|*791q8N%UazGF2Gm$YNLQm&5IQ19?N zMo0fuS57N7mEa|Pft4xyi<`JNL@B={(sV?QS@#8KVgP36xT*jLbDk|q(FH21ls@Tm zX&nPAHFKX7G$;+ip$IhvfLCBD9bT_^I{ZjOSS72o3zgv>l?4cPr> z&5@V0hlSzP3?q38mj>&mHe0st6g*C>_QPECB3Stub*0^tpMfdAXz!F{>yhHPZ9;jE zRe;0?W|>c&6u+N_9J@&Rs;1Wg-u1hzLu*!?;X% zSNZpyYEMBfw3b8u7;>Faiti|A?@K-q1P|-ZDn$gvxl78qDDh>EGnm605Fis-4Ax&5 zm^Zi4(|oX96+H%L0%OA7+<|4>c3#*KY>C0Lv8;>+5rEJ+Ww2Qq5dOn{|B_8 z4sgxYQ(~R#MHP1D`Yg48V-?g})Kq1Z`07fD58dEgLW6kC%eS99k`l+tmYCLXN6WcT zxhJY(49yT4PR2X=J#5?u>_ED2>0NqVq2|-BeKu>^#Vt02C3n+=4h9NK996nlcss_$ zt4pJpipTdb!DW2~nyOO*^6|Hc!v=3#)bjU41GboiapLsv&bsP6b6jBCYK0dgrQIoe zkp0`fagkv+{K%w5dIPIe6S)XIY%1A#+K>4i3VL;~nM-D%fsId6b-Ss z!;@{zvh#(qx&(uI<2W3_J6S>Y<`?rKX<9Azj~Wg-5={Gs?wd}1hFog%y2Y@1q*Qy1 z-bQ*(Pi&0M+)*w7y~+P9+W!-=mgjJ=Y&;O= zJrslF%}}6IRY_DRi%3MZ&dfpALcpF;b4XEKwHnxL8pFK%F6w~nS-T&OeD4KAi5tYh zcEGnWNCu@Nm7z2AnbBHMH-^ohzP79Cr?j8i1p3X)g>cFq|G2%dqMSO?ZAk)KLvls>4~G-DuFYPf5ro6 zC%4HXaLmuZ{+2=S&K94twu@>7ypT{wvKjjgINX2$?U`m;xxfMtOif zOZP9_!F&U%n#lQIJ)LD#R8iZo5v9Akh7cHrZc&kxlyvA0Nnz-eju}E~=Up7-xr>zuXDpKI;2&vjqhTA*2y*-kpeWHlYw3Uv0G`J+(Pm=_?F zAwv!eh5wcX3>iuE{_U0bdFg1XM}S0p-PdJNnJR4VxS!;#zvrh@nyNi~MquY1<|4JK z<)FMgZG@^(oGA0&+F!7vp_XJQkjUO82`Xup)cYncQC=HsWb9ZR{37H@zmLvOw90ml zj3U_7=r5KR_kP^jp*RreB}$Z{dXbTnEBxDnN5tKVR=;xuE7P_ejAb7KEaIc?ve-i6 zB7pRh33AV~*=yqkCU_N%ZLVnB9=B~w``ecIaHN7c9hBnh1!mYL2qgfhe9t=P+pULD z`_{g1Ly(C*?7x)DXe}$=_dV0jJ>%^DCu?7ucF=04(+YzdvyU_BUL&H}B6;+U{!Fh40}5^% zx}QcdDZ&d{QYqeXq;48!+}po;`&0C0E2mD#Y(Ik#_5oLEjSf81vg=!0I7^`-X3;bAU{=G!473u8R4q<> z(Jv9_jT2w?ih@s>&whm)|Kvi!$>V(QZubH3q)U154G41VWaGfp=}wBRz;K)44A5>I zg=e2Ja~VY|E&3R zXaSiztzpaw8;xoKBUEXqyF1MUoWwi!o2IQ;a!vk$ROI2?K{xlHQ&Rgc5rcX&W}MB_ zU+?4f-M@?k4#kR>jNf)03YqCC?%UfHN(!y;XR^Rip?LZgT@fY)h2SFACfu(BBT^Cj zHf1vtdw86Ced*d#wwfgy*D*gar#;UaTcNsm0aFH+qfygn9L!^<(Eb4{KK!*}nf`L$ z?=z^?%tD8Q+XM@}u(oTEnz!DCJ*l6eFbg})qpsSqsr(&^9~=rbEdByAbm>>`|9TuB z0DvkPDirbJQhgDX!HYO1NYsg#BKD}bCz$+w3Hru$tt;92JyAGhI5!QFJRj&Ej$1fD z;(UGb=~;kslsiM)DQ&mmO!eVlsV`a#=#mH6FVCtpp@8PMB@Wv^Ag7f{#IR35 zV!IAu&9owK?oa6B&ZN$Uy8M}O;cCJ;c_(z+abh9k1O3Gm7Ph44(V=~vEw7VQBSG__ zNa77=Ud5+l6xdj|B`5f%l}1!q*H2(H2dm$t^=oiw_t(mRl=_N@kjd9H6e&r=zNz=8 zgYKFGiO&>s&ps2TuCAZ+8TQm7F%hmzit}_+%FI=gVc&%C<2qko;g^v=(d;>Qrhoi8 z<*dakPTMWGacmKad=jf4yk$XgvGqlO(pK(^(c}e{V5sWRh@3sUv> z09`el%I2rif$$wckMa2MMy$l&x{E=u@TLJh=sJX)MuU0Gc>QOGw;YWNDz68>HaayQ4W&w{Fb*UdOj zDyn#O2NtzmkG8N)`ick4bolRd;~SCof1eUQYI1%hPH&S#)b18ArKE0GvLsp0FtMlQ z)`68NmyL~x;M{>-L^WP@T^L{6a0Sm-+e_@CO*bh>FL^I6!XQ&VtgHJGHkH znJM)=UwINDx0n^HvpGIm{i4dSZwV{56M9oqdLSwlgd^x@GOhUH zX2@B;!8-kv@Ao2wxaiUOh~t38+AAw3erYegoPOw?`1d$0f5Zhar z17mzz=5R!pRhvZL*M+9><#y3l?XR@2vGHC`?!^I$`A;IvCj?eNe6$BPdG`Y?TiVW) zg62r_P_BdMdLgTZX+P~exy;0J5m4rIPYn+)YB1lF3}Dnwr=Glz`SZ8BFP%sWWBbQZ z4{Oeem&k^VmW>A%D`w1n+lrTiteF}^$|3V9lLyi?MQe?~I_pS@C}L+(cab1E4ceVK z8gubRZ%Q=DUM>N`adFUf922I+Pq#u977dc_baFo=v>d4NuPaC(uIM_Kpne&;3%|u4 z1*|Z4MwVUE6B>D&Kk=>aLa5rMFEDeypEdj<7P97>wz54Bm)r3nwqw@tNQl{d&5ox> z%!SPJF5{^gm^##d4wAGOJywxNA4rW|G327&N6?zT;_{I71{~SOe#fEIu95wq-d>9F zVQuGSC@*)bVF8kY(Tl5ovt9K=*&%o<(~%7+=Wq?$!a&MoL?$jYS|{ftsc#Rrt$rkk z`FG#OS9&Z-Ked%^`#cSEJ8-e--{}JvN0;G|X#)NzsEQv;ghb;|#z$&?=#uq1T@s+d z4%bM7%-p~n2U4ZIM)EADxgzG1WU{P6pAP~IiOd`!){EFvM(Q`QzcYm+lS~v%JHINe zU15H9EA0Dn2@U75_Zg&KBT3~Q<@aeQ`LX5)^MQQlGqztD97`+;wUO2nZ>N`r4+*@! zR8yul#~5SM{OiuJT}R7=akN`TF5LJLzv1B6J3sXUUIvw$r}F9)|kLI2givS zi0n6wrXY4ONgJ`Q(5i?))NN4j!Q|Hv3C7KwqyMTpYM#JW_2&g#fuI^ahtL{#3y7S0qfDc9YB>z3 zj1G3Ax9#j%VXS&jAcGpH_^9GcW<~VFo%FWSs>6gG<2asNhezKc0EMwljeOv`r51*A z(c-c861tVeQ1i-&Z+l)c@_Rd&M^~x=-eWNtW(4YDI&)QuiEJC;u%j%xuyKip+N1KV zVuiE&Y;=UgJ3}2cvu^#;okAF-Np#}tC~^<*&dsoH+N4!ybk|CoH>97W!KF)bMxs~+ z2KSLK%+(YHNca{PKL;1n6G+1Jq^uuoE6hI_s7ifmyqK>u+mdyu0dJ8I$HU@01pQE} z{$Y79ysaIy-X7IM64`#mfJ-IuVxv*<{@1(}1Y1%M3T%@ulCgTeHmQdu4R2g}{L0T} z@1Flqa=YLepsuTJ7SvS-31JoHO>~^tyj#8vZC82Ql(-eBSWnI25I6G65+Wfdaw_Uj zyXM8CVM=t-Mm$C%pBQtr=SGqNW4F5%Poe&d{!8@hy1@zvZkzIHQ9Nt|~m>5tFE(*P3IetPk{1fJ> z$s|S?v@`u%$^P4K#v#mvtS>9Yh3_(Z+yb*tx{D$Ty56_UH4_SHY;lmy%WiKaI2w~C zAM&#?Vt$#x{5AZm=uO|>qP*&B%0_**rr*1G{=rdwFvEz1x;|g`YB*XiG&h?NEi_X! z(F}CY(q}WhUMDs0UFGX=-x?h2EbPG#Yn8wPBeLzx48T?UA{iEiaX#IgX$puZS*Rk% z&7|PFw>&2XPy?JAkr&k8i0qZS+5TqLEEF0i)W%E1*=;3(cTJlP-%gw#Udps=1`U0~ zHfJBPTeH_UUGyPcK+?+#KSK#H6&=G6T z1SV}^FwM-6J|4=MFifjju+G`jE3+IC4z-iaL6}H26r{(_q6t`eIUW@9s2=SLN zl}|_FI%nZ;gcaTsg~HW&)mJ_~k+9b$*uTIkVW6+c&~!NH%5O9KZrr1l@qEqo5X*6E zg0tRFvf61#lc~VQE7l3K>{M~Ted^(b*Yfc-mh-f6E*Z(q_#|q_M;H8L8yw0+Hm;XT z%pOdm5ju-R5Dl)}qARAn=vU@LESG{l@P5$#Q3) zALaz7Y%gDd_cc`l_1Xy^mnnHP|KKxV$azL+juupsW659StSsU0mzK;3Em8&y4 z$!teMZr@|0qRLgV;cSr_Ii;mRz1J;9A=Sgf2)v32w8klM&wO-q;(>=L2yI{Nl`KkgrqcMw&iP{eK)q zHxw*Td@Ol|F7+mUoaO7RjQhNX(5Sk6s+YjK7&46XMhuJD?tcFKFtRc|fHc$AdSRWk zG9<@{z1tb{T~`4PPFL7-O8bQn*y8R-y~#lpb)_%ArgHt7{C6e(Ck0CT z#A1n2=t<#2*$V(o=5P;W`CQ__{DdF=be?w8HGU5o?LR?g))u={!lvpQ*p$o&!=J<=9`SI2`K*w1tXa{=29eEE z3s&h(jX*Jjie{yB{oU>j;AFt>lJU+goaq%ZU>>26Sthhy*M;hM#XQsM!L?!#Qo~f_ z?lo7hIp^%#hWV@7MALSHYU;e#APCRS4JF(cvb9LwUoDM14+G<5d~U!HeflR_yaMX- za-fX%X-0E~U>8|2b?t&(x} zrGe02KBWC`iw-5_hhF>jQ;xl39XB#~ug7oX*!8A1Z}(QBr{>3nPLR0FU?~VwYL&^v z#+P2L#V)%ELF8-=SSoWYOXcj}{IZwerbwdmNA+`t+6ZC&U7gTC9Fl0+f5G+YngnDw z%6qO{+me%9#=yI&YLWg&)fw_tJ6J|_rtIu_TlA32ohITJ>itnjt`)r!D6p}Ui@yLs z`^|&XpgUJ}XQRrb7ESa@$iw2#eOT6^lRoHq^w=Q8k4oyJs5^eUcF{=`WiGDHP37*S zhlvLjz`{5{^}uPZjv+EGXI7cCJR#vTs@QPy;6UqahEgA8v}$NBkSum3`@zUC5Z746 z+szMVr>|ciuDET+nQ@-vkca*A2p+B>ev9h70KQuLO+kk;TKEam+aq=aCp-Y=?P;Mn zWpM*J?aY?Gi$kW2q9f~WB(F}P@NU!j<9-rEcJBc zS1Ou)o8Y8?%y;cZCd{XT#MK!c6F@Fr^gJ|+@I`1r`y5}+B3DoCq#XI}p_k}UVj*d6bsu1K>Q#Xs^6d$ek7 zhm6}@VhD^%hs50a?J7SP_`ZePs7Q_FR;I+#ChBFONz3DAkW9%B+li(6_a?rO?8!DR zoP085eN6f2c)S{8HKCi>vsK%5o!rtzQc32^VLSQhAibY<0TnNPIV*<0lK5(mxnIAsgB8_K znAwEz^UQvUVKJMHqAEqyeL~fDu>UsEjk>`hZ`~hU5nj`(^^`#pyKL%G#8mI)nVki< z$};F9%1AAz-oSiIqSdOGxtQx?77=kw+na`FD$A#9!^lI)S~YZ;@jnceh7Kn!7%#-M zYnkQ(XFnu${hQYMJEF6pjd}1gwQcmBC%vWz#n~OGIE`LPQMNzkQr`(?I>DPyfh{su zgeTtJ{mLpl-%W2ak@To&F@}P)CxnQuaM>slt)Q#cA^{b=8MGA6c~Lt)i5Cgk~Ut!9j3== z`3`ew%HGNBA;NgFCc(5tXpe$m%*_+sMCmfU>|1bGr%ZEYj&+l-5uWd;kMi4LvmBcr zU)&rQ(+#rITCdpQB*%H*4zJI46=PzktVMO~qPWi9m1Guwa!@Zbd0*L`#tXw~y}Et~U02BbobX(QA`zN@8ooudou)HTGVA;UA0?|*r=Lxu~&0y{N3$p z$hi>IYA`)zRQPD>lxB6GIqhTSfqR^uOkGxoAKr3e=3{R%(EktP$Z$1xNb`>mgHK&g zBO-SJHzD)ukW2m@K-5+G;v0FpZ*a6bv5e!{dKC9A6R|56Ov&{bHVJYIi22wvoWg@t z{dT9GGm0=*TxaQRR`gih{p}wjSve1?G95RuBs-lId+}y@!4fb5^P4<-(|Ol54)rBn zHci;Z(xu5T)69ZGVVc6uO)fyS5lC(!YhFHiOSh)f1;e?uJE@fFhkqD8m|3kkeHlMe zM6kE4of7=5mcHr_W#|bgiw&0iJSkOddLJm~Fl7HZ1H{B^;ZeFSYlQ`3JBg9F&r&IA zmj}wm9Mhe_u&@;}+@(Fp4qBga`@2yG7!=eu&AUcx`yiav6$jh!qo1L$2oVlA6(nJ3 z@?4){fl&5`q$jX&tDp^Xt%zv;|tu+iO2$Zb2 zz6rkoqgTEZ#|cNe7;s%;`K&$^J{crc#+*vWisA7N2Xhx|+(UYY1g4Ci9zO6}sNVd; zh$EWBm^}`Oc18Z-yp)W;U>b_zplS)-hEC%-{;`({#HXvk!%S~U$bVH}IN0Tp=_v5< zXTo*%QCNsD>}ahgMGP|v&exm|0f~Aj&L>V@#XZ;xE$X%!h5Gt{fNo{> zyrE_de5ORDkH>Fv7t3Xwbt7;^_5C2YT!g8HqBUnyDsp19~cvnda6Q@IFn%_eC@r;AI3)9C(jPMR4=GQew443 zU9SGaXxmp>89vE)bw1yDv1=EVz}fQNcLXv@*xq(c`$PEJ5pgkBm_zI+rJfO!w0hsm zuLqaF@y6Y~wg*tom(rQ`D0hV4r^1Sh2=^^9y4>S=+tT@yu&cp7 zKJ#mJWW7{;{ZKY8O0svRTF=~y?oTd*(xqPY=#QOJJlEfiWj8LVpfY-^U6<$>g9>w3grDC*k-dW6y@Sb8Ql- zSur+8>KtkAyuXCU6>XWg<;|tuGJ49V2&Mwd@%Zud+`C^FP9+F?m$SzdwUs_+_2K0% zjM3_MILjdj73zK+I;{KaLOqaj*CvvZr^{xdQ+Ew4a0{V1UOVC7=oZmAIUzH{Wi2VD zX5WtDIc$C`nscyD)vD4&AUWjs`hZ|Ue%0{^;15{&DEdu?FkPP_j>PwLTB~I6>g^bz zR>8jf>uCs$57dlDIrm2_VGVnp?72n62YKA`Fs2tNDvTBGZe@~|7=!bhY0m%=&wCfO zGR_xhJqxE~tQiXb+?-JcI%=lIZ=^#`u7iH|%>KpQgUBbD>_yJoXWJmGB>?Grle)OFMAAHw%15oDrGZKj`#a09=mQKqCanI_}Q5l$oz%4%$> zRvRuLtE9hdT5_y513~~Ctz10V z)Tx*fV!Mr_0P*ycK|v`1$=p!EzN&R0%hZ_I&&lC@uW+N~h;SQOBQ`uX!Uzp4nH{t} z@80Cldf9u5H=PZ#6a<^==`Qj@0eysQTw2>ld$$H(F03<0cA0Mg`rubI@2G8vBjkwV zbd3jJ3yN?LzlK8VH$USlX(t_`%3%sWV;Sy&j^GoH;j;E~LkoD61hcpw%URGx zHAYEyM$&>vMy&<%&jXKkZK5!(trC=bPyGqogZ}GBMzH;t5&sphFfcI50i>*gvRY5s zh2?bq1Bs`vG3Gl*6N@^UTH=fTws2+>kedztcq|C^wF@{|`c`yNYAaG|8<^Dy-HC(a znl0N$ZPDV%gMs7=09dg{BNgX%y%T=XZnsggE1Gksu4$-YnfIXS^{-q%unHN+k2Fs^ zI;F=)nBnu&!mIevgDPZz!4@mYmL^K=x0{L|537Mrd_s3D1tWXG$qa4~~m3Cd8 zx>nDW22!)5`IM=FXi&C)@t9g5%=`bq{6FVn{O8Dw|A6@!1?zvo{5W!oi6OfY)5!Kr zoAECn0B^Y;_qNg=Rb`#XxD=!ET;Eu5`^sl^-6^H>S4P9Qp?k385WeJ0US~&@V6x9A z@A3?d@ha^;IlV%jJ}5|Cx6I?dGD1&fCXG@BO8(+Uj5>Mgmy@=w#SupT8x$g_I>RcS zr#1a$7MrDbdH2^~glZH&%P(f$oBZ)W^BqawqDWpg*6dQKx@TL{T;`-=Y}|yN2|-l! zhBlhf4k8ERIw3DRtcDSqlEzp{&L&m;A|_lW z#+nva*C|vMQ7a2v`S6W{Ci~!yry6&UL0}A zU-Doq(R{@ljEEcmP0KhTC--A@}55Fqel zywrTj9P!C2+c22Ni=FkYwpJJr(|u=C%e~6Yk_*6k{)D`n^4JnqN`)$M(*|VNQlK_7 zwY?NbH!&mWtS9El^bzUQe0l?v?FPD$jFjrWg&rbBrF?Aecl%rhsF?I>^RxvzUv(io zp2P%66DLBU@8wvyI%w5=wmU9;GF2Ki;wrZ1!_r!zY*Bp2W@ASa5!%q_X~e^+ERkZm zY1qEQF>;MeO=BWdJ32HV?XNwzf=kr3bj z;c%o(nDU5Fa4gsWZ7P#!)mPJ~yHNQ}=1eQjj@EMY;f;sXj?yA{KIH?ol&-<5dRG!4 zjp_^T8#}t3T$9dqCEBvV{dYE(i@xRxyxg6fCRrO>x_DF0X>fJK3vFPY)qvD?B5mc& zI6Iq8F}-%gAuvVig@L!lx>_-1jp-)zu~fPRl`U_9yr)ZBv_Cku4EMGgAYCogpsj0v znVedk6FK(-bmr-;85P$)1|?L{7lQh#cBDxJt(FK5bg=8|6&Ve1PqeF)!6?)td29y? zDY^ATI1HMQsP6arSKdvv8c27`NPN=5H@MOCoXT{z*Ncu<8gJ_G-AAY%U$Vmx=4Il) zNT;Ye-!&QsFN;&FA9z)GNw+%tL8^yvqVbplAk8fev5XZe30S_ zga(FoBBrB{-~l4cUwWV}o}xX^{kU{d`P_la-3MTf^yML`5$GW`U%Rn1mtTD_+xh4{ zSVv5W<~4w4T#!=8vj>|g^VdsX_K8nJo@NgbgSS?USgP6P;g3RXV7V?TCmPN54}(gI z^ClCd+Q)x#lnlwaeKC;o$KH+yt=cgp;S(S-E`^`t{%JyqE~3PIxU2uXF>&Jm`0+mn z>->iw|AqEbb}ixmW%ko2kIeo9b8=Pd>k{Ye`r0&85_1Z8Re>_7$3IzNDE4%DOf|Bz zu(Pl(3l0br8Hrfxnep;4gR%+6C6ibv-9hONPJGG(j7!jmm(a_^A{lV%r@|oZ-2w6$ zg+{W(Bt=Sn&OdRFxAKso+CLB2>RC45(kYqmB|s;Ng&4Vu9>IIeC(a48MWm1FRhn?O zw*X4ghp0FVlE@VMcS}-X`26r8)g))NukOzLjgu;ZBcp*MV=-`Ez1GkGy#(~D9xSDW z6@XfRRX09rg*|Oh#l;y6h4No3N`ylmPabwf8e$tBlV5aJ4+Qgl)vvah03S3+ zzN);U0``C9(`#gIVn;Vx9|HRjW>kE}HEPy}pxs7lj<38T;_T?;kAro=O=baJcNgZ0 NFwZ(ZaQ?p^{|^eSS^fY3 literal 0 HcmV?d00001 diff --git a/website/images/photos/martin-choluj.jpg b/website/images/photos/martin-choluj.jpg new file mode 100644 index 0000000000000000000000000000000000000000..200685ab2feccb3d7e6e1eee5f1efee90920a84d GIT binary patch literal 20690 zcmb5Vb95z7*fn}$+vbUFJDGT5+n!*;NiuOxY}>YN+qOB$#GISo`@P>?_uu>UTHR~) zQ(e`)x~qEcUAz9R{@Vqh%1X&d0l>fj0I;tO@NWYk4uFDygoK2E`r1K3K|#YH!oqw7 z3IYNgA{q)hIvNTZ8U{8Y4hAMZ78)8Z87@8%5eW$i1`atTIWZ+6F$wX1m4JO+3Ih#; z3=4})jEROx{Qo=t^#jmg!9>7BAi&T7;AmhFXkh;a0Ym@*I0V>#0r=m5go1#E0SAKx zza7A?@NWvzs@TNqCHa=BU z#96tyAP6cn{=3Gcoi+qHQ!g)n*{djsr{f(7w**UBUCwn9dG~=fN^S`2O@pz?e^{Z+ zvrExIN`)TRT$EDDqzp3vvoc9VtI3_B;#_9|i>eNY@olsFx+@8@+USk?j3U+~bM79v zwR29wXg<3(*x1B2MM^~M(AjrFHAJ|b!pYMHHXA>f$N-jG69kFM1mluHdW}FXZJkOi zGpCnp1~)~`qQNPSPEi{zYPK7>WR)owFOJ)Vh#(H(1>O2n^GyIrfA1Qp!ZVA6@>n`N zG?^hl7Qfe%EFK>9T24XL*>9qPg`DoPB+y6Nv38a>$`^Cj_8x#6)C?d@mJ;WHY>-u6 z7Y(L%K60h;@|=I_9G*75W(53g|6REMeg3TsGj@hafxSUz=K0jIe`^K8Dr<3B+ z?kjVeaz;iAJ53ocwmYQ8Iesaeb1(ZQ8p~QLpg~z5f}k~r1KWr5(xl^big+Dd!bvUl z{X1hly}Y;`XknkvddhX~*FOO5HH1LxQ}avJ-H)tQcZSXm%sak;P2{Rh+T5z(-f3yi zBrK3p9h-$y1!I-hCE#v!qV;DprlquA=r|4W0dJkMRP*8FRFcg=Yn8iXt*>T{>eM~& zj^T{jYd62o%S^j^6|V4Xw`Va9g5nth_X$IHOW(`oh(@`q3NvmcocOt`G?6%S0l zbroq=3$%1wV#j~Dsz0+d={gvp(j}c$s>_bynHq4kU^FAM5Jar*$8E_+k%37JtNNm; zNcB2LDDh|t3jQyrVz}}9_sdOtZq}d5{{YM%-$N5$w;f`pPHe%ZM%o3t2+Aw6k}^_v zVrHk0R)?DTr~+}-yrH!CM($~Wk$D9T1k*xUAl*5=%1A{b$TEGI*QjE9Y%p${;Kj2E zmj*1FxZuM56WffefhF(9-|4=7G*9(cB|;r@Gdm;>hs`!v@}oUT=XK$c1<}Ko%&oHC zSPEs)UF>lE_`h+tYwtJlcsd*{Jky-Ju3n3j4EwB?jnj!#n*rV0j-NbF zu6lSsM;plGi7ZJom*8myas+6D)jvbkZSJOM_0+F+>J@S9N4Cq@k(*|AklKq%cwU_o z7bn${?1A_{j;EBvN=ffvO7~UYi0{t2_80POhmr*m%GOhJ^kab73Z`WBmBkPOveqs# zl!+YOEil4X7O!fc(=ivsW?=PPOXSB;OZa7gcoQpEcm?4uq7tV;g=A$~gD~m6_Fy1w zjfO&MLFo@QjyN2wlI$Nf)|W!U$AS>RtgvM4S#6%3CSe%`)w5-u$0Q}aY%(0i@fgLC zu0Q-f3LotIa;X&`lw$TD3>h_a%S>S4>-~$XC~%lZw%SsJ!H6A}_Um_aHbA7Go7>F` zXWfF+mVU`Up{>5vj)EVTnEn=4rDK4X6NQUua@^vCw$U?~S^*So6!}K^OwxOitGWC# zg)Iv2{G^5sXBr>K&@B*D}3KvCvYi zDEVsR8W&L&Bd1I1+3;?nqQwzUmkaSffL}d-91~+iyfYl0hT{xmSG=?#I?Ifz!&2a`X}^|(euKGDW6S#Ata7CXb?&l* zN23IjW$>6>+R{n%L^H=8(SesPRZ@f++Hy{919lC?q{@|yQx(yA?^SsYpiMxgFNVweqT0nvN*ZZH-=V_WhvBN)$BsIpo6p`CY+$?aQx)nWZ=d6#Ig8!pH{?H@ojO?t~uiN1>2MeRT+*klgd zysFYg@E6W=@3SGX07k6>-BSKx-cp3%xa?(O&;`2AV>r#A%p3}PE`gMkI8Ie;C(dzg z>Ka34WWb{KiLv$}I`#zCl@m`pA=5s>=8B+Vu-_c?5ri2OZc02F_0T$mEJytNF}=d) z+&vhQpJIz~c+pa;a~J2jt73XV%qqH?Jzj^KYeD>+lM2r~H#`g)>}X?gf=zBiw3ZI&+|U=)iL`8N;%?kvbc? zv2(hFce8BN(XKY)+8NBq$fkjxq;k@@WGUq$;+`4p-03wQ(7{*X3ZJH^MLorm%8fC# zNq3&z_U-m_d@lIMC2oHgD-Ly0()erI_mOg{>hTA4|1{a=%HyAlWL=AyMMYlrdrp`< z_+x9|d^M=hH_aBST#dGbzDckI)zLkPV1J9bUtvEuqO9a46vA{YV}b`xcwH#BC#Q#A4Ov9LrYAltVBd z8{DO~+0Ukm6JZ2?88SQg2;CkD#2NEi@$9 zoZzD!ub#i>NV0G|ia$qih!5xtdVg5uAg@Q9z^E4z{xK1PADVxt5JeCevz<}kj05u! zlPDoZO7!=SW&k@5l6E;`E!}i-uoRJ)F=v1TnnYXhT|}{4-SzODaXDWpViISAw2Xn2 z^iub1xF5H`;p5WrrW2flnvy3vA8Re!qOs#r=mPTYJ){|xoENggKS0`BChcm>a8v-@ zY6AcnpEeJ}lq09y8F@|VLN&v6opVQ(nhQ#GLoN+xkx1zUBdRM2cxp> zFRe(w2!apQ!&B8?GLCS2P*`BIOUK-I3EY@PHKm;cw?Ey|;8{bAeBZI{u*FY z11h=<)Y5+HOdH-CyISGe-q#1iIuax^ou|)4encbiEH!Qnn}aAgW?!D`Xxj7?LJ|2UJUvz z<^6M2aDwBw0UbO|8bRM3H<4Ruc2Lp(k+c`^udt_@uzJsl#joM1_>8 z{!n;Knh7$K8s1}ulpJL|+Md!@pW@Q(oVyZT$3scq*!fIl$& zA|ypkr4mx|H>lzwCwtr4jO%TlwA-CIWq2yyFLhvz@7umAN1NnRetVnCWyc#D-aW@d z3#3?;MqwH4=kH$jyE5&J>9bouB*Jbc26$z5#l6!)FNoP5yh<~byqR{mmB;5dUBVsU z#iUkBrV{Q_94`djT?HSQTAskR{=k$0XZcm7&tpXNCc>r0!tF)xy3q^K3(i}?A;J~2 zY{p}&YFE|Cl$cnVNgMzl9;_9!w4-7%skM?OgAZ$SSal5Ewir-S)aeP(sJ@$4TY z2^oo=O#c9K!aNJ=Q&7F(SX=t2y&ZJs=wr&_M{Nx_itk?8Ib4Yb!1=4wMIl}e1xZy- zm*mXnAOz)ei|X*#56z$xSXnwN7DQNF2H4ELjF(tcc>U{ zm;hbu%PiLy^7k|$DgO3$cN=9k$YUkLMf4)@K1)$F%-K0N1o*#Rt3-#-8dq;`*E$}s zue-OtBp|a9vE6%VDLcZ;&Ge}aA~~@!8$_ehq$06cfZSaL4JAO*k%2m%IXCn>k#9%? z34c`Yjjv`jEta9JZ90QR=AV;$7gx5VyN$jJr%Zvu@Uw8oGlR*)c7BNtM$tx2Y6Isp zdLvr{w9#minA?{LHLC1Td%>RX;|VWbavp+^_rK@iDZK9uLYtk6Bq1|u$2f$L$XCzvf%P{f6 zZM3Mau1@}Q-FN7H9i_R_VpGk4XHk{X^N7o>!KyXFYp`wD6 zdPmuofc22fOB9neSPet3A@(=!m|6%>pQMf>uC7E8SLwyqVvCNOJ2pXx|EDRVuOS+EGvUx6rs*NV%`YR$Tm%lQY`UP(~S8aR?$%Ecg66)hsC&S!~BUZCZ7M!b`_k}Q2%rIPkW9rJg!wUxE`^$mRR)QUJ+O>+iIm0pb? z^Ul|^2@Rni+|Z2LTxSvD<#`|_Q3Sm0-E==+N!Y% zJlF1=nEoHYF!M9!d&179K|m4)mJUMr*7qqUEGVe3IQ8K3&a$yMUQywM;;5<9=#2J4 zWb?%Ntyh{_XgQ?har$tw#`y>j-cBDpUnZFTBh99bzhs23=+wkj-_P14=z14b;1GYc zUE*&(EIit1@EJpaEo`Bs!K#ikc_542$h! zwx&+a&+o?qP~o7T*7(6&3kLrIP=sMf$h4!d*es&Pq7Rk+87`Y%MZ@4RfWGvg_IEdlfCp-t=1fx+xM^2z;;B0`>i9F$X9I88A5)g8 z4s!{cgBcY{enHR#zk|LAMVzYgwjs)U0{4$!;TqcNbZt_E$GY0~;a71qQGF{8jr3d| z02)>R(+85Tv23 zL84qevf>{gzLGo&>4-I9rAesX{iUKAT^ z>g(MAd65-WStDpR^}ZONeW?b{3)k`c5_p8i?=PHTp>S_3_zysj)tgEt<}Vi@p=S7n zAjD9=puvAyG$b?}ILv<#;y*ZnhE4`a&MJzkZ0v+V!3ISt=8vfomsi&ZP022Wlx2FCc`$o?yBZ7F*zEfP5}#k}Cpy!IJBHEOjKqbo~DS9!Hqo<|e|C zhkEDHr7h)b=$TIh)XqI}WZJRgKMCW&*Y701Z~vMO=B+(5Y4_>nA;QS{ zm9G|?g062}VojcDPWr>Bz>uuDoGPBQ1cK5d$SobTB3(NZ&)b-)%t$9cxR6l;!z&T) z1LVT2Vhc2`c}e&ODDt$~xojgYw<+Sv2Ztx5D0MemnapUe60@ZiufOK4T*Rlm*tk&| z%o2xeK3QU&PvLzs(B0C_;efEVXTopW$Ylcld~8>QKK;lv;r@ojDzl)^RF3D@UEYPU z-!ZR*&I)czkPF6{#~Dc@OoSO6r@NU>Q*ezbDr$7|**eu+phiM6VH~uTiL{9&X7BnW z)md=p?^@{&pU~*9xJXw{P@s&1(b&&~-X@TQcaj*BZaRQCwdH$?gS~y-W3NnH*#*8K)TSQwXFNOaB_SD=5 zd5}!L(!9Pk@u;0sMX5My!_kY3HjA3v3sa8)vJ&1~b9U~x<`>Z|6cD3JJj@Cm!r=@F zxjU}jf5B;6%gjL)oQ~@%8dH&$93H2%^im}oZEJA0RnzDmhdjbim;1Co;aLqIyRpdsJ~)yL)1wbqy@hh zq|)BdFN-M5n9n3E$Pu?w75#Ih) zFqKjQMuY#u*Umm zO~*mB6mzDc+^7yr+&&OIbd<*QFt5?;o22(*U^zgYVMBqn-7C65X`d?|a=MfaqYl@; z2~4oIuWC)d`06~*)Ujcnp5eyzI&~Y_@w=_UcnL0Zc{!|eF-GDovVn-US@99HK@sG; z4a^EW?A#vgUEvh(5Q)oaD@^yox)=2~&}j@iwX2?NbOh;1Zvhz#RhT?ajPxNdE7m1POm`n&HdBA#uIZVR1nvZ|Jl9n_Qk4nb{h_ zt(ABz_)wd6#|R9m5{Lf$jMHstl}=6|qmr|2K0EyKrhzxWQa!;hd91nqGH7-e$BZ)I zvSiG2sk{z39AZaOGsK6wofpDd{=Me8qEPXC9o2YnO=B^ zL9cX~_ZObPlg7A%N(OqexY=EPzR(GGWzcEKLK|D%(=KFTZwivHTDonsq)d@>N>Njy<3? zX3fa|JwFUU6|LTdV5Uu65jIC!Z=|8%1mldV=ODm5 z|6o#imQ9n0#R-LV-AjT*RVd^B-lPu*npKO&b5j?Go351GU&+7qNeeTR`93i26(nJM zUNI0`F(_a992(LaVBp0dL(W9IO(I&}zh`Duw&jr8XAy)7OL{^|NS$8>-EJ;wS9}E4 zYoQQMl{aXASbHtxm)qYf`AhfA?l>Y-WUfl|Pe!v?dbcQ7x#Xy>vk0Di>5tJBxZ;Es z_$>3ToY7xLhx}$s1&T}xM1@Dy1GV^Jf>p%drXTHknr%ZJa{$sLr*EN>84|!V1*=MT@r$_(r3WO;KB@52YghefB8dbO z49{VGNEohu&Vk&;{D-2<>8Wr{B_J>p(xxOG4>PZ1o_;4mm)hHe{t6;|@7{BoEs^XS z_(^a6wvcM&n#r3N`au%J;gWv1YM)VDhZ{qOMpj~!gP6wKH>l6;>4ISelo}B6hW$SN z)@`IOWmT!U?0HTF}-?sZk@?+*-d zsIm*MSmC9@NQ7K9wJyOM>vMNf{9U0a&Gwdm0=Wa9(*_d-*`jD$9-7tJB3k{N?nyoa z@?J2YylR0JT!22pbv)FN=83DILNOyVHV)P5$}JbHUT%Fg1YQuScTN0?Qiw;+de(=e z6P;IJtWTzbaZKM-!);XPTmK7RDK1>%PmuqK&MlgSWgX?HP)Lw_sEYiv%-qi zQNTRe%Yy4JS@Uo1#*cA%C&7-DGhNo4u>LC%0R1M zMO2gVng0M*7CB{_(KjJc?1JOt^%XZMa5s}BAZKK98xqR(pYq9LSs1uevF4AGS7a9O z1X^p&jue}}X+``ji;|T~R?A#9+10|Tj-(RK}V$>4zNV=ggCm zs1&m391hjFoHJ@TZPFdue8QeV_8|*zy|N*!LXfZ~wL5IoO9*FLmGVAWxD)lGU-z)G zIf*e%`>~KQ1@A%%)e3!``FNdm*8VHe0zPvDY%+-4_%r0grkuoAx<$Y87wPr4{0Wwb zEQFYAPa@e+fm3Frf8%5FnH;QwJ!ysxE`55vFA}qwR z7JlaYKW1&|CUvUdx=7>11b(x;<{FxN!Y-5WPx7`(4j^c~@-`LD6KI8pE0mtpAWIibCXvSQf?mf=v7X8|qK(x(tl`{(Uk{ zXnI27xCOBSC!ZuSXCgIj6R0jAcBAYTHU=2n94qtG%>Z!;&FZ^t9cvR8ghjL*#Xmrq znTB@jM%1j0m9=O(=1H(_EDfx+e?2;m0?RT10@%2jje2mPwC0oz&e^E$!<^ktr z4OH!f6LY2%|06X+bj8}F_06t{!9MK0>T`|tKS2HmGA0`S)&cc-o8LfY~7DQX6(2*-a97X*0M<{h%m7B!(D9z zToeVu_Ox?sjg7zDK2uGVo}=_100DdugMw0LPc7V5P!^F)xlLe6NgsJJ*JLc6s-7U# z&YUfxPGi=d5v~6h6Z-{ctIBra?G#4)!~i$gp05Cul%82cM)_TUwMjm)Iwqe&nQ4B6 zs8MLD>G-pM{#|Emo4=3 zd;BPm#la6}3t1?8)-+Wj(=~Q55L#}t0@a>EDWU#?IBL8-6!boEb$uB}S^f$iLie_@ zGOmDZg1ONmZQ^kj+DfZfl$qjKtPbJbU&-Mzg!03^;yo=j#LyQGJu2s@l?>Xeo#P1P zdGQahxzV=BS(NWCGhU-Sh9Ty0W?%4V_NE{0BXY#so~-)w+$K{66Ou9*=v#Zl*4Y%d zNee=7 z$dIq_?9|(7xw4!P{8hsz--wfU(Iv-#eF*CRM!T8oDFa>q$5z?V1&>X02sL4j(?17e zbbM>h)Yi>1>-%BP*}^=#&&=}9ni(Ox<8(b;&aBg9JzYRipQkNa$UTFdbPFh7!iw36 z&VKKxhH-*b)2xjy;1>z2>2q-_3RYeRnTEDL&*|;*_2$S!Jfke8;2EM?2;hRN+IK1JBhY>ez7=)XI#EokUZ4Gr`oP~jrftI$1 znSr);cmwAgNvIk^qgsy~N}bADzo#|7T^;I!shKAh?&s05Pon0M6w%hbRtcYL$c#Et za&C$Xw?gJUfAJp*#c+rEoYQXV7i;&O;qgr4%lc@XJy#~2J;_B}gL-VdHXVcniVSUa z_;H8-X;8V$)0Rv-*>PJ#w4MH(sky~fF^7}P#)4ZlyCnHEMZI}{&r_w;+U($b4c;w2 zt1*6x!)beB(5Npi0y);J^jSdsW;GdG(j2sB{<~Kf zY^I$o8JoMF-?2CAD$!nT`$u#t>#7_h0i?pW@&1PGXBShdzM4;P^Qbm}^I}yp>bzxJ zKJ{GlB_|kvG=P^qYZW8L3Aejpb7jze_+y`kkUR)BPQIuq_gn!(@;T-rC` zZ1TOz{*sGawUO|lILv}Yk6o}uHIg2~;*a)!8%JDxN%Vs34}rYq+x;#C~R#iiDqr7V;m{E%w+1ECD+zGhw;6?)di zd)!uABr_FDp^eH5(z?Xgdao_)=gUodPaB7xD@`^A7S`^g0+M<#x59V-{sTOmJ?OJK zve4$;x2i0S9L`&9Z8(c`bf^Y)aoyZKqZXZyO>J-}$z8I@$3$;cUv{eOK(i-!It*8k zTKf)?zdBYTy;BbYy{v>|eL0IrKYR$a3+?m+#;~n9UadE`d{a9yh+mQtb1_Mwe(L*r zsP8CRUi|u2itmI2L8TBll9S=8i&mz$`{J2cc3?bkW5NM5SZzk#sQ$Kx_}#m=GdHxz z;)(E4foBk0u0JGbu2{Nmaa(WMY`(;dvz`boHfc85nz+f)sLBmVO(i322H4Wf!_{}=zSdlC$Q zM#d`oC8DAF|EHeK?cLt|-|0AEROvY75Z)+VzurWao>r_>Y{b!6o_**FvSWjzF)0;N zQp&o(#)**9K~`=BTEVIBfgz-F+FCl5I<&Xo@n$Xg|-fh*?zK5HLWZa%^C z(9LgNH*xu-B#v3E^nqv`o~E zzCZ$zQh}GSj0H_~ri*`oxL>smvDhN?1@R#55!vo`I7s@m!eh<(AhN!mOr6REV zHXO1j#g6k(A2Ul42g(%vB|7*F1)ygMCw#t0PS+mvzVir!JeKE_PU!N?Tcpe44S%nq zLCj=*LScD37RR(Y&k*Tq`rl?Q)AT-zbSlC0Ljs>p0DAV__R9q`2P;RU1Izb--S#ar zo{{V^T~gF7aMKR{C{jkyg8UFU_bmXv$K_gKlL{PCIUyCtkw}LEydgK4-bV8E9kDbv zYMAz&eVvpQru)!>ygW^>~|Kmv-{A7eycbgt)#_X2U2z@#iNDTCh)fYYjEdnGVnuo~|&TP+$xx+$sUMKzqw$paWO*9hUxSoZ-SQ zYGl)I05uvLh-xix50zyGBIL7Z1zJ4G-F)h!&kdaG@=H*2;Qp%$e*F3ih&G_dL^k08 zbAX=l4-mZhLsD60X3=&Bihr6y%gABGgUrS$ok;dRukmumCtwN$gnJe0pKyX=agxO!b55@MJs0sqXk}KO2EtdzDpv9YJhhAv8rTjGFyK7XNFc#SC@Gk#Dgwx$ESQZD{cP(T89R8fxs7f#!3`{y>b&LHn&O=LF#6N%Hmjr-+idW1OZN*f zu7UKMxgzbP@6*BR7NAz|qV^Q$I zDbGtH^fCLAS1}p6s90p!FcjlB_8KAex^0?@paM|PUF_gl~lRjY!z0Sjia{VUs^;(w?%)zCf5FX)#)S zDru+`k#RDJLx4(~>1n>_+&Pq_?EHN_&D_9gABFzb<%(f}3r095Gx-gSft}<49mY*{yhsc<=06n$!O#wamV(NEY2j7rQ#byO`)s0^j zkC)tCe#9~HrrA@Q^=yfW3e&0swpc06ejZB1kl}Pl15be~aIxwgJT3NSaB51Hl$};s zEY|D&9{{m=pIpcZAO`?@CpC6Aw?eA4!Bsbq{Ie(l36G9^F;T+Nd0MydrS|m48x`j* z5ydfva9VxH3aE;uUBe+mM9F#>mgX_FA!znu2pt~>m#+LNO=H~cC|`>Ui^5s@2Z(L> zZW*TJ{(GNdcfu#q#IfNj%@oY`BkZ9)Okz+~mJvE3JHMWC2vc5PD_Q!~35D&K-i4rL zNiU-l{(ci;m^H7zD4a3ic_qVoxa*)eZaNPy6W zjQLpzQ>v^;^0YUW=E{J9#O#j;H+8I5XOwDX=+z$3WdSjVq(w*blNAD(DYj z;Fcp*?^lUd7s=PbdRO}U&Vv2nF2-z9yU3fw(wo&y@*bDN+Sh8bRD?^@%|Jqh^x_!&Q+kchbr>l^qtK*2=nJ^(uN^h;g6V;j!2_= zYPPa(k(=EL!ks&#J=!U~C&8WggS*3nvT7=wEc{v`zj@WwgUWr1@JSi_^Q;{|Ck(~0 zCmJ32ccV{`C{G2~FDB{KkkntqOstg$jUx%OB-H16{{c*Ns_LUHs1S!{2$kEE)+PHF zih&%M!8xhBEV$qUVE&poU9MR?hY-c>8z=F}1`AUmM=?#9A zd9ksd2G?<7BT~5nh_8O!u|OjX?JYIYsT6^3y_ldXe$)@(wNCcUTIk=ohqpXNI}~~B zsUAHTnCEG6{N2|gMY}2@7n=n~^bw`!DZc1Ln zaWS61BMM6dX(J;s$Ic9GM;QNw*_smz7(~0}=~@Fjb*-o3q|<)jSBbAoOB-;eGPHn+ z``9pJ-iTgJ+lE#R-{&p{+@LU{C)P4b?M@9g%OsRvPm_`s53V;f>s%d?jHkjcgiH<7 zbg21P+2?NE^9SLDO#IjqugoFkL3yO#{P9f;k%i@fYhs-SKS0)j6?AM;gmlMf$=cY> zI*r@Nz|s1uUdaCBIOi?S1)?c0|zfL4T~ccr}* zRjJym=9Vg@gjF?eVwTpowvo_kl4K1~ChVa31?=Rln_6 zO)FC%uuOck^T#cnxm-i)6x^^4a`)6Z*ym6J%@233a!x;ONe*5MurAme6I)L`pmrcC zOF^rENl~IR0mgv7RIP5H7=X?}-1GohsF>oKPN`6Xztz=S&Y&f14?QAj)&*MQ?*xB{ z3K}n)UrGL&we+)v)%v)^YtiQPxyDd+FO6I){}3Ce-qzpo#MyTN@lXO8^{itp>k_$& zL`Nr6`b}r63$8RvVlHP&u1lK?*9}z6j?NM8ZD#XZzzy_9Sy4)~wlG0J68xX=E`7-w zdWt@8-V0QZgRE_HoYUt6Y-ZMqMS>@GAa9s`sA=h}-*vtiG+KByl#beMYI*c%AiaGt z29~Xr+{<1j-!S>UW|@6l!=SB+9#VTlLzbkZ5kSFsDE@hMv&q2*ArF2d^PBmF+ZWYX z&+YPrQxkqtf|@9=!%ov<{3Tu@FDS@0D z`7IPLCQN1mv;KjJ6CG_{P5cb3nck)j1AzB*tVKJr)V8Apc%K~nOw-A9I#+_?y|PTs z<*r_x_-P2vDi^=Z?mBu-z^B&D@ER`o910=X`2g88E6^=45^=IZA4E3j@lHaJ(-Yaj z81XZ4J$KT565mAq4hp*aS9xnVB>!8PRx9K2Kfs_y3H9(pkydLYU3H?oYN225YQ~p) z=+Nv}pMxU(3k{gjk|pzs%HV+)5@jnX0ptesSs>}@9+N&}Rm0?x3HK1HT|K-$+2Bx> z3dkIL-`FaJ%gD_3nt#l#Uy};g3+y2$jDuoeH~g^v-IHYxFB4Z|ZI02oVKZNHGSDD{ zyuS1T5n5giGx$%LiwhJBh4KQOWwNF_pUtidJmgs=oOj>7bD+&@%7HGQp5^r&qNcPd z{8QB6&qVNQ`c12ljXU4txY@PP%^EO&?7EFf?5m$}8mV_o^wZh4U0>r5bhf=k${bxZ z{1KVQJt>o{Xar=1XVlp`YYeb)ZP(oiW95H$Bo%f)z=x+34A`A#y z1RUesf_UVO$e`rMq@l0Y91Bx z<^7N--GKq^Hshe!(Yijwp(raDRc6!zE8@&L2(y%yk#CEw6ITOOC9Lw!Ig?}@k_M6O zattSdc5?d*9x=WrkPV>SCniXI7EqWoKk=s$^E@E->m|wmP73c~^gWUtkfr}f3s&tY zKR?=u(j7m5Ze}RIW4R`S?{6rXtnQ!6!@54$>stM&VNnIIQY-kSLN zNgl9yPJ;&fPPnB%4&{pFyPwQl^3o4(h2!Ck)&Gel@Wev>>2}mgILdlb3LKgoo7cefBI!Bu((x%SKT*v|Lgl z;Z5cL0HciUmqXh{8GGyr5oEktp7l3gldD!+DkS*`8g*i}Pi-Ox=_EE5<}^-nwH_w4 zBE;n%Q>1&0Z3abHIUiDCb9DSeMz1K8ZRQ5!`%?v!6)Ux!*+7kek2Orr2o`qZsBhAF zfXZpurt}Eoogt`hbF9v9V(x@22!>G8YCbvCY<}Yj`9@KaI)c0Z!(|gEXP+~@NOMs;{g*%doi3$U zRy~2F=dyjxrns6Rl7^2&h?`ouz9kF5%u!@{)2#j?Mu~QmZna(`TKE#^(puUgcKmxM zrYJTV)@)8~7ebXZAN(nW_0VFaKRZnaC*JOXy=1SS(S|`H6SoyN=X9!f(`EHAOu09B z1KV&gKh3?E?kq^)u#-Qc2n<7Qa`7Cnbgg^RqkQmkPK|2rR$U+m58g(uR)KMyD+^^? z(<(~H6H1|XpeadqSQ3m|-*)+x3SjsLn8ZXVi~U+e-D%09Wwo!Sl5E@#VrypT z8n3Rm-}xzVrpKP;+d|`TnK^v2)`)XX+p(Cc>)~dHG`GlbU#lkp7IE<4^{incwE)*g z`?fhItkzI~s^ke`@58J~4f-YJ;m4)b)lbVRQKge+6s+CNy@*ABO|_(7lKd;E&#Us& zL8ao(!ZhW;xOQDB-~vrzL7hZwkowxTCRc<$Otx-6@qO?B2^Vg(i7KZBJ!nu5Yj`z4 z39L25b;boJM3l`4X4OzEzLiIC(diJf;4{Ets{# ziO<{MOn#OCxBFk3F1st9F(MzuSz~yHsp#c#yGWpQwb7WzJGAsKx&NOQ&ODqA~2@1?yHJQ;)o-XC~cM6B91sB2XPde5T(?8x7cmPkvP^7p@Ir=(-` zz7o0n#>2fxec3+kbl~)6?Hg0yZ->rB>|2!XzR-0#bFfea`2BsAwal<7CNqN=*Sp?< z66QZW`0LvtD{o^_Itb6-K=nfOmdtx-`a>5}WPK-G#V@pFaBdXi{tWcQSOYf)ExI2s zC8Jj`=N^xGJGz)+?R6cUnf0UelR4abJ5Q@v9;*86`)_u;$d~6@5Z*Qo$3Q1y3DC63 zyu@*034uUZhzvoEID3Zwakp*E(t%+(6E&4h%rHzuPu(c^uppLnaRJSGCfdqFY_(bT zP+!*myOx^Z7Jb>GMJ4)lpP6lyb&&>YXJN7RCOgqN@05}~xI*le&ZRjm-gP3K^`hg8 zn$#~8q(xXWebq@a=^)hmw*dfXglFxXX`)h&4b-i$$bVA#^4M=RKE3%j;f_&B*8s4A zW3yFquA&Mh`Kft&hFrWRi6Zrx=Hz##{3qCeHKxCLm00L+zc7*%4(DPqk>)Y>2QNZsjq9@84LY^Xab!&8{{)P#6P4&RWTL%p#U zPFw!uVzKHq z)Bmok+K6@;?l&NLxuxN)w9T=h+dmH~$rctjoqLT5?jL|z^TqFz`03qxJF$0StK9t> zavsRnd!#pGEc;=V)`+gBRtjft9^=O_q_0meBE@Wv;mZpL;fY3OJ`=X%;?gY*Msj{w zHEr9*>jY?v?^;J#zDqmp#`7^Q3^1x*RE8oG@9ueJuH5V8YQI38Kx`b3QXPQwBK8=58l{; z)Qcf$0Bzm4jZ^%Cmlp*1gp2SjwQVDS&LuJA*2B8Mwz{XRTYsS`v>&xuQFPO_OO_PH zo{tT)Onv8ETj86S_%E;gkV?g;`@=;wj0(}M8u;e9Au4d|Y?`|EU6H`)iv;fQQ87_J z`rUs&j~tOXDg~514f_d%z``wE4vP^rz?U!)r2n+;sKud!MOuG1dx~C?sUA3lQFbSJ zEp-CDLmVCbJLqh_);coRZwNK|iQ3#J<4E!w=Z^>;S)JY713apZkngmkmdKFKj6Gog zPM?cFfC5D$yxmhUFQ`Q6?X@kSZGpt57QNJ&Xpy4 zh-qT|j2jj|v$>S3J6tqo*J`+<91Ihv-0zH1ZwxYyi%g3vZg48XBuQdr=E~dky-xoR{6^b7k=+efzVvs z*#^F9M5>Y=uEl{y;%BSsY#ahdBkQUuyPg*iZS=y-=KYGL539dtWh~pfU3qbEM1!^z+(z6Cyz0nUufM^DE! zBYa+LxmSb^dFim>aVXY%a!A4w)^~%Uf+Rm~SJWU>)}wDb$(YR@=PUfZ6%Ak7f-|&B zJD1zscQzBcB2-ce7-J86wO;AMx8!3J7u>OA)_`g*C2i9Uz(lV#NlZDCso4=y)kwXh{FIV&97J^UYbiXXV z`!2d7S8;^(?GVErG*K=+C~mg1IYCFO^Jdvd2hFYfy;t5a-EkxRZiA8WSEHFUl22IN zVlCaa5lHx9uWqqGA&QBY#t;oXcJW)8&f>|Ti7#a!I~x=6-J2~@IoMM8I=jyj$g1AH zWdoIgl7|Dvu9^NH{}ZOy z@>&m4Kl$!6upbwaKB)*W>Zl6A4;s(kC?)vjMC!ZzY8}rB87+Q*&f8Z+wQ?;mYMbu( zOgU*m!TQ7*iA3#(ldPFL{g?OI-wvTjKA%_S5G3l+C4=K^y|@ow%lFWP=wnyR; zS9(*P@XPMJ0tU?O%l~1B9JI9>wmnzrQk~>xrqcwOoB%`jCs#j#crmRogXkl!V;F9t zH|m*>^kUmRKaDckhD44_<%#NemkxI|lKJX=nw(A6q}Z-;kF#{@eDZ*_-@~03AGrJd@15B3kK_!Ib9V7h3D7Y=I zk=XvSi)dnfQD$P*a=@T4OK!1`ZJ}k+KjvSJz2-17h(5LaTH9XUECKT~zAf47nVUrR z-fQr-D|K@8UZjCZA=PoKF4$8d_E(6Sa#Cs8YO8_K_M2NP58w4P#D9&R+_+ZEaK9p@ zZ+^LLkNE5nV%Oc$n#)ctFlC>5NWoaQ@Qu4~j|?KkE!dsrua7^&l06hWWS~W) z;yrgC+1F?XhSZ>u!Z@R@PKTI`NPvcvjdArmz}~aOp4UWkQz*K`LlZTk%ELJgEyeYw z2uF)S&m2Ac?Yh>+KgUu-isYGyVl{s4^Daj9-3`B`7FQ={wx*oT%CKR$9hI{}d?4{@3KFWP)xpJe z?RB};H%SLNHEs@ONCN z`jrWO{=P@Q30u-l16Ckar_t9`jywnYz?F*b?!Rc3tz*=+EvO_Au(c@dM6`x+<8lNl zKiYb`U3x_hDYfogPq4*axqhO{x1jIj1q*Y$&SuQGCBEbt+l~od_%aI#1rJW}K%xF4u zO$&^xVKtlC&BEcDn5_Gg;CVu7LqE-#OKp4GZ4x7&WsgJn>G>tZoIkEn)Wzv;D>8A# z;OF(Tx@?q)X+3UpKKR+K(6?HxhB{+&O4%4ntzd}nD!?ND?J;U4%c@*i(m!dM+GuqBZz!68=MBI z$%aV@`&sB)VQH+K@+ch?JA^n0-kQpxAWl{jHrDX9-Q*qllq|+NJ>bjcGS4nQzqCV> z{A3I+W6t8}k;(FuS`Uw2FjsN@+o2PUpF_|Bi0?k>-qzOCATaDZc^#Gg!DM>AcI8f{7KrRw zYO@83YE_pU_SVyG~qf%<~yrIrgd@dVfP3bnYc$DuT+Y@DZQEFEZo o6148TnHLx!fZe7ks#jOH5WShpM#&VPM$*!NJ5OCLkdpCLkgrrJ$!GC8H%LBBExcre$DcW@aX#V&`OI;-F_@ zX8KPMgjZJ#bPON{29Sx2h>Yp~dA;-jfanO)2+~LhKmZ~T0SSokG60|g01y$8{`0v1 zXCWaYq5u$3(a>M56bOJ<+y89^ARr$c-Vh_`P2s zuC4+Os8CvB*s)ns+AGlptB0H$Z}FxQIv^D87}L92 zpt2D<)RDExGhiVsW}p(_)^W5C7ta_deZajgLzD^rzAS?n1(Zz#mKe|#4?W!b2KW*- z8MAPq2Gi4N_x2%13hf<|;2-Iy~i1>6# zI#n@eMHUoA3RQ{f)tO)t>w-La;%)aEaYj+nZBCe&Z5h}C9EqpI=phK&sX7;r7~k}U zKdV#z2D?jV(Q~S7S23>SR30+qu%t~bscEa)&cDxcPLK9J=PEgrL*!Qu>P4oTsYR>T zr9%QBIXsL8gcGtt`B+v9rN{}GZYOdtzV3Uf>=acal&A` zu+CtQ1il}MoSvb?g!x$~Z$5sb6`iFktB0&;fWHUc_gFYuuxFI>!|`uJ;^osBFeO#s2rVX;iQl|r{_oUkB;inEE~<_3X^X`4!IMO&^e9`}_-H_MRbubY|H#c3|CUYP2 zgJ)j2@R89EO*@3C0~qO>Lyi538e(mG7>jt5*Pj&KcD+`iCap|~2i2hmmr1pppdRIK z{HK?`8WH6fh!y;(Y$#h=_(;;ch=Gl)UU@wgddFRY-nqFW_Z6&Ym^}GO@yUUs}8AQ}C%I}1D(d~fT zO#VbYKe@U;IYSP7#YE#=YKc0`W8Rm1&8V>Z|s5Qjdic0H@+55g+#ryV8w5otU9 zrAPbdy?NDJs-ko4DBz|YRfX8J3`(_u&`kZS9kKQvpQFpB5rpJuEi{?3w_swzz~Q)! z$em|A?e&A}XLI{n_gp+FGbK{U*pOIyEj^&fjJU(b;k`WwJL*S?&$h17l~HK>(4yk9 zSnBbogF0H|qfJI>Bs|#}B5EW)w(_ZG6#J`pXcceGK7P&`N_^LXG zR9tAH0v8JPUW~|%!0gfY)DUWCS?Jwiu+OWe&GNsy&n?15MbShlRBjFWg07?fjpw8p zcEDrO>9JyNx-Usp&g$m@&smCOF~RBR^J6NPx7{b1&nCtu4E+(xbRt7KZ~Z_z@vlw$ zB!8@@oG2JQJ2C_bVBdx-z*{u+v;1i6Uv9s`;>s^6uF0R%mo8tAYC_*Nz4>Vc&f5$2 zOjV?ya|ZMrY%W zKcyG*fPU{37xemP>dD4RL{&L|-E7lJJEnPWNrD@gUxNOg1J>`%a^AKG$K34j*zp=( zlz{mmm~P-Kv=^0_=y0dqOI=});$PxOQXIFloS zzguJ!QLxv0RVYuBEQ{>Mn5))YJxt?9nR**gpQgR{(5)C-T_*@S@UzXb-y?58=ldI) zGm@aA5sz_taG96qRHULfxB1~c6 z&;P`ZcuzvYyU6AOvevLgZ7dz z$?3RzByGK7WAsB913gJ05%`*%CZ?_)+`=ZVSg z!Hkf1+eF50_GULfkeb&-j{lN4rXPI#htv+j{&qY-7$qE+lXV4ue62?}R6KE`o#ZY#J5~Z>|#XZNg&UZ3~rDUxW`Kaw{Z|N0W$IjFDG5ar@GXI2r;s7xU*>GbuDp<9U%hWASVxvBjFzbwyhzLvh#wSs& z)K25RRzGqkb&&d4L`?5e1x{ugRDA&`%TE;;akD_bFC+)HZRuY%09ZWf3zX<;D1zZF1kylpCk^Ihum7f z?@=1}-+>1{shYk_o|Cy-f&7%wz-s5^L9k$!2DV(n(wc6o|A^N2WHKtA< z9=Xljt;0qgBIPTwiU&SPO@(KqLpWwrSf)pA_Xyk9ObA+UR0H*d=%RSNSW^l^J=1qx z*Ti$WM&BeOQB~S$e@a-%a8BoWp7SHHqGe}{(1Co7HWqL*=98Wxb)cPKI+=ges-7QyW_Hc{HX)*h(AlrS?L1j9B< zF`MEE*W)+2;@H~DygKeAREu0o?MwvfVY#S%D&?RTa2=+|Bv^g3OMEi)JOP?zvgAU! zke20S%R~bf4Ss7Fg1E^=G1c5dcYJc>I>5^9cE1JmHG5)IhB}iBWY-k*=D4C`kEZIN ztXQswY+djP3;R~i=TplcvbL0v96wP8a9b01_;U;XWWqHCOIXySfE$X#Pb!tz@;q_k zqP>Od_ifFjtxRYzJ*otn< zgKLVV=1kQ{T{C_~QgR&y|AEV=k<=}pww`_Ns90sC4_}Ej_~ILt0+hzg3!OBfRNePY z&Q^Va?-C^*j=yOyK&tJI2$AH(j5=?4uRfJFhZ)pvdC*2@_q2kx;pD3(I8aBIraSh{ zlgf8Oc@r1br5UBHG<%r4ZkPj({&7{Hg?o_hVv&BLCLZdwysJRzt>%&Etz8<@_*G&T zP+(nRWJkAVQPaJdRM4v(!cAZg)BeVBjh;7Po|7@D+*8Qd=k6M-=lDqG<(FC7-dDLH zn$NB>3+54@hrqBOtFh3q;_!oeLha|`H<J}ktWI?2^X_tjiDf=l6cm}0}fOFLmy}B-u>W@fn^f{G_ij?5YVtC6x_Q1Q zumtVYHynrTslNbFX(NZ+%wGVkIb|{G5~HP!;HIY<`OW7E8;QS=$Mx+C++v>cQ)6&l;>?ef?PZjY>6IZj&t0RklsPDEG7==ABSHpSX|ANDyH|U#8O{zX{<5`czDkkikYmKK6p__rA$eMYY(cbUxN)P ze$AIg#h1~Pp3^(KuN&-1M*JQ<6?d=bn8Mqd_)Hp|i}}vB=c(yUupx!#k+K<#ynpMa zX%=NJY6WJO>WW_tGZOl&)MxSpZoFmvZr5i}n3etl0B-#y^3Gw3G^oTgV=x@`F0rk9 zceY1@8nANJakAVyJNCzlM+I~kISkf8$%cI57r9etNsam>{8wd<6BIs*Y0njoh0E-7 z=o1|0*YvwR#I5e@zZTs4 zi3(3n_SM*|EAz+q>HV#5X>Ta8DyUB`M~87Hi-dg238Yb-TjjTllpj-M8pXDTLDlI+ zx-&KUu2SnN1Kv)WXzNG%oH1pRWfPL?Sm7x?LG@@>IDNvF12G0DA7@+71RsiUpl5Ms=D^l>f6gTc)~R*`uuB_&al=rN%#AVQBSc9d}Guy{W5qeOt& z+?H1Ozu)ra12u`=Jy_z{vIoma#}Y_!hW~)FFM!=%+fwF#F)mzXb2D2~+A-TYF*Y;B zz6oYno8@C&zYbNb%P+)2jc*uPg?qyBMh~VQ3%BiM@ThSoiNUKba%7ziJ>2BAtFaL~ z=YP}=>qPXzHA)S4*_Ys$tt}<*1Gf{|32&Kt7Ws!qKP&gY0B({3^xGDf!YWr{59tSo zA)E^B>er3``07O5ob>S8b#V6{%G@wi`R^sTWCbdIWM8WnR`X?XsEok0a@EytL2pwU z$A-&g%d<5Couw8AprA#+vA3?(q*;PhJPZ9h$3M9YvUm0VtmV>H%{dvO%Nb^XeCS@oi?1RcX*<8 z0TO3i{cm3=ghRRPG)BT*JA~&`2f-W*i;eH`#6whxbLBf~va4ZtelO}3bq?U$hFg}! zVRcb62Hvp|Wkc0v6I?MkwOJkM~^lSqU8N?$KgrTlXxV6Wm2B`Mp0y@G&79oje$lBCtdVv-rul>U%CMs zmW#u+qv(+{;oBX@>1YFwQ=-5kEt9jTqEdt{nrhY z8Gh4v#<@O!G#nhS=wnNxd*=rssayIW`;1e+Qf-ZY>2A{!L}`k8Ha(I9gF1{@WAPYCl;L7Yv4(&QkZX>(di0vFm8OKrm$)zn)$o5{E zVbn@p#d(i;+H#-eze~Oq5@z_0@%Tuos?Y^UgF6*wo3K$k$ZixJrC+KvrYY+&kv|O* zr1CN;bdgq-_2>UXG&PL*S4b*k*2lqJkIlQBr^D)#a`bRHHoPn!zE$)7$2m5&X%ylo~o`xB=D7y(n*VzvcfPeVwAULj z9$Vry6-pJK=C0Fbb|g+KqY;wh*3Hvh+3xzV;Ko@!kAr<57OP)!e4Lr})Yz7A4Fu}w zt^~qe6fVv_51T5Tf~my(hkskX0NgvyG|Ob+NR+BVr7Sl$_($x2C=+_MAm36eceUeY z7B1|&(@N}C4c0~}{WMvGyW^StN!}`|js(M53-+v{cb4!l#Nr#k#5TrV$KAVwqp1RL zf~O)FgLpKw$cWsf&_yZrqA|ox`8`lv3d$CGzd0ZNnp#XN*;eY{HwhjN+npNZt}7CH+%w6Jxtt+z`hT zbgjHCEvyy9MZx~S?V6)B|JRiHU9U0TJvimk*71^IB6ox+6fVLbaOvU;W=4l4?ltMh zoHqR2^^KXNS9Y2h*i|V_JBSu3i^oE97OT;@cF*snTvi;iA^upu>hBb=GxrarT`e;D z3cb9YIxMEAkh7k5PJ2)%*+hfqo>nivL%~6X?}(9Jc&WX$#`8a6a`GyYG)X?$A?6~)S<%j`x`^@ zqu7TH;0r*(c~q{0|AyWG!gs{1JxDNM7oaq{pUSiwp_`&97^m`ft=a@ct#&q|%095S zzvD*m0)VAFtHqxyJIUCE)?_S|elQ9UH4PhlWB8*GF;mSBt^>t5@Lo?5b6xn*QBHfM zpRbiW%jS=(ZU-4OvavC~RZvqtNUTv
    J}D|vP&%vir5un*uA`a+!|g20igwVeD; zr>NuiP*vHyh_F-dZ|w$4k;bFZR&au(PWpb(RC=bye5+NYP7bZElaf0!RzxMdul^5V ze6fbuekJi!a+6uQGv=J{$oLJR*)6N4P^tX%BAfgshoMKG?KCweF?z=twz^ zirXxeVxT@|S9ifE7gt*`(?&3yL3m7O=>>ospb%iIYU!u^CJDW$SqUxott_AJJC(+C zTWF;P{IgADF)Zdo=x@IaW%D-^X*4i$n;Z%_**EC)a%0cH{MeNQu>bwLr~vum8&9bf zq0v@vj`W$Ynm3?+NDand9Otx1p6Bl)iLrQVV-H5OQ|7=ywi7s!Tcm=8@nV;-HdMRQR8p0A;Dc7#v0V6W|E03v!({M=w z!Uh`n`&Z733IBDX``>Bse`db_rMv(@da#T(zU6;tF1lX$?ElYg;R{!A<(|v1g`Y8p z{u1NXa{t%R&vSy+{0!yhkgS)eJK^2^*Vr?3F}YnX{+?d0G!~FN&)x8A$Q^%)%PI}# z+wN+d3r82n;jJa@$F(E^+17HDRYvtNM3<=0?43IbXDV^}raY%Of3~3d)=0hD*=E~( z?a}maxc#$HXRH3OIi6jPoO;Rxo6ye6W)#ZcA6ilyAQrkV{QDLFm) z7m}SCDQ5^Ku-KCN2k9`>zvAnhJB%^r zQH<>~Kq+yqN+*km1!_xg1lAh05*87=*P1jDU(YN9LdG19!RaU^>T*<^%dtVw_bx)P z7|sf&OU4)32_!&Tn2XB^o?HF*435^QxG}rnL;8$rQs!+oQW70vZG=V?@wk@4rQ$7| ztmiMFQ*1FZHtK=&4^Qd*ApOeleP!Jcy|tp#(Y4u;h3a(oZ6lN-3zr@HjzBfNeaaGO zMXTU~mQyWV-_D?#KS**fa zExTaI5yL~KZBM_L6k#zcSFbwXn!4TPA$7xixPl?}U=dt9pK+k6EQ(okr{&N6^%(jO7!wGr*$mGv1$vRaoQ! z{QA?!&(avF!9YSnC1h_*@W33vhP_G`0Y9(oM^1_;X5!i&q3BmrO`&Lv*dJhnAoEZ= z6p^V1b~S(?caR9+r~RV@Auraf$<-F#)& z_xV9X<>&?haZ9SjxKVSck;H9!N@mdgG=`%aIjngUlhK{Z#;SDlHGiW{G;;k!#wl_U zf@;_wcrhs7+q6*ND=F=(`2w&tPR%JgC;iq4@IXK$mbxcs3zgky7D%I7ZDElc${q2JIr^Nj z&`|fZX3UhI22*c_awJx{2tH2(LhhU99ks&g3x@Y>(@!`>1Y_ITF`Hjn=zd>50BI1v zRDrO+iY#Rm3?Mf}lws2%eTNEiAtslKX*1QQ=hmByfmaIfcq)Wy$TS1FGi@zDxn%8| zn!6zM?h9;1lOIRd#tNzg*IYmcjQm@oI=!{)Xh^X;k)+`ehY=uStPe2&kD~14Nd%4ozu4z1Dc2(k!;&+KPgl9QY+lftQFVm?VJ(f zjwzj4YPb2l)GJeK@Mcm$C?;yAOY1obF>GM7e%O7TtmIoZ&d~C9=%Fnf1u?6?Jty-q za$LxMp!U;=dk>efnSvtG+*af~9^$dc?!@l_(teX);qlAO17og;TmxY%?)8o_GP<5& z)YKG=Og|F@0mcQt!1dp?#_fWkX8O7>P($9I;70R(&=EF z!6yv&Z!B~MEN7CIG+*&$qk0M@fF}`rsFQIvvn)W!~EkHl3Jj=PaGSBb8W|CN=$CTs<>%aY);egNGWb!Zk2MESh9$!3;&B{)f5ud zTVAHuI;cY)UKU1knkIWiYfD~_q~r>3Lf&mJH+jua%S+xd%M1k3UOZQe;V-DCEKtsZ zCSG9NE@61G&cG*$Ks~?CTox(Uw>g9l@GPzcIck~sgsCEn%%Sdoy&Eq8HWFR}SN788V=NZfP#7 z#H}6 zX(nybF@q$JdDgcvNP9CnRHxmbf8w?Wv;^(aZ+sz{a6n$gn%nRDM5&v*0VIy&EpB|Q zZ@7%5$NDXGys84;SWpGT(KR0_=-QW!+n|JNlo5G%+RP=($U=^j_+jVxQvPfGgb z&mPW>=)y;bI?eYtl9I%v5m;>>RvSJ?73m3Q#2#_Dg-kf`-+ZeG{6|kl(WWHRLS7Yq zkN@3+hof@b#f3>|z;T&sU$~Z(`;ni}M-_qE>J<2u=53@`VOZMh@Wmkhw8Y^=Jjy$i zkoJ=6qd+uw`J=?UW0nZZT#0PTOn}BSk`oJ*A(u&N)nkG`cwdZPMoVzQ@C@)MU8+i| z)TAF(pE?&g_Wo^~0K8jdy>_kq0tmm+M^Y)7=+JXY>}Ox(Ga~uCT-p*Bon^DOrPc)w%xub?y^F!R z&#K`hqAv3s3BFzs7aFDED$6w1$lhk=OY1?%j*#cjvOOn=W)wEbjT==))E4wz)8hH@H9U~@V!(n98(EjMU3?Kfgv){pdY1DtS^9ErqI;cWfZ(#E&1#UWK>M3W`TVc%> zAHu0ia9IUyg{e|aeOgnf;qYloZw-@hFksh9Oa+%)eRK7~*}vt{4(ls8HD9bK`);Bp z%YdXuRljCl_!PD}WI)N6pS^P4csv<|x)n`C8XAwrIr@<=jLr)A*wW`z$?;VamB(La zYQS6V^B7Azrd^p=DetJi#~t;P2dx5;Dn|P|9P&LG+w+duXrVx)H)l5=k)V+Z@6YGI z9gn#jl3?-VGpiJ>R42X4WZS*kYXl`XdsXr^oOEnsn|Q7&ZPjEM!6Vqho=o#7WY~I< zo11NJGyP40B56beS=g+A1F`16kXxnict|^(k%}mMYYOjAt-sREjRw^L#k-8{bADD- zXaiRNjUigB1u9mWgb>+v4eqsVuk|EZn(vKMR)~@2vvk!qP0Rw%cpA~H8^zOrs4Gb} zvVqtq+dSOlDMZ-4@!={5AiK;kEqns0kuf2mWG*%X$xL=`jW-Wde}dn_qD zmEH&lA@^dx)s)E?cNszZ%k`n2&I94cDF_4C^#_#_;^Y_d8jZOWMQy*zJo__=Jbn_! z`e3INlehvB&**=HAGvw92?>Q8@T<*odksat&~)AsRGhuXMfX#v@=ka2PYIf?NELpR zocC?o$I4k0DAn~PR{i&v!yGDAX0c7di>qY68J;tHqgxmuk_Kt^keO(gLqd@+n0l|{ z?I@sBf>)x2#l+F>T7&4hg!)N3fzfHu3}gURH!YcB+|S{J{frtVpLXxAf1^t7RmSEz z1WNehr$X2sc1czz{tp~i>v8=_+LG`toU#rbbIt0b2lCG_(fSWMOqJ5W=(K(>Ib54} zdOR&;mWWqMblcYRJb~)6+kH>+Duc!6l^5Z>u3QSB+3ZmF#e)m|P!8^cj52*v44~=e zneG_klr~L)-ilM%y&!O?jD}eNYTh+{VSzeowZ_uoxnQ>;NF%A_BK{POClsw@1|;*b z0qAr9PDgLs}9nf}AgL0Jws<409bv3ys7x3@7 z8&kvB`e{gir2 zuGtJb2<5X_HRCRBWpK_EP|qB;eRXOh02|A9*wqiS31?iPrHwyVr=c`v<#(`<>COBr zUqJ`_4`1;F-t1Ca>mc%=!EiT=w~AR-_j0v_Q*`}@PE&$rDRw^V%f&wpZ1 zjg%YKdphnKom*ShJ6)VwPAp%4)RMWs8_pb!qlFB+d6M_OMYXIt8ZXu7VY#iz{e_h^ zVZaTp0eJ8>VKO}w3dHO(ukWtxw zjD4`ecKP94HGgj&L#aZ-2!d3krtk1Q$aam=chC{GI3@q6W$RnTZA4o{^ zrOKE!po_rKK~$mo^o|9b6Ha7eBaH1g8`WQROZ`Ch%swBiEkZepndEg5dsi2j#com1R z1gH9RoD>V;ek`dR)TH=PZEZy**z8Bl`=i{1+H;7|aJ%5s$Zm-1_A)QE7ng9oW6gR0 zp{jL%I7u(|>QakR%U&rfktMC7F#7Iz7;hhmi7(%{NlYdB1Gw$ddj%UDV@kHP#*OCR zTu#Jtn|lWy9g@pSC)J$?{pa{x1!XqyYbHy{Ve9^R-5=!A1ECRz_lOIx~)AB4d&U^(QNDL{GZu z1&a!i+5BKy3tq&Y%QQ9i5`K6xvOopGFFcf6G!Ec+q(Mn5DyuC2UDkan9lu@w-mZds z+I^Yl1lBl9w>InB-FCks4R&dIIfH&t8!X#{?ms!h$8CYi7cnl`yzHUxicpesk4Pjh z!#G3>-5^DymGVB7^+h=|ruX*+AWjFQP2FOBjjWZk9!Rk!h5J#{TyZlAwyC?4-dU`? z{T}Zas_iIRlAK++wPZAQ!N^$pO!9%lJp4fro*`5&Dg0EmQ7N5D&nj6x*+-|fHF z2t>f{Cp1C5OTKFPn>UDhGQBT=8M@B%_JMA75M#w(%W}i13BcRP@$^Ve7w&T+Vut zDN*t7BJODFVxVU0%fj4-!qRJ&>A*Xhp}vaFWdPu#oGgOD2Wl@r7|j3$nin#KB)SMc zZy0X8+?Nk>%8euGe>N_H#iS5Whe<6ivZq(__267B?k~ilA@7Yo+aR-MbBA;I0ZU{0g-<7JmP33# zB;-#&eF!8D>n9R%vGCIb(#A<}MX(JpfU!$}(nuN9 zzkF~J{zdo6ZtD>t+5=+j2EOb@BGac2Cn4D~eS%YwUk0PgM9OAbna64Uu<3q47EHBa zN{bo(xSSq(K_2NA0Yx{;ns){;2^XeP@5oWDZ`380f;P0EX)yybY!w(%Qg2l5FZa@^ z+zm+|(Mf4YOx>6t-_nn{7v~H=XkOu+z5ps44c(W9$dm9gR&8DYZ4EN`88NC~ztTvI zEQbM~MxPvmve0c#_<=V-bOve+Yt)-}KSjn5TWWH_&_Jl!njAO@^SdTfAf(BgMX5qb zxN~_%WG}%sT&?1cn@W_QzNMf-r$c$0C#{aV zXe>2xa&g(&Mu(1WVIdIQ42e*PF*F!BqM=d%RYCPI&@GPX0D8LOsAS*cvGW{C9beOo z?KRC1kWgM_tw<>UD>Vo}d^&nW0tQ}bBrQfhSun)H)$M=T_F8OS0CI04OZ?LzH;f-_ z*`L3Kzl%zeNa$*R0pRBk9Q4OiUW*HC96$Lhw>ZtuN4u##knS+tGc?Fu{AS4HwXaT` z9VRsV-J8!upqvF=o71~16ZTdn-j%7yUt?i>n-E#%WYtK*e7W|%?-PDNRJE`h5+u`|ap{(%Fn6(*q;KI7~MG_eS+33HkH_lRJ#RtVZot3?? z3qe9faJlj~>P#Y>zw-;Bh%IW%r^D9xvAb?PcBw4a@?w8kn-d~GT6H)L2hO|Ujb`oi zD9iK5+E${^qD@i&3k%8@ZMR2Y{a zDv0|;e!ic)$P$peYm5=7OaF26*Pkqo!az*!y*qo6%*&nLfxXq?`FK<`N%*i
    + + + +

    + {{ _('Manas Alekar') }} +

    +

    + {{ _('Senior Cloud SWE') }} +

    + +
    +
    + @@ -43,7 +56,7 @@ {{ _('Ivan Blinkov') }}

    - {{ _('VP, Product') }} + {{ _('VP, Technical Program Management') }}

    @@ -99,6 +112,19 @@ {{ _('Software Engineer') }}

    + +
    + + + + +

    + {{ _('Martin Choluj') }} +

    +

    + {{ _('Vice President, Security') }} +

    +
    @@ -125,6 +151,19 @@ {{ _('Core SWE') }}

    +
    +
    + + + + +

    + {{ _('Artur Filatenkov') }} +

    +

    + {{ _('Associate Core Software Engineer') }} +

    +
    @@ -138,6 +177,19 @@ {{ _('Senior Director, Business Technology') }}

    +
    +
    + +
    + +
    +

    + {{ _('Mikhail Fursov') }} +

    +

    + {{ _('Principal UX/UI Engineer') }} +

    +
    @@ -437,6 +489,19 @@ {{ _('Director, Global Learning') }}

    +
    +
    + + + + +

    + {{ _('Marcelo Rodriguez') }} +

    +

    + {{ _('Sr Support Engineer') }} +

    +
    @@ -476,6 +541,19 @@ {{ _('Site Reliability Engineer') }}

    +
    +
    + + + + +

    + {{ _('Bastian Spanneberg') }} +

    +

    + {{ _('Site Reliability Engineer') }} +

    +
    From a23edb489b9ab99d2ef06fce54f7e51699b108b8 Mon Sep 17 00:00:00 2001 From: Mikhail Fursov Date: Wed, 2 Feb 2022 16:45:59 -0500 Subject: [PATCH 117/149] Add 'clickhouse-client --password' comment to the scripts used in Quick Start Guide https://clickhouse.com/#quick-start --- docs/_includes/install/deb.sh | 2 +- docs/_includes/install/rpm.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/_includes/install/deb.sh b/docs/_includes/install/deb.sh index 7dcca601d33..21106e9fc47 100644 --- a/docs/_includes/install/deb.sh +++ b/docs/_includes/install/deb.sh @@ -8,4 +8,4 @@ sudo apt-get update sudo apt-get install -y clickhouse-server clickhouse-client sudo service clickhouse-server start -clickhouse-client +clickhouse-client # or "clickhouse-client --password" if you set up a password. diff --git a/docs/_includes/install/rpm.sh b/docs/_includes/install/rpm.sh index de4a07420f7..e3fd1232047 100644 --- a/docs/_includes/install/rpm.sh +++ b/docs/_includes/install/rpm.sh @@ -4,4 +4,4 @@ sudo yum-config-manager --add-repo https://repo.clickhouse.com/rpm/clickhouse.re sudo yum install clickhouse-server clickhouse-client sudo /etc/init.d/clickhouse-server start -clickhouse-client +clickhouse-client # or "clickhouse-client --password" if you set up a password. From 4742cf307464bafbd0fb3fa62d2666f1e5373999 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 3 Feb 2022 03:03:46 +0300 Subject: [PATCH 118/149] Update CertificateReloader.cpp --- src/Server/CertificateReloader.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Server/CertificateReloader.cpp b/src/Server/CertificateReloader.cpp index 48d4063dbfb..f3f366876da 100644 --- a/src/Server/CertificateReloader.cpp +++ b/src/Server/CertificateReloader.cpp @@ -98,7 +98,7 @@ void CertificateReloader::tryLoad(const Poco::Util::AbstractConfiguration & conf catch (...) { init_was_not_made = true; - LOG_ERROR(log, getCurrentExceptionMessage(false)); + LOG_ERROR(log, fmt::runtime(getCurrentExceptionMessage(false))); } } } From 431611d20528f42dee89c99c379703a4af5510ca Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 3 Feb 2022 03:34:00 +0300 Subject: [PATCH 119/149] Add a test for shebang --- tests/queries/0_stateless/02203_shebang | 3 +++ tests/queries/0_stateless/02203_shebang.reference | 1 + tests/queries/0_stateless/02203_shebang.sh | 8 ++++++++ 3 files changed, 12 insertions(+) create mode 100755 tests/queries/0_stateless/02203_shebang create mode 100644 tests/queries/0_stateless/02203_shebang.reference create mode 100755 tests/queries/0_stateless/02203_shebang.sh diff --git a/tests/queries/0_stateless/02203_shebang b/tests/queries/0_stateless/02203_shebang new file mode 100755 index 00000000000..07686d1aab4 --- /dev/null +++ b/tests/queries/0_stateless/02203_shebang @@ -0,0 +1,3 @@ +#!/usr/bin/clickhouse-local --queries-file + +SELECT 1; diff --git a/tests/queries/0_stateless/02203_shebang.reference b/tests/queries/0_stateless/02203_shebang.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02203_shebang.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02203_shebang.sh b/tests/queries/0_stateless/02203_shebang.sh new file mode 100755 index 00000000000..645a17696b5 --- /dev/null +++ b/tests/queries/0_stateless/02203_shebang.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +sed -i.bak "s!/usr/bin/clickhouse-local!$(command -v ${CLICKHOUSE_LOCAL})!" 02203_shebang +./02203_shebang From f047a28ed82fa827307b64dce66c8fb1eee3be10 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 3 Feb 2022 04:48:52 +0300 Subject: [PATCH 120/149] Fix progress bar width --- src/Common/ProgressIndication.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/ProgressIndication.cpp b/src/Common/ProgressIndication.cpp index b9a8bc923f7..00e2326b0b4 100644 --- a/src/Common/ProgressIndication.cpp +++ b/src/Common/ProgressIndication.cpp @@ -243,7 +243,7 @@ void ProgressIndication::writeProgress() if (width_of_progress_bar > 0) { - size_t bar_width = UnicodeBar::getWidth(current_count, 0, max_count, width_of_progress_bar); + double bar_width = UnicodeBar::getWidth(current_count, 0, max_count, width_of_progress_bar); std::string bar = UnicodeBar::render(bar_width); /// Render profiling_msg at left on top of the progress bar. From 768c71c1ac0144ce487dc4b0d09bbb94fd892c73 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 3 Feb 2022 05:13:48 +0300 Subject: [PATCH 121/149] Update ClientBase.cpp --- src/Client/ClientBase.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 3141ec9d35e..0bc2a161feb 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -609,10 +609,10 @@ void ClientBase::processTextAsSingleQuery(const String & full_query) updateSuggest(*create); } - /// An INSERT query may have the data that follow query text. Remove the - /// Send part of query without data, because data will be sent separately. - /// For asynchronous inserts we don't extract data, because it's needed - /// to be done on server side in that case. + /// An INSERT query may have the data that follows query text. + /// Send part of the query without data, because data will be sent separately. + /// But for asynchronous inserts we don't extract data, because it's needed + /// to be done on server side in that case (for coalescing the data from multiple inserts on server side). const auto * insert = parsed_query->as(); if (insert && isSyncInsertWithData(*insert, global_context)) query_to_execute = full_query.substr(0, insert->data - full_query.data()); From 0131280596002e2358e1ab5421b35a9c0256f672 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 3 Feb 2022 05:16:13 +0300 Subject: [PATCH 122/149] Fix test --- tests/queries/0_stateless/02203_shebang.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02203_shebang.sh b/tests/queries/0_stateless/02203_shebang.sh index 645a17696b5..6c46a15d4f5 100755 --- a/tests/queries/0_stateless/02203_shebang.sh +++ b/tests/queries/0_stateless/02203_shebang.sh @@ -4,5 +4,5 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -sed -i.bak "s!/usr/bin/clickhouse-local!$(command -v ${CLICKHOUSE_LOCAL})!" 02203_shebang -./02203_shebang +sed -i.bak "s!/usr/bin/clickhouse-local!$(command -v ${CLICKHOUSE_LOCAL})!" "${CUR_DIR}/02203_shebang" +"${CUR_DIR}/02203_shebang" From 6d470319aeeae6a2963830729b86519eee427bcb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 3 Feb 2022 05:26:00 +0300 Subject: [PATCH 123/149] Add a test --- .../02204_fractional_progress_bar.reference | 0 .../02204_fractional_progress_bar.sh | 20 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 tests/queries/0_stateless/02204_fractional_progress_bar.reference create mode 100755 tests/queries/0_stateless/02204_fractional_progress_bar.sh diff --git a/tests/queries/0_stateless/02204_fractional_progress_bar.reference b/tests/queries/0_stateless/02204_fractional_progress_bar.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02204_fractional_progress_bar.sh b/tests/queries/0_stateless/02204_fractional_progress_bar.sh new file mode 100755 index 00000000000..6018ee8c96c --- /dev/null +++ b/tests/queries/0_stateless/02204_fractional_progress_bar.sh @@ -0,0 +1,20 @@ +#!/usr/bin/expect -f +# Tags: no-fasttest + +log_user 0 +set timeout 60 +match_max 100000 + +spawn clickhouse-local --progress --query "SELECT sum(number % 100000000 = 12345678 ? sleep(0.1) : 1) FROM numbers(1000000000)" + +expect { + "▏" { exit 0 } + "▎" { exit 0 } + "▍" { exit 0 } + "▌" { exit 0 } + "▋" { exit 0 } + "▋" { exit 0 } + "▊" { exit 0 } + "▉" { exit 0 } + timeout { exit 1 } +} From ae1264e468717a0d3e4e7dcfc600a040ac03327a Mon Sep 17 00:00:00 2001 From: tesw yew isal <278153+cnmade@users.noreply.github.com> Date: Thu, 3 Feb 2022 14:35:22 +0800 Subject: [PATCH 124/149] Translate zh/faq/general/contribute: rename old file --- docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md | 1 - .../zh/faq/general/how-do-i-contribute-code-to-clickhouse.md.bak | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 120000 docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md create mode 100644 docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md.bak diff --git a/docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md b/docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md deleted file mode 120000 index 5ac9a615386..00000000000 --- a/docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/faq/general/how-do-i-contribute-code-to-clickhouse.md \ No newline at end of file diff --git a/docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md.bak b/docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md.bak new file mode 100644 index 00000000000..5ac9a615386 --- /dev/null +++ b/docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md.bak @@ -0,0 +1 @@ +../../../en/faq/general/how-do-i-contribute-code-to-clickhouse.md \ No newline at end of file From a504353e962b6c930a0ef7655bcfb8dd011d12f4 Mon Sep 17 00:00:00 2001 From: tesw yew isal <278153+cnmade@users.noreply.github.com> Date: Thu, 3 Feb 2022 14:36:09 +0800 Subject: [PATCH 125/149] Translate zh/faq/general/contribute: reimport file --- .../how-do-i-contribute-code-to-clickhouse.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md diff --git a/docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md b/docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md new file mode 100644 index 00000000000..dbbd444ca75 --- /dev/null +++ b/docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md @@ -0,0 +1,15 @@ +--- +title: How do I contribute code to ClickHouse? +toc_hidden: true +toc_priority: 120 +--- + +# How do I contribute code to ClickHouse? {#how-do-i-contribute-code-to-clickhouse} + +ClickHouse is an open-source project [developed on GitHub](https://github.com/ClickHouse/ClickHouse). + +As customary, contribution instructions are published in [CONTRIBUTING.md](https://github.com/ClickHouse/ClickHouse/blob/master/CONTRIBUTING.md) file in the root of the source code repository. + +If you want to suggest a substantial change to ClickHouse, consider [opening a GitHub issue](https://github.com/ClickHouse/ClickHouse/issues/new/choose) explaining what you want to do, to discuss it with maintainers and community first. [Examples of such RFC issues](https://github.com/ClickHouse/ClickHouse/issues?q=is%3Aissue+is%3Aopen+rfc). + +If your contributions are security related, please check out [our security policy](https://github.com/ClickHouse/ClickHouse/security/policy/) too. From 474414de3bdef88d917ff4752b2938b71e49c918 Mon Sep 17 00:00:00 2001 From: tesw yew isal <278153+cnmade@users.noreply.github.com> Date: Thu, 3 Feb 2022 14:39:28 +0800 Subject: [PATCH 126/149] Translate zh/faq/general/contribute: translate to zh --- .../how-do-i-contribute-code-to-clickhouse.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md b/docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md index dbbd444ca75..39d2d639229 100644 --- a/docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md +++ b/docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md @@ -1,15 +1,17 @@ --- -title: How do I contribute code to ClickHouse? +title: 我如何为ClickHouse贡献代码? toc_hidden: true toc_priority: 120 --- -# How do I contribute code to ClickHouse? {#how-do-i-contribute-code-to-clickhouse} +# 我如何为ClickHouse贡献代码? {#how-do-i-contribute-code-to-clickhouse} -ClickHouse is an open-source project [developed on GitHub](https://github.com/ClickHouse/ClickHouse). +ClickHouse是一个开源项目[在GitHub上开发](https://github.com/ClickHouse/ClickHouse)。 -As customary, contribution instructions are published in [CONTRIBUTING.md](https://github.com/ClickHouse/ClickHouse/blob/master/CONTRIBUTING.md) file in the root of the source code repository. +按照惯例，贡献指南发布在源代码库根目录的 [CONTRIBUTING.md](https://github.com/ClickHouse/ClickHouse/blob/master/CONTRIBUTING.md)文件中。 + +如果你想对ClickHouse提出实质性的改变建议，可以考虑[在GitHub上发布一个问题](https://github.com/ClickHouse/ClickHouse/issues/new/choose)，解释一下你想做什么，先与维护人员和社区讨论一下。[此类RFC问题的例子](https://github.com/ClickHouse/ClickHouse/issues?q=is%3Aissue+is%3Aopen+rfc)。 + +如果您的贡献与安全相关，也请查看[我们的安全政策](https://github.com/ClickHouse/ClickHouse/security/policy/)。 -If you want to suggest a substantial change to ClickHouse, consider [opening a GitHub issue](https://github.com/ClickHouse/ClickHouse/issues/new/choose) explaining what you want to do, to discuss it with maintainers and community first. [Examples of such RFC issues](https://github.com/ClickHouse/ClickHouse/issues?q=is%3Aissue+is%3Aopen+rfc). -If your contributions are security related, please check out [our security policy](https://github.com/ClickHouse/ClickHouse/security/policy/) too. From 6da1ab27cab721bf4be9346ee5bd2ab53e8b73d6 Mon Sep 17 00:00:00 2001 From: tesw yew isal <278153+cnmade@users.noreply.github.com> Date: Thu, 3 Feb 2022 14:39:51 +0800 Subject: [PATCH 127/149] Translate zh/faq/general/contribute: remove old files --- .../zh/faq/general/how-do-i-contribute-code-to-clickhouse.md.bak | 1 - 1 file changed, 1 deletion(-) delete mode 100644 docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md.bak diff --git a/docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md.bak b/docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md.bak deleted file mode 100644 index 5ac9a615386..00000000000 --- a/docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md.bak +++ /dev/null @@ -1 +0,0 @@ -../../../en/faq/general/how-do-i-contribute-code-to-clickhouse.md \ No newline at end of file From 8f49f62a3c10c00f8f855921c14176acaaddf4de Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 3 Feb 2022 13:53:00 +0300 Subject: [PATCH 128/149] Just a tiny fix. --- .../aggregate-functions/reference/studentttest.md | 3 +-- .../aggregate-functions/reference/welchttest.md | 3 +-- src/AggregateFunctions/AggregateFunctionTTest.h | 10 +++++----- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/docs/en/sql-reference/aggregate-functions/reference/studentttest.md b/docs/en/sql-reference/aggregate-functions/reference/studentttest.md index 0489569146c..7d8d255e15b 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/studentttest.md +++ b/docs/en/sql-reference/aggregate-functions/reference/studentttest.md @@ -28,8 +28,7 @@ The null hypothesis is that means of populations are equal. Normal distribution **Returned values** -[Tuple](../../../sql-reference/data-types/tuple.md) with two elements: -If the optional confidence_level is specified, it returns a [Tuple](../../../sql-reference/data-types/tuple.md) with four elements. +[Tuple](../../../sql-reference/data-types/tuple.md) with two or four elements (if the optional `confidence_level` is specified): - calculated t-statistic. [Float64](../../../sql-reference/data-types/float.md). - calculated p-value. [Float64](../../../sql-reference/data-types/float.md). diff --git a/docs/en/sql-reference/aggregate-functions/reference/welchttest.md b/docs/en/sql-reference/aggregate-functions/reference/welchttest.md index 89a6afffc2c..282d5c04d60 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/welchttest.md +++ b/docs/en/sql-reference/aggregate-functions/reference/welchttest.md @@ -27,8 +27,7 @@ The null hypothesis is that means of populations are equal. Normal distribution **Returned values** -[Tuple](../../../sql-reference/data-types/tuple.md) with two elements: -If the optional confidence_level is specified, it returns a [Tuple](../../../sql-reference/data-types/tuple.md) with four elements. +[Tuple](../../../sql-reference/data-types/tuple.md) with two two or four elements (if the optional `confidence_level` is specified) - calculated t-statistic. [Float64](../../../sql-reference/data-types/float.md). - calculated p-value. [Float64](../../../sql-reference/data-types/float.md). diff --git a/src/AggregateFunctions/AggregateFunctionTTest.h b/src/AggregateFunctions/AggregateFunctionTTest.h index 36605196951..40cf0023878 100644 --- a/src/AggregateFunctions/AggregateFunctionTTest.h +++ b/src/AggregateFunctions/AggregateFunctionTTest.h @@ -86,7 +86,7 @@ class AggregateFunctionTTest : public IAggregateFunctionDataHelper> { private: - bool need_ci = false; + bool need_confidence_interval = false; Float64 confidence_level; public: AggregateFunctionTTest(const DataTypes & arguments, const Array & params) @@ -94,7 +94,7 @@ public: { if (params.size() > 0) { - need_ci = true; + need_confidence_interval = true; confidence_level = params.at(0).safeGet(); if (!std::isfinite(confidence_level)) @@ -117,7 +117,7 @@ public: DataTypePtr getReturnType() const override { - if (need_ci) + if (need_confidence_interval) { DataTypes types { @@ -201,7 +201,7 @@ public: column_stat.getData().push_back(std::numeric_limits::quiet_NaN()); column_value.getData().push_back(std::numeric_limits::quiet_NaN()); - if (need_ci) + if (need_confidence_interval) { auto & column_ci_low = assert_cast &>(column_tuple.getColumn(2)); auto & column_ci_high = assert_cast &>(column_tuple.getColumn(3)); @@ -223,7 +223,7 @@ public: column_stat.getData().push_back(t_statistic); column_value.getData().push_back(p_value); - if (need_ci) + if (need_confidence_interval) { auto [ci_low, ci_high] = data.getConfidenceIntervals(confidence_level, data.getDegreesOfFreedom()); auto & column_ci_low = assert_cast &>(column_tuple.getColumn(2)); From d0bfffa6f2b03ad2793ae4430d71f3eb71011cd4 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 3 Feb 2022 11:05:18 +0000 Subject: [PATCH 129/149] Fix fractional_progress_bar test --- ...bar.reference => 02204_fractional_progress_bar_long.reference} | 0 ...onal_progress_bar.sh => 02204_fractional_progress_bar_long.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{02204_fractional_progress_bar.reference => 02204_fractional_progress_bar_long.reference} (100%) rename tests/queries/0_stateless/{02204_fractional_progress_bar.sh => 02204_fractional_progress_bar_long.sh} (100%) diff --git a/tests/queries/0_stateless/02204_fractional_progress_bar.reference b/tests/queries/0_stateless/02204_fractional_progress_bar_long.reference similarity index 100% rename from tests/queries/0_stateless/02204_fractional_progress_bar.reference rename to tests/queries/0_stateless/02204_fractional_progress_bar_long.reference diff --git a/tests/queries/0_stateless/02204_fractional_progress_bar.sh b/tests/queries/0_stateless/02204_fractional_progress_bar_long.sh similarity index 100% rename from tests/queries/0_stateless/02204_fractional_progress_bar.sh rename to tests/queries/0_stateless/02204_fractional_progress_bar_long.sh From 0755cfa584674dcd13caacee6ffb1a577500cfe6 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 3 Feb 2022 15:04:13 +0300 Subject: [PATCH 130/149] minor fixes --- src/Interpreters/AsynchronousInsertQueue.cpp | 2 ++ tests/queries/0_stateless/02193_async_insert_tcp_client_2.sh | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index c3ec9fa42f1..5321d5f6fd3 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -485,6 +485,8 @@ void AsynchronousInsertQueue::finishWithException( entry->finish(std::make_exception_ptr(exception)); } } + + CurrentMetrics::sub(CurrentMetrics::PendingAsyncInsert, entries.size()); } } diff --git a/tests/queries/0_stateless/02193_async_insert_tcp_client_2.sh b/tests/queries/0_stateless/02193_async_insert_tcp_client_2.sh index e99d0fa69b8..e620b21ac72 100755 --- a/tests/queries/0_stateless/02193_async_insert_tcp_client_2.sh +++ b/tests/queries/0_stateless/02193_async_insert_tcp_client_2.sh @@ -12,8 +12,8 @@ ${CLICKHOUSE_CLIENT} -q "CREATE TABLE t_async_insert_02193_2 (id UInt32, s Strin ${CLICKHOUSE_CLIENT} -q "INSERT INTO t_async_insert_02193_2 FORMAT CSV SETTINGS async_insert = 1 1,aaa" ${CLICKHOUSE_CLIENT} -q "INSERT INTO t_async_insert_02193_2 FORMAT Values SETTINGS async_insert = 1 (2, 'bbb')" -${CLICKHOUSE_CLIENT} -q "INSERT INTO t_async_insert_02193_2 VALUES (3, 'ccc')" -${CLICKHOUSE_CLIENT} -q 'INSERT INTO t_async_insert_02193_2 FORMAT JSONEachRow {"id": 4, "s": "ddd"}' +${CLICKHOUSE_CLIENT} -q "INSERT INTO t_async_insert_02193_2 VALUES (3, 'ccc')" --async_insert=1 +${CLICKHOUSE_CLIENT} -q 'INSERT INTO t_async_insert_02193_2 FORMAT JSONEachRow {"id": 4, "s": "ddd"}' --async_insert=1 ${CLICKHOUSE_CLIENT} -q "SELECT * FROM t_async_insert_02193_2 ORDER BY id" ${CLICKHOUSE_CLIENT} -q "TRUNCATE TABLE t_async_insert_02193_2" From 244bb984e63e0dddc6aaf427b767cb323a9a2e58 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 3 Feb 2022 12:14:33 +0000 Subject: [PATCH 131/149] Wait for pipeline to be destroyed in case of exception --- src/QueryPipeline/BlockIO.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/QueryPipeline/BlockIO.h b/src/QueryPipeline/BlockIO.h index 5918b4b27fc..748e46c3a1e 100644 --- a/src/QueryPipeline/BlockIO.h +++ b/src/QueryPipeline/BlockIO.h @@ -40,10 +40,12 @@ struct BlockIO pipeline.reset(); } - void onException() const + void onException() { if (exception_callback) exception_callback(); + + pipeline.reset(); } private: From c90b1f779493683ce47ebd2c5d08548d3278a147 Mon Sep 17 00:00:00 2001 From: Danila Kutenin Date: Thu, 3 Feb 2022 12:24:33 +0000 Subject: [PATCH 132/149] Optimize quantilesExact{Low,High} to use nth_element instead of sort --- src/AggregateFunctions/QuantileExact.h | 52 ++++++++++++------------ tests/performance/decimal_aggregates.xml | 5 +++ 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/src/AggregateFunctions/QuantileExact.h b/src/AggregateFunctions/QuantileExact.h index c76903f5081..9be24689d12 100644 --- a/src/AggregateFunctions/QuantileExact.h +++ b/src/AggregateFunctions/QuantileExact.h @@ -262,9 +262,7 @@ struct QuantileExactLow : public QuantileExactBase(floor(s / 2))]; + n = static_cast(floor(s / 2)); } else { - return array[static_cast((floor(s / 2)) - 1)]; + n = static_cast((floor(s / 2)) - 1); } } else @@ -284,9 +282,10 @@ struct QuantileExactLow : public QuantileExactBase::quiet_NaN(); } @@ -295,12 +294,11 @@ struct QuantileExactLow : public QuantileExactBase(floor(s / 2))]; + n = static_cast(floor(s / 2)); } else { - result[indices[i]] = array[static_cast(floor((s / 2) - 1))]; + n = static_cast(floor((s / 2) - 1)); } } else { // else quantile is the nth index of the sorted array obtained by multiplying // level and size of array. Example if level = 0.1 and size of array is 10. - size_t n = level < 1 ? level * array.size() : (array.size() - 1); - result[indices[i]] = array[n]; + n = level < 1 ? level * array.size() : (array.size() - 1); } + ::nth_element(array.begin() + prev_n, array.begin() + n, array.end()); + result[indices[i]] = array[n]; + prev_n = n; } } else @@ -344,23 +344,22 @@ struct QuantileExactHigh : public QuantileExactBase(floor(s / 2))]; + n = static_cast(floor(s / 2)); } else { // else quantile is the nth index of the sorted array obtained by multiplying // level and size of array. Example if level = 0.1 and size of array is 10. - size_t n = level < 1 ? level * array.size() : (array.size() - 1); - return array[n]; + n = level < 1 ? level * array.size() : (array.size() - 1); } + ::nth_element(array.begin(), array.begin() + n, array.end()); + return array[n]; } return std::numeric_limits::quiet_NaN(); } @@ -369,26 +368,27 @@ struct QuantileExactHigh : public QuantileExactBase(floor(s / 2))]; + n = static_cast(floor(s / 2)); } else { // else quantile is the nth index of the sorted array obtained by multiplying // level and size of array. Example if level = 0.1 and size of array is 10. - size_t n = level < 1 ? level * array.size() : (array.size() - 1); - result[indices[i]] = array[n]; + n = level < 1 ? level * array.size() : (array.size() - 1); } + ::nth_element(array.begin() + prev_n, array.begin() + n, array.end()); + result[indices[i]] = array[n]; + prev_n = n; } } else diff --git a/tests/performance/decimal_aggregates.xml b/tests/performance/decimal_aggregates.xml index 7078cb16002..ec88be0124f 100644 --- a/tests/performance/decimal_aggregates.xml +++ b/tests/performance/decimal_aggregates.xml @@ -28,6 +28,11 @@ SELECT quantile(d64), quantileExact(d64), quantileExactWeighted(d64, 2) FROM (SELECT * FROM t LIMIT 1000000) SELECT quantile(d128), quantileExact(d128), quantileExactWeighted(d128, 2) FROM (SELECT * FROM t LIMIT 1000000) + SELECT quantilesExactLow(0.5)(d32) FROM (SELECT * FROM t LIMIT 10000000) + SELECT quantilesExactHigh(0.5)(d32) FROM (SELECT * FROM t LIMIT 10000000) + SELECT quantilesExactLow(0.1, 0.5, 0.9)(d32) FROM (SELECT * FROM t LIMIT 10000000) + SELECT quantilesExactHigh(0.1, 0.5, 0.9)(d32) FROM (SELECT * FROM t LIMIT 10000000) + SELECT quantilesExact(0.1, 0.9)(d32), quantilesExactWeighted(0.1, 0.9)(d32, 2) FROM (SELECT * FROM t LIMIT 10000000) SELECT quantilesExact(0.1, 0.9)(d64), quantilesExactWeighted(0.1, 0.9)(d64, 2) FROM (SELECT * FROM t LIMIT 1000000) SELECT quantilesExact(0.1, 0.9)(d128), quantilesExactWeighted(0.1, 0.9)(d128, 2) FROM (SELECT * FROM t LIMIT 1000000) From 1e365111f210649b16dd4dfab176d2430eaa7618 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 3 Feb 2022 12:35:53 +0100 Subject: [PATCH 133/149] Run the StyleCheck always even if tests skipped --- .github/workflows/master.yml | 3 +++ .github/workflows/pull_request.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 91b9ea5bf3d..c32896205d5 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -86,6 +86,7 @@ jobs: StyleCheck: needs: DockerHubPush runs-on: [self-hosted, style-checker] + if: ${{ success() || failure() }} steps: - name: Set envs run: | @@ -93,6 +94,8 @@ jobs: TEMP_PATH=${{ runner.temp }}/style_check EOF - name: Download changed images + # even if artifact does not exist, e.g. on `do not test` label or failed Docker job + continue-on-error: true uses: actions/download-artifact@v2 with: name: changed_images diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index cd8517de8fe..81a0cb68bd9 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -111,6 +111,7 @@ jobs: StyleCheck: needs: DockerHubPush runs-on: [self-hosted, style-checker] + if: ${{ success() || failure() }} steps: - name: Set envs run: | @@ -118,6 +119,8 @@ jobs: TEMP_PATH=${{ runner.temp }}/style_check EOF - name: Download changed images + # even if artifact does not exist, e.g. on `do not test` label or failed Docker job + continue-on-error: true uses: actions/download-artifact@v2 with: name: changed_images From 8243823d744e90ce9f96371cf39b2d3e60469bf3 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 3 Feb 2022 13:27:32 +0100 Subject: [PATCH 134/149] Mark `Run Check` as success for some cases --- tests/ci/run_check.py | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py index 452824d58be..dc3f1da94cd 100644 --- a/tests/ci/run_check.py +++ b/tests/ci/run_check.py @@ -2,8 +2,9 @@ import sys import logging import re -from github import Github +from typing import Tuple +from github import Github from env_helper import GITHUB_RUN_ID, GITHUB_REPOSITORY, GITHUB_SERVER_URL from pr_info import PRInfo from get_robot_token import get_best_robot_token @@ -17,8 +18,10 @@ TRUSTED_ORG_IDS = { 54801242, # clickhouse } -OK_TEST_LABEL = set(["can be tested", "release", "pr-documentation", "pr-doc-fix"]) +OK_SKIP_LABELS = {"release", "pr-documentation", "pr-doc-fix"} +CAN_BE_TESTED_LABEL = "can be tested" DO_NOT_TEST_LABEL = "do not test" +FORCE_TESTS_LABEL = "force tests" # Individual trusted contirbutors who are not in any trusted organization. # Can be changed in runtime: we will append users that we learned to be in @@ -100,29 +103,29 @@ def pr_is_by_trusted_user(pr_user_login, pr_user_orgs): # Returns whether we should look into individual checks for this PR. If not, it # can be skipped entirely. -def should_run_checks_for_pr(pr_info): +# Returns can_run, description, labels_state +def should_run_checks_for_pr(pr_info: PRInfo) -> Tuple[bool, str, str]: # Consider the labels and whether the user is trusted. print("Got labels", pr_info.labels) - force_labels = set(["force tests"]).intersection(pr_info.labels) - if force_labels: - return True, "Labeled '{}'".format(", ".join(force_labels)) + if FORCE_TESTS_LABEL in pr_info.labels: + return True, f"Labeled '{FORCE_TESTS_LABEL}'", "pending" - if "do not test" in pr_info.labels: - return False, "Labeled 'do not test'" + if DO_NOT_TEST_LABEL in pr_info.labels: + return False, f"Labeled '{DO_NOT_TEST_LABEL}'", "success" - if "can be tested" not in pr_info.labels and not pr_is_by_trusted_user( + if CAN_BE_TESTED_LABEL not in pr_info.labels and not pr_is_by_trusted_user( pr_info.user_login, pr_info.user_orgs ): - return False, "Needs 'can be tested' label" + return False, "Needs 'can be tested' label", "failure" - if ( - "release" in pr_info.labels - or "pr-backport" in pr_info.labels - or "pr-cherrypick" in pr_info.labels - ): - return False, "Don't try new checks for release/backports/cherry-picks" + if OK_SKIP_LABELS.intersection(pr_info.labels): + return ( + False, + "Don't try new checks for release/backports/cherry-picks", + "success", + ) - return True, "No special conditions apply" + return True, "No special conditions apply", "pending" def check_pr_description(pr_info): @@ -205,7 +208,7 @@ if __name__ == "__main__": logging.basicConfig(level=logging.INFO) pr_info = PRInfo(need_orgs=True, pr_event_from_api=True) - can_run, description = should_run_checks_for_pr(pr_info) + can_run, description, labels_state = should_run_checks_for_pr(pr_info) gh = Github(get_best_robot_token()) commit = get_commit(gh, pr_info.sha) @@ -231,7 +234,7 @@ if __name__ == "__main__": if not can_run: print("::notice ::Cannot run") commit.create_status( - context=NAME, description=description, state="failure", target_url=url + context=NAME, description=description, state=labels_state, target_url=url ) sys.exit(1) else: From 4f7461fd6fa110043c2e7bf0bd74ec30c60a5665 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 3 Feb 2022 13:32:09 +0100 Subject: [PATCH 135/149] Fix style for tags_stable.yml --- .github/workflows/tags_stable.yml | 42 +++++++++++++++---------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/tags_stable.yml b/.github/workflows/tags_stable.yml index 423fc64c4fc..30b6bfb027e 100644 --- a/.github/workflows/tags_stable.yml +++ b/.github/workflows/tags_stable.yml @@ -14,25 +14,25 @@ jobs: UpdateVersions: runs-on: [self-hosted, style-checker] steps: - - name: Get tag name - run: echo "GITHUB_TAG=${GITHUB_REF#refs/tags/}" >> "$GITHUB_ENV" - - name: Check out repository code - uses: actions/checkout@v2 - with: - ref: master - - name: Generate versions - run: | - git fetch --tags - ./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv - - name: Create Pull Request - uses: peter-evans/create-pull-request@v3 - with: - commit-message: Update version_date.tsv after ${{ env.GITHUB_TAG }} - branch: auto/${{ env.GITHUB_TAG }} - delete-branch: true - title: Update version_date.tsv after ${{ env.GITHUB_TAG }} - body: | - Update version_date.tsv after ${{ env.GITHUB_TAG }} + - name: Get tag name + run: echo "GITHUB_TAG=${GITHUB_REF#refs/tags/}" >> "$GITHUB_ENV" + - name: Check out repository code + uses: actions/checkout@v2 + with: + ref: master + - name: Generate versions + run: | + git fetch --tags + ./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv + - name: Create Pull Request + uses: peter-evans/create-pull-request@v3 + with: + commit-message: Update version_date.tsv after ${{ env.GITHUB_TAG }} + branch: auto/${{ env.GITHUB_TAG }} + delete-branch: true + title: Update version_date.tsv after ${{ env.GITHUB_TAG }} + body: | + Update version_date.tsv after ${{ env.GITHUB_TAG }} - Changelog category (leave one): - - Not for changelog (changelog entry is not required) + Changelog category (leave one): + - Not for changelog (changelog entry is not required) From 94c3d14d68fee11d590b1a8ec64a66c59573e05a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 3 Feb 2022 16:06:33 +0300 Subject: [PATCH 136/149] Update SECURITY.md --- SECURITY.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/SECURITY.md b/SECURITY.md index f002dd53ca9..ca3c8b439fd 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -22,9 +22,10 @@ The following versions of ClickHouse server are currently being supported with s | 21.7 | :x: | | 21.8 | ✅ | | 21.9 | :x: | -| 21.10 | ✅ | +| 21.10 | :x: | | 21.11 | ✅ | | 21.12 | ✅ | +| 22.1 | ✅ | ## Reporting a Vulnerability From 73f579d71c92a649a29c123d14f9531937b4b1a6 Mon Sep 17 00:00:00 2001 From: FArthur-cmd <613623@mail.ru> Date: Thu, 3 Feb 2022 13:31:52 +0000 Subject: [PATCH 137/149] Comment dhParamsFile. It should be uncommented only if it has correct path --- programs/server/config.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/server/config.xml b/programs/server/config.xml index d88773a3fc4..ce0c54f6730 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -244,7 +244,7 @@ openssl dhparam -out /etc/clickhouse-server/dhparam.pem 4096 Only file format with BEGIN DH PARAMETERS is supported. --> - /etc/clickhouse-server/dhparam.pem + none true true From e57320a70afde3b8b615f1a85978a295eef05532 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Thu, 3 Feb 2022 16:10:52 +0000 Subject: [PATCH 138/149] Fixed tests --- tests/queries/0_stateless/02204_fractional_progress_bar_long.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02204_fractional_progress_bar_long.sh b/tests/queries/0_stateless/02204_fractional_progress_bar_long.sh index 6018ee8c96c..aa8aa22be5c 100755 --- a/tests/queries/0_stateless/02204_fractional_progress_bar_long.sh +++ b/tests/queries/0_stateless/02204_fractional_progress_bar_long.sh @@ -1,5 +1,5 @@ #!/usr/bin/expect -f -# Tags: no-fasttest +# Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug, no-fasttest log_user 0 set timeout 60 From 37179bbc9c535d10a47d9fc54c1432aa4e351d73 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 3 Feb 2022 14:06:21 +0100 Subject: [PATCH 139/149] Add verbosity to stylecheck --- docker/test/style/Dockerfile | 1 + docker/test/style/run.sh | 6 ++++++ tests/ci/style_check.py | 10 ++++++++-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index a68b52170e0..85c751edfbe 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -11,6 +11,7 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ curl \ git \ libxml2-utils \ + moreutils \ pylint \ python3-pip \ shellcheck \ diff --git a/docker/test/style/run.sh b/docker/test/style/run.sh index 98bc0053ab9..ce3ea4e50a6 100755 --- a/docker/test/style/run.sh +++ b/docker/test/style/run.sh @@ -3,10 +3,16 @@ # yaml check is not the best one cd /ClickHouse/utils/check-style || echo -e "failure\tRepo not found" > /test_output/check_status.tsv +echo "Check duplicates" | ts ./check-duplicate-includes.sh |& tee /test_output/duplicate_output.txt +echo "Check style" | ts ./check-style -n |& tee /test_output/style_output.txt +echo "Check typos" | ts ./check-typos |& tee /test_output/typos_output.txt +echo "Check whitespaces" | ts ./check-whitespaces -n |& tee /test_output/whitespaces_output.txt +echo "Check sorkflows" | ts ./check-workflows |& tee /test_output/workflows_output.txt +echo "Check shell scripts with shellcheck" | ts ./shellcheck-run.sh |& tee /test_output/shellcheck_output.txt /process_style_check_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 8490b33dd22..1b3037217c8 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -86,12 +86,18 @@ if __name__ == "__main__": docker_image = get_image_with_version(temp_path, "clickhouse/style-test") s3_helper = S3Helper("https://s3.amazonaws.com") - subprocess.check_output( + cmd = ( f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --cap-add=SYS_PTRACE " f"--volume={repo_path}:/ClickHouse --volume={temp_path}:/test_output " - f"{docker_image}", + f"{docker_image}" + ) + + logging.info("Is going to run the command: %s", cmd) + subprocess.check_call( + cmd, shell=True, ) + state, description, test_results, additional_files = process_result(temp_path) ch_helper = ClickHouseHelper() mark_flaky_tests(ch_helper, NAME, test_results) From a7325bbe32118af1d9faf300249322b45cb6637a Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 3 Feb 2022 17:59:29 +0100 Subject: [PATCH 140/149] Fix check-workflows in style check, iterate over checks --- .../test/style/process_style_check_result.py | 84 +++++-------------- utils/check-style/check-workflows | 3 +- 2 files changed, 21 insertions(+), 66 deletions(-) mode change 100644 => 100755 utils/check-style/check-workflows diff --git a/docker/test/style/process_style_check_result.py b/docker/test/style/process_style_check_result.py index b7e00c49e06..655b7d70243 100755 --- a/docker/test/style/process_style_check_result.py +++ b/docker/test/style/process_style_check_result.py @@ -10,72 +10,26 @@ def process_result(result_folder): status = "success" description = "" test_results = [] + checks = ( + ("header duplicates", "duplicate_output.txt"), + ("shellcheck", "shellcheck_output.txt"), + ("style", "style_output.txt"), + ("typos", "typos_output.txt"), + ("whitespaces", "whitespaces_output.txt"), + ("workflows", "workflows_output.txt"), + ) - duplicate_log_path = "{}/duplicate_output.txt".format(result_folder) - if not os.path.exists(duplicate_log_path): - logging.info("No header duplicates check log on path %s", duplicate_log_path) - return "exception", "No header duplicates check log", [] - elif os.stat(duplicate_log_path).st_size != 0: - description += " Header duplicates check failed. " - test_results.append(("Header duplicates check", "FAIL")) - status = "failure" - else: - test_results.append(("Header duplicates check", "OK")) - - shellcheck_log_path = "{}/shellcheck_output.txt".format(result_folder) - if not os.path.exists(shellcheck_log_path): - logging.info("No shellcheck log on path %s", shellcheck_log_path) - return "exception", "No shellcheck log", [] - elif os.stat(shellcheck_log_path).st_size != 0: - description += " Shellcheck check failed. " - test_results.append(("Shellcheck ", "FAIL")) - status = "failure" - else: - test_results.append(("Shellcheck", "OK")) - - style_log_path = "{}/style_output.txt".format(result_folder) - if not os.path.exists(style_log_path): - logging.info("No style check log on path %s", style_log_path) - return "exception", "No style check log", [] - elif os.stat(style_log_path).st_size != 0: - description += "Style check failed. " - test_results.append(("Style check", "FAIL")) - status = "failure" - else: - test_results.append(("Style check", "OK")) - - typos_log_path = "{}/typos_output.txt".format(result_folder) - if not os.path.exists(typos_log_path): - logging.info("No typos check log on path %s", typos_log_path) - return "exception", "No typos check log", [] - elif os.stat(typos_log_path).st_size != 0: - description += "Typos check failed. " - test_results.append(("Typos check", "FAIL")) - status = "failure" - else: - test_results.append(("Typos check", "OK")) - - whitespaces_log_path = "{}/whitespaces_output.txt".format(result_folder) - if not os.path.exists(whitespaces_log_path): - logging.info("No whitespaces check log on path %s", whitespaces_log_path) - return "exception", "No whitespaces check log", [] - elif os.stat(whitespaces_log_path).st_size != 0: - description += "Whitespaces check failed. " - test_results.append(("Whitespaces check", "FAIL")) - status = "failure" - else: - test_results.append(("Whitespaces check", "OK")) - - workflows_log_path = "{}/workflows_output.txt".format(result_folder) - if not os.path.exists(workflows_log_path): - logging.info("No workflows check log on path %s", style_log_path) - return "exception", "No workflows check log", [] - elif os.stat(whitespaces_log_path).st_size != 0: - description += "Workflows check failed. " - test_results.append(("Workflows check", "FAIL")) - status = "failure" - else: - test_results.append(("Workflows check", "OK")) + for name, out_file in checks: + full_path = os.path.join(result_folder, out_file) + if not os.path.exists(full_path): + logging.info("No %s check log on path %s", name, full_path) + return "exception", f"No {name} check log", [] + elif os.stat(full_path).st_size != 0: + description += f"Check {name} failed. " + test_results.append((f"Check {name}", "FAIL")) + status = "failure" + else: + test_results.append((f"Check {name}", "OK")) if not description: description += "Style check success" diff --git a/utils/check-style/check-workflows b/utils/check-style/check-workflows old mode 100644 new mode 100755 index 1a0c0f9ecbe..c0399829c28 --- a/utils/check-style/check-workflows +++ b/utils/check-style/check-workflows @@ -1,5 +1,6 @@ #!/usr/bin/env bash -act --list 1>/dev/null 2>&1 || act --list 2>&1 +GIT_ROOT=$(git rev-parse --show-cdup) +act --list --directory="$GIT_ROOT" 1>/dev/null 2>&1 || act --list --directory="$GIT_ROOT" 2>&1 actionlint From 1600dd9287eae09c48f0396d25c02a1410d3252a Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 3 Feb 2022 18:37:03 +0100 Subject: [PATCH 141/149] Check style of workflows/release.yml --- .github/workflows/release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 77bc285196c..46e36c846d0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,6 @@ jobs: - name: Check out repository code uses: actions/checkout@v2 - name: Download packages and push to Artifactory - env: run: | rm -rf "$TEMP_PATH" && mkdir -p "$REPO_COPY" cp -r "$GITHUB_WORKSPACE" "$REPO_COPY" From d09fb90f181b8ee5ee83bcbc45a7fb18d94418db Mon Sep 17 00:00:00 2001 From: Tom Risse Date: Thu, 3 Feb 2022 11:43:40 -0800 Subject: [PATCH 142/149] Add support policy page content. Add ol.default for li margin. --- website/css/base.css | 8 +++ website/support/policy/index.html | 27 +++++++++ website/templates/support/policy-content.html | 55 +++++++++++++++++++ website/templates/support/policy-hero.html | 10 ++++ 4 files changed, 100 insertions(+) create mode 100644 website/support/policy/index.html create mode 100644 website/templates/support/policy-content.html create mode 100644 website/templates/support/policy-hero.html diff --git a/website/css/base.css b/website/css/base.css index b65c8e869f4..d81b231789a 100644 --- a/website/css/base.css +++ b/website/css/base.css @@ -193,3 +193,11 @@ a.btn-outline-yellow { .height-20 { height: 20px; } + +ol.default{ + margin-top: 10px; +} + +ol.default li{ + margin-bottom: 10px; +} diff --git a/website/support/policy/index.html b/website/support/policy/index.html new file mode 100644 index 00000000000..babff39a747 --- /dev/null +++ b/website/support/policy/index.html @@ -0,0 +1,27 @@ +{% set prefetch_items = [ + ('/docs/en/', 'document') +] %} + +{% extends "templates/base.html" %} + +{% block extra_meta %} + +{% include "templates/common_fonts.html" %} +{% endblock %} + +{% block nav %} + +{% include "templates/global/nav.html" %} + +{% endblock %} + +{% block content %} + +{% include "templates/support/policy-hero.html" %} + +{% include "templates/support/policy-content.html" %} + +{% include "templates/global/newsletter.html" %} +{% include "templates/global/github_stars.html" %} + +{% endblock %} diff --git a/website/templates/support/policy-content.html b/website/templates/support/policy-content.html new file mode 100644 index 00000000000..d6efae2fb67 --- /dev/null +++ b/website/templates/support/policy-content.html @@ -0,0 +1,55 @@ +
    +
    +

    Effective Date: January 28, 2022

    + +

    This ClickHouse Support Services Policy sets forth ClickHouse’s policy for the provision of Support Services to its Subscription customers (each a “Customer”). Capitalized terms used but not defined herein, have the definition set forth in the applicable Subscription Agreement between ClickHouse and Customer (the “Agreement”).

    + +
      +
    1. Defined Terms +
        +
      1. "Business Day" means Monday through Friday other than a day designated from time to time as a national holiday in the place from which Support Services may be provided.
        2. +
      2. “Long-Term Supported Release” means the official release for the Software which has been designated as a release for long term support. ClickHouse designates its as Long-Term Supported Releases by including the letters“lts” in the release number, which are typically March and August releases.
        3. +
      3. “Regular Stable Sequential Release” means the official software release for the Software which has not been designated as a Long Term Supported Release.
        4. +
      4. “Support” means technical support by telephone, chat, or email provided by ClickHouse to a Support Contact concerning a Support Incident.
        5. +
      5. “Support Contact” means a single named individual who is authorized to contact ClickHouse to use the Support Services.
        6. +
      6. "Support Incident" means a single issue or error with the Software that is raised with ClickHouse by a Support Contact. Each Support Incident will be assigned a unique ID by ClickHouse. In the situation where multiple similar or equivalent cases are opened for a single Support Incident, ClickHouse may choose to consolidate these into a single support case, in which case it shall promptly notify Customer.
        7. +
      +
      2. +
    2. Scope and Performance of Support Services. +

      The scope of the Support Services provided to Customer includes general assistance and support regarding the installation of the Software and basic technical configuration of the Software, as well as operational or development assistance on how to use the Software. ClickHouse shall use commercially reasonable efforts to meet the applicable target response times set forth in Section 3 below. Customer acknowledges that the time required for resolution of issues may vary depending on the specific circumstances of each problem, including, without limitation, the nature of the incident/problem, the extent and accuracy of information available about the incident/problem, and the level of Customer's cooperation and responsiveness in providing materials, information, access and support reasonably required by ClickHouse to achieve problem resolution.

      +
        +
      • Normal Business Hours: +
          +
        • 24x7x365 for Severity 1 Issues
          • +
        • Monday 08:00 CEST through Friday 17:00 US Pacific time for Severity 2 and Severity 3 issues
          • +
        +
        • +
      • Number of Support Contacts: unlimited
        • +
      • Emergency Patches: yes
        • +
      • Annual Support Incidents: unlimited
        • +
      +
      3. +
    3. Severity Levels and ClickHouse Efforts. +

      Severity Level 1

      +

      A Severity Level 1 issue is a major production error within the software that severely impacts the Customer's use of the software for production purposes, such as the loss of production data or where production systems are not functioning and no work-around exists. ClickHouse will use continuous efforts during applicable Normal Business Hours to provide a resolution for any Level 1 issues as soon as is commercially reasonable.

      +

      Severity Level 2

      +

      A Severity Level 2 issue is an error within the software where the customer's system is functioning for production purposes but in a reduced capacity, such as a problem that is causing significant impact to portions of the customer's business operations and productivity, or where the software is exposed to potential loss or interruption of service. ClickHouse will use continuous efforts during the Normal Business Hours to provide a resolution for any Severity Level 2 issues.

      +

      Severity Level 3

      +

      A Severity Level 3 issue is a medium-to-low impact error that involves partial and/or non-critical loss of functionality for production purposes or development purposes, such as a problem that impairs some operations but allows the customer's operations to continue to function. Errors for which there is limited or no loss or functionality or impact to the customer's operation and for which there is an easy work-around qualify as Severity Level 3. General questions are also Severity Level issues. ClickHouse will use reasonable efforts to provide a resolution for any Severity Level 3 issues in time for an upcoming release of the software. All inbound production email cases shall have an initial status of Severity Level 3.

      +
      4. +
    4. Customer Obligations. +

      Customer agrees to provide ClickHouse with reasonable: (i) detail of the nature of and circumstances surrounding the issue, (ii) access to Customer's environment as necessary to enable ClickHouse to provide Support Services; and (iii) cooperation in the diagnosis and resolution of any issues.

      +
      5. +
    5. Supported Versions. +

      Notwithstanding anything else, ClickHouse will support the current Regular Stable Release in conjunction with the two (2) prior Regular Stable Releases of the Software or a minimum period of three (3) months from the date of the current Regular Stable Release. ClickHouse will support the current Long-Term Supported Release in conjunction with the one (1) prior Long-Term Supported Release of the Software or a minimum period of one (1) year from the date of the current Long-Term Supported Release.

      + +
      6. +
    6. Support Service Exclusions. +

      ClickHouse will have no obligation to provide Support Services to Customer in the event that (i) the Software has been changed, modified or damaged by Customer or anyone other than ClickHouse, (ii) the problem is caused by Customer's negligence, misconduct, or misuse of the Software, a hardware malfunction, or other causes beyond the reasonable control of ClickHouse, (iii) the problem is due to third party software, (iv) the Software is being hosted by a third party that is offering the Software as a service (v) Customer has not installed or implemented any Software releases made generally available or is not running a then supported version of the Software as provided by ClickHouse or (vi) information requested by Customer could reasonably be expected to assist in the development, deployment, enablement and/or maintenance of any non-ClickHouse software that competes with ClickHouse's commercial software products. The Support Services do not cover the support of any third party software which integrates with the Software or the investigation into a potential or actual security incident in a Customer environment, including but not limited to the analysis and response to security events and signals. In addition, the Support Services do not include the following: (a) use of any version of a Software that is not designated as a production release (such as a milestone or release candidate or code contained in the sandbox or any other repository that is not packaged into a production release distribution); (b) Customer's failure to comply with operating instructions contained in the documentation for the Software; (c) installation, configuration, management and operation of Customer's applications; (d) APIs, interfaces or data formats other than those included with the Software; or (e) any training.

      +
      7. + +
    +
    +
    \ No newline at end of file diff --git a/website/templates/support/policy-hero.html b/website/templates/support/policy-hero.html new file mode 100644 index 00000000000..dc551dc434c --- /dev/null +++ b/website/templates/support/policy-hero.html @@ -0,0 +1,10 @@ +
    +
    +
    + +

    + {{ _('ClickHouse Support Services Policy') }} +

    + +
    +
    \ No newline at end of file From 3e700e854d6a1620bf80da5340206e1ec691b22a Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 3 Feb 2022 23:44:53 +0300 Subject: [PATCH 143/149] cancel merges before acquiring lock for truncate --- src/Interpreters/InterpreterDropQuery.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 90c4311b032..b04342b37f7 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -35,6 +35,10 @@ namespace ErrorCodes extern const int TABLE_IS_READ_ONLY; } +namespace ActionLocks +{ + extern const StorageActionBlockType PartsMerge; +} static DatabasePtr tryGetDatabase(const String & database_name, bool if_exists) { @@ -202,7 +206,15 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQue table->checkTableCanBeDropped(); - auto table_lock = table->lockExclusively(context_->getCurrentQueryId(), context_->getSettingsRef().lock_acquire_timeout); + TableExclusiveLockHolder table_lock; + /// We don't need this lock for ReplicatedMergeTree + if (!table->supportsReplication()) + { + /// And for simple MergeTree we can stop merges before acquiring the lock + auto merges_blocker = table->getActionLock(ActionLocks::PartsMerge); + auto table_lock = table->lockExclusively(context_->getCurrentQueryId(), context_->getSettingsRef().lock_acquire_timeout); + } + auto metadata_snapshot = table->getInMemoryMetadataPtr(); /// Drop table data, don't touch metadata table->truncate(query_ptr, metadata_snapshot, context_, table_lock); From 7c12f5f37abbe4ba6058a8d26bd01511bc786556 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 4 Feb 2022 00:07:31 +0300 Subject: [PATCH 144/149] Fix terribly low performance of `LineAsString` format --- src/IO/ReadHelpers.cpp | 6 +++ src/IO/ReadHelpers.h | 4 +- .../Impl/LineAsStringRowInputFormat.cpp | 46 ++++++++----------- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index 4557f8a4251..e16722dbd90 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -216,6 +216,12 @@ void readStringUntilWhitespaceInto(Vector & s, ReadBuffer & buf) readStringUntilCharsInto<' '>(s, buf); } +template +void readStringUntilNewlineInto(Vector & s, ReadBuffer & buf) +{ + readStringUntilCharsInto<'\n'>(s, buf); +} + template void readNullTerminated(Vector & s, ReadBuffer & buf) { diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 73fe1a28be6..5d580f6b130 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -604,6 +604,9 @@ bool tryReadJSONStringInto(Vector & s, ReadBuffer & buf) template void readStringUntilWhitespaceInto(Vector & s, ReadBuffer & buf); +template +void readStringUntilNewlineInto(Vector & s, ReadBuffer & buf); + /// This could be used as template parameter for functions above, if you want to just skip data. struct NullOutput { @@ -1387,4 +1390,3 @@ void readQuotedFieldIntoString(String & s, ReadBuffer & buf); void readJSONFieldIntoString(String & s, ReadBuffer & buf); } - diff --git a/src/Processors/Formats/Impl/LineAsStringRowInputFormat.cpp b/src/Processors/Formats/Impl/LineAsStringRowInputFormat.cpp index 5983f3170e5..a4654b85a50 100644 --- a/src/Processors/Formats/Impl/LineAsStringRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/LineAsStringRowInputFormat.cpp @@ -2,6 +2,8 @@ #include #include #include +#include + namespace DB { @@ -14,7 +16,9 @@ namespace ErrorCodes LineAsStringRowInputFormat::LineAsStringRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_) : IRowInputFormat(header_, in_, std::move(params_)) { - if (header_.columns() > 1 || header_.getDataTypes()[0]->getTypeId() != TypeIndex::String) + if (header_.columns() != 1 + || header_.getByPosition(0).type->getTypeId() != TypeIndex::String + || !typeid_cast(header_.getByPosition(0).column.get())) { throw Exception("This input format is only suitable for tables with a single column of type String.", ErrorCodes::INCORRECT_QUERY); } @@ -27,28 +31,16 @@ void LineAsStringRowInputFormat::resetParser() void LineAsStringRowInputFormat::readLineObject(IColumn & column) { - DB::Memory<> object; + ColumnString & column_string = assert_cast(column); + auto & chars = column_string.getChars(); + auto & offsets = column_string.getOffsets(); - char * pos = in->position(); - bool need_more_data = true; + readStringUntilNewlineInto(chars, *in); + chars.push_back(0); + offsets.push_back(chars.size()); - while (loadAtPosition(*in, object, pos) && need_more_data) - { - pos = find_first_symbols<'\n'>(pos, in->buffer().end()); - if (pos == in->buffer().end()) - continue; - - if (*pos == '\n') - need_more_data = false; - - ++pos; - } - - saveUpToPosition(*in, object, pos); - loadAtPosition(*in, object, pos); - - /// Last character is always \n. - column.insertData(object.data(), object.size() - 1); + if (!in->eof()) + in->ignore(); /// Skip '\n' } bool LineAsStringRowInputFormat::readRow(MutableColumns & columns, RowReadExtension &) @@ -57,17 +49,16 @@ bool LineAsStringRowInputFormat::readRow(MutableColumns & columns, RowReadExtens return false; readLineObject(*columns[0]); - return true; } void registerInputFormatLineAsString(FormatFactory & factory) { factory.registerInputFormat("LineAsString", []( - ReadBuffer & buf, - const Block & sample, - const RowInputFormatParams & params, - const FormatSettings &) + ReadBuffer & buf, + const Block & sample, + const RowInputFormatParams & params, + const FormatSettings &) { return std::make_shared(sample, buf, params); }); @@ -76,9 +67,10 @@ void registerInputFormatLineAsString(FormatFactory & factory) void registerLineAsStringSchemaReader(FormatFactory & factory) { factory.registerExternalSchemaReader("LineAsString", []( - const FormatSettings &) + const FormatSettings &) { return std::make_shared(); }); } + } From c426f11096c761d0f9760da72038c433ce23f7bf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 4 Feb 2022 00:20:16 +0300 Subject: [PATCH 145/149] Maybe better --- src/Processors/Formats/Impl/LineAsStringRowInputFormat.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Processors/Formats/Impl/LineAsStringRowInputFormat.cpp b/src/Processors/Formats/Impl/LineAsStringRowInputFormat.cpp index a4654b85a50..dc346b4f5f5 100644 --- a/src/Processors/Formats/Impl/LineAsStringRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/LineAsStringRowInputFormat.cpp @@ -17,7 +17,6 @@ LineAsStringRowInputFormat::LineAsStringRowInputFormat(const Block & header_, Re IRowInputFormat(header_, in_, std::move(params_)) { if (header_.columns() != 1 - || header_.getByPosition(0).type->getTypeId() != TypeIndex::String || !typeid_cast(header_.getByPosition(0).column.get())) { throw Exception("This input format is only suitable for tables with a single column of type String.", ErrorCodes::INCORRECT_QUERY); From 4a83dbc514453e66860359fa6db60d42b2000b0a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 4 Feb 2022 00:25:36 +0300 Subject: [PATCH 146/149] Fix linkage --- src/IO/ReadHelpers.cpp | 3 +++ src/Processors/Formats/Impl/RawBLOBRowInputFormat.cpp | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index e16722dbd90..ac002eabf7b 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -222,6 +222,9 @@ void readStringUntilNewlineInto(Vector & s, ReadBuffer & buf) readStringUntilCharsInto<'\n'>(s, buf); } +template void readStringUntilNewlineInto>(PODArray & s, ReadBuffer & buf); +template void readStringUntilNewlineInto(String & s, ReadBuffer & buf); + template void readNullTerminated(Vector & s, ReadBuffer & buf) { diff --git a/src/Processors/Formats/Impl/RawBLOBRowInputFormat.cpp b/src/Processors/Formats/Impl/RawBLOBRowInputFormat.cpp index 91b1cc60fae..8bc9bb5e2a3 100644 --- a/src/Processors/Formats/Impl/RawBLOBRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/RawBLOBRowInputFormat.cpp @@ -61,4 +61,3 @@ void registerRawBLOBSchemaReader(FormatFactory & factory) } } - From e20aec6866fbb40c0967788a60dfc3b395c3f65d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 4 Feb 2022 01:44:43 +0300 Subject: [PATCH 147/149] Fix linkage --- src/IO/ReadHelpers.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index ac002eabf7b..a6125818155 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -222,7 +222,7 @@ void readStringUntilNewlineInto(Vector & s, ReadBuffer & buf) readStringUntilCharsInto<'\n'>(s, buf); } -template void readStringUntilNewlineInto>(PODArray & s, ReadBuffer & buf); +template void readStringUntilNewlineInto>(PaddedPODArray & s, ReadBuffer & buf); template void readStringUntilNewlineInto(String & s, ReadBuffer & buf); template From 78eebd5c7c72956bdc3bdb838e144bb113cfd663 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 4 Feb 2022 02:29:46 +0300 Subject: [PATCH 148/149] Fix parallel loading of data parts --- src/Common/ThreadPool.cpp | 2 ++ src/Storages/MergeTree/MergeTreeData.cpp | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index 9b01987c7cf..db20b64f524 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -54,6 +54,8 @@ void ThreadPoolImpl::setMaxThreads(size_t value) { std::lock_guard lock(mutex); max_threads = value; + queue_size = std::max(queue_size, max_threads); + jobs.reserve(queue_size); } template diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 6a6e2838056..e5771c016e5 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1286,7 +1286,6 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) return; } - DataPartsVector broken_parts_to_detach; DataPartsVector duplicate_parts_to_remove; @@ -1357,7 +1356,6 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) calculateColumnAndSecondaryIndexSizesImpl(); - LOG_DEBUG(log, "Loaded data parts ({} items)", data_parts_indexes.size()); } From a089f9918c08784558da199cd1c1e60da1248e26 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 4 Feb 2022 03:54:33 +0300 Subject: [PATCH 149/149] Update ThreadPool.cpp --- src/Common/ThreadPool.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index db20b64f524..8bfb93c9e94 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -54,6 +54,7 @@ void ThreadPoolImpl::setMaxThreads(size_t value) { std::lock_guard lock(mutex); max_threads = value; + /// We have to also adjust queue size, because it limits the number of scheduled and already running jobs in total. queue_size = std::max(queue_size, max_threads); jobs.reserve(queue_size); }