Merge branch 'master' of github.com:ClickHouse/ClickHouse into full-syntax-highlight

.clang-tidy

@@ -119,7 +119,6 @@ Checks: [
   '-readability-named-parameter',
   '-readability-redundant-declaration',
   '-readability-simplify-boolean-expr',
-  '-readability-static-accessed-through-instance',
   '-readability-suspicious-call-argument',
   '-readability-uppercase-literal-suffix',
   '-readability-use-anyofallof',

.github/workflows/pull_request.yml

@@ -157,7 +157,7 @@ jobs:
   ################################# Stage Final #################################
   #
   FinishCheck:
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !failure() && !cancelled() && github.event_name != 'merge_group' }}
     needs: [Tests_1, Tests_2]
     runs-on: [self-hosted, style-checker]
     steps:

base/poco/Net/include/Poco/Net/HTTPClientSession.h

@@ -213,6 +213,19 @@ namespace Net
         Poco::Timespan getKeepAliveTimeout() const;
         /// Returns the connection timeout for HTTP connections.
 
+        void setKeepAliveMaxRequests(int max_requests);
+
+        int getKeepAliveMaxRequests() const;
+
+        int getKeepAliveRequest() const;
+
+        bool isKeepAliveExpired(double reliability = 1.0) const;
+        /// Returns if the connection is expired with some margin as fraction of timeout as reliability
+
+        double getKeepAliveReliability() const;
+        /// Returns the current fraction of keep alive timeout when connection is considered safe to use
+        /// It helps to avoid situation when a client uses nearly expired connection and receives NoMessageException
+
         virtual std::ostream & sendRequest(HTTPRequest & request);
         /// Sends the header for the given HTTP request to
         /// the server.
@@ -345,6 +358,8 @@ namespace Net
 
         void assign(HTTPClientSession & session);
 
+        void setKeepAliveRequest(int request);
+
         HTTPSessionFactory _proxySessionFactory;
         /// Factory to create HTTPClientSession to proxy.
     private:
@@ -353,6 +368,8 @@ namespace Net
         Poco::UInt16 _port;
         ProxyConfig _proxyConfig;
         Poco::Timespan _keepAliveTimeout;
+        int _keepAliveCurrentRequest = 0;
+        int _keepAliveMaxRequests = 1000;
         Poco::Timestamp _lastRequest;
         bool _reconnect;
         bool _mustReconnect;
@@ -361,6 +378,7 @@ namespace Net
         Poco::SharedPtr<std::ostream> _pRequestStream;
         Poco::SharedPtr<std::istream> _pResponseStream;
 
+        static const double _defaultKeepAliveReliabilityLevel;
         static ProxyConfig _globalProxyConfig;
 
         HTTPClientSession(const HTTPClientSession &);
@@ -450,9 +468,19 @@ namespace Net
         return _lastRequest;
     }
 
-    inline void HTTPClientSession::setLastRequest(Poco::Timestamp time)
+    inline double HTTPClientSession::getKeepAliveReliability() const
     {
-        _lastRequest = time;
+        return _defaultKeepAliveReliabilityLevel;
+    }
+
+    inline int HTTPClientSession::getKeepAliveMaxRequests() const
+    {
+        return _keepAliveMaxRequests;
+    }
+
+    inline int HTTPClientSession::getKeepAliveRequest() const
+    {
+        return _keepAliveCurrentRequest;
     }
 
 }

base/poco/Net/include/Poco/Net/HTTPMessage.h

@@ -120,6 +120,10 @@ namespace Net
         /// The value is set to "Keep-Alive" if keepAlive is
         /// true, or to "Close" otherwise.
 
+        void setKeepAliveTimeout(int timeout, int max_requests);
+        int getKeepAliveTimeout() const;
+        int getKeepAliveMaxRequests() const;
+
         bool getKeepAlive() const;
         /// Returns true if
         ///   * the message has a Connection header field and its value is "Keep-Alive"

base/poco/Net/include/Poco/Net/HTTPServerParams.h

@@ -44,7 +44,7 @@ namespace Net
         ///   - timeout:              60 seconds
         ///   - keepAlive:            true
         ///   - maxKeepAliveRequests: 0
-        ///   - keepAliveTimeout:     10 seconds
+        ///   - keepAliveTimeout:     15 seconds
 
         void setServerName(const std::string & serverName);
         /// Sets the name and port (name:port) that the server uses to identify itself.

base/poco/Net/include/Poco/Net/HTTPServerSession.h

@@ -56,6 +56,8 @@ namespace Net
         SocketAddress serverAddress();
         /// Returns the server's address.
 
+        void setKeepAliveTimeout(Poco::Timespan keepAliveTimeout);
+
     private:
         bool _firstRequest;
         Poco::Timespan _keepAliveTimeout;

base/poco/Net/src/HTTPClientSession.cpp

@@ -37,6 +37,7 @@ namespace Net {
 
 
 HTTPClientSession::ProxyConfig HTTPClientSession::_globalProxyConfig;
+const double HTTPClientSession::_defaultKeepAliveReliabilityLevel = 0.9;
 
 
 HTTPClientSession::HTTPClientSession():
@@ -220,7 +221,41 @@ void HTTPClientSession::setGlobalProxyConfig(const ProxyConfig& config)
 
 void HTTPClientSession::setKeepAliveTimeout(const Poco::Timespan& timeout)
 {
-	_keepAliveTimeout = timeout;
+    if (connected())
+    {
+        throw Poco::IllegalStateException("cannot change keep alive timeout on initiated connection, "
+                                          "That value is managed privately after connection is established.");
+    }
+    _keepAliveTimeout = timeout;
+}
+
+
+void HTTPClientSession::setKeepAliveMaxRequests(int max_requests)
+{
+    if (connected())
+    {
+        throw Poco::IllegalStateException("cannot change keep alive max requests on initiated connection, "
+                                          "That value is managed privately after connection is established.");
+    }
+    _keepAliveMaxRequests = max_requests;
+}
+
+
+void HTTPClientSession::setKeepAliveRequest(int request)
+{
+    _keepAliveCurrentRequest = request;
+}
+
+
+
+void HTTPClientSession::setLastRequest(Poco::Timestamp time)
+{
+    if (connected())
+    {
+        throw Poco::IllegalStateException("cannot change last request on initiated connection, "
+                                          "That value is managed privately after connection is established.");
+    }
+    _lastRequest = time;
 }
 
 
@@ -231,6 +266,8 @@ std::ostream& HTTPClientSession::sendRequest(HTTPRequest& request)
 	clearException();
 	_responseReceived = false;
 
+    _keepAliveCurrentRequest += 1;
+
 	bool keepAlive = getKeepAlive();
 	if (((connected() && !keepAlive) || mustReconnect()) && !_host.empty())
 	{
@@ -241,8 +278,10 @@ std::ostream& HTTPClientSession::sendRequest(HTTPRequest& request)
 	{
 		if (!connected())
 			reconnect();
-		if (!keepAlive)
+        if (!request.has(HTTPMessage::CONNECTION))
-			request.setKeepAlive(false);
+            request.setKeepAlive(keepAlive);
+        if (keepAlive && !request.has(HTTPMessage::CONNECTION_KEEP_ALIVE) && _keepAliveTimeout.totalSeconds() > 0)
+            request.setKeepAliveTimeout(_keepAliveTimeout.totalSeconds(), _keepAliveMaxRequests);
 		if (!request.has(HTTPRequest::HOST) && !_host.empty())
 			request.setHost(_host, _port);
 		if (!_proxyConfig.host.empty() && !bypassProxy())
@@ -324,6 +363,17 @@ std::istream& HTTPClientSession::receiveResponse(HTTPResponse& response)
 
 	_mustReconnect = getKeepAlive() && !response.getKeepAlive();
 
+    if (!_mustReconnect)
+    {
+        /// when server sends its keep alive timeout, client has to follow that value
+        auto timeout = response.getKeepAliveTimeout();
+        if (timeout > 0)
+            _keepAliveTimeout = std::min(_keepAliveTimeout, Poco::Timespan(timeout, 0));
+        auto max_requests = response.getKeepAliveMaxRequests();
+        if (max_requests > 0)
+            _keepAliveMaxRequests = std::min(_keepAliveMaxRequests, max_requests);
+    }
+
 	if (!_expectResponseBody || response.getStatus() < 200 || response.getStatus() == HTTPResponse::HTTP_NO_CONTENT || response.getStatus() == HTTPResponse::HTTP_NOT_MODIFIED)
 		_pResponseStream = new HTTPFixedLengthInputStream(*this, 0);
 	else if (response.getChunkedTransferEncoding())
@@ -430,15 +480,18 @@ std::string HTTPClientSession::proxyRequestPrefix() const
 	return result;
 }
 
+bool HTTPClientSession::isKeepAliveExpired(double reliability) const
+{
+    Poco::Timestamp now;
+    return Timespan(Timestamp::TimeDiff(reliability *_keepAliveTimeout.totalMicroseconds())) <= now - _lastRequest
+            || _keepAliveCurrentRequest > _keepAliveMaxRequests;
+}
 
 bool HTTPClientSession::mustReconnect() const
 {
 	if (!_mustReconnect)
-	{
+        return isKeepAliveExpired(_defaultKeepAliveReliabilityLevel);
-		Poco::Timestamp now;
+    return true;
-		return _keepAliveTimeout <= now - _lastRequest;
-	}
-	else return true;
 }
 
 
@@ -511,14 +564,21 @@ void HTTPClientSession::assign(Poco::Net::HTTPClientSession & session)
     if (buffered())
         throw Poco::LogicException("assign to a session with not empty buffered data");
 
-    attachSocket(session.detachSocket());
+    poco_assert(!connected());
-    setLastRequest(session.getLastRequest());
+
     setResolvedHost(session.getResolvedHost());
-    setKeepAlive(session.getKeepAlive());
+    setProxyConfig(session.getProxyConfig());
 
     setTimeout(session.getConnectionTimeout(), session.getSendTimeout(), session.getReceiveTimeout());
+    setKeepAlive(session.getKeepAlive());
+
+    setLastRequest(session.getLastRequest());
     setKeepAliveTimeout(session.getKeepAliveTimeout());
-    setProxyConfig(session.getProxyConfig());
+
+    _keepAliveMaxRequests = session._keepAliveMaxRequests;
+    _keepAliveCurrentRequest = session._keepAliveCurrentRequest;
+
+    attachSocket(session.detachSocket());
 
     session.reset();
 }

base/poco/Net/src/HTTPMessage.cpp

@@ -17,6 +17,7 @@
 #include "Poco/NumberFormatter.h"
 #include "Poco/NumberParser.h"
 #include "Poco/String.h"
+#include <format>
 
 
 using Poco::NumberFormatter;
@@ -179,4 +180,51 @@ bool HTTPMessage::getKeepAlive() const
 }
 
 
+void HTTPMessage::setKeepAliveTimeout(int timeout, int max_requests)
+{
+    add(HTTPMessage::CONNECTION_KEEP_ALIVE, std::format("timeout={}, max={}", timeout, max_requests));
+}
+
+
+int parseFromHeaderValues(const std::string_view header_value, const std::string_view param_name)
+{
+    auto param_value_pos = header_value.find(param_name);
+    if (param_value_pos == std::string::npos)
+        param_value_pos = header_value.size();
+    if (param_value_pos != header_value.size())
+        param_value_pos += param_name.size();
+
+    auto param_value_end = header_value.find(',', param_value_pos);
+    if (param_value_end == std::string::npos)
+        param_value_end = header_value.size();
+
+    auto timeout_value_substr = header_value.substr(param_value_pos, param_value_end - param_value_pos);
+    if (timeout_value_substr.empty())
+        return -1;
+
+    int value = 0;
+    auto [ptr, ec] = std::from_chars(timeout_value_substr.begin(), timeout_value_substr.end(), value);
+
+    if (ec == std::errc())
+        return value;
+
+    return -1;
+}
+
+
+int HTTPMessage::getKeepAliveTimeout() const
+{
+    const std::string& ka_header = get(HTTPMessage::CONNECTION_KEEP_ALIVE, HTTPMessage::EMPTY);
+    static const std::string_view timeout_param = "timeout=";
+    return parseFromHeaderValues(ka_header, timeout_param);
+}
+
+
+int HTTPMessage::getKeepAliveMaxRequests() const
+{
+    const std::string& ka_header = get(HTTPMessage::CONNECTION_KEEP_ALIVE, HTTPMessage::EMPTY);
+    static const std::string_view timeout_param = "max=";
+    return parseFromHeaderValues(ka_header, timeout_param);
+}
+
 } } // namespace Poco::Net

base/poco/Net/src/HTTPServerConnection.cpp

@@ -88,7 +88,18 @@ void HTTPServerConnection::run()
 					
 						pHandler->handleRequest(request, response);
 						session.setKeepAlive(_pParams->getKeepAlive() && response.getKeepAlive() && session.canKeepAlive());
-					}
+
+                        /// all that fuzz is all about to make session close with less timeout than 15s (set in HTTPServerParams c-tor)
+                        if (_pParams->getKeepAlive() && response.getKeepAlive() && session.canKeepAlive())
+                        {
+                            int value = response.getKeepAliveTimeout();
+                            if (value < 0)
+                                value = request.getKeepAliveTimeout();
+                            if (value > 0)
+                                session.setKeepAliveTimeout(Poco::Timespan(value, 0));
+                        }
+
+                    }
 					else sendErrorResponse(session, HTTPResponse::HTTP_NOT_IMPLEMENTED);
 				}
 				catch (Poco::Exception&)

base/poco/Net/src/HTTPServerSession.cpp

@@ -33,6 +33,12 @@ HTTPServerSession::~HTTPServerSession()
 {
 }
 
+void HTTPServerSession::setKeepAliveTimeout(Poco::Timespan keepAliveTimeout)
+{
+    _keepAliveTimeout = keepAliveTimeout;
+}
+
+
 
 bool HTTPServerSession::hasMoreRequests()
 {

contrib/avro-cmake/CMakeLists.txt

@@ -59,12 +59,3 @@ target_link_libraries (_avrocpp PRIVATE boost::headers_only boost::iostreams)
 target_compile_definitions (_avrocpp PUBLIC SNAPPY_CODEC_AVAILABLE)
 target_include_directories (_avrocpp PRIVATE ${SNAPPY_INCLUDE_DIR})
 target_link_libraries (_avrocpp PRIVATE ch_contrib::snappy)
-
-# create a symlink to include headers with <avro/...>
-set(AVRO_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/include")
-ADD_CUSTOM_TARGET(avro_symlink_headers ALL
-    COMMAND ${CMAKE_COMMAND} -E make_directory "${AVRO_INCLUDE_DIR}"
-    COMMAND ${CMAKE_COMMAND} -E create_symlink "${AVROCPP_ROOT_DIR}/api" "${AVRO_INCLUDE_DIR}/avro"
-)
-add_dependencies(_avrocpp avro_symlink_headers)
-target_include_directories(_avrocpp SYSTEM BEFORE PUBLIC "${AVRO_INCLUDE_DIR}")

contrib/libssh-cmake/CMakeLists.txt

@@ -1,26 +1,18 @@
-option (ENABLE_SSH "Enable support for SSH keys and protocol" ${ENABLE_LIBRARIES})
+option (ENABLE_SSH "Enable support for libssh" ${ENABLE_LIBRARIES})
 
 if (NOT ENABLE_SSH)
-    message(STATUS "Not using SSH")
+    message(STATUS "Not using libssh")
     return()
 endif()
 
+# CMake variables needed by libssh_version.h.cmake, update them when you update libssh
+set(libssh_VERSION_MAJOR 0)
+set(libssh_VERSION_MINOR 9)
+set(libssh_VERSION_PATCH 8)
+
 set(LIB_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libssh")
 set(LIB_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/libssh")
 
-# Set CMake variables which are used in libssh_version.h.cmake
-project(libssh VERSION 0.9.8 LANGUAGES C)
-
-set(LIBRARY_VERSION "4.8.8")
-set(LIBRARY_SOVERSION "4")
-
-set(CMAKE_THREAD_PREFER_PTHREADS ON)
-set(THREADS_PREFER_PTHREAD_FLAG ON)
-
-set(WITH_ZLIB OFF)
-set(WITH_SYMBOL_VERSIONING OFF)
-set(WITH_SERVER ON)
-
 set(libssh_SRCS
     ${LIB_SOURCE_DIR}/src/agent.c
     ${LIB_SOURCE_DIR}/src/auth.c
@@ -28,15 +20,21 @@ set(libssh_SRCS
     ${LIB_SOURCE_DIR}/src/bignum.c
     ${LIB_SOURCE_DIR}/src/buffer.c
     ${LIB_SOURCE_DIR}/src/callbacks.c
+    ${LIB_SOURCE_DIR}/src/chachapoly.c
     ${LIB_SOURCE_DIR}/src/channels.c
     ${LIB_SOURCE_DIR}/src/client.c
     ${LIB_SOURCE_DIR}/src/config.c
+    ${LIB_SOURCE_DIR}/src/config_parser.c
     ${LIB_SOURCE_DIR}/src/connect.c
     ${LIB_SOURCE_DIR}/src/connector.c
     ${LIB_SOURCE_DIR}/src/curve25519.c
     ${LIB_SOURCE_DIR}/src/dh.c
     ${LIB_SOURCE_DIR}/src/ecdh.c
     ${LIB_SOURCE_DIR}/src/error.c
+    ${LIB_SOURCE_DIR}/src/external/bcrypt_pbkdf.c
+    ${LIB_SOURCE_DIR}/src/external/blowfish.c
+    ${LIB_SOURCE_DIR}/src/external/chacha.c
+    ${LIB_SOURCE_DIR}/src/external/poly1305.c
     ${LIB_SOURCE_DIR}/src/getpass.c
     ${LIB_SOURCE_DIR}/src/init.c
     ${LIB_SOURCE_DIR}/src/kdf.c
@@ -55,37 +53,32 @@ set(libssh_SRCS
     ${LIB_SOURCE_DIR}/src/pcap.c
     ${LIB_SOURCE_DIR}/src/pki.c
     ${LIB_SOURCE_DIR}/src/pki_container_openssh.c
+    ${LIB_SOURCE_DIR}/src/pki_ed25519_common.c
     ${LIB_SOURCE_DIR}/src/poll.c
-    ${LIB_SOURCE_DIR}/src/session.c
     ${LIB_SOURCE_DIR}/src/scp.c
+    ${LIB_SOURCE_DIR}/src/session.c
     ${LIB_SOURCE_DIR}/src/socket.c
     ${LIB_SOURCE_DIR}/src/string.c
     ${LIB_SOURCE_DIR}/src/threads.c
-    ${LIB_SOURCE_DIR}/src/wrapper.c
-    ${LIB_SOURCE_DIR}/src/external/bcrypt_pbkdf.c
-    ${LIB_SOURCE_DIR}/src/external/blowfish.c
-    ${LIB_SOURCE_DIR}/src/external/chacha.c
-    ${LIB_SOURCE_DIR}/src/external/poly1305.c
-    ${LIB_SOURCE_DIR}/src/chachapoly.c
-    ${LIB_SOURCE_DIR}/src/config_parser.c
     ${LIB_SOURCE_DIR}/src/token.c
-    ${LIB_SOURCE_DIR}/src/pki_ed25519_common.c
+    ${LIB_SOURCE_DIR}/src/wrapper.c
+    # some files of libssh/src/ are missing - why?
 
     ${LIB_SOURCE_DIR}/src/threads/noop.c
     ${LIB_SOURCE_DIR}/src/threads/pthread.c
+    # files missing - why?
 
     # LIBCRYPT specific
-    ${libssh_SRCS}
+    ${LIB_SOURCE_DIR}/src/dh_crypto.c
-    ${LIB_SOURCE_DIR}/src/threads/libcrypto.c
-    ${LIB_SOURCE_DIR}/src/pki_crypto.c
     ${LIB_SOURCE_DIR}/src/ecdh_crypto.c
     ${LIB_SOURCE_DIR}/src/libcrypto.c
-    ${LIB_SOURCE_DIR}/src/dh_crypto.c
+    ${LIB_SOURCE_DIR}/src/pki_crypto.c
+    ${LIB_SOURCE_DIR}/src/threads/libcrypto.c
 
-    ${LIB_SOURCE_DIR}/src/options.c
-    ${LIB_SOURCE_DIR}/src/server.c
     ${LIB_SOURCE_DIR}/src/bind.c
     ${LIB_SOURCE_DIR}/src/bind_config.c
+    ${LIB_SOURCE_DIR}/src/options.c
+    ${LIB_SOURCE_DIR}/src/server.c
 )
 
 if (NOT (ENABLE_OPENSSL OR ENABLE_OPENSSL_DYNAMIC))
@@ -94,7 +87,7 @@ endif()
 
 configure_file(${LIB_SOURCE_DIR}/include/libssh/libssh_version.h.cmake ${LIB_BINARY_DIR}/include/libssh/libssh_version.h @ONLY)
 
-add_library(_ssh STATIC ${libssh_SRCS})
+add_library(_ssh ${libssh_SRCS})
 add_library(ch_contrib::ssh ALIAS _ssh)
 
 target_link_libraries(_ssh PRIVATE OpenSSL::Crypto)

docker/test/stateless/run.sh

@@ -43,6 +43,8 @@ source /utils.lib
 
 if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
     echo "Azure is disabled"
+elif [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
+    echo "Azure is disabled"
 else
     azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log &
 fi
@@ -139,6 +141,32 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
     MAX_RUN_TIME=$((MAX_RUN_TIME != 0 ? MAX_RUN_TIME : 9000))    # set to 2.5 hours if 0 (unlimited)
 fi
 
+if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
+    sudo cat /etc/clickhouse-server1/config.d/filesystem_caches_path.xml \
+    | sed "s|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches/</filesystem_caches_path>|<filesystem_caches_path>/var/lib/clickhouse/filesystem_caches_1/</filesystem_caches_path>|" \
+    > /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp
+    mv /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp /etc/clickhouse-server1/config.d/filesystem_caches_path.xml
+
+    sudo cat /etc/clickhouse-server1/config.d/filesystem_caches_path.xml \
+    | sed "s|<custom_cached_disks_base_directory replace=\"replace\">/var/lib/clickhouse/filesystem_caches/</custom_cached_disks_base_directory>|<custom_cached_disks_base_directory replace=\"replace\">/var/lib/clickhouse/filesystem_caches_1/</custom_cached_disks_base_directory>|" \
+    > /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp
+    mv /etc/clickhouse-server1/config.d/filesystem_caches_path.xml.tmp /etc/clickhouse-server1/config.d/filesystem_caches_path.xml
+
+    mkdir -p /var/run/clickhouse-server1
+    sudo chown clickhouse:clickhouse /var/run/clickhouse-server1
+    sudo -E -u clickhouse /usr/bin/clickhouse server --config /etc/clickhouse-server1/config.xml --daemon \
+    --pid-file /var/run/clickhouse-server1/clickhouse-server.pid \
+    -- --path /var/lib/clickhouse1/ --logger.stderr /var/log/clickhouse-server/stderr1.log \
+    --logger.log /var/log/clickhouse-server/clickhouse-server1.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server1.err.log \
+    --tcp_port 19000 --tcp_port_secure 19440 --http_port 18123 --https_port 18443 --interserver_http_port 19009 --tcp_with_proxy_port 19010 \
+    --mysql_port 19004 --postgresql_port 19005 \
+    --keeper_server.tcp_port 19181 --keeper_server.server_id 2 \
+    --prometheus.port 19988 \
+    --macros.replica r2   # It doesn't work :(
+
+    MAX_RUN_TIME=$((MAX_RUN_TIME < 9000 ? MAX_RUN_TIME : 9000))  # min(MAX_RUN_TIME, 2.5 hours)
+    MAX_RUN_TIME=$((MAX_RUN_TIME != 0 ? MAX_RUN_TIME : 9000))    # set to 2.5 hours if 0 (unlimited)
+fi
 
 # Wait for the server to start, but not for too long.
 for _ in {1..100}
@@ -185,6 +213,10 @@ function run_tests()
         ADDITIONAL_OPTIONS+=('--s3-storage')
     fi
 
+    if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
+        ADDITIONAL_OPTIONS+=('--shared-catalog')
+    fi
+
     if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
         ADDITIONAL_OPTIONS+=('--replicated-database')
         # Too many tests fail for DatabaseReplicated in parallel.
@@ -266,6 +298,12 @@ do
         echo "$err"
         [[ "0" != "${#err}"  ]] && failed_to_save_logs=1
     fi
+
+    if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
+        err=$( { clickhouse-client --port 19000 -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst; } 2>&1 )
+        echo "$err"
+        [[ "0" != "${#err}"  ]] && failed_to_save_logs=1
+    fi
 done
 
 # Stop server so we can safely read data with clickhouse-local.
@@ -277,6 +315,10 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
     sudo clickhouse stop --pid-path /var/run/clickhouse-server2 ||:
 fi
 
+if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
+    sudo clickhouse stop --pid-path /var/run/clickhouse-server1 ||:
+fi
+
 rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server.log ||:
 rg -A50 -Fa "============" /var/log/clickhouse-server/stderr.log ||:
 zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.zst &
@@ -304,6 +346,10 @@ if [ $failed_to_save_logs -ne 0 ]; then
             clickhouse-local --path /var/lib/clickhouse1/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst ||:
             clickhouse-local --path /var/lib/clickhouse2/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.2.tsv.zst ||:
         fi
+
+        if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
+            clickhouse-local --path /var/lib/clickhouse1/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst ||:
+        fi
     done
 fi
 
@@ -343,3 +389,10 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]
     tar -chf /test_output/coordination1.tar /var/lib/clickhouse1/coordination ||:
     tar -chf /test_output/coordination2.tar /var/lib/clickhouse2/coordination ||:
 fi
+
+if [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
+    rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server1.log ||:
+    zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server1.log > /test_output/clickhouse-server1.log.zst ||:
+    mv /var/log/clickhouse-server/stderr1.log /test_output/ ||:
+    tar -chf /test_output/coordination1.tar /var/lib/clickhouse1/coordination ||:
+fi

docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md

@@ -68,6 +68,12 @@ In the results of `SELECT` query, the values of `AggregateFunction` type have im
 
 ## Example of an Aggregated Materialized View {#example-of-an-aggregated-materialized-view}
 
+The following examples assumes that you have a database named `test` so make sure you create that if it doesn't already exist:
+
+```sql
+CREATE DATABASE test;
+```
+
 We will create the table `test.visits` that contain the raw data:
 
 ``` sql
@@ -80,17 +86,24 @@ CREATE TABLE test.visits
 ) ENGINE = MergeTree ORDER BY (StartDate, CounterID);
 ```
 
+Next, we need to create an `AggregatingMergeTree` table that will store `AggregationFunction`s that keep track of the total number of visits and the number of unique users. 
+
 `AggregatingMergeTree` materialized view that watches the `test.visits` table, and use the `AggregateFunction` type:
 
 ``` sql
-CREATE MATERIALIZED VIEW test.mv_visits
+CREATE TABLE test.agg_visits (
-(
     StartDate DateTime64 NOT NULL,
     CounterID UInt64,
     Visits AggregateFunction(sum, Nullable(Int32)),
     Users AggregateFunction(uniq, Nullable(Int32))
 )
-ENGINE = AggregatingMergeTree() ORDER BY (StartDate, CounterID)
+ENGINE = AggregatingMergeTree() ORDER BY (StartDate, CounterID);
+```
+
+And then let's create a materialized view that populates `test.agg_visits` from `test.visits` :
+
+```sql
+CREATE MATERIALIZED VIEW test.visits_mv TO test.agg_visits
 AS SELECT
     StartDate,
     CounterID,
@@ -104,25 +117,45 @@ Inserting data into the `test.visits` table.
 
 ``` sql
 INSERT INTO test.visits (StartDate, CounterID, Sign, UserID)
- VALUES (1667446031, 1, 3, 4)
+ VALUES (1667446031000, 1, 3, 4), (1667446031000, 1, 6, 3);
-INSERT INTO test.visits (StartDate, CounterID, Sign, UserID)
- VALUES (1667446031, 1, 6, 3)
 ```
 
-The data is inserted in both the table and the materialized view `test.mv_visits`.
+The data is inserted in both `test.visits` and `test.agg_visits`.
 
 To get the aggregated data, we need to execute a query such as `SELECT ... GROUP BY ...` from the materialized view `test.mv_visits`:
 
-``` sql
+```sql
 SELECT
     StartDate,
     sumMerge(Visits) AS Visits,
     uniqMerge(Users) AS Users
-FROM test.mv_visits
+FROM test.agg_visits
 GROUP BY StartDate
 ORDER BY StartDate;
 ```
 
+```text
+┌───────────────StartDate─┬─Visits─┬─Users─┐
+│ 2022-11-03 03:27:11.000 │      9 │     2 │
+└─────────────────────────┴────────┴───────┘
+```
+
+And how about if we add another couple of records to `test.visits`, but this time we'll use a different timestamp for one of the records:
+
+```sql
+INSERT INTO test.visits (StartDate, CounterID, Sign, UserID)
+ VALUES (1669446031000, 2, 5, 10), (1667446031000, 3, 7, 5);
+```
+
+If we then run the `SELECT` query again, we'll see the following output:
+
+```text
+┌───────────────StartDate─┬─Visits─┬─Users─┐
+│ 2022-11-03 03:27:11.000 │     16 │     3 │
+│ 2022-11-26 07:00:31.000 │      5 │     1 │
+└─────────────────────────┴────────┴───────┘
+```
+
 ## Related Content
 
 - Blog: [Using Aggregate Combinators in ClickHouse](https://clickhouse.com/blog/aggregate-functions-combinators-in-clickhouse-for-arrays-maps-and-states)

docs/en/sql-reference/aggregate-functions/reference/uniqcombined.md

@@ -15,9 +15,9 @@ The `uniqCombined` function is a good choice for calculating the number of diffe
 
 **Arguments**
 
-The function takes a variable number of parameters. Parameters can be `Tuple`, `Array`, `Date`, `DateTime`, `String`, or numeric types.
+- `HLL_precision`: The base-2 logarithm of the number of cells in [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog). Optional, you can use the function as `uniqCombined(x[, ...])`. The default value for `HLL_precision` is 17, which is effectively 96 KiB of space (2^17 cells, 6 bits each).
+- `X`: A variable number of parameters. Parameters can be `Tuple`, `Array`, `Date`, `DateTime`, `String`, or numeric types.
 
-`HLL_precision` is the base-2 logarithm of the number of cells in [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog). Optional, you can use the function as `uniqCombined(x[, ...])`. The default value for `HLL_precision` is 17, which is effectively 96 KiB of space (2^17 cells, 6 bits each).
 
 **Returned value**
 
@@ -25,26 +25,43 @@ The function takes a variable number of parameters. Parameters can be `Tuple`, `
 
 **Implementation details**
 
-Function:
+The `uniqCombined` function:
 
 - Calculates a hash (64-bit hash for `String` and 32-bit otherwise) for all parameters in the aggregate, then uses it in calculations.
-
 - Uses a combination of three algorithms: array, hash table, and HyperLogLog with an error correction table.
-
+    - For a small number of distinct elements, an array is used. 
-        For a small number of distinct elements, an array is used. When the set size is larger, a hash table is used. For a larger number of elements, HyperLogLog is used, which will occupy a fixed amount of memory.
+    - When the set size is larger, a hash table is used. 
-
+    - For a larger number of elements, HyperLogLog is used, which will occupy a fixed amount of memory.
 - Provides the result deterministically (it does not depend on the query processing order).
 
 :::note    
-Since it uses 32-bit hash for non-`String` type, the result will have very high error for cardinalities significantly larger than `UINT_MAX` (error will raise quickly after a few tens of billions of distinct values), hence in this case you should use [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md#agg_function-uniqcombined64)
+Since it uses a 32-bit hash for non-`String` types, the result will have very high error for cardinalities significantly larger than `UINT_MAX` (error will raise quickly after a few tens of billions of distinct values), hence in this case you should use [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md#agg_function-uniqcombined64).
 :::
 
-Compared to the [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq) function, the `uniqCombined`:
+Compared to the [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq) function, the `uniqCombined` function:
 
 - Consumes several times less memory.
 - Calculates with several times higher accuracy.
 - Usually has slightly lower performance. In some scenarios, `uniqCombined` can perform better than `uniq`, for example, with distributed queries that transmit a large number of aggregation states over the network.
 
+**Example**
+
+Query:
+
+```sql
+SELECT uniqCombined(number) FROM numbers(1e6);
+```
+
+Result:
+
+```response
+┌─uniqCombined(number)─┐
+│              1001148 │ -- 1.00 million
+└──────────────────────┘
+```
+
+See the example section of [uniqCombined64](../../../sql-reference/aggregate-functions/reference/uniqcombined64.md#agg_function-uniqcombined64) for an example of the difference between `uniqCombined` and `uniqCombined64` for much larger inputs.
+
 **See Also**
 
 - [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq)

docs/en/sql-reference/aggregate-functions/reference/uniqcombined64.md

@@ -5,4 +5,78 @@ sidebar_position: 193
 
 # uniqCombined64
 
-Same as [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniqcombined.md#agg_function-uniqcombined), but uses 64-bit hash for all data types.
+Calculates the approximate number of different argument values. It is the same as [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniqcombined.md#agg_function-uniqcombined), but uses a 64-bit hash for all data types rather than just for the String data type.
+
+``` sql
+uniqCombined64(HLL_precision)(x[, ...])
+```
+
+**Parameters**
+
+- `HLL_precision`: The base-2 logarithm of the number of cells in [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog). Optionally, you can use the function as `uniqCombined64(x[, ...])`. The default value for `HLL_precision` is 17, which is effectively 96 KiB of space (2^17 cells, 6 bits each).
+- `X`: A variable number of parameters. Parameters can be `Tuple`, `Array`, `Date`, `DateTime`, `String`, or numeric types.
+
+**Returned value**
+
+- A number [UInt64](../../../sql-reference/data-types/int-uint.md)-type number.
+
+**Implementation details**
+
+The `uniqCombined64` function:
+- Calculates a hash (64-bit hash for all data types) for all parameters in the aggregate, then uses it in calculations.
+- Uses a combination of three algorithms: array, hash table, and HyperLogLog with an error correction table.
+    - For a small number of distinct elements, an array is used. 
+    - When the set size is larger, a hash table is used. 
+    - For a larger number of elements, HyperLogLog is used, which will occupy a fixed amount of memory.
+- Provides the result deterministically (it does not depend on the query processing order).
+
+:::note
+Since it uses 64-bit hash for all types, the result does not suffer from very high error for cardinalities significantly larger than `UINT_MAX` like [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniqcombined.md) does, which uses a 32-bit hash for non-`String` types.
+:::
+
+Compared to the [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq) function, the `uniqCombined64` function:
+
+- Consumes several times less memory.
+- Calculates with several times higher accuracy.
+
+**Example**
+
+In the example below `uniqCombined64` is run on `1e10` different numbers returning a very close approximation of the number of different argument values. 
+
+Query:
+
+```sql
+SELECT uniqCombined64(number) FROM numbers(1e10);
+```
+
+Result:
+
+```response
+┌─uniqCombined64(number)─┐
+│             9998568925 │ -- 10.00 billion
+└────────────────────────┘
+```
+
+By comparison the `uniqCombined` function returns a rather poor approximation for an input this size.
+
+Query:
+
+```sql
+SELECT uniqCombined(number) FROM numbers(1e10);
+```
+
+Result:
+
+```response
+┌─uniqCombined(number)─┐
+│           5545308725 │ -- 5.55 billion
+└──────────────────────┘
+```
+
+**See Also**
+
+- [uniq](../../../sql-reference/aggregate-functions/reference/uniq.md#agg_function-uniq)
+- [uniqCombined](../../../sql-reference/aggregate-functions/reference/uniqcombined.md)
+- [uniqHLL12](../../../sql-reference/aggregate-functions/reference/uniqhll12.md#agg_function-uniqhll12)
+- [uniqExact](../../../sql-reference/aggregate-functions/reference/uniqexact.md#agg_function-uniqexact)
+- [uniqTheta](../../../sql-reference/aggregate-functions/reference/uniqthetasketch.md#agg_function-uniqthetasketch)

docs/en/sql-reference/data-types/aggregatefunction.md

@@ -1,6 +1,6 @@
 ---
 slug: /en/sql-reference/data-types/aggregatefunction
-sidebar_position: 53
+sidebar_position: 46
 sidebar_label: AggregateFunction
 ---
 

docs/en/sql-reference/data-types/array.md

@@ -1,6 +1,6 @@
 ---
 slug: /en/sql-reference/data-types/array
-sidebar_position: 52
+sidebar_position: 32
 sidebar_label: Array(T)
 ---
 

docs/en/sql-reference/data-types/boolean.md

@@ -1,6 +1,6 @@
 ---
 slug: /en/sql-reference/data-types/boolean
-sidebar_position: 43
+sidebar_position: 22
 sidebar_label: Boolean
 ---
 

docs/en/sql-reference/data-types/date.md

@@ -1,6 +1,6 @@
 ---
 slug: /en/sql-reference/data-types/date
-sidebar_position: 47
+sidebar_position: 12
 sidebar_label: Date
 ---
 

docs/en/sql-reference/data-types/date32.md

@@ -1,6 +1,6 @@
 ---
 slug: /en/sql-reference/data-types/date32
-sidebar_position: 48
+sidebar_position: 14
 sidebar_label: Date32
 ---
 

docs/en/sql-reference/data-types/datetime.md

@@ -1,6 +1,6 @@
 ---
 slug: /en/sql-reference/data-types/datetime
-sidebar_position: 48
+sidebar_position: 16
 sidebar_label: DateTime
 ---
 

docs/en/sql-reference/data-types/datetime64.md

@@ -1,6 +1,6 @@
 ---
 slug: /en/sql-reference/data-types/datetime64
-sidebar_position: 49
+sidebar_position: 18
 sidebar_label: DateTime64
 ---
 

docs/en/sql-reference/data-types/decimal.md

@@ -1,6 +1,6 @@
 ---
 slug: /en/sql-reference/data-types/decimal
-sidebar_position: 42
+sidebar_position: 6
 sidebar_label: Decimal
 ---
 

docs/en/sql-reference/data-types/enum.md

@@ -1,6 +1,6 @@
 ---
 slug: /en/sql-reference/data-types/enum
-sidebar_position: 50
+sidebar_position: 20
 sidebar_label: Enum
 ---
 

docs/en/sql-reference/data-types/fixedstring.md

@@ -1,10 +1,10 @@
 ---
 slug: /en/sql-reference/data-types/fixedstring
-sidebar_position: 45
+sidebar_position: 10
 sidebar_label: FixedString(N)
 ---
 
-# FixedString
+# FixedString(N)
 
 A fixed-length string of `N` bytes (neither characters nor code points).
 

docs/en/sql-reference/data-types/float.md

@@ -1,6 +1,6 @@
 ---
 slug: /en/sql-reference/data-types/float
-sidebar_position: 41
+sidebar_position: 4
 sidebar_label: Float32, Float64
 ---
 

docs/en/sql-reference/data-types/geo.md

@@ -1,8 +1,8 @@
 ---
 slug: /en/sql-reference/data-types/geo
-sidebar_position: 62
+sidebar_position: 54
 sidebar_label: Geo
-title: "Geo Data Types"
+title: "Geometric"
 ---
 
 ClickHouse supports data types for representing geographical objects — locations, lands, etc.

docs/en/sql-reference/data-types/index.md

@@ -1,10 +1,10 @@
 ---
 slug: /en/sql-reference/data-types/
 sidebar_label: List of data types
-sidebar_position: 37
+sidebar_position: 1
 ---
 
-# ClickHouse Data Types
+# Data Types in ClickHouse
 
 ClickHouse can store various kinds of data in table cells. This section describes the supported data types and special considerations for using and/or implementing them if any.
 

docs/en/sql-reference/data-types/int-uint.md

@@ -1,6 +1,6 @@
 ---
 slug: /en/sql-reference/data-types/int-uint
-sidebar_position: 40
+sidebar_position: 2
 sidebar_label: UInt8, UInt16, UInt32, UInt64, UInt128, UInt256, Int8, Int16, Int32, Int64, Int128, Int256
 ---
 

docs/en/sql-reference/data-types/ipv4.md

@@ -1,6 +1,6 @@
 ---
 slug: /en/sql-reference/data-types/ipv4
-sidebar_position: 59
+sidebar_position: 28
 sidebar_label: IPv4
 ---
 

docs/en/sql-reference/data-types/ipv6.md

@@ -1,6 +1,6 @@
 ---
 slug: /en/sql-reference/data-types/ipv6
-sidebar_position: 60
+sidebar_position: 30
 sidebar_label: IPv6
 ---
 

docs/en/sql-reference/data-types/json.md

@@ -1,6 +1,6 @@
 ---
 slug: /en/sql-reference/data-types/json
-sidebar_position: 54
+sidebar_position: 26
 sidebar_label: JSON
 ---
 

docs/en/sql-reference/data-types/lowcardinality.md

@@ -1,10 +1,10 @@
 ---
 slug: /en/sql-reference/data-types/lowcardinality
-sidebar_position: 51
+sidebar_position: 42
-sidebar_label: LowCardinality
+sidebar_label: LowCardinality(T)
 ---
 
-# LowCardinality
+# LowCardinality(T)
 
 Changes the internal representation of other data types to be dictionary-encoded.
 

docs/en/sql-reference/data-types/map.md

@@ -1,12 +1,12 @@
 ---
 slug: /en/sql-reference/data-types/map
-sidebar_position: 65
+sidebar_position: 36
-sidebar_label: Map(key, value)
+sidebar_label: Map(K, V)
 ---
 
-# Map(key, value)
+# Map(K, V)
 
-`Map(key, value)` data type stores `key:value` pairs.
+`Map(K, V)` data type stores `key:value` pairs.
 
 **Parameters**
 

docs/en/sql-reference/data-types/multiword-types.md

@@ -1,27 +0,0 @@
----
-slug: /en/sql-reference/data-types/multiword-types
-sidebar_position: 61
-sidebar_label: Multiword Type Names
-title: "Multiword Types"
----
-
-When creating tables, you can use data types with a name consisting of several words. This is implemented for better SQL compatibility.
-
-## Multiword Types Support
-
-| Multiword types                  | Simple types                                                 |
-|----------------------------------|--------------------------------------------------------------|
-| DOUBLE PRECISION                 | [Float64](../../sql-reference/data-types/float.md)           |
-| CHAR LARGE OBJECT                | [String](../../sql-reference/data-types/string.md)           |
-| CHAR VARYING                     | [String](../../sql-reference/data-types/string.md)           |
-| CHARACTER LARGE OBJECT           | [String](../../sql-reference/data-types/string.md)           |
-| CHARACTER VARYING                | [String](../../sql-reference/data-types/string.md)           |
-| NCHAR LARGE OBJECT               | [String](../../sql-reference/data-types/string.md)           |
-| NCHAR VARYING                    | [String](../../sql-reference/data-types/string.md)           |
-| NATIONAL CHARACTER LARGE OBJECT  | [String](../../sql-reference/data-types/string.md)           |
-| NATIONAL CHARACTER VARYING       | [String](../../sql-reference/data-types/string.md)           |
-| NATIONAL CHAR VARYING            | [String](../../sql-reference/data-types/string.md)           |
-| NATIONAL CHARACTER               | [String](../../sql-reference/data-types/string.md)           |
-| NATIONAL CHAR                    | [String](../../sql-reference/data-types/string.md)           |
-| BINARY LARGE OBJECT              | [String](../../sql-reference/data-types/string.md)           |
-| BINARY VARYING                   | [String](../../sql-reference/data-types/string.md)           |

docs/en/sql-reference/data-types/nullable.md

@@ -1,7 +1,7 @@
 ---
 slug: /en/sql-reference/data-types/nullable
-sidebar_position: 55
+sidebar_position: 44
-sidebar_label: Nullable
+sidebar_label: Nullable(T)
 ---
 
 # Nullable(T)

docs/en/sql-reference/data-types/simpleaggregatefunction.md

@@ -1,5 +1,7 @@
 ---
 slug: /en/sql-reference/data-types/simpleaggregatefunction
+sidebar_position: 48
+sidebar_label: SimpleAggregateFunction
 ---
 # SimpleAggregateFunction
 

docs/en/sql-reference/data-types/string.md

@@ -1,6 +1,6 @@
 ---
 slug: /en/sql-reference/data-types/string
-sidebar_position: 44
+sidebar_position: 8
 sidebar_label: String
 ---
 
@@ -13,7 +13,7 @@ When creating tables, numeric parameters for string fields can be set (e.g. `VAR
 
 Aliases:
 
-- `String` — `LONGTEXT`, `MEDIUMTEXT`, `TINYTEXT`, `TEXT`, `LONGBLOB`, `MEDIUMBLOB`, `TINYBLOB`, `BLOB`, `VARCHAR`, `CHAR`.
+- `String` — `LONGTEXT`, `MEDIUMTEXT`, `TINYTEXT`, `TEXT`, `LONGBLOB`, `MEDIUMBLOB`, `TINYBLOB`, `BLOB`, `VARCHAR`, `CHAR`, `CHAR LARGE OBJECT`, `CHAR VARYING`, `CHARACTER LARGE OBJECT`, `CHARACTER VARYING`, `NCHAR LARGE OBJECT`, `NCHAR VARYING`, `NATIONAL CHARACTER LARGE OBJECT`, `NATIONAL CHARACTER VARYING`, `NATIONAL CHAR VARYING`, `NATIONAL CHARACTER`, `NATIONAL CHAR`, `BINARY LARGE OBJECT`, `BINARY VARYING`,
 
 ## Encodings
 

docs/en/sql-reference/data-types/tuple.md

@@ -1,10 +1,10 @@
 ---
 slug: /en/sql-reference/data-types/tuple
-sidebar_position: 54
+sidebar_position: 34
 sidebar_label: Tuple(T1, T2, ...)
 ---
 
-# Tuple(T1, T2, …)
+# Tuple(T1, T2, ...)
 
 A tuple of elements, each having an individual [type](../../sql-reference/data-types/index.md#data_types). Tuple must contain at least one element.
 

docs/en/sql-reference/data-types/uuid.md

@@ -1,6 +1,6 @@
 ---
 slug: /en/sql-reference/data-types/uuid
-sidebar_position: 46
+sidebar_position: 24
 sidebar_label: UUID
 ---
 

docs/en/sql-reference/data-types/variant.md

@@ -1,10 +1,10 @@
 ---
 slug: /en/sql-reference/data-types/variant
-sidebar_position: 55
+sidebar_position: 40
-sidebar_label: Variant
+sidebar_label: Variant(T1, T2, ...)
 ---
 
-# Variant(T1, T2, T3, ...)
+# Variant(T1, T2, ...)
 
 This type represents a union of other data types. Type `Variant(T1, T2, ..., TN)` means that each row of this type 
 has a value of either type `T1` or `T2` or ... or `TN` or none of them (`NULL` value).

docs/en/sql-reference/functions/distance-functions.md

@@ -81,6 +81,43 @@ Result:
 │ 2.23606797749979 │
 └──────────────────┘
 ```
+## L2SquaredNorm
+
+Calculates the square root of the sum of the squares of the vector values (the [L2Norm](#l2norm)) squared.
+
+**Syntax**
+
+```sql
+L2SquaredNorm(vector)
+```
+
+Alias: `normL2Squared`.
+
+***Arguments**
+
+- `vector` — [Tuple](../../sql-reference/data-types/tuple.md) or [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+- L2-norm squared.
+
+Type: [Float](../../sql-reference/data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT L2SquaredNorm((1, 2));
+```
+
+Result:
+
+```text
+┌─L2SquaredNorm((1, 2))─┐
+│                     5 │
+└───────────────────────┘
+```
 
 ## LinfNorm
 

docs/en/sql-reference/functions/hash-functions.md

@@ -594,6 +594,45 @@ Calculates JumpConsistentHash form a UInt64.
 Accepts two arguments: a UInt64-type key and the number of buckets. Returns Int32.
 For more information, see the link: [JumpConsistentHash](https://arxiv.org/pdf/1406.2294.pdf)
 
+## kostikConsistentHash
+
+An O(1) time and space consistent hash algorithm by Konstantin 'kostik' Oblakov. Previously `yandexConsistentHash`. 
+
+**Syntax**
+
+```sql
+kostikConsistentHash(input, n)
+```
+
+Alias: `yandexConsistentHash` (left for backwards compatibility sake).
+
+**Parameters**
+
+- `input`: A UInt64-type key [UInt64](/docs/en/sql-reference/data-types/int-uint.md).
+- `n`: Number of buckets. [UInt16](/docs/en/sql-reference/data-types/int-uint.md).
+
+**Returned value**
+
+- A [UInt16](/docs/en/sql-reference/data-types/int-uint.md) data type hash value.
+
+**Implementation details**
+
+It is efficient only if n <= 32768. 
+
+**Example**
+
+Query:
+
+```sql
+SELECT kostikConsistentHash(16045690984833335023, 2);
+```
+
+```response
+┌─kostikConsistentHash(16045690984833335023, 2)─┐
+│                                             1 │
+└───────────────────────────────────────────────┘
+```
+
 ## murmurHash2_32, murmurHash2_64
 
 Produces a [MurmurHash2](https://github.com/aappleby/smhasher) hash value.
@@ -1153,6 +1192,42 @@ Result:
 └────────────┘
 ```
 
+## wyHash64
+
+Produces a 64-bit [wyHash64](https://github.com/wangyi-fudan/wyhash) hash value.
+
+**Syntax**
+
+```sql
+wyHash64(string)
+```
+
+**Arguments**
+
+- `string` — String. [String](/docs/en/sql-reference/data-types/string.md).
+
+**Returned value**
+
+- Hash value.
+
+Type: [UInt64](/docs/en/sql-reference/data-types/int-uint.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT wyHash64('ClickHouse') AS Hash;
+```
+
+Result:
+
+```response
+┌─────────────────Hash─┐
+│ 12336419557878201794 │
+└──────────────────────┘
+```
+
 ## ngramMinHash
 
 Splits a ASCII string into n-grams of `ngramsize` symbols and calculates hash values for each n-gram. Uses `hashnum` minimum hashes to calculate the minimum hash and `hashnum` maximum hashes to calculate the maximum hash. Returns a tuple with these hashes. Is case sensitive.

docs/en/sql-reference/functions/string-search-functions.md

@@ -6,14 +6,17 @@ sidebar_label: Searching in Strings
 
 # Functions for Searching in Strings
 
-All functions in this section search by default case-sensitively. Case-insensitive search is usually provided by separate function variants.
+All functions in this section search case-sensitively by default. Case-insensitive search is usually provided by separate function variants.
-Note that case-insensitive search follows the lowercase-uppercase rules of the English language. E.g. Uppercased `i` in English language is
-`I` whereas in Turkish language it is `İ` - results for languages other than English may be unexpected.
 
-Functions in this section also assume that the searched string and the search string are single-byte encoded text. If this assumption is
+:::note
+Case-insensitive search follows the lowercase-uppercase rules of the English language. E.g. Uppercased `i` in the English language is
+`I` whereas in the Turkish language it is `İ` - results for languages other than English may be unexpected.
+:::
+
+Functions in this section also assume that the searched string (referred to in this section as `haystack`) and the search string (referred to in this section as `needle`) are single-byte encoded text. If this assumption is
 violated, no exception is thrown and results are undefined. Search with UTF-8 encoded strings is usually provided by separate function
 variants. Likewise, if a UTF-8 function variant is used and the input strings are not UTF-8 encoded text, no exception is thrown and the
-results are undefined. Note that no automatic Unicode normalization is performed, you can use the
+results are undefined. Note that no automatic Unicode normalization is performed, however you can use the
 [normalizeUTF8*()](https://clickhouse.com/docs/en/sql-reference/functions/string-functions/) functions for that.
 
 [General strings functions](string-functions.md) and [functions for replacing in strings](string-replace-functions.md) are described separately.
@@ -54,6 +57,8 @@ Type: `Integer`.
 
 **Examples**
 
+Query:
+
 ``` sql
 SELECT position('Hello, world!', '!');
 ```
@@ -68,6 +73,8 @@ Result:
 
 Example with `start_pos` argument:
 
+Query:
+
 ``` sql
 SELECT
     position('Hello, world!', 'o', 1),
@@ -84,6 +91,8 @@ Result:
 
 Example for `needle IN haystack` syntax:
 
+Query:
+
 ```sql
 SELECT 6 = position('/' IN s) FROM (SELECT 'Hello/World' AS s);
 ```
@@ -98,6 +107,8 @@ Result:
 
 Examples with empty `needle` substring:
 
+Query:
+
 ``` sql
 SELECT
     position('abc', ''),
@@ -109,6 +120,8 @@ SELECT
     position('abc', '', 5)
 ```
 
+Result:
+
 ``` text
 ┌─position('abc', '')─┬─position('abc', '', 0)─┬─position('abc', '', 1)─┬─position('abc', '', 2)─┬─position('abc', '', 3)─┬─position('abc', '', 4)─┬─position('abc', '', 5)─┐
 │                   1 │                      1 │                      1 │                      2 │                      3 │                      4 │                      0 │
@@ -132,7 +145,23 @@ locate(needle, haystack[, start_pos])
 
 ## positionCaseInsensitive
 
-Like [position](#position) but searches case-insensitively.
+A case insensitive invariant of [position](#position).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT position('Hello, world!', 'hello');
+```
+
+Result:
+
+``` text
+┌─position('Hello, world!', 'hello')─┐
+│                                  0 │
+└────────────────────────────────────┘
+```
 
 ## positionUTF8
 
@@ -142,6 +171,8 @@ Like [position](#position) but assumes `haystack` and `needle` are UTF-8 encoded
 
 Function `positionUTF8` correctly counts character `ö` (represented by two points) as a single Unicode codepoint:
 
+Query:
+
 ``` sql
 SELECT positionUTF8('Motörhead', 'r');
 ```
@@ -175,14 +206,17 @@ multiSearchAllPositions(haystack, [needle1, needle2, ..., needleN])
 **Arguments**
 
 - `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal).
-- `needle` — Substrings to be searched. Array
+- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md).
 
 **Returned values**
 
-- Array of the starting position in bytes and counting from 1 (if the substring was found) or 0 (if the substring was not found)
+- Array of the starting position in bytes and counting from 1, if the substring was found.
+- 0, if the substring was not found.
 
 **Example**
 
+Query:
+
 ``` sql
 SELECT multiSearchAllPositions('Hello, World!', ['hello', '!', 'world']);
 ```
@@ -194,45 +228,535 @@ Result:
 │ [0,13,0]                                                          │
 └───────────────────────────────────────────────────────────────────┘
 ```
+## multiSearchAllPositionsCaseInsensitive
 
-## multiSearchAllPositionsUTF8
+Like [multiSearchAllPositions](#multisearchallpositions) but ignores case.
-
-Like [multiSearchAllPositions](#multiSearchAllPositions) but assumes `haystack` and the `needle`-s are UTF-8 encoded strings.
-
-## multiSearchFirstPosition
-
-Like `position` but returns the leftmost offset in a `haystack` string which matches any of multiple `needle` strings.
-
-Functions `multiSearchFirstPositionCaseInsensitive`, `multiSearchFirstPositionUTF8` and `multiSearchFirstPositionCaseInsensitiveUTF8` provide case-insensitive and/or UTF-8 variants of this function.
 
 **Syntax**
 
 ```sql
-multiSearchFirstPosition(haystack, \[needle<sub>1</sub>, needle<sub>2</sub>, …, needle<sub>n</sub>\])
+multiSearchAllPositionsCaseInsensitive(haystack, [needle1, needle2, ..., needleN])
+```
+
+**Parameters**
+
+- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal).
+- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+- Array of the starting position in bytes and counting from 1 (if the substring was found).
+- 0 if the substring was not found.
+
+**Example**
+
+Query:
+
+```sql
+SELECT multiSearchAllPositionsCaseInsensitive('ClickHouse',['c','h']);
+```
+
+Result:
+
+```response
+["1","6"]
+```
+
+## multiSearchAllPositionsUTF8
+
+Like [multiSearchAllPositions](#multiSearchAllPositions) but assumes `haystack` and the `needle` substrings are UTF-8 encoded strings.
+
+**Syntax**
+
+```sql
+multiSearchAllPositionsUTF8(haystack, [needle1, needle2, ..., needleN])
+```
+
+**Parameters**
+
+- `haystack` — UTF-8 encoded string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal).
+- `needle` — UTF-8 encoded substrings to be searched. [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+- Array of the starting position in bytes and counting from 1 (if the substring was found).
+- 0 if the substring was not found.
+
+**Example**
+
+Given `ClickHouse` as a UTF-8 string, find the positions of `C` (`\x43`) and `H` (`\x48`).
+
+Query:
+
+```sql
+SELECT multiSearchAllPositionsUTF8('\x43\x6c\x69\x63\x6b\x48\x6f\x75\x73\x65',['\x43','\x48']);
+```
+
+Result:
+
+```response
+["1","6"]
+```
+
+## multiSearchAllPositionsCaseInsensitiveUTF8
+
+Like [multiSearchAllPositionsUTF8](#multisearchallpositionsutf8) but ignores case.
+
+**Syntax**
+
+```sql
+multiSearchAllPositionsCaseInsensitiveUTF8(haystack, [needle1, needle2, ..., needleN])
+```
+
+**Parameters**
+
+- `haystack` — UTF-8 encoded string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal).
+- `needle` — UTF-8 encoded substrings to be searched. [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+- Array of the starting position in bytes and counting from 1 (if the substring was found).
+- 0 if the substring was not found.
+
+**Example**
+
+Given `ClickHouse` as a UTF-8 string, find the positions of `c` (`\x63`) and `h` (`\x68`).
+
+Query:
+
+```sql
+SELECT multiSearchAllPositionsCaseInsensitiveUTF8('\x43\x6c\x69\x63\x6b\x48\x6f\x75\x73\x65',['\x63','\x68']);
+```
+
+Result:
+
+```response
+["1","6"]
+```
+
+## multiSearchFirstPosition
+
+Like [`position`](#position) but returns the leftmost offset in a `haystack` string which matches any of multiple `needle` strings.
+
+Functions [`multiSearchFirstPositionCaseInsensitive`](#multiSearchFirstPositionCaseInsensitive), [`multiSearchFirstPositionUTF8`](#multiSearchFirstPositionUTF8) and [`multiSearchFirstPositionCaseInsensitiveUTF8`](#multiSearchFirstPositionCaseInsensitiveUTF8) provide case-insensitive and/or UTF-8 variants of this function.
+
+**Syntax**
+
+```sql
+multiSearchFirstPosition(haystack, [needle1, needle2, ..., needleN])
+```
+
+**Parameters**
+
+- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal).
+- `needle` —  Substrings to be searched. [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+- Leftmost offset in a `haystack` string which matches any of multiple `needle` strings.
+- 0, if there was no match.
+
+**Example**
+
+Query:
+
+```sql
+SELECT multiSearchFirstPosition('Hello World',['llo', 'Wor', 'ld']);
+```
+
+Result:
+
+```response
+3
+```
+
+## multiSearchFirstPositionCaseInsensitive
+
+Like [`multiSearchFirstPosition`](#multiSearchFirstPosition) but ignores case.
+
+**Syntax**
+
+```sql
+multiSearchFirstPositionCaseInsensitive(haystack, [needle1, needle2, ..., needleN])
+```
+
+**Parameters**
+
+- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal).
+- `needle` — Array of substrings to be searched. [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+- Leftmost offset in a `haystack` string which matches any of multiple `needle` strings.
+- 0, if there was no match.
+
+**Example**
+
+Query:
+
+```sql
+SELECT multiSearchFirstPositionCaseInsensitive('HELLO WORLD',['wor', 'ld', 'ello']);
+```
+
+Result:
+
+```response
+2
+```
+
+## multiSearchFirstPositionUTF8
+
+Like [`multiSearchFirstPosition`](#multiSearchFirstPosition) but assumes `haystack` and `needle` to be UTF-8 strings.
+
+**Syntax**
+
+```sql
+multiSearchFirstPositionUTF8(haystack, [needle1, needle2, ..., needleN])
+```
+
+**Parameters**
+
+- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal).
+- `needle` — Array of UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+- Leftmost offset in a `haystack` string which matches any of multiple `needle` strings.
+- 0, if there was no match.
+
+**Example**
+
+Find the leftmost offset in UTF-8 string `hello world` which matches any of the given needles.
+
+Query:
+
+```sql
+SELECT multiSearchFirstPositionUTF8('\x68\x65\x6c\x6c\x6f\x20\x77\x6f\x72\x6c\x64',['wor', 'ld', 'ello']);
+```
+
+Result:
+
+```response
+2
+```
+
+## multiSearchFirstPositionCaseInsensitiveUTF8
+
+Like [`multiSearchFirstPosition`](#multiSearchFirstPosition) but assumes `haystack` and `needle` to be UTF-8 strings and ignores case.
+
+**Syntax**
+
+```sql
+multiSearchFirstPositionCaseInsensitiveUTF8(haystack, [needle1, needle2, ..., needleN])
+```
+
+**Parameters**
+
+- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal).
+- `needle` — Array of UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md)
+
+**Returned value**
+
+- Leftmost offset in a `haystack` string which matches any of multiple `needle` strings, ignoring case.
+- 0, if there was no match.
+
+**Example**
+
+Find the leftmost offset in UTF-8 string `HELLO WORLD` which matches any of the given needles.
+
+Query:
+
+```sql
+SELECT multiSearchFirstPositionCaseInsensitiveUTF8('\x48\x45\x4c\x4c\x4f\x20\x57\x4f\x52\x4c\x44',['wor', 'ld', 'ello']);
+```
+
+Result:
+
+```response
+2
 ```
 
 ## multiSearchFirstIndex
 
 Returns the index `i` (starting from 1) of the leftmost found needle<sub>i</sub> in the string `haystack` and 0 otherwise.
 
-Functions `multiSearchFirstIndexCaseInsensitive`, `multiSearchFirstIndexUTF8` and `multiSearchFirstIndexCaseInsensitiveUTF8` provide case-insensitive and/or UTF-8 variants of this function.
+Functions [`multiSearchFirstIndexCaseInsensitive`](#multiSearchFirstIndexCaseInsensitive), [`multiSearchFirstIndexUTF8`](#multiSearchFirstIndexUTF8) and [`multiSearchFirstIndexCaseInsensitiveUTF8`](#multiSearchFirstIndexCaseInsensitiveUTF8) provide case-insensitive and/or UTF-8 variants of this function.
 
 **Syntax**
 
 ```sql
-multiSearchFirstIndex(haystack, \[needle<sub>1</sub>, needle<sub>2</sub>, …, needle<sub>n</sub>\])
+multiSearchFirstIndex(haystack, [needle1, needle2, ..., needleN])
+```
+**Parameters**
+
+- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal).
+- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+- index (starting from 1) of the leftmost found needle.
+- 0, if there was no match.
+
+**Example**
+
+Query:
+
+```sql
+SELECT multiSearchFirstIndex('Hello World',['World','Hello']);
 ```
 
-## multiSearchAny {#multisearchany}
+Result:
+
+```response
+1
+```
+
+## multiSearchFirstIndexCaseInsensitive
+
+Returns the index `i` (starting from 1) of the leftmost found needle<sub>i</sub> in the string `haystack` and 0 otherwise. Ignores case.
+
+**Syntax**
+
+```sql
+multiSearchFirstIndexCaseInsensitive(haystack, [needle1, needle2, ..., needleN])
+```
+
+**Parameters**
+
+- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal).
+- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+- index (starting from 1) of the leftmost found needle.
+- 0, if there was no match.
+
+**Example**
+
+Query:
+
+```sql
+SELECT multiSearchFirstIndexCaseInsensitive('hElLo WoRlD',['World','Hello']);
+```
+
+Result:
+
+```response
+1
+```
+
+## multiSearchFirstIndexUTF8
+
+Returns the index `i` (starting from 1) of the leftmost found needle<sub>i</sub> in the string `haystack` and 0 otherwise. Assumes `haystack` and `needle` are UTF-8 encoded strings.
+
+**Syntax**
+
+```sql
+multiSearchFirstIndexUTF8(haystack, [needle1, needle2, ..., needleN])
+```
+
+**Parameters**
+
+- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal).
+- `needle` — Array of UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md)
+
+**Returned value**
+
+- index (starting from 1) of the leftmost found needle.
+- 0, if there was no match.
+
+**Example**
+
+Given `Hello World` as a UTF-8 string, find the first index of UTF-8 strings `Hello` and `World`.
+
+Query:
+
+```sql
+SELECT multiSearchFirstIndexUTF8('\x48\x65\x6c\x6c\x6f\x20\x57\x6f\x72\x6c\x64',['\x57\x6f\x72\x6c\x64','\x48\x65\x6c\x6c\x6f']);
+```
+
+Result:
+
+```response
+1
+```
+
+## multiSearchFirstIndexCaseInsensitiveUTF8
+
+Returns the index `i` (starting from 1) of the leftmost found needle<sub>i</sub> in the string `haystack` and 0 otherwise. Assumes `haystack` and `needle` are UTF-8 encoded strings. Ignores case.
+
+**Syntax**
+
+```sql
+multiSearchFirstIndexCaseInsensitiveUTF8(haystack, [needle1, needle2, ..., needleN])
+```
+
+**Parameters**
+
+- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal).
+- `needle` — Array of UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+- index (starting from 1) of the leftmost found needle.
+- 0, if there was no match.
+
+**Example**
+
+Given `HELLO WORLD` as a UTF-8 string, find the first index of UTF-8 strings `hello` and `world`.
+
+Query:
+
+```sql
+SELECT multiSearchFirstIndexCaseInsensitiveUTF8('\x48\x45\x4c\x4c\x4f\x20\x57\x4f\x52\x4c\x44',['\x68\x65\x6c\x6c\x6f','\x77\x6f\x72\x6c\x64']);
+```
+
+Result:
+
+```response
+1
+```
+
+## multiSearchAny
 
 Returns 1, if at least one string needle<sub>i</sub> matches the string `haystack` and 0 otherwise.
 
-Functions `multiSearchAnyCaseInsensitive`, `multiSearchAnyUTF8` and `multiSearchAnyCaseInsensitiveUTF8` provide case-insensitive and/or UTF-8 variants of this function.
+Functions [`multiSearchAnyCaseInsensitive`](#multiSearchAnyCaseInsensitive), [`multiSearchAnyUTF8`](#multiSearchAnyUTF8) and []`multiSearchAnyCaseInsensitiveUTF8`](#multiSearchAnyCaseInsensitiveUTF8) provide case-insensitive and/or UTF-8 variants of this function.
 
 **Syntax**
 
 ```sql
-multiSearchAny(haystack, \[needle<sub>1</sub>, needle<sub>2</sub>, …, needle<sub>n</sub>\])
+multiSearchAny(haystack, [needle1, needle2, ..., needleN])
+```
+
+**Parameters**
+
+- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal).
+- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+- 1, if there was at least one match.
+- 0, if there was not at least one match.
+
+**Example**
+
+Query:
+
+```sql
+SELECT multiSearchAny('ClickHouse',['C','H']);
+```
+
+Result:
+
+```response
+1
+```
+
+## multiSearchAnyCaseInsensitive
+
+Like [multiSearchAny](#multisearchany) but ignores case.
+
+**Syntax**
+
+```sql
+multiSearchAnyCaseInsensitive(haystack, [needle1, needle2, ..., needleN])
+```
+
+**Parameters**
+
+- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal).
+- `needle` — Substrings to be searched. [Array](../../sql-reference/data-types/array.md)
+
+**Returned value**
+
+- 1, if there was at least one case-insensitive match.
+- 0, if there was not at least one case-insensitive match.
+
+**Example**
+
+Query:
+
+```sql
+SELECT multiSearchAnyCaseInsensitive('ClickHouse',['c','h']);
+```
+
+Result:
+
+```response
+1
+```
+
+## multiSearchAnyUTF8
+
+Like [multiSearchAny](#multisearchany) but assumes `haystack` and the `needle` substrings are UTF-8 encoded strings.
+
+*Syntax**
+
+```sql
+multiSearchAnyUTF8(haystack, [needle1, needle2, ..., needleN])
+```
+
+**Parameters**
+
+- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal).
+- `needle` — UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md).
+
+**Returned value**
+
+- 1, if there was at least one match.
+- 0, if there was not at least one match.
+
+**Example**
+
+Given `ClickHouse` as a UTF-8 string, check if there are any `C` ('\x43') or `H` ('\x48') letters in the word.
+
+Query:
+
+```sql
+SELECT multiSearchAnyUTF8('\x43\x6c\x69\x63\x6b\x48\x6f\x75\x73\x65',['\x43','\x48']);
+```
+
+Result:
+
+```response
+1
+```
+
+## multiSearchAnyCaseInsensitiveUTF8
+
+Like [multiSearchAnyUTF8](#multiSearchAnyUTF8) but ignores case.
+
+*Syntax**
+
+```sql
+multiSearchAnyCaseInsensitiveUTF8(haystack, [needle1, needle2, ..., needleN])
+```
+
+**Parameters**
+
+- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal).
+- `needle` — UTF-8 substrings to be searched. [Array](../../sql-reference/data-types/array.md)
+
+**Returned value**
+
+- 1, if there was at least one case-insensitive match.
+- 0, if there was not at least one case-insensitive match.
+
+**Example**
+
+Given `ClickHouse` as a UTF-8 string, check if there is any letter `h`(`\x68`) in the word, ignoring case.
+
+Query:
+
+```sql
+SELECT multiSearchAnyCaseInsensitiveUTF8('\x43\x6c\x69\x63\x6b\x48\x6f\x75\x73\x65',['\x68']);
+```
+
+Result:
+
+```response
+1
 ```
 
 ## match {#match}

docs/en/sql-reference/functions/tuple-functions.md

@@ -584,6 +584,278 @@ SELECT tupleConcat((1, 2), (3, 4), (true, false)) AS res
 └──────────────────────┘
 ```
 
+## tupleIntDiv
+
+Does integer division of a tuple of numerators and a tuple of denominators, and returns a tuple of the quotients.
+
+**Syntax**
+
+```sql
+tupleIntDiv(tuple_num, tuple_div)
+```
+
+**Parameters**
+
+- `tuple_num`: Tuple of numerator values. [Tuple](../data-types/tuple) of numeric type.
+- `tuple_div`: Tuple of divisor values. [Tuple](../data-types/tuple) of numeric type.
+
+**Returned value**
+
+- Tuple of the quotients of `tuple_num` and `tuple_div`. [Tuple](../data-types/tuple) of integer values.
+
+**Implementation details**
+
+- If either `tuple_num` or `tuple_div` contain non-integer values then the result is calculated by rounding to the nearest integer for each non-integer numerator or divisor.
+- An error will be thrown for division by 0. 
+
+**Examples**
+
+Query:
+
+``` sql
+SELECT tupleIntDiv((15, 10, 5), (5, 5, 5));
+```
+
+Result:
+
+``` text
+┌─tupleIntDiv((15, 10, 5), (5, 5, 5))─┐
+│ (3,2,1)                             │
+└─────────────────────────────────────┘
+```
+
+Query:
+
+``` sql
+SELECT tupleIntDiv((15, 10, 5), (5.5, 5.5, 5.5));
+```
+
+Result:
+
+``` text
+┌─tupleIntDiv((15, 10, 5), (5.5, 5.5, 5.5))─┐
+│ (2,1,0)                                   │
+└───────────────────────────────────────────┘
+```
+
+## tupleIntDivOrZero
+
+Like [tupleIntDiv](#tupleintdiv) it does integer division of a tuple of numerators and a tuple of denominators, and returns a tuple of the quotients. It does not throw an error for 0 divisors, but rather returns the quotient as 0. 
+
+**Syntax**
+
+```sql
+tupleIntDivOrZero(tuple_num, tuple_div)
+```
+
+- `tuple_num`: Tuple of numerator values. [Tuple](../data-types/tuple) of numeric type.
+- `tuple_div`: Tuple of divisor values. [Tuple](../data-types/tuple) of numeric type.
+
+**Returned value**
+
+- Tuple of the quotients of `tuple_num` and `tuple_div`. [Tuple](../data-types/tuple) of integer values.
+- Returns 0 for quotients where the divisor is 0.
+
+**Implementation details**
+
+- If either `tuple_num` or `tuple_div` contain non-integer values then the result is calculated by rounding to the nearest integer for each non-integer numerator or divisor as in [tupleIntDiv](#tupleintdiv).
+
+**Examples**
+
+Query:
+
+``` sql
+SELECT tupleIntDivOrZero((5, 10, 15), (0, 0, 0));
+```
+
+Result:
+
+``` text
+┌─tupleIntDivOrZero((5, 10, 15), (0, 0, 0))─┐
+│ (0,0,0)                                   │
+└───────────────────────────────────────────┘
+```
+
+## tupleIntDivByNumber
+
+Does integer division of a tuple of numerators by a given denominator, and returns a tuple of the quotients.
+
+**Syntax**
+
+```sql
+tupleIntDivByNumber(tuple_num, div)
+```
+
+**Parameters**
+
+- `tuple_num`: Tuple of numerator values. [Tuple](../data-types/tuple) of numeric type.
+- `div`: The divisor value. [Numeric](../data-types/int-uint.md) type.
+
+**Returned value**
+
+- Tuple of the quotients of `tuple_num` and `div`. [Tuple](../data-types/tuple) of integer values.
+
+**Implementation details**
+
+- If either `tuple_num` or `div` contain non-integer values then the result is calculated by rounding to the nearest integer for each non-integer numerator or divisor.
+- An error will be thrown for division by 0. 
+
+**Examples**
+
+Query:
+
+``` sql
+SELECT tupleIntDivByNumber((15, 10, 5), 5);
+```
+
+Result:
+
+``` text
+┌─tupleIntDivByNumber((15, 10, 5), 5)─┐
+│ (3,2,1)                             │
+└─────────────────────────────────────┘
+```
+
+Query:
+
+``` sql
+SELECT tupleIntDivByNumber((15.2, 10.7, 5.5), 5.8);
+```
+
+Result:
+
+``` text
+┌─tupleIntDivByNumber((15.2, 10.7, 5.5), 5.8)─┐
+│ (2,1,0)                                     │
+└─────────────────────────────────────────────┘
+```
+
+## tupleIntDivOrZeroByNumber
+
+Like [tupleIntDivByNumber](#tupleintdivbynumber) it does integer division of a tuple of numerators by a given denominator, and returns a tuple of the quotients. It does not throw an error for 0 divisors, but rather returns the quotient as 0.
+
+**Syntax**
+
+```sql
+tupleIntDivOrZeroByNumber(tuple_num, div)
+```
+
+**Parameters**
+
+- `tuple_num`: Tuple of numerator values. [Tuple](../data-types/tuple) of numeric type.
+- `div`: The divisor value. [Numeric](../data-types/int-uint.md) type.
+
+**Returned value**
+
+- Tuple of the quotients of `tuple_num` and `div`. [Tuple](../data-types/tuple) of integer values.
+- Returns 0 for quotients where the divisor is 0.
+
+**Implementation details**
+
+- If either `tuple_num` or `div` contain non-integer values then the result is calculated by rounding to the nearest integer for each non-integer numerator or divisor as in [tupleIntDivByNumber](#tupleintdivbynumber).
+
+**Examples**
+
+Query:
+
+``` sql
+SELECT tupleIntDivOrZeroByNumber((15, 10, 5), 5);
+```
+
+Result:
+
+``` text
+┌─tupleIntDivOrZeroByNumber((15, 10, 5), 5)─┐
+│ (3,2,1)                                   │
+└───────────────────────────────────────────┘
+```
+
+Query:
+
+``` sql
+SELECT tupleIntDivOrZeroByNumber((15, 10, 5), 0)
+```
+
+Result:
+
+``` text
+┌─tupleIntDivOrZeroByNumber((15, 10, 5), 0)─┐
+│ (0,0,0)                                   │
+└───────────────────────────────────────────┘
+```
+
+## tupleModulo
+
+Returns a tuple of the moduli (remainders) of division operations of two tuples.
+
+**Syntax**
+
+```sql
+tupleModulo(tuple_num, tuple_mod)
+```
+
+**Parameters**
+
+- `tuple_num`: Tuple of numerator values. [Tuple](../data-types/tuple) of numeric type.
+- `tuple_div`: Tuple of modulus values. [Tuple](../data-types/tuple) of numeric type.
+
+**Returned value**
+
+- Tuple of the remainders of division of `tuple_num` and `tuple_div`. [Tuple](../data-types/tuple) of non-zero integer values.
+- An error is thrown for division by zero.
+
+**Examples**
+
+Query:
+
+``` sql
+SELECT tupleModulo((15, 10, 5), (5, 3, 2));
+```
+
+Result:
+
+``` text
+┌─tupleModulo((15, 10, 5), (5, 3, 2))─┐
+│ (0,1,1)                             │
+└─────────────────────────────────────┘
+```
+
+## tupleModuloByNumber
+
+Returns a tuple of the moduli (remainders) of division operations of a tuple and a given divisor.
+
+**Syntax**
+
+```sql
+tupleModuloByNumber(tuple_num, div)
+```
+
+**Parameters**
+
+- `tuple_num`: Tuple of numerator values. [Tuple](../data-types/tuple) of numeric type.
+- `div`: The divisor value. [Numeric](../data-types/int-uint.md) type.
+
+**Returned value**
+
+- Tuple of the remainders of division of `tuple_num` and `div`. [Tuple](../data-types/tuple) of non-zero integer values.
+- An error is thrown for division by zero.
+
+**Examples**
+
+Query:
+
+``` sql
+SELECT tupleModuloByNumber((15, 10, 5), 2);
+```
+
+Result:
+
+``` text
+┌─tupleModuloByNumber((15, 10, 5), 2)─┐
+│ (1,0,1)                             │
+└─────────────────────────────────────┘
+```
+
 ## Distance functions
 
 All supported functions are described in [distance functions documentation](../../sql-reference/functions/distance-functions.md).

docs/en/sql-reference/statements/drop.md

@@ -20,11 +20,10 @@ DROP DATABASE [IF EXISTS] db [ON CLUSTER cluster] [SYNC]
 
 ## DROP TABLE
 
-Deletes the table.
+Deletes one or more tables.
-In case when `IF EMPTY` clause is specified server will check if table is empty only on replica that received initial query.  
 
 :::tip
-Also see [UNDROP TABLE](/docs/en/sql-reference/statements/undrop.md)
+To undo the deletion of a table, please see [UNDROP TABLE](/docs/en/sql-reference/statements/undrop.md)
 :::
 
 Syntax:
@@ -33,7 +32,9 @@ Syntax:
 DROP [TEMPORARY] TABLE [IF EXISTS] [IF EMPTY]  [db1.]name_1[, [db2.]name_2, ...] [ON CLUSTER cluster] [SYNC]
 ```
 
-Note that deleting multiple tables at the same time is a non-atomic deletion. If a table fails to be deleted, subsequent tables will not be deleted.
+Limitations:
+- If the clause `IF EMPTY` is specified, the server checks the emptiness of the table only on the replica which received the query.  
+- Deleting multiple tables at once is not an atomic operation, i.e. if the deletion of a table fails, subsequent tables will not be deleted.
 
 ## DROP DICTIONARY
 

docs/en/sql-reference/window-functions/index.md

@@ -12,25 +12,23 @@ Some of the calculations that you can do are similar to those that can be done w
 
 ClickHouse supports the standard grammar for defining windows and window functions. The table below indicates whether a feature is currently supported.
 
-| Feature                                                                            | Support or workaround                                                                                                                                                                       |
+| Feature                                                                            | Supported?                                                                                                                                                                       |
 |------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| ad hoc window specification (`count(*) over (partition by id order by time desc)`) | supported                                                                                                                                                                                   |
+| ad hoc window specification (`count(*) over (partition by id order by time desc)`) | ✅                                                                                                                                                                                  |
-| expressions involving window functions, e.g. `(count(*) over ()) / 2)`             | supported                                                                                                                                                                                   |
+| expressions involving window functions, e.g. `(count(*) over ()) / 2)`             | ✅                                                                                                                                                                                   |
-| `WINDOW` clause (`select ... from table window w as (partition by id)`)            | supported                                                                                                                                                                                   |
+| `WINDOW` clause (`select ... from table window w as (partition by id)`)            | ✅                                                                                                                                                                                   |
-| `ROWS` frame                                                                       | supported                                                                                                                                                                                   |
+| `ROWS` frame                                                                       | ✅                                                                                                                                                                                   |
-| `RANGE` frame                                                                      | supported, the default                                                                                                                                                                      |
+| `RANGE` frame                                                                      | ✅ (the default)                                                                                                                                                                      |
-| `INTERVAL` syntax for `DateTime` `RANGE OFFSET` frame                              | not supported, specify the number of seconds instead (`RANGE` works with any numeric type).                                                                                                                                 |
+| `INTERVAL` syntax for `DateTime` `RANGE OFFSET` frame                              | ❌ (specify the number of seconds instead (`RANGE` works with any numeric type).)                                                                                                                                 |
-| `GROUPS` frame                                                                     | not supported                                                                                                                                                                               |
+| `GROUPS` frame                                                                     | ❌                                                                                                                                                                               |
-| Calculating aggregate functions over a frame (`sum(value) over (order by time)`)   | all aggregate functions are supported                                                                                                                                                       |
+| Calculating aggregate functions over a frame (`sum(value) over (order by time)`)   | ✅ (All aggregate functions are supported)                                                                                                                                                       |
-| `rank()`, `dense_rank()`, `row_number()`                                           | supported                                                                                                                                                                                   |
+| `rank()`, `dense_rank()`, `row_number()`                                           | ✅                                                                                                                                                                                   |
-| `lag/lead(value, offset)`                                                          | Not supported. Workarounds:                                                                                                                                                                 |
+| `lag/lead(value, offset)`                                                          | ❌ <br/> You can use one of the following workarounds:<br/> 1) `any(value) over (.... rows between <offset> preceding and <offset> preceding)`, or `following` for `lead` <br/> 2) `lagInFrame/leadInFrame`, which are analogous, but respect the window frame. To get behavior identical to `lag/lead`, use `rows between unbounded preceding and unbounded following`                                                                 |
-|                                                                                    | 1) replace with `any(value) over (.... rows between <offset> preceding and <offset> preceding)`, or `following` for `lead`                                                                  |
+| ntile(buckets) | ✅ <br/> Specify window like, (partition by x order by y rows between unbounded preceding and unrounded following). |
-|                                                                                    | 2) use `lagInFrame/leadInFrame`, which are analogous, but respect the window frame. To get behavior identical to `lag/lead`, use `rows between unbounded preceding and unbounded following` |
-| ntile(buckets) | Supported. Specify window like, (partition by x order by y rows between unbounded preceding and unrounded following). |
 
 ## ClickHouse-specific Window Functions
 
-There are also the following window function that's specific to ClickHouse:
+There is also the following ClickHouse specific window function:
 
 ### nonNegativeDerivative(metric_column, timestamp_column[, INTERVAL X UNITS])
 
@@ -89,6 +87,102 @@ These functions can be used only as a window function.
 
 Let's have a look at some examples of how window functions can be used.
 
+### Numbering rows
+
+```sql
+CREATE TABLE salaries
+(
+    `team` String,
+    `player` String,
+    `salary` UInt32,
+    `position` String
+)
+Engine = Memory;
+
+INSERT INTO salaries FORMAT Values
+    ('Port Elizabeth Barbarians', 'Gary Chen', 195000, 'F'),
+    ('New Coreystad Archdukes', 'Charles Juarez', 190000, 'F'),
+    ('Port Elizabeth Barbarians', 'Michael Stanley', 150000, 'D'),
+    ('New Coreystad Archdukes', 'Scott Harrison', 150000, 'D'),
+    ('Port Elizabeth Barbarians', 'Robert George', 195000, 'M');
+```
+
+```sql
+SELECT player, salary, 
+       row_number() OVER (ORDER BY salary) AS row
+FROM salaries;
+```
+
+```text
+┌─player──────────┬─salary─┬─row─┐
+│ Michael Stanley │ 150000 │   1 │
+│ Scott Harrison  │ 150000 │   2 │
+│ Charles Juarez  │ 190000 │   3 │
+│ Gary Chen       │ 195000 │   4 │
+│ Robert George   │ 195000 │   5 │
+└─────────────────┴────────┴─────┘
+```
+
+```sql
+SELECT player, salary, 
+       row_number() OVER (ORDER BY salary) AS row,
+       rank() OVER (ORDER BY salary) AS rank,
+       dense_rank() OVER (ORDER BY salary) AS denseRank
+FROM salaries;
+```
+
+```text
+┌─player──────────┬─salary─┬─row─┬─rank─┬─denseRank─┐
+│ Michael Stanley │ 150000 │   1 │    1 │         1 │
+│ Scott Harrison  │ 150000 │   2 │    1 │         1 │
+│ Charles Juarez  │ 190000 │   3 │    3 │         2 │
+│ Gary Chen       │ 195000 │   4 │    4 │         3 │
+│ Robert George   │ 195000 │   5 │    4 │         3 │
+└─────────────────┴────────┴─────┴──────┴───────────┘
+```
+
+### Aggregation functions
+
+Compare each player's salary to the average for their team.
+
+```sql
+SELECT player, salary, team,
+       avg(salary) OVER (PARTITION BY team) AS teamAvg,
+       salary - teamAvg AS diff
+FROM salaries;
+```
+
+```text
+┌─player──────────┬─salary─┬─team──────────────────────┬─teamAvg─┬───diff─┐
+│ Charles Juarez  │ 190000 │ New Coreystad Archdukes   │  170000 │  20000 │
+│ Scott Harrison  │ 150000 │ New Coreystad Archdukes   │  170000 │ -20000 │
+│ Gary Chen       │ 195000 │ Port Elizabeth Barbarians │  180000 │  15000 │
+│ Michael Stanley │ 150000 │ Port Elizabeth Barbarians │  180000 │ -30000 │
+│ Robert George   │ 195000 │ Port Elizabeth Barbarians │  180000 │  15000 │
+└─────────────────┴────────┴───────────────────────────┴─────────┴────────┘
+```
+
+Compare each player's salary to the maximum for their team.
+
+```sql
+SELECT player, salary, team,
+       max(salary) OVER (PARTITION BY team) AS teamAvg,
+       salary - teamAvg AS diff
+FROM salaries;
+```
+
+```text
+┌─player──────────┬─salary─┬─team──────────────────────┬─teamAvg─┬───diff─┐
+│ Charles Juarez  │ 190000 │ New Coreystad Archdukes   │  190000 │      0 │
+│ Scott Harrison  │ 150000 │ New Coreystad Archdukes   │  190000 │ -40000 │
+│ Gary Chen       │ 195000 │ Port Elizabeth Barbarians │  195000 │      0 │
+│ Michael Stanley │ 150000 │ Port Elizabeth Barbarians │  195000 │ -45000 │
+│ Robert George   │ 195000 │ Port Elizabeth Barbarians │  195000 │      0 │
+└─────────────────┴────────┴───────────────────────────┴─────────┴────────┘
+```
+
+### Partitioning by column
+
 ```sql
 CREATE TABLE wf_partition
 (
@@ -120,6 +214,8 @@ ORDER BY
 └──────────┴───────┴───────┴──────────────┘
 ```
 
+### Frame bounding
+
 ```sql
 CREATE TABLE wf_frame
 (
@@ -131,14 +227,19 @@ ENGINE = Memory;
 
 INSERT INTO wf_frame FORMAT Values
    (1,1,1), (1,2,2), (1,3,3), (1,4,4), (1,5,5);
+```
 
--- frame is bounded by bounds of a partition (BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)
+```sql
+-- Frame is bounded by bounds of a partition (BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)
 SELECT
     part_key,
     value,
     order,
-    groupArray(value) OVER (PARTITION BY part_key ORDER BY order ASC
+    groupArray(value) OVER (
-         Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS frame_values
+        PARTITION BY part_key 
+        ORDER BY order ASC
+        Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
+    ) AS frame_values
 FROM wf_frame
 ORDER BY
     part_key ASC,
@@ -151,7 +252,9 @@ ORDER BY
 │        1 │     4 │     4 │ [1,2,3,4,5]  │
 │        1 │     5 │     5 │ [1,2,3,4,5]  │
 └──────────┴───────┴───────┴──────────────┘
+```
 
+```sql
 -- short form - no bound expression, no order by
 SELECT
     part_key,
@@ -169,14 +272,19 @@ ORDER BY
 │        1 │     4 │     4 │ [1,2,3,4,5]  │
 │        1 │     5 │     5 │ [1,2,3,4,5]  │
 └──────────┴───────┴───────┴──────────────┘
+```
 
--- frame is bounded by the beggining of a partition and the current row
+```sql
+-- frame is bounded by the beginning of a partition and the current row
 SELECT
     part_key,
     value,
     order,
-    groupArray(value) OVER (PARTITION BY part_key ORDER BY order ASC
+    groupArray(value) OVER (
-          Rows BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS frame_values
+        PARTITION BY part_key 
+        ORDER BY order ASC
+        Rows BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
+    ) AS frame_values
 FROM wf_frame
 ORDER BY
     part_key ASC,
@@ -189,8 +297,10 @@ ORDER BY
 │        1 │     4 │     4 │ [1,2,3,4]    │
 │        1 │     5 │     5 │ [1,2,3,4,5]  │
 └──────────┴───────┴───────┴──────────────┘
+```
 
--- short form (frame is bounded by the beggining of a partition and the current row)
+```sql
+-- short form (frame is bounded by the beginning of a partition and the current row)
 SELECT
     part_key,
     value,
@@ -207,8 +317,10 @@ ORDER BY
 │        1 │     4 │     4 │ [1,2,3,4]    │
 │        1 │     5 │     5 │ [1,2,3,4,5]  │
 └──────────┴───────┴───────┴──────────────┘
+```
 
--- frame is bounded by the beggining of a partition and the current row, but order is backward
+```sql
+-- frame is bounded by the beginning of a partition and the current row, but order is backward
 SELECT
     part_key,
     value,
@@ -225,14 +337,19 @@ ORDER BY
 │        1 │     4 │     4 │ [5,4]        │
 │        1 │     5 │     5 │ [5]          │
 └──────────┴───────┴───────┴──────────────┘
+```
 
+```sql
 -- sliding frame - 1 PRECEDING ROW AND CURRENT ROW
 SELECT
     part_key,
     value,
     order,
-    groupArray(value) OVER (PARTITION BY part_key ORDER BY order ASC
+    groupArray(value) OVER (
-          Rows BETWEEN 1 PRECEDING AND CURRENT ROW) AS frame_values
+        PARTITION BY part_key 
+        ORDER BY order ASC
+        Rows BETWEEN 1 PRECEDING AND CURRENT ROW
+    ) AS frame_values
 FROM wf_frame
 ORDER BY
     part_key ASC,
@@ -245,14 +362,19 @@ ORDER BY
 │        1 │     4 │     4 │ [3,4]        │
 │        1 │     5 │     5 │ [4,5]        │
 └──────────┴───────┴───────┴──────────────┘
+```
 
+```sql
 -- sliding frame - Rows BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING 
 SELECT
     part_key,
     value,
     order,
-    groupArray(value) OVER (PARTITION BY part_key ORDER BY order ASC
+    groupArray(value) OVER (
-          Rows BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) AS frame_values
+        PARTITION BY part_key 
+        ORDER BY order ASC
+        Rows BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING
+    ) AS frame_values
 FROM wf_frame
 ORDER BY
     part_key ASC,
@@ -264,7 +386,9 @@ ORDER BY
 │        1 │     4 │     4 │ [3,4,5]      │
 │        1 │     5 │     5 │ [4,5]        │
 └──────────┴───────┴───────┴──────────────┘
+```
 
+```sql
 -- row_number does not respect the frame, so rn_1 = rn_2 = rn_3 != rn_4
 SELECT
     part_key,
@@ -278,8 +402,11 @@ SELECT
 FROM wf_frame
 WINDOW
     w1 AS (PARTITION BY part_key ORDER BY order DESC),
-    w2 AS (PARTITION BY part_key ORDER BY order DESC 
+    w2 AS (
-                   Rows BETWEEN 1 PRECEDING AND CURRENT ROW)
+        PARTITION BY part_key 
+        ORDER BY order DESC 
+        Rows BETWEEN 1 PRECEDING AND CURRENT ROW
+    )
 ORDER BY
     part_key ASC,
     value ASC;
@@ -290,7 +417,9 @@ ORDER BY
 │        1 │     4 │     4 │ [5,4]        │    2 │    2 │    2 │    2 │
 │        1 │     5 │     5 │ [5]          │    1 │    1 │    1 │    1 │
 └──────────┴───────┴───────┴──────────────┴──────┴──────┴──────┴──────┘
+```
 
+```sql
 -- first_value and last_value respect the frame
 SELECT
     groupArray(value) OVER w1 AS frame_values_1,
@@ -313,7 +442,9 @@ ORDER BY
 │ [1,2,3,4]      │             1 │            4 │ [3,4]          │             3 │            4 │
 │ [1,2,3,4,5]    │             1 │            5 │ [4,5]          │             4 │            5 │
 └────────────────┴───────────────┴──────────────┴────────────────┴───────────────┴──────────────┘
+```
 
+```sql
 -- second value within the frame
 SELECT
     groupArray(value) OVER w1 AS frame_values_1,
@@ -330,7 +461,9 @@ ORDER BY
 │ [1,2,3,4]      │            2 │
 │ [2,3,4,5]      │            3 │
 └────────────────┴──────────────┘
+```
 
+```sql
 -- second value within the frame + Null for missing values
 SELECT
     groupArray(value) OVER w1 AS frame_values_1,
@@ -351,7 +484,9 @@ ORDER BY
 
 ## Real world examples
 
-### Maximum/total salary per department.
+The following examples solve common real-world problems.
+
+### Maximum/total salary per department
 
 ```sql
 CREATE TABLE employees
@@ -369,7 +504,9 @@ INSERT INTO employees FORMAT Values
    ('IT', 'Tim', 200),
    ('IT', 'Anna', 300),
    ('IT', 'Elen', 500);
+```
 
+```sql
 SELECT
     department,
     employee_name AS emp,
@@ -386,8 +523,10 @@ FROM
         max(salary) OVER wndw AS max_salary_per_dep,
         sum(salary) OVER wndw AS total_salary_per_dep
     FROM employees
-    WINDOW wndw AS (PARTITION BY department
+    WINDOW wndw AS (
-          rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)
+        PARTITION BY department
+        rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
+    )
     ORDER BY
         department ASC,
         employee_name ASC
@@ -403,7 +542,7 @@ FROM
 └────────────┴──────┴────────┴────────────────────┴──────────────────────┴──────────────────┘
 ```
 
-### Cumulative sum.
+### Cumulative sum
 
 ```sql
 CREATE TABLE warehouse
@@ -421,7 +560,9 @@ INSERT INTO warehouse VALUES
     ('sku1', '2020-01-01', 1),
     ('sku1', '2020-02-01', 1),
     ('sku1', '2020-03-01', 1);
+```    
 
+```sql
 SELECT
     item,
     ts,
@@ -461,13 +602,18 @@ insert into sensors values('cpu_temp', '2020-01-01 00:00:00', 87),
                           ('cpu_temp', '2020-01-01 00:00:05', 87),
                           ('cpu_temp', '2020-01-01 00:00:06', 87),
                           ('cpu_temp', '2020-01-01 00:00:07', 87);
+```
+
+```sql
 SELECT
     metric,
     ts,
     value,
-    avg(value) OVER
+    avg(value) OVER (
-       (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN 2 PRECEDING AND CURRENT ROW)
+        PARTITION BY metric 
-         AS moving_avg_temp
+        ORDER BY ts ASC 
+        Rows BETWEEN 2 PRECEDING AND CURRENT ROW
+    ) AS moving_avg_temp
 FROM sensors
 ORDER BY
     metric ASC,
@@ -536,7 +682,9 @@ insert into sensors values('ambient_temp', '2020-01-01 00:00:00', 16),
                           ('ambient_temp', '2020-03-01 12:00:00', 16),
                           ('ambient_temp', '2020-03-01 12:00:00', 16),
                           ('ambient_temp', '2020-03-01 12:00:00', 16);
+```
 
+```sql
 SELECT
     metric,
     ts,

docs/ru/sql-reference/data-types/multiword-types.md

@@ -1,28 +0,0 @@
----
-slug: /ru/sql-reference/data-types/multiword-types
-sidebar_position: 61
-sidebar_label: Составные типы
----
-
-# Составные типы {#multiword-types}
-
-При создании таблиц вы можете использовать типы данных с названием, состоящим из нескольких слов. Такие названия поддерживаются для лучшей совместимости с SQL.
-
-## Поддержка составных типов {#multiword-types-support}
-
-| Составные типы                      | Обычные типы                                              |
-|-------------------------------------|-----------------------------------------------------------|
-| DOUBLE PRECISION                    | [Float64](../../sql-reference/data-types/float.md)        |
-| CHAR LARGE OBJECT                   | [String](../../sql-reference/data-types/string.md)        |
-| CHAR VARYING                        | [String](../../sql-reference/data-types/string.md)        |
-| CHARACTER LARGE OBJECT              | [String](../../sql-reference/data-types/string.md)        |
-| CHARACTER VARYING                   | [String](../../sql-reference/data-types/string.md)        |
-| NCHAR LARGE OBJECT                  | [String](../../sql-reference/data-types/string.md)        |
-| NCHAR VARYING                       | [String](../../sql-reference/data-types/string.md)        |
-| NATIONAL CHARACTER LARGE OBJECT     | [String](../../sql-reference/data-types/string.md)        |
-| NATIONAL CHARACTER VARYING          | [String](../../sql-reference/data-types/string.md)        |
-| NATIONAL CHAR VARYING               | [String](../../sql-reference/data-types/string.md)        |
-| NATIONAL CHARACTER                  | [String](../../sql-reference/data-types/string.md)        |
-| NATIONAL CHAR                       | [String](../../sql-reference/data-types/string.md)        |
-| BINARY LARGE OBJECT                 | [String](../../sql-reference/data-types/string.md)        |
-| BINARY VARYING                      | [String](../../sql-reference/data-types/string.md)        |

docs/zh/sql-reference/data-types/multiword-types.mdx

@@ -1,10 +0,0 @@
----
-slug: /zh/sql-reference/data-types/multiword-types
-sidebar_position: 61
-sidebar_label: Multiword Type Names
-title: "Multiword Types"
----
-
-import Content from '@site/docs/en/sql-reference/data-types/multiword-types.md';
-
-<Content />

programs/bash-completion/completions/clickhouse

@@ -3,7 +3,7 @@
 function _clickhouse_get_utils()
 {
     local cmd=$1 && shift
-    "$cmd" --help |& awk '/^clickhouse.*args/ { print $2 }'
+    "$cmd" help |& awk '/^clickhouse.*args/ { print $2 }'
 }
 
 function _complete_for_clickhouse_entrypoint_bin()

programs/client/Client.cpp

@@ -934,8 +934,8 @@ void Client::addOptions(OptionsDescription & options_description)
         ("user,u", po::value<std::string>()->default_value("default"), "user")
         ("password", po::value<std::string>(), "password")
         ("ask-password", "ask-password")
-        ("ssh-key-file", po::value<std::string>(), "File containing ssh private key needed for authentication. If not set does password authentication.")
+        ("ssh-key-file", po::value<std::string>(), "File containing the SSH private key for authenticate with the server.")
-        ("ssh-key-passphrase", po::value<std::string>(), "Passphrase for imported ssh key.")
+        ("ssh-key-passphrase", po::value<std::string>(), "Passphrase for the SSH private key specified by --ssh-key-file.")
         ("quota_key", po::value<std::string>(), "A string to differentiate quotas when the user have keyed quotas configured on server")
 
         ("max_client_network_bandwidth", po::value<int>(), "the maximum speed of data exchange over the network for the client in bytes per second.")

programs/disks/DisksApp.cpp

@@ -166,7 +166,7 @@ int DisksApp::main(const std::vector<String> & /*args*/)
     {
         String config_path = config().getString("config-file", getDefaultConfigFileName());
         ConfigProcessor config_processor(config_path, false, false);
-        config_processor.setConfigPath(fs::path(config_path).parent_path());
+        ConfigProcessor::setConfigPath(fs::path(config_path).parent_path());
         auto loaded_config = config_processor.loadConfig();
         config().add(loaded_config.configuration.duplicate(), false, false);
     }

programs/keeper-client/KeeperClient.cpp

@@ -368,7 +368,7 @@ int KeeperClient::main(const std::vector<String> & /* args */)
     DB::ConfigProcessor config_processor(config().getString("config-file", "config.xml"));
 
     /// This will handle a situation when clickhouse is running on the embedded config, but config.d folder is also present.
-    config_processor.registerEmbeddedConfig("config.xml", "<clickhouse/>");
+    ConfigProcessor::registerEmbeddedConfig("config.xml", "<clickhouse/>");
     auto clickhouse_config = config_processor.loadConfig();
 
     Poco::Util::AbstractConfiguration::Keys keys;

programs/local/LocalServer.cpp

@@ -122,7 +122,7 @@ void LocalServer::initialize(Poco::Util::Application & self)
     {
         const auto config_path = config().getString("config-file", "config.xml");
         ConfigProcessor config_processor(config_path, false, true);
-        config_processor.setConfigPath(fs::path(config_path).parent_path());
+        ConfigProcessor::setConfigPath(fs::path(config_path).parent_path());
         auto loaded_config = config_processor.loadConfig();
         config().add(loaded_config.configuration.duplicate(), PRIO_DEFAULT, false);
     }

programs/main.cpp

@@ -487,7 +487,7 @@ int main(int argc_, char ** argv_)
     /// Interpret binary without argument or with arguments starts with dash
     /// ('-') as clickhouse-local for better usability:
     ///
-    ///     clickhouse # dumps help
+    ///     clickhouse help # dumps help
     ///     clickhouse -q 'select 1' # use local
     ///     clickhouse # spawn local
     ///     clickhouse local # spawn local

src/Access/Authentication.cpp

@@ -4,11 +4,12 @@
 #include <Access/ExternalAuthenticators.h>
 #include <Access/LDAPClient.h>
 #include <Access/GSSAcceptor.h>
-#include <Common/Exception.h>
 #include <Poco/SHA1Engine.h>
+#include <Common/Exception.h>
+#include <Common/SSHWrapper.h>
 #include <Common/typeid_cast.h>
-#include <Common/SSH/Wrappers.h>
 
+#include "config.h"
 
 namespace DB
 {
@@ -74,7 +75,7 @@ namespace
     }
 
 #if USE_SSH
-    bool checkSshSignature(const std::vector<ssh::SSHKey> & keys, std::string_view signature, std::string_view original)
+    bool checkSshSignature(const std::vector<SSHKey> & keys, std::string_view signature, std::string_view original)
     {
         for (const auto & key: keys)
             if (key.isPublic() && key.verifySignature(signature, original))
@@ -114,7 +115,11 @@ bool Authentication::areCredentialsValid(
                 throw Authentication::Require<BasicCredentials>("ClickHouse X.509 Authentication");
 
             case AuthenticationType::SSH_KEY:
-                throw Authentication::Require<SshCredentials>("Ssh Keys Authentication");
+#if USE_SSH
+                throw Authentication::Require<SshCredentials>("SSH Keys Authentication");
+#else
+                throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh");
+#endif
 
             case AuthenticationType::MAX:
                 break;
@@ -145,7 +150,11 @@ bool Authentication::areCredentialsValid(
                 throw Authentication::Require<BasicCredentials>("ClickHouse X.509 Authentication");
 
             case AuthenticationType::SSH_KEY:
-                throw Authentication::Require<SshCredentials>("Ssh Keys Authentication");
+#if USE_SSH
+                throw Authentication::Require<SshCredentials>("SSH Keys Authentication");
+#else
+                throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh");
+#endif
 
             case AuthenticationType::MAX:
                 break;
@@ -178,7 +187,11 @@ bool Authentication::areCredentialsValid(
                 throw Authentication::Require<BasicCredentials>("ClickHouse X.509 Authentication");
 
             case AuthenticationType::SSH_KEY:
-                throw Authentication::Require<SshCredentials>("Ssh Keys Authentication");
+#if USE_SSH
+                throw Authentication::Require<SshCredentials>("SSH Keys Authentication");
+#else
+                throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh");
+#endif
 
             case AuthenticationType::BCRYPT_PASSWORD:
                 return checkPasswordBcrypt(basic_credentials->getPassword(), auth_data.getPasswordHashBinary());
@@ -216,13 +229,18 @@ bool Authentication::areCredentialsValid(
                 return auth_data.getSSLCertificateCommonNames().contains(ssl_certificate_credentials->getCommonName());
 
             case AuthenticationType::SSH_KEY:
-                throw Authentication::Require<SshCredentials>("Ssh Keys Authentication");
+#if USE_SSH
+                throw Authentication::Require<SshCredentials>("SSH Keys Authentication");
+#else
+                throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh");
+#endif
 
             case AuthenticationType::MAX:
                 break;
         }
     }
 
+#if USE_SSH
     if (const auto * ssh_credentials = typeid_cast<const SshCredentials *>(&credentials))
     {
         switch (auth_data.getType())
@@ -243,15 +261,12 @@ bool Authentication::areCredentialsValid(
                 throw Authentication::Require<SSLCertificateCredentials>("ClickHouse X.509 Authentication");
 
             case AuthenticationType::SSH_KEY:
-#if USE_SSH
                 return checkSshSignature(auth_data.getSSHKeys(), ssh_credentials->getSignature(), ssh_credentials->getOriginal());
-#else
-                throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without OpenSSL");
-#endif
             case AuthenticationType::MAX:
                 break;
         }
     }
+#endif
 
     if ([[maybe_unused]] const auto * always_allow_credentials = typeid_cast<const AlwaysAllowCredentials *>(&credentials))
         return true;

src/Access/AuthenticationData.cpp

@@ -105,7 +105,10 @@ bool operator ==(const AuthenticationData & lhs, const AuthenticationData & rhs)
     return (lhs.type == rhs.type) && (lhs.password_hash == rhs.password_hash)
         && (lhs.ldap_server_name == rhs.ldap_server_name) && (lhs.kerberos_realm == rhs.kerberos_realm)
         && (lhs.ssl_certificate_common_names == rhs.ssl_certificate_common_names)
-        && (lhs.ssh_keys == rhs.ssh_keys) && (lhs.http_auth_scheme == rhs.http_auth_scheme)
+#if USE_SSH
+        && (lhs.ssh_keys == rhs.ssh_keys)
+#endif
+        && (lhs.http_auth_scheme == rhs.http_auth_scheme)
         && (lhs.http_auth_server_name == rhs.http_auth_server_name);
 }
 
@@ -326,7 +329,7 @@ std::shared_ptr<ASTAuthenticationData> AuthenticationData::toAST() const
 
             break;
 #else
-            throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without OpenSSL");
+            throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh");
 #endif
         }
         case AuthenticationType::HTTP:
@@ -355,7 +358,7 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que
     {
 #if USE_SSH
         AuthenticationData auth_data(*query.type);
-        std::vector<ssh::SSHKey> keys;
+        std::vector<SSHKey> keys;
 
         size_t args_size = query.children.size();
         for (size_t i = 0; i < args_size; ++i)
@@ -366,7 +369,7 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que
 
             try
             {
-                keys.emplace_back(ssh::SSHKeyFactory::makePublicFromBase64(key_base64, type));
+                keys.emplace_back(SSHKeyFactory::makePublicKeyFromBase64(key_base64, type));
             }
             catch (const std::invalid_argument &)
             {
@@ -377,7 +380,7 @@ AuthenticationData AuthenticationData::fromAST(const ASTAuthenticationData & que
         auth_data.setSSHKeys(std::move(keys));
         return auth_data;
 #else
-        throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without OpenSSL");
+        throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh");
 #endif
     }
 

src/Access/AuthenticationData.h

@@ -2,14 +2,16 @@
 
 #include <Access/Common/AuthenticationType.h>
 #include <Access/Common/HTTPAuthenticationScheme.h>
+#include <Common/SSHWrapper.h>
 #include <Interpreters/Context_fwd.h>
 #include <Parsers/Access/ASTAuthenticationData.h>
-#include <Common/SSH/Wrappers.h>
 
 #include <vector>
 #include <base/types.h>
 #include <boost/container/flat_set.hpp>
 
+#include "config.h"
+
 namespace DB
 {
 
@@ -59,8 +61,10 @@ public:
     const boost::container::flat_set<String> & getSSLCertificateCommonNames() const { return ssl_certificate_common_names; }
     void setSSLCertificateCommonNames(boost::container::flat_set<String> common_names_);
 
-    const std::vector<ssh::SSHKey> & getSSHKeys() const { return ssh_keys; }
+#if USE_SSH
-    void setSSHKeys(std::vector<ssh::SSHKey> && ssh_keys_) { ssh_keys = std::forward<std::vector<ssh::SSHKey>>(ssh_keys_); }
+    const std::vector<SSHKey> & getSSHKeys() const { return ssh_keys; }
+    void setSSHKeys(std::vector<SSHKey> && ssh_keys_) { ssh_keys = std::forward<std::vector<SSHKey>>(ssh_keys_); }
+#endif
 
     HTTPAuthenticationScheme getHTTPAuthenticationScheme() const { return http_auth_scheme; }
     void setHTTPAuthenticationScheme(HTTPAuthenticationScheme scheme) { http_auth_scheme = scheme; }
@@ -94,7 +98,9 @@ private:
     String kerberos_realm;
     boost::container::flat_set<String> ssl_certificate_common_names;
     String salt;
-    std::vector<ssh::SSHKey> ssh_keys;
+#if USE_SSH
+    std::vector<SSHKey> ssh_keys;
+#endif
     /// HTTP authentication properties
     String http_auth_server_name;
     HTTPAuthenticationScheme http_auth_scheme = HTTPAuthenticationScheme::BASIC;

src/Access/Common/AuthenticationType.h

@@ -34,8 +34,8 @@ enum class AuthenticationType
     /// Password is encrypted in bcrypt hash.
     BCRYPT_PASSWORD,
 
-    /// Server sends a random string named `challenge` which client needs to encrypt with private key.
+    /// Server sends a random string named `challenge` to the client. The client encrypts it with its SSH private key.
-    /// The check is performed on server side by decrypting the data and comparing with the original string.
+    /// The server decrypts the result using the SSH public key registered for the user and compares with the original string.
     SSH_KEY,
 
     /// Authentication through HTTP protocol

src/Access/Credentials.h

@@ -3,6 +3,7 @@
 #include <base/types.h>
 #include <memory>
 
+#include "config.h"
 
 namespace DB
 {
@@ -86,10 +87,11 @@ class MySQLNative41Credentials : public CredentialsWithScramble
     using CredentialsWithScramble::CredentialsWithScramble;
 };
 
+#if USE_SSH
 class SshCredentials : public Credentials
 {
 public:
-    explicit SshCredentials(const String& user_name_, const String& signature_, const String& original_)
+    SshCredentials(const String & user_name_, const String & signature_, const String & original_)
         : Credentials(user_name_), signature(signature_), original(original_)
     {
         is_ready = true;
@@ -117,5 +119,6 @@ private:
     String signature;
     String original;
 };
+#endif
 
 }

src/Access/User.cpp

@@ -31,7 +31,7 @@ void User::setName(const String & name_)
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "User name is empty");
     if (name_ == EncodedUserInfo::USER_INTERSERVER_MARKER)
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "User name '{}' is reserved", name_);
-    if (startsWith(name_, EncodedUserInfo::SSH_KEY_AUTHENTICAION_MARKER))
+    if (name_.starts_with(EncodedUserInfo::SSH_KEY_AUTHENTICAION_MARKER))
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "User name '{}' is reserved", name_);
     name = name_;
 }

src/Access/UsersConfigAccessStorage.cpp

@@ -1,6 +1,5 @@
 #include <Access/UsersConfigAccessStorage.h>
 #include <Access/Quota.h>
-#include <Common/SSH/Wrappers.h>
 #include <Access/RowPolicy.h>
 #include <Access/User.h>
 #include <Access/Role.h>
@@ -10,6 +9,7 @@
 #include <Access/AccessChangesNotifier.h>
 #include <Dictionaries/IDictionary.h>
 #include <Common/Config/ConfigReloader.h>
+#include <Common/SSHWrapper.h>
 #include <Common/StringUtils/StringUtils.h>
 #include <Common/quoteString.h>
 #include <Common/transformEndianness.h>
@@ -214,7 +214,7 @@ namespace
 
             Poco::Util::AbstractConfiguration::Keys entries;
             config.keys(ssh_keys_config, entries);
-            std::vector<ssh::SSHKey> keys;
+            std::vector<SSHKey> keys;
             for (const String& entry : entries)
             {
                 const auto conf_pref = ssh_keys_config + "." + entry + ".";
@@ -237,7 +237,7 @@ namespace
 
                     try
                     {
-                        keys.emplace_back(ssh::SSHKeyFactory::makePublicFromBase64(base64_key, type));
+                        keys.emplace_back(SSHKeyFactory::makePublicKeyFromBase64(base64_key, type));
                     }
                     catch (const std::invalid_argument &)
                     {
@@ -249,7 +249,7 @@ namespace
             }
             user->auth_data.setSSHKeys(std::move(keys));
 #else
-            throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without OpenSSL");
+            throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh");
 #endif
         }
         else if (has_http_auth)

src/AggregateFunctions/AggregateFunctionExponentialMovingAverage.cpp

@@ -54,30 +54,30 @@ public:
     {
         const auto & value = columns[0]->getFloat64(row_num);
         const auto & time = columns[1]->getFloat64(row_num);
-        this->data(place).add(value, time, half_decay);
+        data(place).add(value, time, half_decay);
     }
 
     void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena *) const override
     {
-        this->data(place).merge(this->data(rhs), half_decay);
+        data(place).merge(data(rhs), half_decay);
     }
 
     void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional<size_t> /* version */) const override
     {
-        writeBinary(this->data(place).value, buf);
+        writeBinary(data(place).value, buf);
-        writeBinary(this->data(place).time, buf);
+        writeBinary(data(place).time, buf);
     }
 
     void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional<size_t> /* version */, Arena *) const override
     {
-        readBinary(this->data(place).value, buf);
+        readBinary(data(place).value, buf);
-        readBinary(this->data(place).time, buf);
+        readBinary(data(place).time, buf);
     }
 
     void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override
     {
         auto & column = assert_cast<ColumnVector<Float64> &>(to);
-        column.getData().push_back(this->data(place).get(half_decay));
+        column.getData().push_back(data(place).get(half_decay));
     }
 };
 

src/AggregateFunctions/AggregateFunctionKolmogorovSmirnovTest.cpp

@@ -293,32 +293,32 @@ public:
         Float64 value = columns[0]->getFloat64(row_num);
         UInt8 is_second = columns[1]->getUInt(row_num);
         if (is_second)
-            this->data(place).addY(value, arena);
+            data(place).addY(value, arena);
         else
-            this->data(place).addX(value, arena);
+            data(place).addX(value, arena);
     }
 
     void merge(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena) const override
     {
-        this->data(place).merge(this->data(rhs), arena);
+        data(place).merge(data(rhs), arena);
     }
 
     void serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional<size_t> /* version */) const override
     {
-        this->data(place).write(buf);
+        data(place).write(buf);
     }
 
     void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional<size_t> /* version */, Arena * arena) const override
     {
-        this->data(place).read(buf, arena);
+        data(place).read(buf, arena);
     }
 
     void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override
     {
-        if (!this->data(place).size_x || !this->data(place).size_y)
+        if (!data(place).size_x || !data(place).size_y)
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "Aggregate function {} require both samples to be non empty", getName());
 
-        auto [d_statistic, p_value] = this->data(place).getResult(alternative, method);
+        auto [d_statistic, p_value] = data(place).getResult(alternative, method);
 
         /// Because p-value is a probability.
         p_value = std::min(1.0, std::max(0.0, p_value));

src/AggregateFunctions/DDSketch.h

@@ -147,6 +147,8 @@ public:
         negative_store->merge(other.negative_store.get());
     }
 
+    /// NOLINTBEGIN(readability-static-accessed-through-instance)
+
     void serialize(WriteBuffer& buf) const
     {
         // Write the mapping
@@ -201,6 +203,8 @@ public:
         count = static_cast<Float64>(negative_store->count + zero_count + store->count);
     }
 
+    /// NOLINTEND(readability-static-accessed-through-instance)
+
 private:
     std::unique_ptr<DDSketchLogarithmicMapping> mapping;
     std::unique_ptr<DDSketchDenseStore> store;

src/AggregateFunctions/DDSketch/Store.h

@@ -87,6 +87,8 @@ public:
         count += other->count;
     }
 
+    /// NOLINTBEGIN(readability-static-accessed-through-instance)
+
     void serialize(WriteBuffer& buf) const
     {
 
@@ -179,6 +181,8 @@ public:
         }
     }
 
+    /// NOLINTEND(readability-static-accessed-through-instance)
+
 private:
     UInt32 chunk_size;
     DDSketchEncoding enc;

src/Analyzer/Passes/QueryAnalysisPass.cpp

@@ -5799,7 +5799,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
         return result_projection_names;
     }
 
-    FunctionOverloadResolverPtr function = UserDefinedExecutableFunctionFactory::instance().tryGet(function_name, scope.context, parameters);
+    FunctionOverloadResolverPtr function = UserDefinedExecutableFunctionFactory::instance().tryGet(function_name, scope.context, parameters); /// NOLINT(readability-static-accessed-through-instance)
     bool is_executable_udf = true;
 
     IdentifierResolveScope::ResolvedFunctionsCache * function_cache = nullptr;
@@ -5829,7 +5829,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
         {
             std::vector<std::string> possible_function_names;
 
-            auto function_names = UserDefinedExecutableFunctionFactory::instance().getRegisteredNames(scope.context);
+            auto function_names = UserDefinedExecutableFunctionFactory::instance().getRegisteredNames(scope.context); /// NOLINT(readability-static-accessed-through-instance)
             possible_function_names.insert(possible_function_names.end(), function_names.begin(), function_names.end());
 
             function_names = UserDefinedSQLFunctionFactory::instance().getAllRegisteredNames();
@@ -5847,8 +5847,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
                     possible_function_names.push_back(name);
             }
 
-            NamePrompter<2> name_prompter;
+            auto hints = NamePrompter<2>::getHints(function_name, possible_function_names);
-            auto hints = name_prompter.getHints(function_name, possible_function_names);
 
             throw Exception(ErrorCodes::UNKNOWN_FUNCTION,
                 "Function with name '{}' does not exists. In scope {}{}",

src/CMakeLists.txt

@@ -85,7 +85,6 @@ add_headers_and_sources(clickhouse_common_io Common)
 add_headers_and_sources(clickhouse_common_io Common/HashTable)
 add_headers_and_sources(clickhouse_common_io Common/Scheduler)
 add_headers_and_sources(clickhouse_common_io Common/Scheduler/Nodes)
-add_headers_and_sources(clickhouse_common_io Common/SSH)
 add_headers_and_sources(clickhouse_common_io IO)
 add_headers_and_sources(clickhouse_common_io IO/Archives)
 add_headers_and_sources(clickhouse_common_io IO/S3)
@@ -99,7 +98,6 @@ add_headers_and_sources(clickhouse_compression Core)
 #Included these specific files to avoid linking grpc
 add_glob(clickhouse_compression_headers Server/ServerType.h)
 add_glob(clickhouse_compression_sources Server/ServerType.cpp)
-add_headers_and_sources(clickhouse_compression Common/SSH)
 add_library(clickhouse_compression ${clickhouse_compression_headers} ${clickhouse_compression_sources})
 
 
@@ -370,8 +368,7 @@ if (TARGET ch_contrib::crc32-vpmsum)
  endif()
 
 if (TARGET ch_contrib::ssh)
- target_link_libraries(clickhouse_common_io PUBLIC ch_contrib::ssh)
+    target_link_libraries(clickhouse_common_io PUBLIC ch_contrib::ssh)
- target_link_libraries(clickhouse_compression PUBLIC ch_contrib::ssh)
 endif()
 
 dbms_target_link_libraries(PUBLIC ch_contrib::abseil_swiss_tables)

src/Client/ClientBase.cpp

@@ -1964,7 +1964,7 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
         }
 
         /// INSERT query for which data transfer is needed (not an INSERT SELECT or input()) is processed separately.
-        if (insert && (!insert->select || input_function) && !insert->watch && !is_async_insert_with_inlined_data)
+        if (insert && (!insert->select || input_function) && !is_async_insert_with_inlined_data)
         {
             if (input_function && insert->format.empty())
                 throw Exception(ErrorCodes::INVALID_USAGE_OF_INPUT, "FORMAT must be specified for function input()");

src/Client/Connection.cpp

@@ -67,7 +67,7 @@ Connection::~Connection() = default;
 Connection::Connection(const String & host_, UInt16 port_,
     const String & default_database_,
     const String & user_, const String & password_,
-    const ssh::SSHKey & ssh_private_key_,
+    [[maybe_unused]] const SSHKey & ssh_private_key_,
     const String & quota_key_,
     const String & cluster_,
     const String & cluster_secret_,
@@ -76,7 +76,9 @@ Connection::Connection(const String & host_, UInt16 port_,
     Protocol::Secure secure_)
     : host(host_), port(port_), default_database(default_database_)
     , user(user_), password(password_)
+#if USE_SSH
     , ssh_private_key(ssh_private_key_)
+#endif
     , quota_key(quota_key_)
     , cluster(cluster_)
     , cluster_secret(cluster_secret_)
@@ -141,7 +143,7 @@ void Connection::connect(const ConnectionTimeouts & timeouts)
                         async_callback(socket->impl()->sockfd(), connection_timeout, AsyncEventTimeoutType::CONNECT, description, AsyncTaskExecutor::READ | AsyncTaskExecutor::WRITE | AsyncTaskExecutor::ERROR);
 
                     if (auto err = socket->impl()->socketError())
-                        socket->impl()->error(err); // Throws an exception
+                        socket->impl()->error(err); // Throws an exception /// NOLINT(readability-static-accessed-through-instance)
 
                     socket->setBlocking(true);
                 }
@@ -276,17 +278,6 @@ void Connection::disconnect()
 }
 
 
-String Connection::packStringForSshSign(String challenge)
-{
-    String message;
-    message.append(std::to_string(DBMS_TCP_PROTOCOL_VERSION));
-    message.append(default_database);
-    message.append(user);
-    message.append(challenge);
-    return message;
-}
-
-
 void Connection::sendHello()
 {
     /** Disallow control characters in user controlled parameters
@@ -334,10 +325,10 @@ void Connection::sendHello()
 #endif
     }
 #if USE_SSH
-    /// Just inform server that we will authenticate using SSH keys.
     else if (!ssh_private_key.isEmpty())
     {
-        writeStringBinary(fmt::format("{}{}", EncodedUserInfo::SSH_KEY_AUTHENTICAION_MARKER, user), *out);
+        /// Inform server that we will authenticate using SSH keys.
+        writeStringBinary(String(EncodedUserInfo::SSH_KEY_AUTHENTICAION_MARKER) + user, *out);
         writeStringBinary(password, *out);
 
         performHandshakeForSSHAuth();
@@ -361,9 +352,9 @@ void Connection::sendAddendum()
 }
 
 
+#if USE_SSH
 void Connection::performHandshakeForSSHAuth()
 {
-#if USE_SSH
     String challenge;
     {
         writeVarUInt(Protocol::Client::SSHChallengeRequest, *out);
@@ -388,12 +379,23 @@ void Connection::performHandshakeForSSHAuth()
     }
 
     writeVarUInt(Protocol::Client::SSHChallengeResponse, *out);
-    String to_sign = packStringForSshSign(challenge);
+
+    auto pack_string_for_ssh_sign = [&](String challenge_)
+    {
+        String message;
+        message.append(std::to_string(DBMS_TCP_PROTOCOL_VERSION));
+        message.append(default_database);
+        message.append(user);
+        message.append(challenge_);
+        return message;
+    };
+
+    String to_sign = pack_string_for_ssh_sign(challenge);
     String signature = ssh_private_key.signString(to_sign);
     writeStringBinary(signature, *out);
     out->next();
-#endif
 }
+#endif
 
 
 void Connection::receiveHello(const Poco::Timespan & handshake_timeout)

src/Client/Connection.h

@@ -1,10 +1,9 @@
 #pragma once
 
-
 #include <Poco/Net/StreamSocket.h>
 
-#include <Common/SSH/Wrappers.h>
 #include <Common/callOnce.h>
+#include <Common/SSHWrapper.h>
 #include <Client/IServerConnection.h>
 #include <Core/Defines.h>
 
@@ -53,7 +52,7 @@ public:
     Connection(const String & host_, UInt16 port_,
         const String & default_database_,
         const String & user_, const String & password_,
-        const ssh::SSHKey & ssh_private_key_,
+        const SSHKey & ssh_private_key_,
         const String & quota_key_,
         const String & cluster_,
         const String & cluster_secret_,
@@ -170,7 +169,9 @@ private:
     String default_database;
     String user;
     String password;
-    ssh::SSHKey ssh_private_key;
+#if USE_SSH
+    SSHKey ssh_private_key;
+#endif
     String quota_key;
 
     /// For inter-server authorization
@@ -265,9 +266,10 @@ private:
 
     void connect(const ConnectionTimeouts & timeouts);
     void sendHello();
-    String packStringForSshSign(String challenge);
 
+#if USE_SSH
     void performHandshakeForSSHAuth();
+#endif
 
     void sendAddendum();
     void receiveHello(const Poco::Timespan & handshake_timeout);

src/Client/ConnectionParameters.cpp

@@ -1,11 +1,10 @@
 #include "ConnectionParameters.h"
-#include <fstream>
+
 #include <Core/Defines.h>
 #include <Core/Protocol.h>
 #include <Core/Types.h>
 #include <IO/ConnectionTimeouts.h>
 #include <Poco/Util/AbstractConfiguration.h>
-#include <Common/SSH/Wrappers.h>
 #include <Common/Exception.h>
 #include <Common/isLocalAddress.h>
 #include <Common/DNSResolver.h>
@@ -88,19 +87,19 @@ ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfigurati
         }
         else
         {
-            std::string prompt{"Enter your private key passphrase (leave empty for no passphrase): "};
+            std::string prompt{"Enter your SSH private key passphrase (leave empty for no passphrase): "};
             char buf[1000] = {};
             if (auto * result = readpassphrase(prompt.c_str(), buf, sizeof(buf), 0))
                 passphrase = result;
         }
 
-        ssh::SSHKey key = ssh::SSHKeyFactory::makePrivateFromFile(filename, passphrase);
+        SSHKey key = SSHKeyFactory::makePrivateKeyFromFile(filename, passphrase);
         if (!key.isPrivate())
-            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Found public key in file: {} but expected private", filename);
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "File {} did not contain a private key (is it a public key?)", filename);
 
         ssh_private_key = std::move(key);
 #else
-        throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without OpenSSL");
+        throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh");
 #endif
     }
 

src/Client/ConnectionParameters.h

@@ -1,9 +1,10 @@
 #pragma once
 
-#include <string>
+#include <Common/SSHWrapper.h>
 #include <Core/Protocol.h>
 #include <IO/ConnectionTimeouts.h>
-#include <Common/SSH/Wrappers.h>
+
+#include <string>
 
 namespace Poco::Util
 {
@@ -20,7 +21,7 @@ struct ConnectionParameters
     std::string user;
     std::string password;
     std::string quota_key;
-    ssh::SSHKey ssh_private_key;
+    SSHKey ssh_private_key;
     Protocol::Secure security = Protocol::Secure::Disable;
     Protocol::Compression compression = Protocol::Compression::Enable;
     ConnectionTimeouts timeouts;

src/Client/ConnectionPool.h

@@ -123,7 +123,7 @@ protected:
     {
         return std::make_shared<Connection>(
             host, port,
-            default_database, user, password, ssh::SSHKey(), quota_key,
+            default_database, user, password, SSHKey(), quota_key,
             cluster, cluster_secret,
             client_name, compression, secure);
     }

src/Columns/ColumnObject.cpp

@@ -940,7 +940,7 @@ void ColumnObject::addNestedSubcolumn(const PathInData & key, const FieldInfo &
     if (nested_node)
     {
         /// Find any leaf of Nested subcolumn.
-        const auto * leaf = subcolumns.findLeaf(nested_node, [&](const auto &) { return true; });
+        const auto * leaf = Subcolumns::findLeaf(nested_node, [&](const auto &) { return true; });
         assert(leaf);
 
         /// Recreate subcolumn with default values and the same sizes of arrays.
@@ -983,7 +983,7 @@ const ColumnObject::Subcolumns::Node * ColumnObject::getLeafOfTheSameNested(cons
     while (current_node)
     {
         /// Try to find the first Nested up to the current node.
-        const auto * node_nested = subcolumns.findParent(current_node,
+        const auto * node_nested = Subcolumns::findParent(current_node,
             [](const auto & candidate) { return candidate.isNested(); });
 
         if (!node_nested)
@@ -993,7 +993,7 @@ const ColumnObject::Subcolumns::Node * ColumnObject::getLeafOfTheSameNested(cons
         /// for the last rows.
         /// If there are no leaves, skip current node and find
         /// the next node up to the current.
-        leaf = subcolumns.findLeaf(node_nested,
+        leaf = Subcolumns::findLeaf(node_nested,
             [&](const auto & candidate)
             {
                 return candidate.data.size() > old_size;

src/Common/HTTPConnectionPool.cpp

@@ -16,6 +16,7 @@
 #include <Poco/Net/HTTPRequest.h>
 #include <Poco/Net/HTTPResponse.h>
 #include <Poco/Net/HTTPStream.h>
+#include <Poco/Net/NetException.h>
 #include <Poco/Timespan.h>
 
 #include <queue>
@@ -83,17 +84,15 @@ namespace
     }
 
 
-    size_t roundUp(size_t x, size_t rounding)
+    constexpr size_t roundUp(size_t x, size_t rounding)
     {
         chassert(rounding > 0);
-        return (x + (rounding - 1)) / rounding * rounding;
+        return (x + rounding) / rounding * rounding;
-    }
-
-
-    Poco::Timespan divide(const Poco::Timespan span, int divisor)
-    {
-        return Poco::Timespan(Poco::Timestamp::TimeDiff(span.totalMicroseconds() / divisor));
     }
+    static_assert(roundUp(10000, 100) == 10100);
+    static_assert(roundUp(10001, 100) == 10100);
+    static_assert(roundUp(10099, 100) == 10100);
+    static_assert(roundUp(10100, 100) == 10200);
 }
 
 namespace DB
@@ -202,8 +201,9 @@ public:
 
         if (total_connections_in_group >= limits.warning_limit && total_connections_in_group >= mute_warning_until)
         {
-            LOG_WARNING(log, "Too many active sessions in group {}, count {}, warning limit {}", type, total_connections_in_group, limits.warning_limit);
             mute_warning_until = roundUp(total_connections_in_group, HTTPConnectionPools::Limits::warning_step);
+            LOG_WARNING(log, "Too many active sessions in group {}, count {}, warning limit {}, next warning at {}",
+                        type, total_connections_in_group, limits.warning_limit, mute_warning_until);
         }
     }
 
@@ -213,7 +213,8 @@ public:
 
         --total_connections_in_group;
 
-        const size_t reduced_warning_limit = limits.warning_limit > 10 ? limits.warning_limit - 10 : 1;
+        const size_t gap = 20;
+        const size_t reduced_warning_limit = limits.warning_limit > gap ? limits.warning_limit - gap : 1;
         if (mute_warning_until > 0 && total_connections_in_group < reduced_warning_limit)
         {
             LOG_WARNING(log, "Sessions count is OK in the group {}, count {}", type, total_connections_in_group);
@@ -273,9 +274,15 @@ private:
     public:
         using Ptr = std::shared_ptr<PooledConnection>;
 
+        using Session::mustReconnect;
+
+        void markAsExpired()
+        {
+            isExpired = true;
+        }
+
         void reconnect() override
         {
-            ProfileEvents::increment(metrics.reset);
             Session::close();
 
             if (auto lock = pool.lock())
@@ -283,6 +290,7 @@ private:
                 auto timeouts = getTimeouts(*this);
                 auto new_connection = lock->getConnection(timeouts);
                 Session::assign(*new_connection);
+                Session::setKeepAliveRequest(Session::getKeepAliveRequest() + 1);
             }
             else
             {
@@ -304,6 +312,12 @@ private:
                                Session::getPort());
         }
 
+        Poco::Timespan idleTime()
+        {
+            Poco::Timestamp now;
+            return now - Session::getLastRequest();
+        }
+
         void flushRequest() override
         {
             if (bool(request_stream))
@@ -335,6 +349,7 @@ private:
 
         std::ostream & sendRequest(Poco::Net::HTTPRequest & request) override
         {
+            auto idle = idleTime();
             std::ostream & result = Session::sendRequest(request);
             result.exceptions(std::ios::badbit);
 
@@ -392,10 +407,11 @@ private:
             }
             response_stream = nullptr;
 
-            if (auto lock = pool.lock())
+            group->atConnectionDestroy();
-                lock->atConnectionDestroy(*this);
+
-            else
+            if (!isExpired)
-                ProfileEvents::increment(metrics.reset);
+                if (auto lock = pool.lock())
+                    lock->atConnectionDestroy(*this);
 
             CurrentMetrics::sub(metrics.active_count);
         }
@@ -404,10 +420,18 @@ private:
         friend class EndpointConnectionPool;
 
         template <class... Args>
-        explicit PooledConnection(EndpointConnectionPool::WeakPtr pool_, IHTTPConnectionPoolForEndpoint::Metrics metrics_, Args &&... args)
+        explicit PooledConnection(
-            : Session(args...), pool(std::move(pool_)), metrics(std::move(metrics_))
+                EndpointConnectionPool::WeakPtr pool_,
+                ConnectionGroup::Ptr group_,
+                IHTTPConnectionPoolForEndpoint::Metrics metrics_,
+                Args &&... args)
+            : Session(std::forward<Args>(args)...)
+            , pool(std::move(pool_))
+            , group(group_)
+            , metrics(std::move(metrics_))
         {
             CurrentMetrics::add(metrics.active_count);
+            group->atConnectionCreate();
         }
 
         template <class... Args>
@@ -433,10 +457,12 @@ private:
             return request_stream_completed && response_stream_completed;
         }
 
-        WeakPtr pool;
+        EndpointConnectionPool::WeakPtr pool;
+        ConnectionGroup::Ptr group;
         IHTTPConnectionPoolForEndpoint::Metrics metrics;
+        bool isExpired = false;
 
-        Poco::Logger * log = &Poco::Logger::get("PooledConnection");
+        LoggerPtr log = getLogger("PooledConnection");
 
         std::ostream * request_stream = nullptr;
         std::istream * response_stream = nullptr;
@@ -484,7 +510,6 @@ public:
 
     IHTTPConnectionPoolForEndpoint::ConnectionPtr getConnection(const ConnectionTimeouts & timeouts) override
     {
-        Poco::Timestamp now;
         std::vector<ConnectionPtr> expired_connections;
 
         SCOPE_EXIT({
@@ -494,8 +519,9 @@ public:
 
         {
             std::lock_guard lock(mutex);
+            expired_connections.reserve(stored_connections.size());
 
-            wipeExpiredImpl(expired_connections, now);
+            wipeExpiredImpl(expired_connections);
 
             if (!stored_connections.empty())
             {
@@ -526,7 +552,6 @@ public:
 
     size_t wipeExpired() override
     {
-        Poco::Timestamp now;
         std::vector<ConnectionPtr> expired_connections;
 
         SCOPE_EXIT({
@@ -535,25 +560,29 @@ public:
         });
 
         std::lock_guard lock(mutex);
-        return wipeExpiredImpl(expired_connections, now);
+        return wipeExpiredImpl(expired_connections);
     }
 
-    size_t wipeExpiredImpl(std::vector<ConnectionPtr> & expired_connections, Poco::Timestamp now) TSA_REQUIRES(mutex)
+    size_t wipeExpiredImpl(std::vector<ConnectionPtr> & expired_connections) TSA_REQUIRES(mutex)
     {
+        SCOPE_EXIT({
+            CurrentMetrics::sub(getMetrics().stored_count, expired_connections.size());
+            ProfileEvents::increment(getMetrics().expired, expired_connections.size());
+        });
+
+        auto isSoftLimitReached = group->isSoftLimitReached();
         while (!stored_connections.empty())
         {
             auto connection = stored_connections.top();
 
-            if (!isExpired(now, connection))
+            if (!isExpired(connection, isSoftLimitReached))
                 return stored_connections.size();
 
             stored_connections.pop();
+            connection->markAsExpired();
             expired_connections.push_back(connection);
         }
 
-        CurrentMetrics::sub(getMetrics().stored_count, expired_connections.size());
-        ProfileEvents::increment(getMetrics().expired, expired_connections.size());
-
         return stored_connections.size();
     }
 
@@ -569,57 +598,53 @@ private:
 
     WeakPtr getWeakFromThis() { return EndpointConnectionPool::weak_from_this(); }
 
-    bool isExpired(Poco::Timestamp & now, ConnectionPtr connection)
+    bool isExpired(ConnectionPtr connection, bool isSoftLimitReached) TSA_REQUIRES(mutex)
     {
-        if (group->isSoftLimitReached())
+        if (isSoftLimitReached)
-            return now > (connection->getLastRequest() + divide(connection->getKeepAliveTimeout(), 10));
+            return connection->isKeepAliveExpired(0.1);
-        return now > connection->getLastRequest() + connection->getKeepAliveTimeout();
+        return connection->isKeepAliveExpired(0.8);
     }
 
-    ConnectionPtr allocateNewConnection()
+
+    ConnectionPtr prepareNewConnection(const ConnectionTimeouts & timeouts)
     {
-        ConnectionPtr connection = PooledConnection::create(this->getWeakFromThis(), getMetrics(), host, port);
+        auto connection = PooledConnection::create(this->getWeakFromThis(), group, getMetrics(), host, port);
+
         connection->setKeepAlive(true);
+        setTimeouts(*connection, timeouts);
 
         if (!proxy_configuration.isEmpty())
         {
             connection->setProxyConfig(proxyConfigurationToPocoProxyConfig(proxy_configuration));
         }
 
-        group->atConnectionCreate();
-
-        return connection;
-    }
-
-    ConnectionPtr prepareNewConnection(const ConnectionTimeouts & timeouts)
-    {
         auto address = HostResolversPool::instance().getResolver(host)->resolve();
-
+        connection->setResolvedHost(*address);
-        auto session = allocateNewConnection();
-
-        setTimeouts(*session, timeouts);
-        session->setResolvedHost(*address);
 
         try
         {
             auto timer = CurrentThread::getProfileEvents().timer(getMetrics().elapsed_microseconds);
-            session->doConnect();
+            connection->doConnect();
         }
         catch (...)
         {
             address.setFail();
             ProfileEvents::increment(getMetrics().errors);
-            session->reset();
+            connection->reset();
             throw;
         }
 
         ProfileEvents::increment(getMetrics().created);
-        return session;
+        return connection;
     }
 
     void atConnectionDestroy(PooledConnection & connection)
     {
-        group->atConnectionDestroy();
+        if (connection.getKeepAliveRequest() >= connection.getKeepAliveMaxRequests())
+        {
+            ProfileEvents::increment(getMetrics().expired, 1);
+            return;
+        }
 
         if (!connection.connected() || connection.mustReconnect() || !connection.isCompleted() || connection.buffered()
             || group->isStoreLimitReached())
@@ -628,17 +653,17 @@ private:
             return;
         }
 
-        auto connection_to_store = allocateNewConnection();
+        auto connection_to_store = PooledConnection::create(this->getWeakFromThis(), group, getMetrics(), host, port);
         connection_to_store->assign(connection);
 
-        CurrentMetrics::add(getMetrics().stored_count, 1);
-        ProfileEvents::increment(getMetrics().preserved, 1);
-
         {
             MemoryTrackerSwitcher switcher{&total_memory_tracker};
             std::lock_guard lock(mutex);
             stored_connections.push(connection_to_store);
         }
+
+        CurrentMetrics::add(getMetrics().stored_count, 1);
+        ProfileEvents::increment(getMetrics().preserved, 1);
     }
 
 
@@ -726,14 +751,13 @@ createConnectionPool(ConnectionGroup::Ptr group, std::string host, UInt16 port,
 class HTTPConnectionPools::Impl
 {
 private:
-    const size_t DEFAULT_WIPE_TIMEOUT_SECONDS = 5 * 60;
+    const size_t DEFAULT_WIPE_TIMEOUT_SECONDS = 10 * 60;
     const Poco::Timespan wipe_timeout = Poco::Timespan(DEFAULT_WIPE_TIMEOUT_SECONDS, 0);
 
     ConnectionGroup::Ptr disk_group = std::make_shared<ConnectionGroup>(HTTPConnectionGroupType::DISK);
     ConnectionGroup::Ptr storage_group = std::make_shared<ConnectionGroup>(HTTPConnectionGroupType::STORAGE);
     ConnectionGroup::Ptr http_group = std::make_shared<ConnectionGroup>(HTTPConnectionGroupType::HTTP);
 
-
     /// If multiple mutexes are held simultaneously,
     /// they should be locked in this order:
     /// HTTPConnectionPools::mutex, then EndpointConnectionPool::mutex, then ConnectionGroup::mutex.

src/Common/SSHWrapper.cpp

@@ -1,4 +1,5 @@
-#include <Common/SSH/Wrappers.h>
+#include <Common/SSHWrapper.h>
+
 # if USE_SSH
 #    include <stdexcept>
 
@@ -10,6 +11,14 @@
 
 #    pragma clang diagnostic pop
 
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int LIBSSH_ERROR;
+}
+
 namespace
 {
 
@@ -18,17 +27,19 @@ class SSHString
 public:
     explicit SSHString(std::string_view input)
     {
-        string = ssh_string_new(input.size());
+        if (string = ssh_string_new(input.size()); string == nullptr)
-        ssh_string_fill(string, input.data(), input.size());
+            throw Exception(ErrorCodes::LIBSSH_ERROR, "Can't create SSHString");
+        if (int rc = ssh_string_fill(string, input.data(), input.size()); rc != SSH_OK)
+            throw Exception(ErrorCodes::LIBSSH_ERROR, "Can't create SSHString");
     }
 
-    explicit SSHString(ssh_string c_other) { string = c_other; }
+    explicit SSHString(ssh_string other) { string = other; }
 
     ssh_string get() { return string; }
 
     String toString()
     {
-        return String(ssh_string_get_char(string), ssh_string_len(string));
+        return {ssh_string_get_char(string), ssh_string_len(string)};
     }
 
     ~SSHString()
@@ -42,46 +53,28 @@ private:
 
 }
 
-namespace DB
+SSHKey SSHKeyFactory::makePrivateKeyFromFile(String filename, String passphrase)
-{
-
-namespace ErrorCodes
-{
-    extern const int LIBSSH_ERROR;
-}
-
-namespace ssh
-{
-
-SSHKey SSHKeyFactory::makePrivateFromFile(String filename, String passphrase)
 {
     ssh_key key;
-    int rc = ssh_pki_import_privkey_file(filename.c_str(), passphrase.c_str(), nullptr, nullptr, &key);
+    if (int rc = ssh_pki_import_privkey_file(filename.c_str(), passphrase.c_str(), nullptr, nullptr, &key); rc != SSH_OK)
-    if (rc != SSH_OK)
-    {
         throw Exception(ErrorCodes::LIBSSH_ERROR, "Can't import SSH private key from file");
-    }
     return SSHKey(key);
 }
 
-SSHKey SSHKeyFactory::makePublicFromFile(String filename)
+SSHKey SSHKeyFactory::makePublicKeyFromFile(String filename)
 {
     ssh_key key;
-    int rc = ssh_pki_import_pubkey_file(filename.c_str(), &key);
+    if (int rc = ssh_pki_import_pubkey_file(filename.c_str(), &key); rc != SSH_OK)
-    if (rc != SSH_OK)
         throw Exception(ErrorCodes::LIBSSH_ERROR, "Can't import SSH public key from file");
-
     return SSHKey(key);
 }
 
-SSHKey SSHKeyFactory::makePublicFromBase64(String base64_key, String type_name)
+SSHKey SSHKeyFactory::makePublicKeyFromBase64(String base64_key, String type_name)
 {
     ssh_key key;
     auto key_type = ssh_key_type_from_name(type_name.c_str());
-    int rc = ssh_pki_import_pubkey_base64(base64_key.c_str(), key_type, &key);
+    if (int rc = ssh_pki_import_pubkey_base64(base64_key.c_str(), key_type, &key); rc != SSH_OK)
-    if (rc != SSH_OK)
         throw Exception(ErrorCodes::LIBSSH_ERROR, "Bad SSH public key provided");
-
     return SSHKey(key);
 }
 
@@ -90,6 +83,12 @@ SSHKey::SSHKey(const SSHKey & other)
     key = ssh_key_dup(other.key);
 }
 
+SSHKey::SSHKey(SSHKey && other) noexcept
+{
+    key = other.key;
+    other.key = nullptr;
+}
+
 SSHKey & SSHKey::operator=(const SSHKey & other)
 {
     ssh_key_free(key);
@@ -119,13 +118,11 @@ bool SSHKey::isEqual(const SSHKey & other) const
 String SSHKey::signString(std::string_view input) const
 {
     SSHString input_str(input);
-    ssh_string c_output = nullptr;
+    ssh_string output = nullptr;
-    int rc = pki_sign_string(key, input_str.get(), &c_output);
+    if (int rc = pki_sign_string(key, input_str.get(), &output); rc != SSH_OK)
-    if (rc != SSH_OK)
         throw Exception(ErrorCodes::LIBSSH_ERROR, "Error singing with ssh key");
-
+    SSHString output_str(output);
-    SSHString output(c_output);
+    return output_str.toString();
-    return output.toString();
 }
 
 bool SSHKey::verifySignature(std::string_view signature, std::string_view original) const
@@ -149,18 +146,15 @@ namespace
 {
     struct CStringDeleter
     {
-        [[maybe_unused]] void operator()(char * ptr) const { std::free(ptr); }
+        void operator()(char * ptr) const { std::free(ptr); }
     };
 }
 
 String SSHKey::getBase64() const
 {
     char * buf = nullptr;
-    int rc = ssh_pki_export_pubkey_base64(key, &buf);
+    if (int rc = ssh_pki_export_pubkey_base64(key, &buf); rc != SSH_OK)
-
-    if (rc != SSH_OK)
         throw DB::Exception(DB::ErrorCodes::LIBSSH_ERROR, "Failed to export public key to base64");
-
     /// Create a String from cstring, which makes a copy of the first one and requires freeing memory after it
     /// This is to safely manage buf memory
     std::unique_ptr<char, CStringDeleter> buf_ptr(buf);
@@ -177,7 +171,6 @@ SSHKey::~SSHKey()
     ssh_key_free(key); // it's safe free from libssh
 }
 
-}
 }
 
 #endif

src/Common/SSHWrapper.h

@@ -1,20 +1,18 @@
 #pragma once
-#include <Common/Exception.h>
-#include "config.h"
-#if USE_SSH
-#    include <string_view>
-#    include <base/types.h>
 
+#include <Common/Exception.h>
+
+#include <string_view>
+#include <base/types.h>
+
+#include "config.h"
+
+#if USE_SSH
 using ssh_key = struct ssh_key_struct *;
 
 namespace DB
 {
 
-namespace ssh
-{
-
-class SSHKeyFactory;
-
 class SSHKey
 {
 public:
@@ -22,11 +20,7 @@ public:
     ~SSHKey();
 
     SSHKey(const SSHKey & other);
-    SSHKey(SSHKey && other) noexcept
+    SSHKey(SSHKey && other) noexcept;
-    {
-        key = other.key;
-        other.key = nullptr;
-    }
     SSHKey & operator=(const SSHKey & other);
     SSHKey & operator=(SSHKey && other) noexcept;
 
@@ -43,7 +37,7 @@ public:
     String getBase64() const;
     String getKeyType() const;
 
-    friend SSHKeyFactory;
+    friend class SSHKeyFactory;
 private:
     explicit SSHKey(ssh_key key_) : key(key_) { }
     ssh_key key = nullptr;
@@ -56,17 +50,14 @@ public:
     /// The check whether the path is allowed to read for ClickHouse has
     /// (e.g. a file is inside `user_files` directory)
     /// to be done outside of this functions.
-    static SSHKey makePrivateFromFile(String filename, String passphrase);
+    static SSHKey makePrivateKeyFromFile(String filename, String passphrase);
-    static SSHKey makePublicFromFile(String filename);
+    static SSHKey makePublicKeyFromFile(String filename);
-    static SSHKey makePublicFromBase64(String base64_key, String type_name);
+    static SSHKey makePublicKeyFromBase64(String base64_key, String type_name);
 };
 
-}
 }
 
 #else
-namespace ssh
-{
 class SSHKey
 {
 public:
@@ -74,5 +65,4 @@ public:
     [[ noreturn ]] bool isEmpty() { std::terminate(); }
     [[ noreturn ]] String signString(std::string_view) const { std::terminate(); }
 };
-}
 #endif

src/Common/StackTrace.cpp

@@ -18,13 +18,10 @@
 #include <filesystem>
 #include <map>
 #include <mutex>
-#include <sstream>
 #include <unordered_map>
 #include <fmt/format.h>
 #include <libunwind.h>
 
-#include "config.h"
-
 #include <boost/algorithm/string/split.hpp>
 
 #if defined(OS_DARWIN)
@@ -481,7 +478,17 @@ void StackTrace::toStringEveryLine(void ** frame_pointers_raw, size_t offset, si
     toStringEveryLineImpl(true, {frame_pointers, offset, size}, std::move(callback));
 }
 
-using StackTraceCache = std::map<StackTraceTriple, String, std::less<>>;
+struct CacheEntry
+{
+    std::optional<String> stacktrace_string;
+    bool to_string_in_progress = false;
+
+    std::condition_variable cv;
+};
+
+using CacheEntryPtr = std::shared_ptr<CacheEntry>;
+
+using StackTraceCache = std::map<StackTraceTriple, CacheEntryPtr, std::less<>>;
 
 static StackTraceCache & cacheInstance()
 {
@@ -493,23 +500,63 @@ static std::mutex stacktrace_cache_mutex;
 
 String toStringCached(const StackTrace::FramePointers & pointers, size_t offset, size_t size)
 {
+    const StackTraceRefTriple key{pointers, offset, size};
+
     /// Calculation of stack trace text is extremely slow.
     /// We use simple cache because otherwise the server could be overloaded by trash queries.
     /// Note that this cache can grow unconditionally, but practically it should be small.
-    std::lock_guard lock{stacktrace_cache_mutex};
+    std::unique_lock lock{stacktrace_cache_mutex};
-
+    CacheEntryPtr cache_entry;
     StackTraceCache & cache = cacheInstance();
-    const StackTraceRefTriple key{pointers, offset, size};
-
     if (auto it = cache.find(key); it != cache.end())
-        return it->second;
+    {
+        cache_entry = it->second;
+    }
     else
+    {
+        auto [new_it, inserted] = cache.emplace(StackTraceTriple{pointers, offset, size}, std::make_shared<CacheEntry>());
+        chassert(inserted);
+        cache_entry = new_it->second;
+    }
+
+    if (!cache_entry->to_string_in_progress && cache_entry->stacktrace_string.has_value())
+        return *cache_entry->stacktrace_string;
+
+    if (cache_entry->to_string_in_progress)
+    {
+        cache_entry->cv.wait(lock, [&]{ return !cache_entry->to_string_in_progress; });
+
+        if (cache_entry->stacktrace_string.has_value())
+            return *cache_entry->stacktrace_string;
+    }
+
+    cache_entry->to_string_in_progress = true;
+
+    lock.unlock();
+
+    String stacktrace_string;
+    try
     {
         DB::WriteBufferFromOwnString out;
         toStringEveryLineImpl(false, key, [&](std::string_view str) { out << str << '\n'; });
-
+        stacktrace_string = out.str();
-        return cache.emplace(StackTraceTriple{pointers, offset, size}, out.str()).first->second;
     }
+    catch (...)
+    {
+        lock.lock();
+        cache_entry->to_string_in_progress = false;
+        lock.unlock();
+        cache_entry->cv.notify_one();
+        throw;
+    }
+
+    lock.lock();
+    cache_entry->to_string_in_progress = false;
+    cache_entry->stacktrace_string = stacktrace_string;
+    lock.unlock();
+
+    cache_entry->cv.notify_all();
+    return stacktrace_string;
 }
 
 std::string StackTrace::toString() const

src/Common/tests/gtest_connection_pool.cpp

@@ -2,7 +2,6 @@
 #include <Common/HTTPConnectionPool.h>
 
 #include <Poco/URI.h>
-#include <Poco/Net/ServerSocket.h>
 #include <Poco/Net/MessageHeader.h>
 #include <Poco/Net/HTTPServerRequest.h>
 #include <Poco/Net/HTTPServerResponse.h>
@@ -17,6 +16,40 @@
 namespace
 {
 
+template<class T>
+class SafeHandler
+{
+public:
+    using Ptr = std::shared_ptr<SafeHandler<T>>;
+
+    SafeHandler() = default;
+    SafeHandler(SafeHandler<T>&) = delete;
+    SafeHandler& operator=(SafeHandler<T>&) = delete;
+
+    T get()
+    {
+        std::lock_guard lock(mutex);
+        return obj;
+    }
+
+    void set(T && options_)
+    {
+        std::lock_guard lock(mutex);
+        obj = std::move(options_);
+    }
+
+protected:
+    std::mutex mutex;
+    T obj = {};
+};
+
+struct RequestOptions
+{
+    size_t slowdown_receive = 0;
+    int overwrite_keep_alive_timeout = 0;
+    int overwrite_keep_alive_max_requests = 10;
+};
+
 size_t stream_copy_n(std::istream & in, std::ostream & out, std::size_t count = std::numeric_limits<size_t>::max())
 {
     const size_t buffer_size = 4096;
@@ -47,13 +80,21 @@ size_t stream_copy_n(std::istream & in, std::ostream & out, std::size_t count =
 class MockRequestHandler : public Poco::Net::HTTPRequestHandler
 {
 public:
-    explicit MockRequestHandler(std::shared_ptr<std::atomic<size_t>> slowdown_)
+    explicit MockRequestHandler(SafeHandler<RequestOptions>::Ptr options_)
-        : slowdown(std::move(slowdown_))
+        : options(options_)
     {
     }
 
     void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override
     {
+        int value = request.getKeepAliveTimeout();
+        ASSERT_GT(value, 0);
+
+        auto params = options->get();
+
+        if (params.overwrite_keep_alive_timeout > 0)
+            response.setKeepAliveTimeout(params.overwrite_keep_alive_timeout, params.overwrite_keep_alive_max_requests);
+
         response.setStatus(Poco::Net::HTTPResponse::HTTP_OK);
         auto size = request.getContentLength();
         if (size > 0)
@@ -61,28 +102,29 @@ public:
         else
             response.setChunkedTransferEncoding(true); // or chunk encoding
 
-        sleepForSeconds(*slowdown);
+        if (params.slowdown_receive > 0)
+            sleepForSeconds(params.slowdown_receive);
 
         stream_copy_n(request.stream(), response.send(), size);
     }
 
-    std::shared_ptr<std::atomic<size_t>> slowdown;
+    SafeHandler<RequestOptions>::Ptr options;
 };
 
 class HTTPRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory
 {
 public:
-    explicit HTTPRequestHandlerFactory(std::shared_ptr<std::atomic<size_t>> slowdown_)
+    explicit HTTPRequestHandlerFactory(SafeHandler<RequestOptions>::Ptr options_)
-        : slowdown(std::move(slowdown_))
+        : options(options_)
     {
     }
 
     Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest &) override
     {
-        return new MockRequestHandler(slowdown);
+        return new MockRequestHandler(options);
     }
 
-    std::shared_ptr<std::atomic<size_t>> slowdown;
+    SafeHandler<RequestOptions>::Ptr options;
 };
 
 }
@@ -94,6 +136,8 @@ class ConnectionPoolTest : public testing::Test {
 protected:
     ConnectionPoolTest()
     {
+        options = std::make_shared<SafeHandler<RequestOptions>>();
+
         startServer();
     }
 
@@ -102,7 +146,7 @@ protected:
         DB::HTTPConnectionPools::Limits def_limits{};
         DB::HTTPConnectionPools::instance().setLimits(def_limits, def_limits, def_limits);
 
-        setSlowDown(0);
+        options->set(RequestOptions());
 
         DB::HTTPConnectionPools::instance().dropCache();
         DB::CurrentThread::getProfileEvents().reset();
@@ -129,7 +173,7 @@ protected:
     void startServer()
     {
         server_data.reset();
-        server_data.handler_factory = new HTTPRequestHandlerFactory(slowdown_receive);
+        server_data.handler_factory = new HTTPRequestHandlerFactory(options);
         server_data.server =  std::make_unique<Poco::Net::HTTPServer>(
             server_data.handler_factory, server_data.port);
 
@@ -143,11 +187,21 @@ protected:
 
     void setSlowDown(size_t seconds)
     {
-        *slowdown_receive = seconds;
+        auto opt = options->get();
+        opt.slowdown_receive = seconds;
+        options->set(std::move(opt));
+    }
+
+    void setOverWriteKeepAlive(size_t seconds, int max_requests)
+    {
+        auto opt = options->get();
+        opt.overwrite_keep_alive_timeout = int(seconds);
+        opt.overwrite_keep_alive_max_requests= max_requests;
+        options->set(std::move(opt));
     }
 
     DB::ConnectionTimeouts timeouts;
-    std::shared_ptr<std::atomic<size_t>> slowdown_receive = std::make_shared<std::atomic<size_t>>(0);
+    SafeHandler<RequestOptions>::Ptr options;
 
     struct ServerData
     {
@@ -182,7 +236,7 @@ protected:
 void wait_until(std::function<bool()> pred)
 {
     while (!pred())
-        sleepForMilliseconds(250);
+        sleepForMilliseconds(10);
 }
 
 void echoRequest(String data, HTTPSession & session)
@@ -245,45 +299,52 @@ TEST_F(ConnectionPoolTest, CanRequest)
     ASSERT_EQ(0, getServer().currentConnections());
     ASSERT_EQ(1, getServer().totalConnections());
 
-    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
+    auto metrics = pool->getMetrics();
+
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
+
+    ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count));
+    ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
 }
 
 TEST_F(ConnectionPoolTest, CanPreserve)
 {
     auto pool = getPool();
+    auto metrics = pool->getMetrics();
 
     {
         auto connection = pool->getConnection(timeouts);
     }
 
-    ASSERT_EQ(1, CurrentMetrics::get(pool->getMetrics().active_count));
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]);
-    ASSERT_EQ(1, CurrentMetrics::get(pool->getMetrics().stored_count));
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
+
+    ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count));
+    ASSERT_EQ(1, CurrentMetrics::get(metrics.stored_count));
 
     wait_until([&] () { return getServer().currentConnections() == 1; });
     ASSERT_EQ(1, getServer().currentConnections());
-
-    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
-    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
-    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]);
 }
 
 TEST_F(ConnectionPoolTest, CanReuse)
 {
     auto pool = getPool();
+    auto metrics = pool->getMetrics();
 
     {
         auto connection = pool->getConnection(timeouts);
-        // DB::setReuseTag(*connection);
     }
 
-    ASSERT_EQ(1, CurrentMetrics::get(pool->getMetrics().active_count));
-    ASSERT_EQ(1, CurrentMetrics::get(pool->getMetrics().stored_count));
-
     {
         auto connection = pool->getConnection(timeouts);
 
-        ASSERT_EQ(1, CurrentMetrics::get(pool->getMetrics().active_count));
+        ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count));
-        ASSERT_EQ(0, CurrentMetrics::get(pool->getMetrics().stored_count));
+        ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
 
         wait_until([&] () { return getServer().currentConnections() == 1; });
         ASSERT_EQ(1, getServer().currentConnections());
@@ -293,6 +354,11 @@ TEST_F(ConnectionPoolTest, CanReuse)
         ASSERT_EQ(1, getServer().totalConnections());
         ASSERT_EQ(1, getServer().currentConnections());
 
+        ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]);
+        ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
+        ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reused]);
+        ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
+
         connection->reset();
     }
 
@@ -303,15 +369,16 @@ TEST_F(ConnectionPoolTest, CanReuse)
     ASSERT_EQ(0, getServer().currentConnections());
     ASSERT_EQ(1, getServer().totalConnections());
 
-    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]);
-    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
-    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]);
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reused]);
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]);
 }
 
 TEST_F(ConnectionPoolTest, CanReuse10)
 {
     auto pool = getPool();
-
+    auto metrics = pool->getMetrics();
 
     for (int i = 0; i < 10; ++i)
     {
@@ -328,16 +395,23 @@ TEST_F(ConnectionPoolTest, CanReuse10)
     ASSERT_EQ(0, getServer().currentConnections());
     ASSERT_EQ(1, getServer().totalConnections());
 
-    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
+
-    ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]);
-    ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]);
+    ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
+    ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[metrics.reused]);
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]);
+
+    ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count));
+    ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
 }
 
 TEST_F(ConnectionPoolTest, CanReuse5)
 {
-    timeouts.withHTTPKeepAliveTimeout(1);
+    auto ka = Poco::Timespan(1, 0); // 1 seconds
+    timeouts.withHTTPKeepAliveTimeout(ka);
 
     auto pool = getPool();
+    auto metrics = pool->getMetrics();
 
     std::vector<DB::HTTPSessionPtr> connections;
     connections.reserve(5);
@@ -347,11 +421,14 @@ TEST_F(ConnectionPoolTest, CanReuse5)
     }
     connections.clear();
 
-    ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
+    ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[metrics.created]);
-    ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
+    ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
-    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
-    ASSERT_EQ(5, CurrentMetrics::get(pool->getMetrics().active_count));
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
-    ASSERT_EQ(5, CurrentMetrics::get(pool->getMetrics().stored_count));
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
+
+    ASSERT_EQ(5, CurrentMetrics::get(metrics.active_count));
+    ASSERT_EQ(5, CurrentMetrics::get(metrics.stored_count));
 
     wait_until([&] () { return getServer().currentConnections() == 5; });
     ASSERT_EQ(5, getServer().currentConnections());
@@ -363,35 +440,56 @@ TEST_F(ConnectionPoolTest, CanReuse5)
         echoRequest("Hello", *connection);
     }
 
-    ASSERT_EQ(5, getServer().totalConnections());
+    ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[metrics.created]);
+    ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
+    ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[metrics.reused]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
 
-    ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
+    ASSERT_EQ(5, CurrentMetrics::get(metrics.active_count));
-    ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
+    ASSERT_EQ(5, CurrentMetrics::get(metrics.stored_count));
-    ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]);
+
-    ASSERT_EQ(5, CurrentMetrics::get(pool->getMetrics().active_count));
+    /// wait until all connections are timeouted
-    ASSERT_EQ(5, CurrentMetrics::get(pool->getMetrics().stored_count));
+    wait_until([&] () { return getServer().currentConnections() == 0; });
+
+    {
+        // just to trigger pool->wipeExpired();
+        auto connection = pool->getConnection(timeouts);
+        connection->reset();
+    }
+
+    ASSERT_EQ(6, DB::CurrentThread::getProfileEvents()[metrics.created]);
+    ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
+    ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[metrics.reused]);
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]);
+    ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[metrics.expired]);
+
+    ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count));
+    ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
 }
 
 TEST_F(ConnectionPoolTest, CanReconnectAndCreate)
 {
     auto pool = getPool();
+    auto metrics = pool->getMetrics();
 
     std::vector<HTTPSessionPtr> in_use;
 
-    const size_t count = 2;
+    const size_t count = 3;
     for (int i = 0; i < count; ++i)
     {
         auto connection = pool->getConnection(timeouts);
-        // DB::setReuseTag(*connection);
         in_use.push_back(connection);
     }
 
-    ASSERT_EQ(count, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
+    ASSERT_EQ(count, DB::CurrentThread::getProfileEvents()[metrics.created]);
-    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
-    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
 
-    ASSERT_EQ(count, CurrentMetrics::get(pool->getMetrics().active_count));
+    ASSERT_EQ(count, CurrentMetrics::get(metrics.active_count));
-    ASSERT_EQ(0, CurrentMetrics::get(pool->getMetrics().stored_count));
+    ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
 
     auto connection = std::move(in_use.back());
     in_use.pop_back();
@@ -402,28 +500,39 @@ TEST_F(ConnectionPoolTest, CanReconnectAndCreate)
 
     echoRequest("Hello", *connection);
 
-    connection->reset();
+    ASSERT_EQ(count+1, DB::CurrentThread::getProfileEvents()[metrics.created]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
 
-    wait_until([&] () { return getServer().currentConnections() == 1; });
+    ASSERT_EQ(count, CurrentMetrics::get(metrics.active_count));
-    ASSERT_EQ(1, getServer().currentConnections());
+    ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
-    ASSERT_EQ(count+1, getServer().totalConnections());
-
-    ASSERT_EQ(count+1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
-    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
-    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]);
 }
 
 TEST_F(ConnectionPoolTest, CanReconnectAndReuse)
 {
+    auto ka = Poco::Timespan(1, 0); // 1 seconds
+    timeouts.withHTTPKeepAliveTimeout(ka);
+
     auto pool = getPool();
+    auto metrics = pool->getMetrics();
 
     std::vector<HTTPSessionPtr> in_use;
 
-    const size_t count = 2;
+    const size_t count = 3;
+    for (int i = 0; i < count; ++i)
+    {
+        auto connection = pool->getConnection(timeouts);
+        /// make some request in order to show to the server the keep alive headers
+        echoRequest("Hello", *connection);
+        in_use.push_back(std::move(connection));
+    }
+    in_use.clear();
+
     for (int i = 0; i < count; ++i)
     {
         auto connection = pool->getConnection(timeouts);
-        // DB::setReuseTag(*connection);
         in_use.push_back(std::move(connection));
     }
 
@@ -441,11 +550,16 @@ TEST_F(ConnectionPoolTest, CanReconnectAndReuse)
 
     wait_until([&] () { return getServer().currentConnections() == 0; });
     ASSERT_EQ(0, getServer().currentConnections());
-    ASSERT_EQ(2, getServer().totalConnections());
+    ASSERT_EQ(count, getServer().totalConnections());
 
-    ASSERT_EQ(count, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
+    ASSERT_EQ(count, DB::CurrentThread::getProfileEvents()[metrics.created]);
-    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
+    ASSERT_EQ(count + count - 1, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
-    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]);
+    ASSERT_EQ(count + 1, DB::CurrentThread::getProfileEvents()[metrics.reused]);
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
+
+    ASSERT_EQ(count-1, CurrentMetrics::get(metrics.active_count));
+    ASSERT_EQ(count-2, CurrentMetrics::get(metrics.stored_count));
 }
 
 TEST_F(ConnectionPoolTest, ReceiveTimeout)
@@ -454,6 +568,7 @@ TEST_F(ConnectionPoolTest, ReceiveTimeout)
     timeouts.withReceiveTimeout(1);
 
     auto pool = getPool();
+    auto metrics = pool->getMetrics();
 
     {
         auto connection = pool->getConnection(timeouts);
@@ -462,10 +577,14 @@ TEST_F(ConnectionPoolTest, ReceiveTimeout)
         );
     }
 
-    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]);
-    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
-    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
-    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reset]);
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
+
+    ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count));
+    ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
 
     {
         timeouts.withReceiveTimeout(3);
@@ -475,10 +594,14 @@ TEST_F(ConnectionPoolTest, ReceiveTimeout)
         );
     }
 
-    ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
+    ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[metrics.created]);
-    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
-    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
-    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reset]);
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
+
+    ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count));
+    ASSERT_EQ(1, CurrentMetrics::get(metrics.stored_count));
 
     {
         /// timeouts have effect for reused session
@@ -489,10 +612,14 @@ TEST_F(ConnectionPoolTest, ReceiveTimeout)
         );
     }
 
-    ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
+    ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[metrics.created]);
-    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
-    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]);
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reused]);
-    ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reset]);
+    ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[metrics.reset]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
+
+    ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count));
+    ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
 }
 
 TEST_F(ConnectionPoolTest, ReadWriteBufferFromHTTP)
@@ -500,6 +627,7 @@ TEST_F(ConnectionPoolTest, ReadWriteBufferFromHTTP)
     std::string_view message = "Hello ReadWriteBufferFromHTTP";
     auto uri = Poco::URI(getServerUrl());
     auto metrics = DB::HTTPConnectionPools::instance().getPool(DB::HTTPConnectionGroupType::HTTP, uri, DB::ProxyConfiguration{})->getMetrics();
+
     Poco::Net::HTTPBasicCredentials empty_creds;
     auto buf_from_http = DB::BuilderRWBufferFromHTTP(uri)
                              .withConnectionGroup(DB::HTTPConnectionGroupType::HTTP)
@@ -527,6 +655,7 @@ TEST_F(ConnectionPoolTest, ReadWriteBufferFromHTTP)
     ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
     ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
     ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
 
     ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count));
     ASSERT_EQ(1, CurrentMetrics::get(metrics.stored_count));
@@ -538,23 +667,26 @@ TEST_F(ConnectionPoolTest, HardLimit)
     DB::HTTPConnectionPools::instance().setLimits(zero_limits, zero_limits, zero_limits);
 
     auto pool = getPool();
+    auto metrics = pool->getMetrics();
 
     {
         auto connection = pool->getConnection(timeouts);
     }
 
-    ASSERT_EQ(0, CurrentMetrics::get(pool->getMetrics().active_count));
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]);
-    ASSERT_EQ(0, CurrentMetrics::get(pool->getMetrics().stored_count));
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
 
-
+    ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count));
-    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
+    ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
-    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
-    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reset]);
 }
 
 TEST_F(ConnectionPoolTest, NoReceiveCall)
 {
     auto pool = getPool();
+    auto metrics = pool->getMetrics();
 
     {
         auto connection = pool->getConnection(timeouts);
@@ -570,11 +702,209 @@ TEST_F(ConnectionPoolTest, NoReceiveCall)
         connection->flushRequest();
     }
 
-    ASSERT_EQ(0, CurrentMetrics::get(pool->getMetrics().active_count));
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]);
-    ASSERT_EQ(0, CurrentMetrics::get(pool->getMetrics().stored_count));
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
 
-
+    ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count));
-    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
+    ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
-    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
+}
-    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reset]);
+
+TEST_F(ConnectionPoolTest, ReconnectedWhenConnectionIsHoldTooLong)
+{
+    auto ka = Poco::Timespan(1, 0); // 1 seconds
+    timeouts.withHTTPKeepAliveTimeout(ka);
+
+    auto pool = getPool();
+    auto metrics = pool->getMetrics();
+
+    {
+        auto connection = pool->getConnection(timeouts);
+
+        echoRequest("Hello", *connection);
+
+        auto fake_ka = Poco::Timespan(30 * 1000 * 1000); // 30 seconds
+        timeouts.withHTTPKeepAliveTimeout(fake_ka);
+        DB::setTimeouts(*connection, timeouts); // new keep alive timeout has no effect
+
+        wait_until([&] () { return getServer().currentConnections() == 0; });
+
+        ASSERT_EQ(1, connection->connected());
+        ASSERT_EQ(1, connection->getKeepAlive());
+        ASSERT_EQ(1000, connection->getKeepAliveTimeout().totalMilliseconds());
+        ASSERT_EQ(1, connection->isKeepAliveExpired(connection->getKeepAliveReliability()));
+
+        echoRequest("Hello", *connection);
+    }
+
+
+    ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[metrics.created]);
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
+
+    ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count));
+    ASSERT_EQ(1, CurrentMetrics::get(metrics.stored_count));
+}
+
+TEST_F(ConnectionPoolTest, ReconnectedWhenConnectionIsNearlyExpired)
+{
+    auto ka = Poco::Timespan(1, 0); // 1 seconds
+    timeouts.withHTTPKeepAliveTimeout(ka);
+
+    auto pool = getPool();
+    auto metrics = pool->getMetrics();
+
+    {
+        {
+            auto connection = pool->getConnection(timeouts);
+            echoRequest("Hello", *connection);
+        }
+
+        sleepForMilliseconds(900);
+
+        {
+            auto connection = pool->getConnection(timeouts);
+            echoRequest("Hello", *connection);
+        }
+    }
+
+    ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[metrics.created]);
+    ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.expired]);
+
+    ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count));
+    ASSERT_EQ(1, CurrentMetrics::get(metrics.stored_count));
+}
+
+TEST_F(ConnectionPoolTest, ServerOverwriteKeepAlive)
+{
+    auto ka = Poco::Timespan(30, 0); // 30 seconds
+    timeouts.withHTTPKeepAliveTimeout(ka);
+
+    auto pool = getPool();
+    auto metrics = pool->getMetrics();
+
+    {
+        auto connection = pool->getConnection(timeouts);
+        echoRequest("Hello", *connection);
+        ASSERT_EQ(30, timeouts.http_keep_alive_timeout.totalSeconds());
+        ASSERT_EQ(30, connection->getKeepAliveTimeout().totalSeconds());
+    }
+
+    {
+        setOverWriteKeepAlive(1, 10);
+        auto connection = pool->getConnection(timeouts);
+        echoRequest("Hello", *connection);
+        ASSERT_EQ(30, timeouts.http_keep_alive_timeout.totalSeconds());
+        ASSERT_EQ(1, connection->getKeepAliveTimeout().totalSeconds());
+    }
+
+    {
+        // server do not overwrite it in the following requests but client has to remember last agreed value
+        setOverWriteKeepAlive(0, 0);
+        auto connection = pool->getConnection(timeouts);
+        echoRequest("Hello", *connection);
+        ASSERT_EQ(30, timeouts.http_keep_alive_timeout.totalSeconds());
+        ASSERT_EQ(1, connection->getKeepAliveTimeout().totalSeconds());
+    }
+
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]);
+    ASSERT_EQ(3, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
+    ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[metrics.reused]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
+
+    ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count));
+    ASSERT_EQ(1, CurrentMetrics::get(metrics.stored_count));
+}
+
+TEST_F(ConnectionPoolTest, MaxRequests)
+{
+    auto ka = Poco::Timespan(30, 0); // 30 seconds
+    timeouts.withHTTPKeepAliveTimeout(ka);
+    auto max_requests = 5;
+    timeouts.http_keep_alive_max_requests = max_requests;
+
+    auto pool = getPool();
+    auto metrics = pool->getMetrics();
+
+    for (int i = 1; i <= max_requests - 1; ++i)
+    {
+        auto connection = pool->getConnection(timeouts);
+        echoRequest("Hello", *connection);
+        ASSERT_EQ(30, connection->getKeepAliveTimeout().totalSeconds());
+        ASSERT_EQ(max_requests, connection->getKeepAliveMaxRequests());
+        ASSERT_EQ(i, connection->getKeepAliveRequest());
+    }
+
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]);
+    ASSERT_EQ(max_requests-1, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
+    ASSERT_EQ(max_requests-2, DB::CurrentThread::getProfileEvents()[metrics.reused]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
+
+    ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count));
+    ASSERT_EQ(1, CurrentMetrics::get(metrics.stored_count));
+
+    {
+        auto connection = pool->getConnection(timeouts);
+        echoRequest("Hello", *connection);
+        ASSERT_EQ(30, connection->getKeepAliveTimeout().totalSeconds());
+        ASSERT_EQ(max_requests, connection->getKeepAliveMaxRequests());
+        ASSERT_EQ(max_requests, connection->getKeepAliveRequest());
+    }
+
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]);
+    ASSERT_EQ(max_requests-1, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
+    ASSERT_EQ(max_requests-1, DB::CurrentThread::getProfileEvents()[metrics.reused]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
+    ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.expired]);
+
+    ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count));
+    ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
+}
+
+
+TEST_F(ConnectionPoolTest, ServerOverwriteMaxRequests)
+{
+    auto ka = Poco::Timespan(30, 0); // 30 seconds
+    timeouts.withHTTPKeepAliveTimeout(ka);
+
+    auto pool = getPool();
+    auto metrics = pool->getMetrics();
+
+    {
+        auto connection = pool->getConnection(timeouts);
+        echoRequest("Hello", *connection);
+        ASSERT_EQ(30, connection->getKeepAliveTimeout().totalSeconds());
+        ASSERT_EQ(1000, connection->getKeepAliveMaxRequests());
+        ASSERT_EQ(1, connection->getKeepAliveRequest());
+    }
+
+    auto max_requests = 3;
+    setOverWriteKeepAlive(5, max_requests);
+
+    for (int i = 2; i <= 10*max_requests; ++i)
+    {
+        auto connection = pool->getConnection(timeouts);
+        echoRequest("Hello", *connection);
+        ASSERT_EQ(5, connection->getKeepAliveTimeout().totalSeconds());
+        ASSERT_EQ(max_requests, connection->getKeepAliveMaxRequests());
+        ASSERT_EQ(((i-1) % max_requests) + 1, connection->getKeepAliveRequest());
+    }
+
+    ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[metrics.created]);
+    ASSERT_EQ(10*max_requests-10, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
+    ASSERT_EQ(10*max_requests-10, DB::CurrentThread::getProfileEvents()[metrics.reused]);
+    ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
+    ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[metrics.expired]);
+
+    ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count));
+    ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
 }

src/Compression/tests/gtest_compressionCodec.cpp

@@ -483,7 +483,7 @@ void testTranscoding(Timer & timer, ICompressionCodec & codec, const CodecTestSe
 
     ASSERT_TRUE(EqualByteContainers(test_sequence.data_type->getSizeOfValueInMemory(), source_data, decoded));
 
-    const auto header_size = codec.getHeaderSize();
+    const auto header_size = ICompressionCodec::getHeaderSize();
     const auto compression_ratio = (encoded_size - header_size) / (source_data.size() * 1.0);
 
     if (expected_compression_ratio)

src/Core/Defines.h

@@ -54,6 +54,7 @@ static constexpr auto DEFAULT_COUNT_OF_HTTP_CONNECTIONS_PER_ENDPOINT = 15;
 
 static constexpr auto DEFAULT_TCP_KEEP_ALIVE_TIMEOUT = 290;
 static constexpr auto DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT = 30;
+static constexpr auto DEFAULT_HTTP_KEEP_ALIVE_MAX_REQUEST = 1000;
 
 static constexpr auto DBMS_DEFAULT_PATH = "/var/lib/clickhouse/";
 

src/Core/Protocol.h

@@ -56,10 +56,11 @@ namespace DB
 namespace EncodedUserInfo
 {
 
-/// Marker of the inter-server secret (passed in the user name)
+/// Marker for the inter-server secret (passed as the user name)
 /// (anyway user cannot be started with a whitespace)
 const char USER_INTERSERVER_MARKER[] = " INTERSERVER SECRET ";
-/// Marker of the SSH keys based authentication (passed in the user name)
+
+/// Marker for SSH-keys-based authentication (passed as the user name)
 const char SSH_KEY_AUTHENTICAION_MARKER[] = " SSH KEY AUTHENTICATION ";
 
 };
@@ -160,8 +161,8 @@ namespace Protocol
             ReadTaskResponse = 9,           /// A filename to read from s3 (used in s3Cluster)
             MergeTreeReadTaskResponse = 10, /// Coordinator's decision with a modified set of mark ranges allowed to read
 
-            SSHChallengeRequest = 11,       /// Request for SSH signature challenge
+            SSHChallengeRequest = 11,       /// Request SSH signature challenge
-            SSHChallengeResponse = 12,       /// Request for SSH signature challenge
+            SSHChallengeResponse = 12,      /// Reply to SSH signature challenge
             MAX = SSHChallengeResponse,
         };
 

src/Core/ServerSettings.h

@@ -128,9 +128,9 @@ namespace DB
     M(Bool, format_alter_operations_with_parentheses, false, "If enabled, each operation in alter queries will be surrounded with parentheses in formatted queries to make them less ambiguous.", 0) \
     M(String, default_replica_path, "/clickhouse/tables/{uuid}/{shard}", "The path to the table in ZooKeeper", 0) \
     M(String, default_replica_name, "{replica}", "The replica name in ZooKeeper", 0) \
-    M(UInt64, disk_connections_soft_limit, 1000, "Connections above this limit have significantly shorter time to live. The limit applies to the disks connections.", 0) \
+    M(UInt64, disk_connections_soft_limit, 5000, "Connections above this limit have significantly shorter time to live. The limit applies to the disks connections.", 0) \
     M(UInt64, disk_connections_warn_limit, 10000, "Warning massages are written to the logs if number of in-use connections are higher than this limit. The limit applies to the disks connections.", 0) \
-    M(UInt64, disk_connections_store_limit, 12000, "Connections above this limit reset after use. Set to 0 to turn connection cache off. The limit applies to the disks connections.", 0) \
+    M(UInt64, disk_connections_store_limit, 30000, "Connections above this limit reset after use. Set to 0 to turn connection cache off. The limit applies to the disks connections.", 0) \
     M(UInt64, storage_connections_soft_limit, 100, "Connections above this limit have significantly shorter time to live. The limit applies to the storages connections.", 0) \
     M(UInt64, storage_connections_warn_limit, 1000, "Warning massages are written to the logs if number of in-use connections are higher than this limit. The limit applies to the storages connections.", 0) \
     M(UInt64, storage_connections_store_limit, 5000, "Connections above this limit reset after use. Set to 0 to turn connection cache off. The limit applies to the storages connections.", 0) \

src/Daemon/BaseDaemon.cpp

@@ -332,6 +332,7 @@ private:
         const std::vector<StackTrace::FramePointers> & thread_frame_pointers,
         UInt32 thread_num,
         ThreadStatus * thread_ptr) const
+    try
     {
         ThreadStatus thread_status;
 
@@ -519,7 +520,7 @@ private:
             }
         }
 
-        /// ClickHouse Keeper does not link to some part of Settings.
+        /// ClickHouse Keeper does not link to some parts of Settings.
 #ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
         /// List changed settings.
         if (!query_id.empty())
@@ -537,12 +538,18 @@ private:
         }
 #endif
 
-        /// When everything is done, we will try to send these error messages to client.
+        /// When everything is done, we will try to send these error messages to the client.
         if (thread_ptr)
             thread_ptr->onFatalError();
 
         fatal_error_printed.test_and_set();
     }
+    catch (...)
+    {
+        /// onFault is called from the std::thread, and it should catch all exceptions; otherwise, you can get unrelated fatal errors.
+        PreformattedMessage message = getCurrentExceptionMessageAndPattern(true);
+        LOG_FATAL(getLogger(__PRETTY_FUNCTION__), message);
+    }
 };
 
 
@@ -665,7 +672,7 @@ void BaseDaemon::reloadConfiguration()
       */
     config_path = config().getString("config-file", getDefaultConfigFileName());
     ConfigProcessor config_processor(config_path, false, true);
-    config_processor.setConfigPath(fs::path(config_path).parent_path());
+    ConfigProcessor::setConfigPath(fs::path(config_path).parent_path());
     loaded_config = config_processor.loadConfig(/* allow_zk_includes = */ true);
 
     if (last_configuration != nullptr)

src/DataTypes/DataTypeDate32.cpp

@@ -18,7 +18,7 @@ SerializationPtr DataTypeDate32::doGetDefaultSerialization() const
 
 Field DataTypeDate32::getDefault() const
 {
-    return -static_cast<Int64>(DateLUT::instance().getDayNumOffsetEpoch());
+    return -static_cast<Int64>(DateLUT::instance().getDayNumOffsetEpoch()); /// NOLINT(readability-static-accessed-through-instance)
 }
 
 void registerDataTypeDate32(DataTypeFactory & factory)

src/DataTypes/Serializations/tests/gtest_json_parser.cpp

@@ -34,7 +34,7 @@ TEST(JSONDataParser, ReadJSON)
         JSONDataParser<SimdJSONParser> parser;
         ReadBufferFromString buf(json_bad);
         String res;
-        parser.readJSON(res, buf);
+        JSONDataParser<SimdJSONParser>::readJSON(res, buf);
         ASSERT_EQ(json1, res);
     }
 
@@ -44,7 +44,7 @@ TEST(JSONDataParser, ReadJSON)
         JSONDataParser<SimdJSONParser> parser;
         ReadBufferFromString buf(json_bad);
         String res;
-        parser.readJSON(res, buf);
+        JSONDataParser<SimdJSONParser>::readJSON(res, buf);
         ASSERT_EQ(json2, res);
     }
 }

src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp

@@ -346,7 +346,7 @@ CachedOnDiskReadBufferFromFile::getReadBufferForFileSegment(FileSegment & file_s
                 }
 
                 auto downloader_id = file_segment.getOrSetDownloader();
-                if (downloader_id == file_segment.getCallerId())
+                if (downloader_id == FileSegment::getCallerId())
                 {
                     if (canStartFromCache(file_offset_of_buffer_end, file_segment))
                     {

src/Disks/ObjectStorages/S3/diskSettings.cpp

@@ -76,6 +76,9 @@ std::unique_ptr<S3::Client> getClient(
     client_configuration.connectTimeoutMs = config.getUInt(config_prefix + ".connect_timeout_ms", S3::DEFAULT_CONNECT_TIMEOUT_MS);
     client_configuration.requestTimeoutMs = config.getUInt(config_prefix + ".request_timeout_ms", S3::DEFAULT_REQUEST_TIMEOUT_MS);
     client_configuration.maxConnections = config.getUInt(config_prefix + ".max_connections", S3::DEFAULT_MAX_CONNECTIONS);
+    client_configuration.http_keep_alive_timeout = config.getUInt(config_prefix + ".http_keep_alive_timeout", S3::DEFAULT_KEEP_ALIVE_TIMEOUT);
+    client_configuration.http_keep_alive_max_requests = config.getUInt(config_prefix + ".http_keep_alive_max_requests", S3::DEFAULT_KEEP_ALIVE_MAX_REQUESTS);
+
     client_configuration.endpointOverride = uri.endpoint;
     client_configuration.s3_use_adaptive_timeouts = config.getBool(
         config_prefix + ".use_adaptive_timeouts", client_configuration.s3_use_adaptive_timeouts);

src/Functions/FunctionsConversion.cpp

@@ -279,7 +279,7 @@ struct ToDate32Transform32Or64Signed
 
     static NO_SANITIZE_UNDEFINED Int32 execute(const FromType & from, const DateLUTImpl & time_zone)
     {
-        static const Int32 daynum_min_offset = -static_cast<Int32>(time_zone.getDayNumOffsetEpoch());
+        static const Int32 daynum_min_offset = -static_cast<Int32>(DateLUTImpl::getDayNumOffsetEpoch());
 
         if constexpr (date_time_overflow_behavior == FormatSettings::DateTimeOverflowBehavior::Throw)
         {
@@ -1092,7 +1092,7 @@ struct ConvertThroughParsing
                 {
                     if constexpr (std::is_same_v<ToDataType, DataTypeDate32>)
                     {
-                        vec_to[i] = -static_cast<Int32>(DateLUT::instance().getDayNumOffsetEpoch());
+                        vec_to[i] = -static_cast<Int32>(DateLUT::instance().getDayNumOffsetEpoch()); /// NOLINT(readability-static-accessed-through-instance)
                     }
                     else
                     {

src/Functions/UserDefined/UserDefinedSQLFunctionFactory.cpp

@@ -106,7 +106,7 @@ void UserDefinedSQLFunctionFactory::checkCanBeRegistered(const ContextPtr & cont
     if (AggregateFunctionFactory::instance().hasNameOrAlias(function_name))
         throw Exception(ErrorCodes::FUNCTION_ALREADY_EXISTS, "The aggregate function '{}' already exists", function_name);
 
-    if (UserDefinedExecutableFunctionFactory::instance().has(function_name, context))
+    if (UserDefinedExecutableFunctionFactory::instance().has(function_name, context)) /// NOLINT(readability-static-accessed-through-instance)
         throw Exception(ErrorCodes::FUNCTION_ALREADY_EXISTS, "User defined executable function '{}' already exists", function_name);
 
     validateFunction(assert_cast<const ASTCreateFunctionQuery &>(create_function_query).function_core, function_name);
@@ -118,7 +118,7 @@ void UserDefinedSQLFunctionFactory::checkCanBeUnregistered(const ContextPtr & co
         AggregateFunctionFactory::instance().hasNameOrAlias(function_name))
         throw Exception(ErrorCodes::CANNOT_DROP_FUNCTION, "Cannot drop system function '{}'", function_name);
 
-    if (UserDefinedExecutableFunctionFactory::instance().has(function_name, context))
+    if (UserDefinedExecutableFunctionFactory::instance().has(function_name, context)) /// NOLINT(readability-static-accessed-through-instance)
         throw Exception(ErrorCodes::CANNOT_DROP_FUNCTION, "Cannot drop user defined executable function '{}'", function_name);
 }
 

src/Functions/array/arrayDotProduct.cpp

@@ -66,13 +66,13 @@ struct DotProduct
     };
 
     template <typename Type>
-    static void accumulate(State<Type> & state, Type x, Type y)
+    static NO_SANITIZE_UNDEFINED void accumulate(State<Type> & state, Type x, Type y)
     {
         state.sum += x * y;
     }
 
     template <typename Type>
-    static void combine(State<Type> & state, const State<Type> & other_state)
+    static NO_SANITIZE_UNDEFINED void combine(State<Type> & state, const State<Type> & other_state)
     {
         state.sum += other_state.sum;
     }

src/IO/ConnectionTimeouts.cpp

@@ -144,7 +144,12 @@ ConnectionTimeouts ConnectionTimeouts::getAdaptiveTimeouts(const String & method
 void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts)
 {
     session.setTimeout(timeouts.connection_timeout, timeouts.send_timeout, timeouts.receive_timeout);
-    session.setKeepAliveTimeout(timeouts.http_keep_alive_timeout);
+    /// we can not change keep alive timeout for already initiated connections
+    if (!session.connected())
+    {
+        session.setKeepAliveTimeout(timeouts.http_keep_alive_timeout);
+        session.setKeepAliveMaxRequests(int(timeouts.http_keep_alive_max_requests));
+    }
 }
 
 ConnectionTimeouts getTimeouts(const Poco::Net::HTTPClientSession & session)

src/IO/ConnectionTimeouts.h

@@ -35,6 +35,8 @@ struct ConnectionTimeouts
 
     Poco::Timespan tcp_keep_alive_timeout = Poco::Timespan(DEFAULT_TCP_KEEP_ALIVE_TIMEOUT, 0);
     Poco::Timespan http_keep_alive_timeout = Poco::Timespan(DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT, 0);
+    size_t http_keep_alive_max_requests = DEFAULT_HTTP_KEEP_ALIVE_MAX_REQUEST;
+
 
     /// Timeouts for HedgedConnections
     Poco::Timespan hedged_connection_timeout = Poco::Timespan(DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC, 0);
@@ -69,6 +71,7 @@ APPLY_FOR_ALL_CONNECTION_TIMEOUT_MEMBERS(DECLARE_BUILDER_FOR_MEMBER)
 
     ConnectionTimeouts & withConnectionTimeout(size_t seconds);
     ConnectionTimeouts & withConnectionTimeout(Poco::Timespan span);
+    ConnectionTimeouts & withHTTPKeepAliveMaxRequests(size_t requests);
 };
 
 /// NOLINTBEGIN(bugprone-macro-parentheses)
@@ -114,6 +117,12 @@ inline ConnectionTimeouts & ConnectionTimeouts::withConnectionTimeout(Poco::Time
     return *this;
 }
 
+inline ConnectionTimeouts & ConnectionTimeouts::withHTTPKeepAliveMaxRequests(size_t requests)
+{
+    http_keep_alive_max_requests = requests;
+    return *this;
+}
+
 void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts);
 ConnectionTimeouts getTimeouts(const Poco::Net::HTTPClientSession & session);
 

src/IO/MMapReadBufferFromFileWithCache.cpp

@@ -37,7 +37,7 @@ MMapReadBufferFromFileWithCache::MMapReadBufferFromFileWithCache(
 MMapReadBufferFromFileWithCache::MMapReadBufferFromFileWithCache(
     MMappedFileCache & cache, const std::string & file_name, size_t offset)
 {
-    mapped = cache.getOrSet(cache.hash(file_name, offset, -1), [&]
+    mapped = cache.getOrSet(MMappedFileCache::hash(file_name, offset, -1), [&]
     {
         return std::make_shared<MMappedFile>(file_name, offset);
     });

src/IO/ReadWriteBufferFromHTTP.cpp

@@ -345,7 +345,7 @@ void ReadWriteBufferFromHTTP::doWithRetries(std::function<void()> && callable,
         if (last_attempt || !is_retriable)
         {
             if (!mute_logging)
-                LOG_ERROR(log,
+                LOG_DEBUG(log,
                           "Failed to make request to '{}'{}. "
                           "Error: '{}'. "
                           "Failed at try {}/{}.",
@@ -361,7 +361,7 @@ void ReadWriteBufferFromHTTP::doWithRetries(std::function<void()> && callable,
                 on_retry();
 
             if (!mute_logging)
-                LOG_INFO(log,
+                LOG_TRACE(log,
                          "Failed to make request to '{}'{}. "
                          "Error: {}. "
                          "Failed at try {}/{}. "

src/IO/S3/Client.h

@@ -96,9 +96,9 @@ bool isS3ExpressEndpoint(const std::string & endpoint);
 
 struct ClientSettings
 {
-    bool use_virtual_addressing;
+    bool use_virtual_addressing = false;
     /// Disable checksum to avoid extra read of the input stream
-    bool disable_checksum;
+    bool disable_checksum = false;
     /// Should client send ComposeObject request after upload to GCS.
     ///
     /// Previously ComposeObject request was required to make Copy possible,
@@ -108,8 +108,8 @@ struct ClientSettings
     ///
     /// Ability to enable it preserved since likely it is required for old
     /// files.
-    bool gcs_issue_compose_request;
+    bool gcs_issue_compose_request = false;
-    bool is_s3express_bucket;
+    bool is_s3express_bucket = false;
 };
 
 /// Client that improves the client from the AWS SDK

src/IO/S3/Credentials.h

@@ -22,6 +22,8 @@ inline static constexpr uint64_t DEFAULT_EXPIRATION_WINDOW_SECONDS = 120;
 inline static constexpr uint64_t DEFAULT_CONNECT_TIMEOUT_MS = 1000;
 inline static constexpr uint64_t DEFAULT_REQUEST_TIMEOUT_MS = 30000;
 inline static constexpr uint64_t DEFAULT_MAX_CONNECTIONS = 100;
+inline static constexpr uint64_t DEFAULT_KEEP_ALIVE_TIMEOUT = 5;
+inline static constexpr uint64_t DEFAULT_KEEP_ALIVE_MAX_REQUESTS = 100;
 
 /// In GCP metadata service can be accessed via DNS regardless of IPv4 or IPv6.
 static inline constexpr char GCP_METADATA_SERVICE_ENDPOINT[] = "http://metadata.google.internal";

src/IO/S3/PocoHTTPClient.cpp

@@ -146,7 +146,9 @@ ConnectionTimeouts getTimeoutsFromConfiguration(const PocoHTTPClientConfiguratio
         .withSendTimeout(Poco::Timespan(client_configuration.requestTimeoutMs * 1000))
         .withReceiveTimeout(Poco::Timespan(client_configuration.requestTimeoutMs * 1000))
         .withTCPKeepAliveTimeout(Poco::Timespan(
-            client_configuration.enableTcpKeepAlive ? client_configuration.tcpKeepAliveIntervalMs * 1000 : 0));
+            client_configuration.enableTcpKeepAlive ? client_configuration.tcpKeepAliveIntervalMs * 1000 : 0))
+        .withHTTPKeepAliveTimeout(Poco::Timespan(client_configuration.http_keep_alive_timeout, 0))
+        .withHTTPKeepAliveMaxRequests(client_configuration.http_keep_alive_max_requests);
 }
 
 PocoHTTPClient::PocoHTTPClient(const PocoHTTPClientConfiguration & client_configuration)