mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-28 02:21:59 +00:00
Merge remote-tracking branch 'origin/master' into add-part-name-in-check-part-exception
This commit is contained in:
commit
4c106b2d0c
2
.github/workflows/pull_request.yml
vendored
2
.github/workflows/pull_request.yml
vendored
@ -157,7 +157,7 @@ jobs:
|
||||
################################# Stage Final #################################
|
||||
#
|
||||
FinishCheck:
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
if: ${{ !failure() && !cancelled() && github.event_name != 'merge_group' }}
|
||||
needs: [Tests_1, Tests_2]
|
||||
runs-on: [self-hosted, style-checker]
|
||||
steps:
|
||||
|
@ -213,6 +213,19 @@ namespace Net
|
||||
Poco::Timespan getKeepAliveTimeout() const;
|
||||
/// Returns the connection timeout for HTTP connections.
|
||||
|
||||
void setKeepAliveMaxRequests(int max_requests);
|
||||
|
||||
int getKeepAliveMaxRequests() const;
|
||||
|
||||
int getKeepAliveRequest() const;
|
||||
|
||||
bool isKeepAliveExpired(double reliability = 1.0) const;
|
||||
/// Returns if the connection is expired with some margin as fraction of timeout as reliability
|
||||
|
||||
double getKeepAliveReliability() const;
|
||||
/// Returns the current fraction of keep alive timeout when connection is considered safe to use
|
||||
/// It helps to avoid situation when a client uses nearly expired connection and receives NoMessageException
|
||||
|
||||
virtual std::ostream & sendRequest(HTTPRequest & request);
|
||||
/// Sends the header for the given HTTP request to
|
||||
/// the server.
|
||||
@ -345,6 +358,8 @@ namespace Net
|
||||
|
||||
void assign(HTTPClientSession & session);
|
||||
|
||||
void setKeepAliveRequest(int request);
|
||||
|
||||
HTTPSessionFactory _proxySessionFactory;
|
||||
/// Factory to create HTTPClientSession to proxy.
|
||||
private:
|
||||
@ -353,6 +368,8 @@ namespace Net
|
||||
Poco::UInt16 _port;
|
||||
ProxyConfig _proxyConfig;
|
||||
Poco::Timespan _keepAliveTimeout;
|
||||
int _keepAliveCurrentRequest = 0;
|
||||
int _keepAliveMaxRequests = 1000;
|
||||
Poco::Timestamp _lastRequest;
|
||||
bool _reconnect;
|
||||
bool _mustReconnect;
|
||||
@ -361,6 +378,7 @@ namespace Net
|
||||
Poco::SharedPtr<std::ostream> _pRequestStream;
|
||||
Poco::SharedPtr<std::istream> _pResponseStream;
|
||||
|
||||
static const double _defaultKeepAliveReliabilityLevel;
|
||||
static ProxyConfig _globalProxyConfig;
|
||||
|
||||
HTTPClientSession(const HTTPClientSession &);
|
||||
@ -450,9 +468,19 @@ namespace Net
|
||||
return _lastRequest;
|
||||
}
|
||||
|
||||
inline void HTTPClientSession::setLastRequest(Poco::Timestamp time)
|
||||
inline double HTTPClientSession::getKeepAliveReliability() const
|
||||
{
|
||||
_lastRequest = time;
|
||||
return _defaultKeepAliveReliabilityLevel;
|
||||
}
|
||||
|
||||
inline int HTTPClientSession::getKeepAliveMaxRequests() const
|
||||
{
|
||||
return _keepAliveMaxRequests;
|
||||
}
|
||||
|
||||
inline int HTTPClientSession::getKeepAliveRequest() const
|
||||
{
|
||||
return _keepAliveCurrentRequest;
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -120,6 +120,10 @@ namespace Net
|
||||
/// The value is set to "Keep-Alive" if keepAlive is
|
||||
/// true, or to "Close" otherwise.
|
||||
|
||||
void setKeepAliveTimeout(int timeout, int max_requests);
|
||||
int getKeepAliveTimeout() const;
|
||||
int getKeepAliveMaxRequests() const;
|
||||
|
||||
bool getKeepAlive() const;
|
||||
/// Returns true if
|
||||
/// * the message has a Connection header field and its value is "Keep-Alive"
|
||||
|
@ -44,7 +44,7 @@ namespace Net
|
||||
/// - timeout: 60 seconds
|
||||
/// - keepAlive: true
|
||||
/// - maxKeepAliveRequests: 0
|
||||
/// - keepAliveTimeout: 10 seconds
|
||||
/// - keepAliveTimeout: 15 seconds
|
||||
|
||||
void setServerName(const std::string & serverName);
|
||||
/// Sets the name and port (name:port) that the server uses to identify itself.
|
||||
|
@ -56,6 +56,8 @@ namespace Net
|
||||
SocketAddress serverAddress();
|
||||
/// Returns the server's address.
|
||||
|
||||
void setKeepAliveTimeout(Poco::Timespan keepAliveTimeout);
|
||||
|
||||
private:
|
||||
bool _firstRequest;
|
||||
Poco::Timespan _keepAliveTimeout;
|
||||
|
@ -37,6 +37,7 @@ namespace Net {
|
||||
|
||||
|
||||
HTTPClientSession::ProxyConfig HTTPClientSession::_globalProxyConfig;
|
||||
const double HTTPClientSession::_defaultKeepAliveReliabilityLevel = 0.9;
|
||||
|
||||
|
||||
HTTPClientSession::HTTPClientSession():
|
||||
@ -220,7 +221,41 @@ void HTTPClientSession::setGlobalProxyConfig(const ProxyConfig& config)
|
||||
|
||||
void HTTPClientSession::setKeepAliveTimeout(const Poco::Timespan& timeout)
|
||||
{
|
||||
_keepAliveTimeout = timeout;
|
||||
if (connected())
|
||||
{
|
||||
throw Poco::IllegalStateException("cannot change keep alive timeout on initiated connection, "
|
||||
"That value is managed privately after connection is established.");
|
||||
}
|
||||
_keepAliveTimeout = timeout;
|
||||
}
|
||||
|
||||
|
||||
void HTTPClientSession::setKeepAliveMaxRequests(int max_requests)
|
||||
{
|
||||
if (connected())
|
||||
{
|
||||
throw Poco::IllegalStateException("cannot change keep alive max requests on initiated connection, "
|
||||
"That value is managed privately after connection is established.");
|
||||
}
|
||||
_keepAliveMaxRequests = max_requests;
|
||||
}
|
||||
|
||||
|
||||
void HTTPClientSession::setKeepAliveRequest(int request)
|
||||
{
|
||||
_keepAliveCurrentRequest = request;
|
||||
}
|
||||
|
||||
|
||||
|
||||
void HTTPClientSession::setLastRequest(Poco::Timestamp time)
|
||||
{
|
||||
if (connected())
|
||||
{
|
||||
throw Poco::IllegalStateException("cannot change last request on initiated connection, "
|
||||
"That value is managed privately after connection is established.");
|
||||
}
|
||||
_lastRequest = time;
|
||||
}
|
||||
|
||||
|
||||
@ -231,6 +266,8 @@ std::ostream& HTTPClientSession::sendRequest(HTTPRequest& request)
|
||||
clearException();
|
||||
_responseReceived = false;
|
||||
|
||||
_keepAliveCurrentRequest += 1;
|
||||
|
||||
bool keepAlive = getKeepAlive();
|
||||
if (((connected() && !keepAlive) || mustReconnect()) && !_host.empty())
|
||||
{
|
||||
@ -241,8 +278,10 @@ std::ostream& HTTPClientSession::sendRequest(HTTPRequest& request)
|
||||
{
|
||||
if (!connected())
|
||||
reconnect();
|
||||
if (!keepAlive)
|
||||
request.setKeepAlive(false);
|
||||
if (!request.has(HTTPMessage::CONNECTION))
|
||||
request.setKeepAlive(keepAlive);
|
||||
if (keepAlive && !request.has(HTTPMessage::CONNECTION_KEEP_ALIVE) && _keepAliveTimeout.totalSeconds() > 0)
|
||||
request.setKeepAliveTimeout(_keepAliveTimeout.totalSeconds(), _keepAliveMaxRequests);
|
||||
if (!request.has(HTTPRequest::HOST) && !_host.empty())
|
||||
request.setHost(_host, _port);
|
||||
if (!_proxyConfig.host.empty() && !bypassProxy())
|
||||
@ -324,6 +363,17 @@ std::istream& HTTPClientSession::receiveResponse(HTTPResponse& response)
|
||||
|
||||
_mustReconnect = getKeepAlive() && !response.getKeepAlive();
|
||||
|
||||
if (!_mustReconnect)
|
||||
{
|
||||
/// when server sends its keep alive timeout, client has to follow that value
|
||||
auto timeout = response.getKeepAliveTimeout();
|
||||
if (timeout > 0)
|
||||
_keepAliveTimeout = std::min(_keepAliveTimeout, Poco::Timespan(timeout, 0));
|
||||
auto max_requests = response.getKeepAliveMaxRequests();
|
||||
if (max_requests > 0)
|
||||
_keepAliveMaxRequests = std::min(_keepAliveMaxRequests, max_requests);
|
||||
}
|
||||
|
||||
if (!_expectResponseBody || response.getStatus() < 200 || response.getStatus() == HTTPResponse::HTTP_NO_CONTENT || response.getStatus() == HTTPResponse::HTTP_NOT_MODIFIED)
|
||||
_pResponseStream = new HTTPFixedLengthInputStream(*this, 0);
|
||||
else if (response.getChunkedTransferEncoding())
|
||||
@ -430,15 +480,18 @@ std::string HTTPClientSession::proxyRequestPrefix() const
|
||||
return result;
|
||||
}
|
||||
|
||||
bool HTTPClientSession::isKeepAliveExpired(double reliability) const
|
||||
{
|
||||
Poco::Timestamp now;
|
||||
return Timespan(Timestamp::TimeDiff(reliability *_keepAliveTimeout.totalMicroseconds())) <= now - _lastRequest
|
||||
|| _keepAliveCurrentRequest > _keepAliveMaxRequests;
|
||||
}
|
||||
|
||||
bool HTTPClientSession::mustReconnect() const
|
||||
{
|
||||
if (!_mustReconnect)
|
||||
{
|
||||
Poco::Timestamp now;
|
||||
return _keepAliveTimeout <= now - _lastRequest;
|
||||
}
|
||||
else return true;
|
||||
return isKeepAliveExpired(_defaultKeepAliveReliabilityLevel);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
@ -511,14 +564,21 @@ void HTTPClientSession::assign(Poco::Net::HTTPClientSession & session)
|
||||
if (buffered())
|
||||
throw Poco::LogicException("assign to a session with not empty buffered data");
|
||||
|
||||
attachSocket(session.detachSocket());
|
||||
setLastRequest(session.getLastRequest());
|
||||
poco_assert(!connected());
|
||||
|
||||
setResolvedHost(session.getResolvedHost());
|
||||
setKeepAlive(session.getKeepAlive());
|
||||
setProxyConfig(session.getProxyConfig());
|
||||
|
||||
setTimeout(session.getConnectionTimeout(), session.getSendTimeout(), session.getReceiveTimeout());
|
||||
setKeepAlive(session.getKeepAlive());
|
||||
|
||||
setLastRequest(session.getLastRequest());
|
||||
setKeepAliveTimeout(session.getKeepAliveTimeout());
|
||||
setProxyConfig(session.getProxyConfig());
|
||||
|
||||
_keepAliveMaxRequests = session._keepAliveMaxRequests;
|
||||
_keepAliveCurrentRequest = session._keepAliveCurrentRequest;
|
||||
|
||||
attachSocket(session.detachSocket());
|
||||
|
||||
session.reset();
|
||||
}
|
||||
|
@ -17,6 +17,7 @@
|
||||
#include "Poco/NumberFormatter.h"
|
||||
#include "Poco/NumberParser.h"
|
||||
#include "Poco/String.h"
|
||||
#include <format>
|
||||
|
||||
|
||||
using Poco::NumberFormatter;
|
||||
@ -179,4 +180,51 @@ bool HTTPMessage::getKeepAlive() const
|
||||
}
|
||||
|
||||
|
||||
void HTTPMessage::setKeepAliveTimeout(int timeout, int max_requests)
|
||||
{
|
||||
add(HTTPMessage::CONNECTION_KEEP_ALIVE, std::format("timeout={}, max={}", timeout, max_requests));
|
||||
}
|
||||
|
||||
|
||||
int parseFromHeaderValues(const std::string_view header_value, const std::string_view param_name)
|
||||
{
|
||||
auto param_value_pos = header_value.find(param_name);
|
||||
if (param_value_pos == std::string::npos)
|
||||
param_value_pos = header_value.size();
|
||||
if (param_value_pos != header_value.size())
|
||||
param_value_pos += param_name.size();
|
||||
|
||||
auto param_value_end = header_value.find(',', param_value_pos);
|
||||
if (param_value_end == std::string::npos)
|
||||
param_value_end = header_value.size();
|
||||
|
||||
auto timeout_value_substr = header_value.substr(param_value_pos, param_value_end - param_value_pos);
|
||||
if (timeout_value_substr.empty())
|
||||
return -1;
|
||||
|
||||
int value = 0;
|
||||
auto [ptr, ec] = std::from_chars(timeout_value_substr.begin(), timeout_value_substr.end(), value);
|
||||
|
||||
if (ec == std::errc())
|
||||
return value;
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
||||
int HTTPMessage::getKeepAliveTimeout() const
|
||||
{
|
||||
const std::string& ka_header = get(HTTPMessage::CONNECTION_KEEP_ALIVE, HTTPMessage::EMPTY);
|
||||
static const std::string_view timeout_param = "timeout=";
|
||||
return parseFromHeaderValues(ka_header, timeout_param);
|
||||
}
|
||||
|
||||
|
||||
int HTTPMessage::getKeepAliveMaxRequests() const
|
||||
{
|
||||
const std::string& ka_header = get(HTTPMessage::CONNECTION_KEEP_ALIVE, HTTPMessage::EMPTY);
|
||||
static const std::string_view timeout_param = "max=";
|
||||
return parseFromHeaderValues(ka_header, timeout_param);
|
||||
}
|
||||
|
||||
} } // namespace Poco::Net
|
||||
|
@ -88,7 +88,18 @@ void HTTPServerConnection::run()
|
||||
|
||||
pHandler->handleRequest(request, response);
|
||||
session.setKeepAlive(_pParams->getKeepAlive() && response.getKeepAlive() && session.canKeepAlive());
|
||||
}
|
||||
|
||||
/// all that fuzz is all about to make session close with less timeout than 15s (set in HTTPServerParams c-tor)
|
||||
if (_pParams->getKeepAlive() && response.getKeepAlive() && session.canKeepAlive())
|
||||
{
|
||||
int value = response.getKeepAliveTimeout();
|
||||
if (value < 0)
|
||||
value = request.getKeepAliveTimeout();
|
||||
if (value > 0)
|
||||
session.setKeepAliveTimeout(Poco::Timespan(value, 0));
|
||||
}
|
||||
|
||||
}
|
||||
else sendErrorResponse(session, HTTPResponse::HTTP_NOT_IMPLEMENTED);
|
||||
}
|
||||
catch (Poco::Exception&)
|
||||
|
@ -33,6 +33,12 @@ HTTPServerSession::~HTTPServerSession()
|
||||
{
|
||||
}
|
||||
|
||||
void HTTPServerSession::setKeepAliveTimeout(Poco::Timespan keepAliveTimeout)
|
||||
{
|
||||
_keepAliveTimeout = keepAliveTimeout;
|
||||
}
|
||||
|
||||
|
||||
|
||||
bool HTTPServerSession::hasMoreRequests()
|
||||
{
|
||||
|
@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \
|
||||
# lts / testing / prestable / etc
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
|
||||
ARG VERSION="24.3.1.2672"
|
||||
ARG VERSION="24.3.2.23"
|
||||
ARG PACKAGES="clickhouse-keeper"
|
||||
ARG DIRECT_DOWNLOAD_URLS=""
|
||||
|
||||
|
@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \
|
||||
# lts / testing / prestable / etc
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
|
||||
ARG VERSION="24.3.1.2672"
|
||||
ARG VERSION="24.3.2.23"
|
||||
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
|
||||
ARG DIRECT_DOWNLOAD_URLS=""
|
||||
|
||||
|
@ -27,7 +27,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
|
||||
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
|
||||
ARG VERSION="24.3.1.2672"
|
||||
ARG VERSION="24.3.2.23"
|
||||
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
|
||||
|
||||
# set non-empty deb_location_url url to create a docker image
|
||||
|
29
docs/changelogs/v24.3.2.23-lts.md
Normal file
29
docs/changelogs/v24.3.2.23-lts.md
Normal file
@ -0,0 +1,29 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2024
|
||||
---
|
||||
|
||||
# 2024 Changelog
|
||||
|
||||
### ClickHouse release v24.3.2.23-lts (8b7d910960c) FIXME as compared to v24.3.1.2672-lts (2c5c589a882)
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in an official stable release)
|
||||
|
||||
* Fix logical error in group_by_use_nulls + grouping set + analyzer + materialize/constant [#61567](https://github.com/ClickHouse/ClickHouse/pull/61567) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Fix external table cannot parse data type Bool [#62115](https://github.com/ClickHouse/ClickHouse/pull/62115) ([Duc Canh Le](https://github.com/canhld94)).
|
||||
* Revert "Merge pull request [#61564](https://github.com/ClickHouse/ClickHouse/issues/61564) from liuneng1994/optimize_in_single_value" [#62135](https://github.com/ClickHouse/ClickHouse/pull/62135) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
|
||||
#### CI Fix or Improvement (changelog entry is not required)
|
||||
|
||||
* Backported in [#62030](https://github.com/ClickHouse/ClickHouse/issues/62030):. [#61869](https://github.com/ClickHouse/ClickHouse/pull/61869) ([Nikita Fomichev](https://github.com/fm4v)).
|
||||
* Backported in [#62057](https://github.com/ClickHouse/ClickHouse/issues/62057): ... [#62044](https://github.com/ClickHouse/ClickHouse/pull/62044) ([Max K.](https://github.com/maxknv)).
|
||||
* Backported in [#62204](https://github.com/ClickHouse/ClickHouse/issues/62204):. [#62190](https://github.com/ClickHouse/ClickHouse/pull/62190) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Fix some crashes with analyzer and group_by_use_nulls. [#61933](https://github.com/ClickHouse/ClickHouse/pull/61933) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix scalars create as select [#61998](https://github.com/ClickHouse/ClickHouse/pull/61998) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Ignore IfChainToMultiIfPass if returned type changed. [#62059](https://github.com/ClickHouse/ClickHouse/pull/62059) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix type for ConvertInToEqualPass [#62066](https://github.com/ClickHouse/ClickHouse/pull/62066) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Revert output Pretty in tty [#62090](https://github.com/ClickHouse/ClickHouse/pull/62090) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
|
@ -68,6 +68,12 @@ In the results of `SELECT` query, the values of `AggregateFunction` type have im
|
||||
|
||||
## Example of an Aggregated Materialized View {#example-of-an-aggregated-materialized-view}
|
||||
|
||||
The following examples assumes that you have a database named `test` so make sure you create that if it doesn't already exist:
|
||||
|
||||
```sql
|
||||
CREATE DATABASE test;
|
||||
```
|
||||
|
||||
We will create the table `test.visits` that contain the raw data:
|
||||
|
||||
``` sql
|
||||
@ -80,17 +86,24 @@ CREATE TABLE test.visits
|
||||
) ENGINE = MergeTree ORDER BY (StartDate, CounterID);
|
||||
```
|
||||
|
||||
Next, we need to create an `AggregatingMergeTree` table that will store `AggregationFunction`s that keep track of the total number of visits and the number of unique users.
|
||||
|
||||
`AggregatingMergeTree` materialized view that watches the `test.visits` table, and use the `AggregateFunction` type:
|
||||
|
||||
``` sql
|
||||
CREATE MATERIALIZED VIEW test.mv_visits
|
||||
(
|
||||
CREATE TABLE test.agg_visits (
|
||||
StartDate DateTime64 NOT NULL,
|
||||
CounterID UInt64,
|
||||
Visits AggregateFunction(sum, Nullable(Int32)),
|
||||
Users AggregateFunction(uniq, Nullable(Int32))
|
||||
)
|
||||
ENGINE = AggregatingMergeTree() ORDER BY (StartDate, CounterID)
|
||||
ENGINE = AggregatingMergeTree() ORDER BY (StartDate, CounterID);
|
||||
```
|
||||
|
||||
And then let's create a materialized view that populates `test.agg_visits` from `test.visits` :
|
||||
|
||||
```sql
|
||||
CREATE MATERIALIZED VIEW test.visits_mv TO test.agg_visits
|
||||
AS SELECT
|
||||
StartDate,
|
||||
CounterID,
|
||||
@ -104,25 +117,45 @@ Inserting data into the `test.visits` table.
|
||||
|
||||
``` sql
|
||||
INSERT INTO test.visits (StartDate, CounterID, Sign, UserID)
|
||||
VALUES (1667446031, 1, 3, 4)
|
||||
INSERT INTO test.visits (StartDate, CounterID, Sign, UserID)
|
||||
VALUES (1667446031, 1, 6, 3)
|
||||
VALUES (1667446031000, 1, 3, 4), (1667446031000, 1, 6, 3);
|
||||
```
|
||||
|
||||
The data is inserted in both the table and the materialized view `test.mv_visits`.
|
||||
The data is inserted in both `test.visits` and `test.agg_visits`.
|
||||
|
||||
To get the aggregated data, we need to execute a query such as `SELECT ... GROUP BY ...` from the materialized view `test.mv_visits`:
|
||||
|
||||
``` sql
|
||||
```sql
|
||||
SELECT
|
||||
StartDate,
|
||||
sumMerge(Visits) AS Visits,
|
||||
uniqMerge(Users) AS Users
|
||||
FROM test.mv_visits
|
||||
FROM test.agg_visits
|
||||
GROUP BY StartDate
|
||||
ORDER BY StartDate;
|
||||
```
|
||||
|
||||
```text
|
||||
┌───────────────StartDate─┬─Visits─┬─Users─┐
|
||||
│ 2022-11-03 03:27:11.000 │ 9 │ 2 │
|
||||
└─────────────────────────┴────────┴───────┘
|
||||
```
|
||||
|
||||
And how about if we add another couple of records to `test.visits`, but this time we'll use a different timestamp for one of the records:
|
||||
|
||||
```sql
|
||||
INSERT INTO test.visits (StartDate, CounterID, Sign, UserID)
|
||||
VALUES (1669446031000, 2, 5, 10), (1667446031000, 3, 7, 5);
|
||||
```
|
||||
|
||||
If we then run the `SELECT` query again, we'll see the following output:
|
||||
|
||||
```text
|
||||
┌───────────────StartDate─┬─Visits─┬─Users─┐
|
||||
│ 2022-11-03 03:27:11.000 │ 16 │ 3 │
|
||||
│ 2022-11-26 07:00:31.000 │ 5 │ 1 │
|
||||
└─────────────────────────┴────────┴───────┘
|
||||
```
|
||||
|
||||
## Related Content
|
||||
|
||||
- Blog: [Using Aggregate Combinators in ClickHouse](https://clickhouse.com/blog/aggregate-functions-combinators-in-clickhouse-for-arrays-maps-and-states)
|
||||
|
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/aggregatefunction
|
||||
sidebar_position: 53
|
||||
sidebar_position: 46
|
||||
sidebar_label: AggregateFunction
|
||||
---
|
||||
|
||||
|
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/array
|
||||
sidebar_position: 52
|
||||
sidebar_position: 32
|
||||
sidebar_label: Array(T)
|
||||
---
|
||||
|
||||
|
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/boolean
|
||||
sidebar_position: 43
|
||||
sidebar_position: 22
|
||||
sidebar_label: Boolean
|
||||
---
|
||||
|
||||
|
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/date
|
||||
sidebar_position: 47
|
||||
sidebar_position: 12
|
||||
sidebar_label: Date
|
||||
---
|
||||
|
||||
|
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/date32
|
||||
sidebar_position: 48
|
||||
sidebar_position: 14
|
||||
sidebar_label: Date32
|
||||
---
|
||||
|
||||
|
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/datetime
|
||||
sidebar_position: 48
|
||||
sidebar_position: 16
|
||||
sidebar_label: DateTime
|
||||
---
|
||||
|
||||
|
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/datetime64
|
||||
sidebar_position: 49
|
||||
sidebar_position: 18
|
||||
sidebar_label: DateTime64
|
||||
---
|
||||
|
||||
|
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/decimal
|
||||
sidebar_position: 42
|
||||
sidebar_position: 6
|
||||
sidebar_label: Decimal
|
||||
---
|
||||
|
||||
|
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/enum
|
||||
sidebar_position: 50
|
||||
sidebar_position: 20
|
||||
sidebar_label: Enum
|
||||
---
|
||||
|
||||
|
@ -1,10 +1,10 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/fixedstring
|
||||
sidebar_position: 45
|
||||
sidebar_position: 10
|
||||
sidebar_label: FixedString(N)
|
||||
---
|
||||
|
||||
# FixedString
|
||||
# FixedString(N)
|
||||
|
||||
A fixed-length string of `N` bytes (neither characters nor code points).
|
||||
|
||||
|
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/float
|
||||
sidebar_position: 41
|
||||
sidebar_position: 4
|
||||
sidebar_label: Float32, Float64
|
||||
---
|
||||
|
||||
|
@ -1,8 +1,8 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/geo
|
||||
sidebar_position: 62
|
||||
sidebar_position: 54
|
||||
sidebar_label: Geo
|
||||
title: "Geo Data Types"
|
||||
title: "Geometric"
|
||||
---
|
||||
|
||||
ClickHouse supports data types for representing geographical objects — locations, lands, etc.
|
||||
|
@ -1,10 +1,10 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/
|
||||
sidebar_label: List of data types
|
||||
sidebar_position: 37
|
||||
sidebar_position: 1
|
||||
---
|
||||
|
||||
# ClickHouse Data Types
|
||||
# Data Types in ClickHouse
|
||||
|
||||
ClickHouse can store various kinds of data in table cells. This section describes the supported data types and special considerations for using and/or implementing them if any.
|
||||
|
||||
|
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/int-uint
|
||||
sidebar_position: 40
|
||||
sidebar_position: 2
|
||||
sidebar_label: UInt8, UInt16, UInt32, UInt64, UInt128, UInt256, Int8, Int16, Int32, Int64, Int128, Int256
|
||||
---
|
||||
|
||||
|
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/ipv4
|
||||
sidebar_position: 59
|
||||
sidebar_position: 28
|
||||
sidebar_label: IPv4
|
||||
---
|
||||
|
||||
|
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/ipv6
|
||||
sidebar_position: 60
|
||||
sidebar_position: 30
|
||||
sidebar_label: IPv6
|
||||
---
|
||||
|
||||
|
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/json
|
||||
sidebar_position: 54
|
||||
sidebar_position: 26
|
||||
sidebar_label: JSON
|
||||
---
|
||||
|
||||
|
@ -1,10 +1,10 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/lowcardinality
|
||||
sidebar_position: 51
|
||||
sidebar_label: LowCardinality
|
||||
sidebar_position: 42
|
||||
sidebar_label: LowCardinality(T)
|
||||
---
|
||||
|
||||
# LowCardinality
|
||||
# LowCardinality(T)
|
||||
|
||||
Changes the internal representation of other data types to be dictionary-encoded.
|
||||
|
||||
|
@ -1,12 +1,12 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/map
|
||||
sidebar_position: 65
|
||||
sidebar_label: Map(key, value)
|
||||
sidebar_position: 36
|
||||
sidebar_label: Map(K, V)
|
||||
---
|
||||
|
||||
# Map(key, value)
|
||||
# Map(K, V)
|
||||
|
||||
`Map(key, value)` data type stores `key:value` pairs.
|
||||
`Map(K, V)` data type stores `key:value` pairs.
|
||||
|
||||
**Parameters**
|
||||
|
||||
|
@ -1,27 +0,0 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/multiword-types
|
||||
sidebar_position: 61
|
||||
sidebar_label: Multiword Type Names
|
||||
title: "Multiword Types"
|
||||
---
|
||||
|
||||
When creating tables, you can use data types with a name consisting of several words. This is implemented for better SQL compatibility.
|
||||
|
||||
## Multiword Types Support
|
||||
|
||||
| Multiword types | Simple types |
|
||||
|----------------------------------|--------------------------------------------------------------|
|
||||
| DOUBLE PRECISION | [Float64](../../sql-reference/data-types/float.md) |
|
||||
| CHAR LARGE OBJECT | [String](../../sql-reference/data-types/string.md) |
|
||||
| CHAR VARYING | [String](../../sql-reference/data-types/string.md) |
|
||||
| CHARACTER LARGE OBJECT | [String](../../sql-reference/data-types/string.md) |
|
||||
| CHARACTER VARYING | [String](../../sql-reference/data-types/string.md) |
|
||||
| NCHAR LARGE OBJECT | [String](../../sql-reference/data-types/string.md) |
|
||||
| NCHAR VARYING | [String](../../sql-reference/data-types/string.md) |
|
||||
| NATIONAL CHARACTER LARGE OBJECT | [String](../../sql-reference/data-types/string.md) |
|
||||
| NATIONAL CHARACTER VARYING | [String](../../sql-reference/data-types/string.md) |
|
||||
| NATIONAL CHAR VARYING | [String](../../sql-reference/data-types/string.md) |
|
||||
| NATIONAL CHARACTER | [String](../../sql-reference/data-types/string.md) |
|
||||
| NATIONAL CHAR | [String](../../sql-reference/data-types/string.md) |
|
||||
| BINARY LARGE OBJECT | [String](../../sql-reference/data-types/string.md) |
|
||||
| BINARY VARYING | [String](../../sql-reference/data-types/string.md) |
|
@ -1,7 +1,7 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/nullable
|
||||
sidebar_position: 55
|
||||
sidebar_label: Nullable
|
||||
sidebar_position: 44
|
||||
sidebar_label: Nullable(T)
|
||||
---
|
||||
|
||||
# Nullable(T)
|
||||
|
@ -1,5 +1,7 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/simpleaggregatefunction
|
||||
sidebar_position: 48
|
||||
sidebar_label: SimpleAggregateFunction
|
||||
---
|
||||
# SimpleAggregateFunction
|
||||
|
||||
|
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/string
|
||||
sidebar_position: 44
|
||||
sidebar_position: 8
|
||||
sidebar_label: String
|
||||
---
|
||||
|
||||
@ -13,7 +13,7 @@ When creating tables, numeric parameters for string fields can be set (e.g. `VAR
|
||||
|
||||
Aliases:
|
||||
|
||||
- `String` — `LONGTEXT`, `MEDIUMTEXT`, `TINYTEXT`, `TEXT`, `LONGBLOB`, `MEDIUMBLOB`, `TINYBLOB`, `BLOB`, `VARCHAR`, `CHAR`.
|
||||
- `String` — `LONGTEXT`, `MEDIUMTEXT`, `TINYTEXT`, `TEXT`, `LONGBLOB`, `MEDIUMBLOB`, `TINYBLOB`, `BLOB`, `VARCHAR`, `CHAR`, `CHAR LARGE OBJECT`, `CHAR VARYING`, `CHARACTER LARGE OBJECT`, `CHARACTER VARYING`, `NCHAR LARGE OBJECT`, `NCHAR VARYING`, `NATIONAL CHARACTER LARGE OBJECT`, `NATIONAL CHARACTER VARYING`, `NATIONAL CHAR VARYING`, `NATIONAL CHARACTER`, `NATIONAL CHAR`, `BINARY LARGE OBJECT`, `BINARY VARYING`,
|
||||
|
||||
## Encodings
|
||||
|
||||
|
@ -1,10 +1,10 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/tuple
|
||||
sidebar_position: 54
|
||||
sidebar_position: 34
|
||||
sidebar_label: Tuple(T1, T2, ...)
|
||||
---
|
||||
|
||||
# Tuple(T1, T2, …)
|
||||
# Tuple(T1, T2, ...)
|
||||
|
||||
A tuple of elements, each having an individual [type](../../sql-reference/data-types/index.md#data_types). Tuple must contain at least one element.
|
||||
|
||||
|
@ -1,6 +1,6 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/uuid
|
||||
sidebar_position: 46
|
||||
sidebar_position: 24
|
||||
sidebar_label: UUID
|
||||
---
|
||||
|
||||
|
@ -1,10 +1,10 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/variant
|
||||
sidebar_position: 55
|
||||
sidebar_label: Variant
|
||||
sidebar_position: 40
|
||||
sidebar_label: Variant(T1, T2, ...)
|
||||
---
|
||||
|
||||
# Variant(T1, T2, T3, ...)
|
||||
# Variant(T1, T2, ...)
|
||||
|
||||
This type represents a union of other data types. Type `Variant(T1, T2, ..., TN)` means that each row of this type
|
||||
has a value of either type `T1` or `T2` or ... or `TN` or none of them (`NULL` value).
|
||||
|
@ -8,7 +8,7 @@ sidebar_label: VIEW
|
||||
|
||||
You can modify `SELECT` query that was specified when a [materialized view](../create/view.md#materialized) was created with the `ALTER TABLE … MODIFY QUERY` statement without interrupting ingestion process.
|
||||
|
||||
This command is created to change materialized view created with `TO [db.]name` clause. It does not change the structure of the underling storage table and it does not change the columns' definition of the materialized view, because of this the application of this command is very limited for materialized views are created without `TO [db.]name` clause.
|
||||
This command is created to change materialized view created with `TO [db.]name` clause. It does not change the structure of the underlying storage table and it does not change the columns' definition of the materialized view, because of this the application of this command is very limited for materialized views are created without `TO [db.]name` clause.
|
||||
|
||||
**Example with TO table**
|
||||
|
||||
|
@ -20,11 +20,10 @@ DROP DATABASE [IF EXISTS] db [ON CLUSTER cluster] [SYNC]
|
||||
|
||||
## DROP TABLE
|
||||
|
||||
Deletes the table.
|
||||
In case when `IF EMPTY` clause is specified server will check if table is empty only on replica that received initial query.
|
||||
Deletes one or more tables.
|
||||
|
||||
:::tip
|
||||
Also see [UNDROP TABLE](/docs/en/sql-reference/statements/undrop.md)
|
||||
To undo the deletion of a table, please see [UNDROP TABLE](/docs/en/sql-reference/statements/undrop.md)
|
||||
:::
|
||||
|
||||
Syntax:
|
||||
@ -33,7 +32,9 @@ Syntax:
|
||||
DROP [TEMPORARY] TABLE [IF EXISTS] [IF EMPTY] [db1.]name_1[, [db2.]name_2, ...] [ON CLUSTER cluster] [SYNC]
|
||||
```
|
||||
|
||||
Note that deleting multiple tables at the same time is a non-atomic deletion. If a table fails to be deleted, subsequent tables will not be deleted.
|
||||
Limitations:
|
||||
- If the clause `IF EMPTY` is specified, the server checks the emptiness of the table only on the replica which received the query.
|
||||
- Deleting multiple tables at once is not an atomic operation, i.e. if the deletion of a table fails, subsequent tables will not be deleted.
|
||||
|
||||
## DROP DICTIONARY
|
||||
|
||||
|
@ -12,25 +12,23 @@ Some of the calculations that you can do are similar to those that can be done w
|
||||
|
||||
ClickHouse supports the standard grammar for defining windows and window functions. The table below indicates whether a feature is currently supported.
|
||||
|
||||
| Feature | Support or workaround |
|
||||
| Feature | Supported? |
|
||||
|------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| ad hoc window specification (`count(*) over (partition by id order by time desc)`) | supported |
|
||||
| expressions involving window functions, e.g. `(count(*) over ()) / 2)` | supported |
|
||||
| `WINDOW` clause (`select ... from table window w as (partition by id)`) | supported |
|
||||
| `ROWS` frame | supported |
|
||||
| `RANGE` frame | supported, the default |
|
||||
| `INTERVAL` syntax for `DateTime` `RANGE OFFSET` frame | not supported, specify the number of seconds instead (`RANGE` works with any numeric type). |
|
||||
| `GROUPS` frame | not supported |
|
||||
| Calculating aggregate functions over a frame (`sum(value) over (order by time)`) | all aggregate functions are supported |
|
||||
| `rank()`, `dense_rank()`, `row_number()` | supported |
|
||||
| `lag/lead(value, offset)` | Not supported. Workarounds: |
|
||||
| | 1) replace with `any(value) over (.... rows between <offset> preceding and <offset> preceding)`, or `following` for `lead` |
|
||||
| | 2) use `lagInFrame/leadInFrame`, which are analogous, but respect the window frame. To get behavior identical to `lag/lead`, use `rows between unbounded preceding and unbounded following` |
|
||||
| ntile(buckets) | Supported. Specify window like, (partition by x order by y rows between unbounded preceding and unrounded following). |
|
||||
| ad hoc window specification (`count(*) over (partition by id order by time desc)`) | ✅ |
|
||||
| expressions involving window functions, e.g. `(count(*) over ()) / 2)` | ✅ |
|
||||
| `WINDOW` clause (`select ... from table window w as (partition by id)`) | ✅ |
|
||||
| `ROWS` frame | ✅ |
|
||||
| `RANGE` frame | ✅ (the default) |
|
||||
| `INTERVAL` syntax for `DateTime` `RANGE OFFSET` frame | ❌ (specify the number of seconds instead (`RANGE` works with any numeric type).) |
|
||||
| `GROUPS` frame | ❌ |
|
||||
| Calculating aggregate functions over a frame (`sum(value) over (order by time)`) | ✅ (All aggregate functions are supported) |
|
||||
| `rank()`, `dense_rank()`, `row_number()` | ✅ |
|
||||
| `lag/lead(value, offset)` | ❌ <br/> You can use one of the following workarounds:<br/> 1) `any(value) over (.... rows between <offset> preceding and <offset> preceding)`, or `following` for `lead` <br/> 2) `lagInFrame/leadInFrame`, which are analogous, but respect the window frame. To get behavior identical to `lag/lead`, use `rows between unbounded preceding and unbounded following` |
|
||||
| ntile(buckets) | ✅ <br/> Specify window like, (partition by x order by y rows between unbounded preceding and unrounded following). |
|
||||
|
||||
## ClickHouse-specific Window Functions
|
||||
|
||||
There are also the following window function that's specific to ClickHouse:
|
||||
There is also the following ClickHouse specific window function:
|
||||
|
||||
### nonNegativeDerivative(metric_column, timestamp_column[, INTERVAL X UNITS])
|
||||
|
||||
@ -89,6 +87,102 @@ These functions can be used only as a window function.
|
||||
|
||||
Let's have a look at some examples of how window functions can be used.
|
||||
|
||||
### Numbering rows
|
||||
|
||||
```sql
|
||||
CREATE TABLE salaries
|
||||
(
|
||||
`team` String,
|
||||
`player` String,
|
||||
`salary` UInt32,
|
||||
`position` String
|
||||
)
|
||||
Engine = Memory;
|
||||
|
||||
INSERT INTO salaries FORMAT Values
|
||||
('Port Elizabeth Barbarians', 'Gary Chen', 195000, 'F'),
|
||||
('New Coreystad Archdukes', 'Charles Juarez', 190000, 'F'),
|
||||
('Port Elizabeth Barbarians', 'Michael Stanley', 150000, 'D'),
|
||||
('New Coreystad Archdukes', 'Scott Harrison', 150000, 'D'),
|
||||
('Port Elizabeth Barbarians', 'Robert George', 195000, 'M');
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT player, salary,
|
||||
row_number() OVER (ORDER BY salary) AS row
|
||||
FROM salaries;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─player──────────┬─salary─┬─row─┐
|
||||
│ Michael Stanley │ 150000 │ 1 │
|
||||
│ Scott Harrison │ 150000 │ 2 │
|
||||
│ Charles Juarez │ 190000 │ 3 │
|
||||
│ Gary Chen │ 195000 │ 4 │
|
||||
│ Robert George │ 195000 │ 5 │
|
||||
└─────────────────┴────────┴─────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT player, salary,
|
||||
row_number() OVER (ORDER BY salary) AS row,
|
||||
rank() OVER (ORDER BY salary) AS rank,
|
||||
dense_rank() OVER (ORDER BY salary) AS denseRank
|
||||
FROM salaries;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─player──────────┬─salary─┬─row─┬─rank─┬─denseRank─┐
|
||||
│ Michael Stanley │ 150000 │ 1 │ 1 │ 1 │
|
||||
│ Scott Harrison │ 150000 │ 2 │ 1 │ 1 │
|
||||
│ Charles Juarez │ 190000 │ 3 │ 3 │ 2 │
|
||||
│ Gary Chen │ 195000 │ 4 │ 4 │ 3 │
|
||||
│ Robert George │ 195000 │ 5 │ 4 │ 3 │
|
||||
└─────────────────┴────────┴─────┴──────┴───────────┘
|
||||
```
|
||||
|
||||
### Aggregation functions
|
||||
|
||||
Compare each player's salary to the average for their team.
|
||||
|
||||
```sql
|
||||
SELECT player, salary, team,
|
||||
avg(salary) OVER (PARTITION BY team) AS teamAvg,
|
||||
salary - teamAvg AS diff
|
||||
FROM salaries;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─player──────────┬─salary─┬─team──────────────────────┬─teamAvg─┬───diff─┐
|
||||
│ Charles Juarez │ 190000 │ New Coreystad Archdukes │ 170000 │ 20000 │
|
||||
│ Scott Harrison │ 150000 │ New Coreystad Archdukes │ 170000 │ -20000 │
|
||||
│ Gary Chen │ 195000 │ Port Elizabeth Barbarians │ 180000 │ 15000 │
|
||||
│ Michael Stanley │ 150000 │ Port Elizabeth Barbarians │ 180000 │ -30000 │
|
||||
│ Robert George │ 195000 │ Port Elizabeth Barbarians │ 180000 │ 15000 │
|
||||
└─────────────────┴────────┴───────────────────────────┴─────────┴────────┘
|
||||
```
|
||||
|
||||
Compare each player's salary to the maximum for their team.
|
||||
|
||||
```sql
|
||||
SELECT player, salary, team,
|
||||
max(salary) OVER (PARTITION BY team) AS teamAvg,
|
||||
salary - teamAvg AS diff
|
||||
FROM salaries;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─player──────────┬─salary─┬─team──────────────────────┬─teamAvg─┬───diff─┐
|
||||
│ Charles Juarez │ 190000 │ New Coreystad Archdukes │ 190000 │ 0 │
|
||||
│ Scott Harrison │ 150000 │ New Coreystad Archdukes │ 190000 │ -40000 │
|
||||
│ Gary Chen │ 195000 │ Port Elizabeth Barbarians │ 195000 │ 0 │
|
||||
│ Michael Stanley │ 150000 │ Port Elizabeth Barbarians │ 195000 │ -45000 │
|
||||
│ Robert George │ 195000 │ Port Elizabeth Barbarians │ 195000 │ 0 │
|
||||
└─────────────────┴────────┴───────────────────────────┴─────────┴────────┘
|
||||
```
|
||||
|
||||
### Partitioning by column
|
||||
|
||||
```sql
|
||||
CREATE TABLE wf_partition
|
||||
(
|
||||
@ -120,6 +214,8 @@ ORDER BY
|
||||
└──────────┴───────┴───────┴──────────────┘
|
||||
```
|
||||
|
||||
### Frame bounding
|
||||
|
||||
```sql
|
||||
CREATE TABLE wf_frame
|
||||
(
|
||||
@ -131,14 +227,19 @@ ENGINE = Memory;
|
||||
|
||||
INSERT INTO wf_frame FORMAT Values
|
||||
(1,1,1), (1,2,2), (1,3,3), (1,4,4), (1,5,5);
|
||||
```
|
||||
|
||||
-- frame is bounded by bounds of a partition (BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)
|
||||
```sql
|
||||
-- Frame is bounded by bounds of a partition (BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)
|
||||
SELECT
|
||||
part_key,
|
||||
value,
|
||||
order,
|
||||
groupArray(value) OVER (PARTITION BY part_key ORDER BY order ASC
|
||||
Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS frame_values
|
||||
groupArray(value) OVER (
|
||||
PARTITION BY part_key
|
||||
ORDER BY order ASC
|
||||
Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
|
||||
) AS frame_values
|
||||
FROM wf_frame
|
||||
ORDER BY
|
||||
part_key ASC,
|
||||
@ -151,7 +252,9 @@ ORDER BY
|
||||
│ 1 │ 4 │ 4 │ [1,2,3,4,5] │
|
||||
│ 1 │ 5 │ 5 │ [1,2,3,4,5] │
|
||||
└──────────┴───────┴───────┴──────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
-- short form - no bound expression, no order by
|
||||
SELECT
|
||||
part_key,
|
||||
@ -169,14 +272,19 @@ ORDER BY
|
||||
│ 1 │ 4 │ 4 │ [1,2,3,4,5] │
|
||||
│ 1 │ 5 │ 5 │ [1,2,3,4,5] │
|
||||
└──────────┴───────┴───────┴──────────────┘
|
||||
```
|
||||
|
||||
-- frame is bounded by the beggining of a partition and the current row
|
||||
```sql
|
||||
-- frame is bounded by the beginning of a partition and the current row
|
||||
SELECT
|
||||
part_key,
|
||||
value,
|
||||
order,
|
||||
groupArray(value) OVER (PARTITION BY part_key ORDER BY order ASC
|
||||
Rows BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS frame_values
|
||||
groupArray(value) OVER (
|
||||
PARTITION BY part_key
|
||||
ORDER BY order ASC
|
||||
Rows BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW
|
||||
) AS frame_values
|
||||
FROM wf_frame
|
||||
ORDER BY
|
||||
part_key ASC,
|
||||
@ -189,8 +297,10 @@ ORDER BY
|
||||
│ 1 │ 4 │ 4 │ [1,2,3,4] │
|
||||
│ 1 │ 5 │ 5 │ [1,2,3,4,5] │
|
||||
└──────────┴───────┴───────┴──────────────┘
|
||||
```
|
||||
|
||||
-- short form (frame is bounded by the beggining of a partition and the current row)
|
||||
```sql
|
||||
-- short form (frame is bounded by the beginning of a partition and the current row)
|
||||
SELECT
|
||||
part_key,
|
||||
value,
|
||||
@ -207,8 +317,10 @@ ORDER BY
|
||||
│ 1 │ 4 │ 4 │ [1,2,3,4] │
|
||||
│ 1 │ 5 │ 5 │ [1,2,3,4,5] │
|
||||
└──────────┴───────┴───────┴──────────────┘
|
||||
```
|
||||
|
||||
-- frame is bounded by the beggining of a partition and the current row, but order is backward
|
||||
```sql
|
||||
-- frame is bounded by the beginning of a partition and the current row, but order is backward
|
||||
SELECT
|
||||
part_key,
|
||||
value,
|
||||
@ -225,14 +337,19 @@ ORDER BY
|
||||
│ 1 │ 4 │ 4 │ [5,4] │
|
||||
│ 1 │ 5 │ 5 │ [5] │
|
||||
└──────────┴───────┴───────┴──────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
-- sliding frame - 1 PRECEDING ROW AND CURRENT ROW
|
||||
SELECT
|
||||
part_key,
|
||||
value,
|
||||
order,
|
||||
groupArray(value) OVER (PARTITION BY part_key ORDER BY order ASC
|
||||
Rows BETWEEN 1 PRECEDING AND CURRENT ROW) AS frame_values
|
||||
groupArray(value) OVER (
|
||||
PARTITION BY part_key
|
||||
ORDER BY order ASC
|
||||
Rows BETWEEN 1 PRECEDING AND CURRENT ROW
|
||||
) AS frame_values
|
||||
FROM wf_frame
|
||||
ORDER BY
|
||||
part_key ASC,
|
||||
@ -245,14 +362,19 @@ ORDER BY
|
||||
│ 1 │ 4 │ 4 │ [3,4] │
|
||||
│ 1 │ 5 │ 5 │ [4,5] │
|
||||
└──────────┴───────┴───────┴──────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
-- sliding frame - Rows BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING
|
||||
SELECT
|
||||
part_key,
|
||||
value,
|
||||
order,
|
||||
groupArray(value) OVER (PARTITION BY part_key ORDER BY order ASC
|
||||
Rows BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) AS frame_values
|
||||
groupArray(value) OVER (
|
||||
PARTITION BY part_key
|
||||
ORDER BY order ASC
|
||||
Rows BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING
|
||||
) AS frame_values
|
||||
FROM wf_frame
|
||||
ORDER BY
|
||||
part_key ASC,
|
||||
@ -264,7 +386,9 @@ ORDER BY
|
||||
│ 1 │ 4 │ 4 │ [3,4,5] │
|
||||
│ 1 │ 5 │ 5 │ [4,5] │
|
||||
└──────────┴───────┴───────┴──────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
-- row_number does not respect the frame, so rn_1 = rn_2 = rn_3 != rn_4
|
||||
SELECT
|
||||
part_key,
|
||||
@ -278,8 +402,11 @@ SELECT
|
||||
FROM wf_frame
|
||||
WINDOW
|
||||
w1 AS (PARTITION BY part_key ORDER BY order DESC),
|
||||
w2 AS (PARTITION BY part_key ORDER BY order DESC
|
||||
Rows BETWEEN 1 PRECEDING AND CURRENT ROW)
|
||||
w2 AS (
|
||||
PARTITION BY part_key
|
||||
ORDER BY order DESC
|
||||
Rows BETWEEN 1 PRECEDING AND CURRENT ROW
|
||||
)
|
||||
ORDER BY
|
||||
part_key ASC,
|
||||
value ASC;
|
||||
@ -290,7 +417,9 @@ ORDER BY
|
||||
│ 1 │ 4 │ 4 │ [5,4] │ 2 │ 2 │ 2 │ 2 │
|
||||
│ 1 │ 5 │ 5 │ [5] │ 1 │ 1 │ 1 │ 1 │
|
||||
└──────────┴───────┴───────┴──────────────┴──────┴──────┴──────┴──────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
-- first_value and last_value respect the frame
|
||||
SELECT
|
||||
groupArray(value) OVER w1 AS frame_values_1,
|
||||
@ -313,7 +442,9 @@ ORDER BY
|
||||
│ [1,2,3,4] │ 1 │ 4 │ [3,4] │ 3 │ 4 │
|
||||
│ [1,2,3,4,5] │ 1 │ 5 │ [4,5] │ 4 │ 5 │
|
||||
└────────────────┴───────────────┴──────────────┴────────────────┴───────────────┴──────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
-- second value within the frame
|
||||
SELECT
|
||||
groupArray(value) OVER w1 AS frame_values_1,
|
||||
@ -330,7 +461,9 @@ ORDER BY
|
||||
│ [1,2,3,4] │ 2 │
|
||||
│ [2,3,4,5] │ 3 │
|
||||
└────────────────┴──────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
-- second value within the frame + Null for missing values
|
||||
SELECT
|
||||
groupArray(value) OVER w1 AS frame_values_1,
|
||||
@ -351,7 +484,9 @@ ORDER BY
|
||||
|
||||
## Real world examples
|
||||
|
||||
### Maximum/total salary per department.
|
||||
The following examples solve common real-world problems.
|
||||
|
||||
### Maximum/total salary per department
|
||||
|
||||
```sql
|
||||
CREATE TABLE employees
|
||||
@ -369,7 +504,9 @@ INSERT INTO employees FORMAT Values
|
||||
('IT', 'Tim', 200),
|
||||
('IT', 'Anna', 300),
|
||||
('IT', 'Elen', 500);
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
department,
|
||||
employee_name AS emp,
|
||||
@ -386,8 +523,10 @@ FROM
|
||||
max(salary) OVER wndw AS max_salary_per_dep,
|
||||
sum(salary) OVER wndw AS total_salary_per_dep
|
||||
FROM employees
|
||||
WINDOW wndw AS (PARTITION BY department
|
||||
rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)
|
||||
WINDOW wndw AS (
|
||||
PARTITION BY department
|
||||
rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING
|
||||
)
|
||||
ORDER BY
|
||||
department ASC,
|
||||
employee_name ASC
|
||||
@ -403,7 +542,7 @@ FROM
|
||||
└────────────┴──────┴────────┴────────────────────┴──────────────────────┴──────────────────┘
|
||||
```
|
||||
|
||||
### Cumulative sum.
|
||||
### Cumulative sum
|
||||
|
||||
```sql
|
||||
CREATE TABLE warehouse
|
||||
@ -421,7 +560,9 @@ INSERT INTO warehouse VALUES
|
||||
('sku1', '2020-01-01', 1),
|
||||
('sku1', '2020-02-01', 1),
|
||||
('sku1', '2020-03-01', 1);
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
item,
|
||||
ts,
|
||||
@ -461,13 +602,18 @@ insert into sensors values('cpu_temp', '2020-01-01 00:00:00', 87),
|
||||
('cpu_temp', '2020-01-01 00:00:05', 87),
|
||||
('cpu_temp', '2020-01-01 00:00:06', 87),
|
||||
('cpu_temp', '2020-01-01 00:00:07', 87);
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
metric,
|
||||
ts,
|
||||
value,
|
||||
avg(value) OVER
|
||||
(PARTITION BY metric ORDER BY ts ASC Rows BETWEEN 2 PRECEDING AND CURRENT ROW)
|
||||
AS moving_avg_temp
|
||||
avg(value) OVER (
|
||||
PARTITION BY metric
|
||||
ORDER BY ts ASC
|
||||
Rows BETWEEN 2 PRECEDING AND CURRENT ROW
|
||||
) AS moving_avg_temp
|
||||
FROM sensors
|
||||
ORDER BY
|
||||
metric ASC,
|
||||
@ -536,7 +682,9 @@ insert into sensors values('ambient_temp', '2020-01-01 00:00:00', 16),
|
||||
('ambient_temp', '2020-03-01 12:00:00', 16),
|
||||
('ambient_temp', '2020-03-01 12:00:00', 16),
|
||||
('ambient_temp', '2020-03-01 12:00:00', 16);
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
metric,
|
||||
ts,
|
||||
|
@ -1,28 +0,0 @@
|
||||
---
|
||||
slug: /ru/sql-reference/data-types/multiword-types
|
||||
sidebar_position: 61
|
||||
sidebar_label: Составные типы
|
||||
---
|
||||
|
||||
# Составные типы {#multiword-types}
|
||||
|
||||
При создании таблиц вы можете использовать типы данных с названием, состоящим из нескольких слов. Такие названия поддерживаются для лучшей совместимости с SQL.
|
||||
|
||||
## Поддержка составных типов {#multiword-types-support}
|
||||
|
||||
| Составные типы | Обычные типы |
|
||||
|-------------------------------------|-----------------------------------------------------------|
|
||||
| DOUBLE PRECISION | [Float64](../../sql-reference/data-types/float.md) |
|
||||
| CHAR LARGE OBJECT | [String](../../sql-reference/data-types/string.md) |
|
||||
| CHAR VARYING | [String](../../sql-reference/data-types/string.md) |
|
||||
| CHARACTER LARGE OBJECT | [String](../../sql-reference/data-types/string.md) |
|
||||
| CHARACTER VARYING | [String](../../sql-reference/data-types/string.md) |
|
||||
| NCHAR LARGE OBJECT | [String](../../sql-reference/data-types/string.md) |
|
||||
| NCHAR VARYING | [String](../../sql-reference/data-types/string.md) |
|
||||
| NATIONAL CHARACTER LARGE OBJECT | [String](../../sql-reference/data-types/string.md) |
|
||||
| NATIONAL CHARACTER VARYING | [String](../../sql-reference/data-types/string.md) |
|
||||
| NATIONAL CHAR VARYING | [String](../../sql-reference/data-types/string.md) |
|
||||
| NATIONAL CHARACTER | [String](../../sql-reference/data-types/string.md) |
|
||||
| NATIONAL CHAR | [String](../../sql-reference/data-types/string.md) |
|
||||
| BINARY LARGE OBJECT | [String](../../sql-reference/data-types/string.md) |
|
||||
| BINARY VARYING | [String](../../sql-reference/data-types/string.md) |
|
@ -1,10 +0,0 @@
|
||||
---
|
||||
slug: /zh/sql-reference/data-types/multiword-types
|
||||
sidebar_position: 61
|
||||
sidebar_label: Multiword Type Names
|
||||
title: "Multiword Types"
|
||||
---
|
||||
|
||||
import Content from '@site/docs/en/sql-reference/data-types/multiword-types.md';
|
||||
|
||||
<Content />
|
@ -3992,9 +3992,15 @@ IdentifierResolveResult QueryAnalyzer::tryResolveIdentifierInParentScopes(const
|
||||
}
|
||||
else if (resolved_identifier->as<ConstantNode>())
|
||||
{
|
||||
lookup_result.resolved_identifier = resolved_identifier;
|
||||
return lookup_result;
|
||||
}
|
||||
else if (auto * resolved_function = resolved_identifier->as<FunctionNode>())
|
||||
{
|
||||
/// Special case: scalar subquery was executed and replaced by __getScalar function.
|
||||
/// Handle it as a constant.
|
||||
if (resolved_function->getFunctionName() == "__getScalar")
|
||||
return lookup_result;
|
||||
}
|
||||
|
||||
throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
|
||||
"Resolve identifier '{}' from parent scope only supported for constants and CTE. Actual {} node type {}. In scope {}",
|
||||
|
@ -29,7 +29,8 @@ NamesAndTypes extractProjectionColumnsForGroupBy(const QueryNode * query_node)
|
||||
return {};
|
||||
|
||||
NamesAndTypes result;
|
||||
for (const auto & group_by_ele : query_node->getGroupByNode()->getChildren())
|
||||
const auto & group_by_elements = query_node->getGroupByNode()->getChildren();
|
||||
for (const auto & group_by_element : group_by_elements)
|
||||
{
|
||||
const auto & projection_columns = query_node->getProjectionColumns();
|
||||
const auto & projection_nodes = query_node->getProjection().getNodes();
|
||||
@ -38,10 +39,18 @@ NamesAndTypes extractProjectionColumnsForGroupBy(const QueryNode * query_node)
|
||||
|
||||
for (size_t i = 0; i < projection_columns.size(); i++)
|
||||
{
|
||||
if (projection_nodes[i]->isEqual(*group_by_ele))
|
||||
if (projection_nodes[i]->isEqual(*group_by_element))
|
||||
{
|
||||
result.push_back(projection_columns[i]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
/// If some group by keys are not matched, we cannot apply optimization,
|
||||
/// because prefix of group by keys may not be unique.
|
||||
if (result.size() != group_by_elements.size())
|
||||
return {};
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
@ -1964,7 +1964,7 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
|
||||
}
|
||||
|
||||
/// INSERT query for which data transfer is needed (not an INSERT SELECT or input()) is processed separately.
|
||||
if (insert && (!insert->select || input_function) && !insert->watch && !is_async_insert_with_inlined_data)
|
||||
if (insert && (!insert->select || input_function) && !is_async_insert_with_inlined_data)
|
||||
{
|
||||
if (input_function && insert->format.empty())
|
||||
throw Exception(ErrorCodes::INVALID_USAGE_OF_INPUT, "FORMAT must be specified for function input()");
|
||||
|
@ -16,6 +16,7 @@
|
||||
#include <Poco/Net/HTTPRequest.h>
|
||||
#include <Poco/Net/HTTPResponse.h>
|
||||
#include <Poco/Net/HTTPStream.h>
|
||||
#include <Poco/Net/NetException.h>
|
||||
#include <Poco/Timespan.h>
|
||||
|
||||
#include <queue>
|
||||
@ -83,17 +84,15 @@ namespace
|
||||
}
|
||||
|
||||
|
||||
size_t roundUp(size_t x, size_t rounding)
|
||||
constexpr size_t roundUp(size_t x, size_t rounding)
|
||||
{
|
||||
chassert(rounding > 0);
|
||||
return (x + (rounding - 1)) / rounding * rounding;
|
||||
}
|
||||
|
||||
|
||||
Poco::Timespan divide(const Poco::Timespan span, int divisor)
|
||||
{
|
||||
return Poco::Timespan(Poco::Timestamp::TimeDiff(span.totalMicroseconds() / divisor));
|
||||
return (x + rounding) / rounding * rounding;
|
||||
}
|
||||
static_assert(roundUp(10000, 100) == 10100);
|
||||
static_assert(roundUp(10001, 100) == 10100);
|
||||
static_assert(roundUp(10099, 100) == 10100);
|
||||
static_assert(roundUp(10100, 100) == 10200);
|
||||
}
|
||||
|
||||
namespace DB
|
||||
@ -202,8 +201,9 @@ public:
|
||||
|
||||
if (total_connections_in_group >= limits.warning_limit && total_connections_in_group >= mute_warning_until)
|
||||
{
|
||||
LOG_WARNING(log, "Too many active sessions in group {}, count {}, warning limit {}", type, total_connections_in_group, limits.warning_limit);
|
||||
mute_warning_until = roundUp(total_connections_in_group, HTTPConnectionPools::Limits::warning_step);
|
||||
LOG_WARNING(log, "Too many active sessions in group {}, count {}, warning limit {}, next warning at {}",
|
||||
type, total_connections_in_group, limits.warning_limit, mute_warning_until);
|
||||
}
|
||||
}
|
||||
|
||||
@ -213,7 +213,8 @@ public:
|
||||
|
||||
--total_connections_in_group;
|
||||
|
||||
const size_t reduced_warning_limit = limits.warning_limit > 10 ? limits.warning_limit - 10 : 1;
|
||||
const size_t gap = 20;
|
||||
const size_t reduced_warning_limit = limits.warning_limit > gap ? limits.warning_limit - gap : 1;
|
||||
if (mute_warning_until > 0 && total_connections_in_group < reduced_warning_limit)
|
||||
{
|
||||
LOG_WARNING(log, "Sessions count is OK in the group {}, count {}", type, total_connections_in_group);
|
||||
@ -273,9 +274,15 @@ private:
|
||||
public:
|
||||
using Ptr = std::shared_ptr<PooledConnection>;
|
||||
|
||||
using Session::mustReconnect;
|
||||
|
||||
void markAsExpired()
|
||||
{
|
||||
isExpired = true;
|
||||
}
|
||||
|
||||
void reconnect() override
|
||||
{
|
||||
ProfileEvents::increment(metrics.reset);
|
||||
Session::close();
|
||||
|
||||
if (auto lock = pool.lock())
|
||||
@ -283,6 +290,7 @@ private:
|
||||
auto timeouts = getTimeouts(*this);
|
||||
auto new_connection = lock->getConnection(timeouts);
|
||||
Session::assign(*new_connection);
|
||||
Session::setKeepAliveRequest(Session::getKeepAliveRequest() + 1);
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -304,6 +312,12 @@ private:
|
||||
Session::getPort());
|
||||
}
|
||||
|
||||
Poco::Timespan idleTime()
|
||||
{
|
||||
Poco::Timestamp now;
|
||||
return now - Session::getLastRequest();
|
||||
}
|
||||
|
||||
void flushRequest() override
|
||||
{
|
||||
if (bool(request_stream))
|
||||
@ -335,6 +349,7 @@ private:
|
||||
|
||||
std::ostream & sendRequest(Poco::Net::HTTPRequest & request) override
|
||||
{
|
||||
auto idle = idleTime();
|
||||
std::ostream & result = Session::sendRequest(request);
|
||||
result.exceptions(std::ios::badbit);
|
||||
|
||||
@ -392,10 +407,11 @@ private:
|
||||
}
|
||||
response_stream = nullptr;
|
||||
|
||||
if (auto lock = pool.lock())
|
||||
lock->atConnectionDestroy(*this);
|
||||
else
|
||||
ProfileEvents::increment(metrics.reset);
|
||||
group->atConnectionDestroy();
|
||||
|
||||
if (!isExpired)
|
||||
if (auto lock = pool.lock())
|
||||
lock->atConnectionDestroy(*this);
|
||||
|
||||
CurrentMetrics::sub(metrics.active_count);
|
||||
}
|
||||
@ -404,10 +420,18 @@ private:
|
||||
friend class EndpointConnectionPool;
|
||||
|
||||
template <class... Args>
|
||||
explicit PooledConnection(EndpointConnectionPool::WeakPtr pool_, IHTTPConnectionPoolForEndpoint::Metrics metrics_, Args &&... args)
|
||||
: Session(args...), pool(std::move(pool_)), metrics(std::move(metrics_))
|
||||
explicit PooledConnection(
|
||||
EndpointConnectionPool::WeakPtr pool_,
|
||||
ConnectionGroup::Ptr group_,
|
||||
IHTTPConnectionPoolForEndpoint::Metrics metrics_,
|
||||
Args &&... args)
|
||||
: Session(std::forward<Args>(args)...)
|
||||
, pool(std::move(pool_))
|
||||
, group(group_)
|
||||
, metrics(std::move(metrics_))
|
||||
{
|
||||
CurrentMetrics::add(metrics.active_count);
|
||||
group->atConnectionCreate();
|
||||
}
|
||||
|
||||
template <class... Args>
|
||||
@ -433,10 +457,12 @@ private:
|
||||
return request_stream_completed && response_stream_completed;
|
||||
}
|
||||
|
||||
WeakPtr pool;
|
||||
EndpointConnectionPool::WeakPtr pool;
|
||||
ConnectionGroup::Ptr group;
|
||||
IHTTPConnectionPoolForEndpoint::Metrics metrics;
|
||||
bool isExpired = false;
|
||||
|
||||
Poco::Logger * log = &Poco::Logger::get("PooledConnection");
|
||||
LoggerPtr log = getLogger("PooledConnection");
|
||||
|
||||
std::ostream * request_stream = nullptr;
|
||||
std::istream * response_stream = nullptr;
|
||||
@ -484,7 +510,6 @@ public:
|
||||
|
||||
IHTTPConnectionPoolForEndpoint::ConnectionPtr getConnection(const ConnectionTimeouts & timeouts) override
|
||||
{
|
||||
Poco::Timestamp now;
|
||||
std::vector<ConnectionPtr> expired_connections;
|
||||
|
||||
SCOPE_EXIT({
|
||||
@ -494,8 +519,9 @@ public:
|
||||
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
expired_connections.reserve(stored_connections.size());
|
||||
|
||||
wipeExpiredImpl(expired_connections, now);
|
||||
wipeExpiredImpl(expired_connections);
|
||||
|
||||
if (!stored_connections.empty())
|
||||
{
|
||||
@ -526,7 +552,6 @@ public:
|
||||
|
||||
size_t wipeExpired() override
|
||||
{
|
||||
Poco::Timestamp now;
|
||||
std::vector<ConnectionPtr> expired_connections;
|
||||
|
||||
SCOPE_EXIT({
|
||||
@ -535,25 +560,29 @@ public:
|
||||
});
|
||||
|
||||
std::lock_guard lock(mutex);
|
||||
return wipeExpiredImpl(expired_connections, now);
|
||||
return wipeExpiredImpl(expired_connections);
|
||||
}
|
||||
|
||||
size_t wipeExpiredImpl(std::vector<ConnectionPtr> & expired_connections, Poco::Timestamp now) TSA_REQUIRES(mutex)
|
||||
size_t wipeExpiredImpl(std::vector<ConnectionPtr> & expired_connections) TSA_REQUIRES(mutex)
|
||||
{
|
||||
SCOPE_EXIT({
|
||||
CurrentMetrics::sub(getMetrics().stored_count, expired_connections.size());
|
||||
ProfileEvents::increment(getMetrics().expired, expired_connections.size());
|
||||
});
|
||||
|
||||
auto isSoftLimitReached = group->isSoftLimitReached();
|
||||
while (!stored_connections.empty())
|
||||
{
|
||||
auto connection = stored_connections.top();
|
||||
|
||||
if (!isExpired(now, connection))
|
||||
if (!isExpired(connection, isSoftLimitReached))
|
||||
return stored_connections.size();
|
||||
|
||||
stored_connections.pop();
|
||||
connection->markAsExpired();
|
||||
expired_connections.push_back(connection);
|
||||
}
|
||||
|
||||
CurrentMetrics::sub(getMetrics().stored_count, expired_connections.size());
|
||||
ProfileEvents::increment(getMetrics().expired, expired_connections.size());
|
||||
|
||||
return stored_connections.size();
|
||||
}
|
||||
|
||||
@ -569,57 +598,53 @@ private:
|
||||
|
||||
WeakPtr getWeakFromThis() { return EndpointConnectionPool::weak_from_this(); }
|
||||
|
||||
bool isExpired(Poco::Timestamp & now, ConnectionPtr connection)
|
||||
bool isExpired(ConnectionPtr connection, bool isSoftLimitReached) TSA_REQUIRES(mutex)
|
||||
{
|
||||
if (group->isSoftLimitReached())
|
||||
return now > (connection->getLastRequest() + divide(connection->getKeepAliveTimeout(), 10));
|
||||
return now > connection->getLastRequest() + connection->getKeepAliveTimeout();
|
||||
if (isSoftLimitReached)
|
||||
return connection->isKeepAliveExpired(0.1);
|
||||
return connection->isKeepAliveExpired(0.8);
|
||||
}
|
||||
|
||||
ConnectionPtr allocateNewConnection()
|
||||
|
||||
ConnectionPtr prepareNewConnection(const ConnectionTimeouts & timeouts)
|
||||
{
|
||||
ConnectionPtr connection = PooledConnection::create(this->getWeakFromThis(), getMetrics(), host, port);
|
||||
auto connection = PooledConnection::create(this->getWeakFromThis(), group, getMetrics(), host, port);
|
||||
|
||||
connection->setKeepAlive(true);
|
||||
setTimeouts(*connection, timeouts);
|
||||
|
||||
if (!proxy_configuration.isEmpty())
|
||||
{
|
||||
connection->setProxyConfig(proxyConfigurationToPocoProxyConfig(proxy_configuration));
|
||||
}
|
||||
|
||||
group->atConnectionCreate();
|
||||
|
||||
return connection;
|
||||
}
|
||||
|
||||
ConnectionPtr prepareNewConnection(const ConnectionTimeouts & timeouts)
|
||||
{
|
||||
auto address = HostResolversPool::instance().getResolver(host)->resolve();
|
||||
|
||||
auto session = allocateNewConnection();
|
||||
|
||||
setTimeouts(*session, timeouts);
|
||||
session->setResolvedHost(*address);
|
||||
connection->setResolvedHost(*address);
|
||||
|
||||
try
|
||||
{
|
||||
auto timer = CurrentThread::getProfileEvents().timer(getMetrics().elapsed_microseconds);
|
||||
session->doConnect();
|
||||
connection->doConnect();
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
address.setFail();
|
||||
ProfileEvents::increment(getMetrics().errors);
|
||||
session->reset();
|
||||
connection->reset();
|
||||
throw;
|
||||
}
|
||||
|
||||
ProfileEvents::increment(getMetrics().created);
|
||||
return session;
|
||||
return connection;
|
||||
}
|
||||
|
||||
void atConnectionDestroy(PooledConnection & connection)
|
||||
{
|
||||
group->atConnectionDestroy();
|
||||
if (connection.getKeepAliveRequest() >= connection.getKeepAliveMaxRequests())
|
||||
{
|
||||
ProfileEvents::increment(getMetrics().expired, 1);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!connection.connected() || connection.mustReconnect() || !connection.isCompleted() || connection.buffered()
|
||||
|| group->isStoreLimitReached())
|
||||
@ -628,17 +653,17 @@ private:
|
||||
return;
|
||||
}
|
||||
|
||||
auto connection_to_store = allocateNewConnection();
|
||||
auto connection_to_store = PooledConnection::create(this->getWeakFromThis(), group, getMetrics(), host, port);
|
||||
connection_to_store->assign(connection);
|
||||
|
||||
CurrentMetrics::add(getMetrics().stored_count, 1);
|
||||
ProfileEvents::increment(getMetrics().preserved, 1);
|
||||
|
||||
{
|
||||
MemoryTrackerSwitcher switcher{&total_memory_tracker};
|
||||
std::lock_guard lock(mutex);
|
||||
stored_connections.push(connection_to_store);
|
||||
}
|
||||
|
||||
CurrentMetrics::add(getMetrics().stored_count, 1);
|
||||
ProfileEvents::increment(getMetrics().preserved, 1);
|
||||
}
|
||||
|
||||
|
||||
@ -726,14 +751,13 @@ createConnectionPool(ConnectionGroup::Ptr group, std::string host, UInt16 port,
|
||||
class HTTPConnectionPools::Impl
|
||||
{
|
||||
private:
|
||||
const size_t DEFAULT_WIPE_TIMEOUT_SECONDS = 5 * 60;
|
||||
const size_t DEFAULT_WIPE_TIMEOUT_SECONDS = 10 * 60;
|
||||
const Poco::Timespan wipe_timeout = Poco::Timespan(DEFAULT_WIPE_TIMEOUT_SECONDS, 0);
|
||||
|
||||
ConnectionGroup::Ptr disk_group = std::make_shared<ConnectionGroup>(HTTPConnectionGroupType::DISK);
|
||||
ConnectionGroup::Ptr storage_group = std::make_shared<ConnectionGroup>(HTTPConnectionGroupType::STORAGE);
|
||||
ConnectionGroup::Ptr http_group = std::make_shared<ConnectionGroup>(HTTPConnectionGroupType::HTTP);
|
||||
|
||||
|
||||
/// If multiple mutexes are held simultaneously,
|
||||
/// they should be locked in this order:
|
||||
/// HTTPConnectionPools::mutex, then EndpointConnectionPool::mutex, then ConnectionGroup::mutex.
|
||||
|
@ -2,7 +2,6 @@
|
||||
#include <Common/HTTPConnectionPool.h>
|
||||
|
||||
#include <Poco/URI.h>
|
||||
#include <Poco/Net/ServerSocket.h>
|
||||
#include <Poco/Net/MessageHeader.h>
|
||||
#include <Poco/Net/HTTPServerRequest.h>
|
||||
#include <Poco/Net/HTTPServerResponse.h>
|
||||
@ -17,6 +16,40 @@
|
||||
namespace
|
||||
{
|
||||
|
||||
template<class T>
|
||||
class SafeHandler
|
||||
{
|
||||
public:
|
||||
using Ptr = std::shared_ptr<SafeHandler<T>>;
|
||||
|
||||
SafeHandler() = default;
|
||||
SafeHandler(SafeHandler<T>&) = delete;
|
||||
SafeHandler& operator=(SafeHandler<T>&) = delete;
|
||||
|
||||
T get()
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
return obj;
|
||||
}
|
||||
|
||||
void set(T && options_)
|
||||
{
|
||||
std::lock_guard lock(mutex);
|
||||
obj = std::move(options_);
|
||||
}
|
||||
|
||||
protected:
|
||||
std::mutex mutex;
|
||||
T obj = {};
|
||||
};
|
||||
|
||||
struct RequestOptions
|
||||
{
|
||||
size_t slowdown_receive = 0;
|
||||
int overwrite_keep_alive_timeout = 0;
|
||||
int overwrite_keep_alive_max_requests = 10;
|
||||
};
|
||||
|
||||
size_t stream_copy_n(std::istream & in, std::ostream & out, std::size_t count = std::numeric_limits<size_t>::max())
|
||||
{
|
||||
const size_t buffer_size = 4096;
|
||||
@ -47,13 +80,21 @@ size_t stream_copy_n(std::istream & in, std::ostream & out, std::size_t count =
|
||||
class MockRequestHandler : public Poco::Net::HTTPRequestHandler
|
||||
{
|
||||
public:
|
||||
explicit MockRequestHandler(std::shared_ptr<std::atomic<size_t>> slowdown_)
|
||||
: slowdown(std::move(slowdown_))
|
||||
explicit MockRequestHandler(SafeHandler<RequestOptions>::Ptr options_)
|
||||
: options(options_)
|
||||
{
|
||||
}
|
||||
|
||||
void handleRequest(Poco::Net::HTTPServerRequest & request, Poco::Net::HTTPServerResponse & response) override
|
||||
{
|
||||
int value = request.getKeepAliveTimeout();
|
||||
ASSERT_GT(value, 0);
|
||||
|
||||
auto params = options->get();
|
||||
|
||||
if (params.overwrite_keep_alive_timeout > 0)
|
||||
response.setKeepAliveTimeout(params.overwrite_keep_alive_timeout, params.overwrite_keep_alive_max_requests);
|
||||
|
||||
response.setStatus(Poco::Net::HTTPResponse::HTTP_OK);
|
||||
auto size = request.getContentLength();
|
||||
if (size > 0)
|
||||
@ -61,28 +102,29 @@ public:
|
||||
else
|
||||
response.setChunkedTransferEncoding(true); // or chunk encoding
|
||||
|
||||
sleepForSeconds(*slowdown);
|
||||
if (params.slowdown_receive > 0)
|
||||
sleepForSeconds(params.slowdown_receive);
|
||||
|
||||
stream_copy_n(request.stream(), response.send(), size);
|
||||
}
|
||||
|
||||
std::shared_ptr<std::atomic<size_t>> slowdown;
|
||||
SafeHandler<RequestOptions>::Ptr options;
|
||||
};
|
||||
|
||||
class HTTPRequestHandlerFactory : public Poco::Net::HTTPRequestHandlerFactory
|
||||
{
|
||||
public:
|
||||
explicit HTTPRequestHandlerFactory(std::shared_ptr<std::atomic<size_t>> slowdown_)
|
||||
: slowdown(std::move(slowdown_))
|
||||
explicit HTTPRequestHandlerFactory(SafeHandler<RequestOptions>::Ptr options_)
|
||||
: options(options_)
|
||||
{
|
||||
}
|
||||
|
||||
Poco::Net::HTTPRequestHandler * createRequestHandler(const Poco::Net::HTTPServerRequest &) override
|
||||
{
|
||||
return new MockRequestHandler(slowdown);
|
||||
return new MockRequestHandler(options);
|
||||
}
|
||||
|
||||
std::shared_ptr<std::atomic<size_t>> slowdown;
|
||||
SafeHandler<RequestOptions>::Ptr options;
|
||||
};
|
||||
|
||||
}
|
||||
@ -94,6 +136,8 @@ class ConnectionPoolTest : public testing::Test {
|
||||
protected:
|
||||
ConnectionPoolTest()
|
||||
{
|
||||
options = std::make_shared<SafeHandler<RequestOptions>>();
|
||||
|
||||
startServer();
|
||||
}
|
||||
|
||||
@ -102,7 +146,7 @@ protected:
|
||||
DB::HTTPConnectionPools::Limits def_limits{};
|
||||
DB::HTTPConnectionPools::instance().setLimits(def_limits, def_limits, def_limits);
|
||||
|
||||
setSlowDown(0);
|
||||
options->set(RequestOptions());
|
||||
|
||||
DB::HTTPConnectionPools::instance().dropCache();
|
||||
DB::CurrentThread::getProfileEvents().reset();
|
||||
@ -129,7 +173,7 @@ protected:
|
||||
void startServer()
|
||||
{
|
||||
server_data.reset();
|
||||
server_data.handler_factory = new HTTPRequestHandlerFactory(slowdown_receive);
|
||||
server_data.handler_factory = new HTTPRequestHandlerFactory(options);
|
||||
server_data.server = std::make_unique<Poco::Net::HTTPServer>(
|
||||
server_data.handler_factory, server_data.port);
|
||||
|
||||
@ -143,11 +187,21 @@ protected:
|
||||
|
||||
void setSlowDown(size_t seconds)
|
||||
{
|
||||
*slowdown_receive = seconds;
|
||||
auto opt = options->get();
|
||||
opt.slowdown_receive = seconds;
|
||||
options->set(std::move(opt));
|
||||
}
|
||||
|
||||
void setOverWriteKeepAlive(size_t seconds, int max_requests)
|
||||
{
|
||||
auto opt = options->get();
|
||||
opt.overwrite_keep_alive_timeout = int(seconds);
|
||||
opt.overwrite_keep_alive_max_requests= max_requests;
|
||||
options->set(std::move(opt));
|
||||
}
|
||||
|
||||
DB::ConnectionTimeouts timeouts;
|
||||
std::shared_ptr<std::atomic<size_t>> slowdown_receive = std::make_shared<std::atomic<size_t>>(0);
|
||||
SafeHandler<RequestOptions>::Ptr options;
|
||||
|
||||
struct ServerData
|
||||
{
|
||||
@ -182,7 +236,7 @@ protected:
|
||||
void wait_until(std::function<bool()> pred)
|
||||
{
|
||||
while (!pred())
|
||||
sleepForMilliseconds(250);
|
||||
sleepForMilliseconds(10);
|
||||
}
|
||||
|
||||
void echoRequest(String data, HTTPSession & session)
|
||||
@ -245,45 +299,52 @@ TEST_F(ConnectionPoolTest, CanRequest)
|
||||
ASSERT_EQ(0, getServer().currentConnections());
|
||||
ASSERT_EQ(1, getServer().totalConnections());
|
||||
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
|
||||
auto metrics = pool->getMetrics();
|
||||
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
|
||||
|
||||
ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count));
|
||||
ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
|
||||
}
|
||||
|
||||
TEST_F(ConnectionPoolTest, CanPreserve)
|
||||
{
|
||||
auto pool = getPool();
|
||||
auto metrics = pool->getMetrics();
|
||||
|
||||
{
|
||||
auto connection = pool->getConnection(timeouts);
|
||||
}
|
||||
|
||||
ASSERT_EQ(1, CurrentMetrics::get(pool->getMetrics().active_count));
|
||||
ASSERT_EQ(1, CurrentMetrics::get(pool->getMetrics().stored_count));
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
|
||||
|
||||
ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count));
|
||||
ASSERT_EQ(1, CurrentMetrics::get(metrics.stored_count));
|
||||
|
||||
wait_until([&] () { return getServer().currentConnections() == 1; });
|
||||
ASSERT_EQ(1, getServer().currentConnections());
|
||||
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]);
|
||||
}
|
||||
|
||||
TEST_F(ConnectionPoolTest, CanReuse)
|
||||
{
|
||||
auto pool = getPool();
|
||||
auto metrics = pool->getMetrics();
|
||||
|
||||
{
|
||||
auto connection = pool->getConnection(timeouts);
|
||||
// DB::setReuseTag(*connection);
|
||||
}
|
||||
|
||||
ASSERT_EQ(1, CurrentMetrics::get(pool->getMetrics().active_count));
|
||||
ASSERT_EQ(1, CurrentMetrics::get(pool->getMetrics().stored_count));
|
||||
|
||||
{
|
||||
auto connection = pool->getConnection(timeouts);
|
||||
|
||||
ASSERT_EQ(1, CurrentMetrics::get(pool->getMetrics().active_count));
|
||||
ASSERT_EQ(0, CurrentMetrics::get(pool->getMetrics().stored_count));
|
||||
ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count));
|
||||
ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
|
||||
|
||||
wait_until([&] () { return getServer().currentConnections() == 1; });
|
||||
ASSERT_EQ(1, getServer().currentConnections());
|
||||
@ -293,6 +354,11 @@ TEST_F(ConnectionPoolTest, CanReuse)
|
||||
ASSERT_EQ(1, getServer().totalConnections());
|
||||
ASSERT_EQ(1, getServer().currentConnections());
|
||||
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reused]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
|
||||
|
||||
connection->reset();
|
||||
}
|
||||
|
||||
@ -303,15 +369,16 @@ TEST_F(ConnectionPoolTest, CanReuse)
|
||||
ASSERT_EQ(0, getServer().currentConnections());
|
||||
ASSERT_EQ(1, getServer().totalConnections());
|
||||
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reused]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]);
|
||||
}
|
||||
|
||||
TEST_F(ConnectionPoolTest, CanReuse10)
|
||||
{
|
||||
auto pool = getPool();
|
||||
|
||||
auto metrics = pool->getMetrics();
|
||||
|
||||
for (int i = 0; i < 10; ++i)
|
||||
{
|
||||
@ -328,16 +395,23 @@ TEST_F(ConnectionPoolTest, CanReuse10)
|
||||
ASSERT_EQ(0, getServer().currentConnections());
|
||||
ASSERT_EQ(1, getServer().totalConnections());
|
||||
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
|
||||
ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
|
||||
ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]);
|
||||
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]);
|
||||
ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
|
||||
ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[metrics.reused]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]);
|
||||
|
||||
ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count));
|
||||
ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
|
||||
}
|
||||
|
||||
TEST_F(ConnectionPoolTest, CanReuse5)
|
||||
{
|
||||
timeouts.withHTTPKeepAliveTimeout(1);
|
||||
auto ka = Poco::Timespan(1, 0); // 1 seconds
|
||||
timeouts.withHTTPKeepAliveTimeout(ka);
|
||||
|
||||
auto pool = getPool();
|
||||
auto metrics = pool->getMetrics();
|
||||
|
||||
std::vector<DB::HTTPSessionPtr> connections;
|
||||
connections.reserve(5);
|
||||
@ -347,11 +421,14 @@ TEST_F(ConnectionPoolTest, CanReuse5)
|
||||
}
|
||||
connections.clear();
|
||||
|
||||
ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
|
||||
ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]);
|
||||
ASSERT_EQ(5, CurrentMetrics::get(pool->getMetrics().active_count));
|
||||
ASSERT_EQ(5, CurrentMetrics::get(pool->getMetrics().stored_count));
|
||||
ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[metrics.created]);
|
||||
ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
|
||||
|
||||
ASSERT_EQ(5, CurrentMetrics::get(metrics.active_count));
|
||||
ASSERT_EQ(5, CurrentMetrics::get(metrics.stored_count));
|
||||
|
||||
wait_until([&] () { return getServer().currentConnections() == 5; });
|
||||
ASSERT_EQ(5, getServer().currentConnections());
|
||||
@ -363,35 +440,56 @@ TEST_F(ConnectionPoolTest, CanReuse5)
|
||||
echoRequest("Hello", *connection);
|
||||
}
|
||||
|
||||
ASSERT_EQ(5, getServer().totalConnections());
|
||||
ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[metrics.created]);
|
||||
ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
|
||||
ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[metrics.reused]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
|
||||
|
||||
ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
|
||||
ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
|
||||
ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]);
|
||||
ASSERT_EQ(5, CurrentMetrics::get(pool->getMetrics().active_count));
|
||||
ASSERT_EQ(5, CurrentMetrics::get(pool->getMetrics().stored_count));
|
||||
ASSERT_EQ(5, CurrentMetrics::get(metrics.active_count));
|
||||
ASSERT_EQ(5, CurrentMetrics::get(metrics.stored_count));
|
||||
|
||||
/// wait until all connections are timeouted
|
||||
wait_until([&] () { return getServer().currentConnections() == 0; });
|
||||
|
||||
{
|
||||
// just to trigger pool->wipeExpired();
|
||||
auto connection = pool->getConnection(timeouts);
|
||||
connection->reset();
|
||||
}
|
||||
|
||||
ASSERT_EQ(6, DB::CurrentThread::getProfileEvents()[metrics.created]);
|
||||
ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
|
||||
ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[metrics.reused]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]);
|
||||
ASSERT_EQ(5, DB::CurrentThread::getProfileEvents()[metrics.expired]);
|
||||
|
||||
ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count));
|
||||
ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
|
||||
}
|
||||
|
||||
TEST_F(ConnectionPoolTest, CanReconnectAndCreate)
|
||||
{
|
||||
auto pool = getPool();
|
||||
auto metrics = pool->getMetrics();
|
||||
|
||||
std::vector<HTTPSessionPtr> in_use;
|
||||
|
||||
const size_t count = 2;
|
||||
const size_t count = 3;
|
||||
for (int i = 0; i < count; ++i)
|
||||
{
|
||||
auto connection = pool->getConnection(timeouts);
|
||||
// DB::setReuseTag(*connection);
|
||||
in_use.push_back(connection);
|
||||
}
|
||||
|
||||
ASSERT_EQ(count, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]);
|
||||
ASSERT_EQ(count, DB::CurrentThread::getProfileEvents()[metrics.created]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
|
||||
|
||||
ASSERT_EQ(count, CurrentMetrics::get(pool->getMetrics().active_count));
|
||||
ASSERT_EQ(0, CurrentMetrics::get(pool->getMetrics().stored_count));
|
||||
ASSERT_EQ(count, CurrentMetrics::get(metrics.active_count));
|
||||
ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
|
||||
|
||||
auto connection = std::move(in_use.back());
|
||||
in_use.pop_back();
|
||||
@ -402,28 +500,39 @@ TEST_F(ConnectionPoolTest, CanReconnectAndCreate)
|
||||
|
||||
echoRequest("Hello", *connection);
|
||||
|
||||
connection->reset();
|
||||
ASSERT_EQ(count+1, DB::CurrentThread::getProfileEvents()[metrics.created]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
|
||||
|
||||
wait_until([&] () { return getServer().currentConnections() == 1; });
|
||||
ASSERT_EQ(1, getServer().currentConnections());
|
||||
ASSERT_EQ(count+1, getServer().totalConnections());
|
||||
|
||||
ASSERT_EQ(count+1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]);
|
||||
ASSERT_EQ(count, CurrentMetrics::get(metrics.active_count));
|
||||
ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
|
||||
}
|
||||
|
||||
TEST_F(ConnectionPoolTest, CanReconnectAndReuse)
|
||||
{
|
||||
auto ka = Poco::Timespan(1, 0); // 1 seconds
|
||||
timeouts.withHTTPKeepAliveTimeout(ka);
|
||||
|
||||
auto pool = getPool();
|
||||
auto metrics = pool->getMetrics();
|
||||
|
||||
std::vector<HTTPSessionPtr> in_use;
|
||||
|
||||
const size_t count = 2;
|
||||
const size_t count = 3;
|
||||
for (int i = 0; i < count; ++i)
|
||||
{
|
||||
auto connection = pool->getConnection(timeouts);
|
||||
/// make some request in order to show to the server the keep alive headers
|
||||
echoRequest("Hello", *connection);
|
||||
in_use.push_back(std::move(connection));
|
||||
}
|
||||
in_use.clear();
|
||||
|
||||
for (int i = 0; i < count; ++i)
|
||||
{
|
||||
auto connection = pool->getConnection(timeouts);
|
||||
// DB::setReuseTag(*connection);
|
||||
in_use.push_back(std::move(connection));
|
||||
}
|
||||
|
||||
@ -441,11 +550,16 @@ TEST_F(ConnectionPoolTest, CanReconnectAndReuse)
|
||||
|
||||
wait_until([&] () { return getServer().currentConnections() == 0; });
|
||||
ASSERT_EQ(0, getServer().currentConnections());
|
||||
ASSERT_EQ(2, getServer().totalConnections());
|
||||
ASSERT_EQ(count, getServer().totalConnections());
|
||||
|
||||
ASSERT_EQ(count, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]);
|
||||
ASSERT_EQ(count, DB::CurrentThread::getProfileEvents()[metrics.created]);
|
||||
ASSERT_EQ(count + count - 1, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
|
||||
ASSERT_EQ(count + 1, DB::CurrentThread::getProfileEvents()[metrics.reused]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
|
||||
|
||||
ASSERT_EQ(count-1, CurrentMetrics::get(metrics.active_count));
|
||||
ASSERT_EQ(count-2, CurrentMetrics::get(metrics.stored_count));
|
||||
}
|
||||
|
||||
TEST_F(ConnectionPoolTest, ReceiveTimeout)
|
||||
@ -454,6 +568,7 @@ TEST_F(ConnectionPoolTest, ReceiveTimeout)
|
||||
timeouts.withReceiveTimeout(1);
|
||||
|
||||
auto pool = getPool();
|
||||
auto metrics = pool->getMetrics();
|
||||
|
||||
{
|
||||
auto connection = pool->getConnection(timeouts);
|
||||
@ -462,10 +577,14 @@ TEST_F(ConnectionPoolTest, ReceiveTimeout)
|
||||
);
|
||||
}
|
||||
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reset]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
|
||||
|
||||
ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count));
|
||||
ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
|
||||
|
||||
{
|
||||
timeouts.withReceiveTimeout(3);
|
||||
@ -475,10 +594,14 @@ TEST_F(ConnectionPoolTest, ReceiveTimeout)
|
||||
);
|
||||
}
|
||||
|
||||
ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reset]);
|
||||
ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[metrics.created]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
|
||||
|
||||
ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count));
|
||||
ASSERT_EQ(1, CurrentMetrics::get(metrics.stored_count));
|
||||
|
||||
{
|
||||
/// timeouts have effect for reused session
|
||||
@ -489,10 +612,14 @@ TEST_F(ConnectionPoolTest, ReceiveTimeout)
|
||||
);
|
||||
}
|
||||
|
||||
ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reused]);
|
||||
ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reset]);
|
||||
ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[metrics.created]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reused]);
|
||||
ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[metrics.reset]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
|
||||
|
||||
ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count));
|
||||
ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
|
||||
}
|
||||
|
||||
TEST_F(ConnectionPoolTest, ReadWriteBufferFromHTTP)
|
||||
@ -500,6 +627,7 @@ TEST_F(ConnectionPoolTest, ReadWriteBufferFromHTTP)
|
||||
std::string_view message = "Hello ReadWriteBufferFromHTTP";
|
||||
auto uri = Poco::URI(getServerUrl());
|
||||
auto metrics = DB::HTTPConnectionPools::instance().getPool(DB::HTTPConnectionGroupType::HTTP, uri, DB::ProxyConfiguration{})->getMetrics();
|
||||
|
||||
Poco::Net::HTTPBasicCredentials empty_creds;
|
||||
auto buf_from_http = DB::BuilderRWBufferFromHTTP(uri)
|
||||
.withConnectionGroup(DB::HTTPConnectionGroupType::HTTP)
|
||||
@ -527,6 +655,7 @@ TEST_F(ConnectionPoolTest, ReadWriteBufferFromHTTP)
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
|
||||
|
||||
ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count));
|
||||
ASSERT_EQ(1, CurrentMetrics::get(metrics.stored_count));
|
||||
@ -538,23 +667,26 @@ TEST_F(ConnectionPoolTest, HardLimit)
|
||||
DB::HTTPConnectionPools::instance().setLimits(zero_limits, zero_limits, zero_limits);
|
||||
|
||||
auto pool = getPool();
|
||||
auto metrics = pool->getMetrics();
|
||||
|
||||
{
|
||||
auto connection = pool->getConnection(timeouts);
|
||||
}
|
||||
|
||||
ASSERT_EQ(0, CurrentMetrics::get(pool->getMetrics().active_count));
|
||||
ASSERT_EQ(0, CurrentMetrics::get(pool->getMetrics().stored_count));
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
|
||||
|
||||
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reset]);
|
||||
ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count));
|
||||
ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
|
||||
}
|
||||
|
||||
TEST_F(ConnectionPoolTest, NoReceiveCall)
|
||||
{
|
||||
auto pool = getPool();
|
||||
auto metrics = pool->getMetrics();
|
||||
|
||||
{
|
||||
auto connection = pool->getConnection(timeouts);
|
||||
@ -570,11 +702,209 @@ TEST_F(ConnectionPoolTest, NoReceiveCall)
|
||||
connection->flushRequest();
|
||||
}
|
||||
|
||||
ASSERT_EQ(0, CurrentMetrics::get(pool->getMetrics().active_count));
|
||||
ASSERT_EQ(0, CurrentMetrics::get(pool->getMetrics().stored_count));
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
|
||||
|
||||
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().created]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[pool->getMetrics().preserved]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[pool->getMetrics().reset]);
|
||||
ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count));
|
||||
ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
|
||||
}
|
||||
|
||||
TEST_F(ConnectionPoolTest, ReconnectedWhenConnectionIsHoldTooLong)
|
||||
{
|
||||
auto ka = Poco::Timespan(1, 0); // 1 seconds
|
||||
timeouts.withHTTPKeepAliveTimeout(ka);
|
||||
|
||||
auto pool = getPool();
|
||||
auto metrics = pool->getMetrics();
|
||||
|
||||
{
|
||||
auto connection = pool->getConnection(timeouts);
|
||||
|
||||
echoRequest("Hello", *connection);
|
||||
|
||||
auto fake_ka = Poco::Timespan(30 * 1000 * 1000); // 30 seconds
|
||||
timeouts.withHTTPKeepAliveTimeout(fake_ka);
|
||||
DB::setTimeouts(*connection, timeouts); // new keep alive timeout has no effect
|
||||
|
||||
wait_until([&] () { return getServer().currentConnections() == 0; });
|
||||
|
||||
ASSERT_EQ(1, connection->connected());
|
||||
ASSERT_EQ(1, connection->getKeepAlive());
|
||||
ASSERT_EQ(1000, connection->getKeepAliveTimeout().totalMilliseconds());
|
||||
ASSERT_EQ(1, connection->isKeepAliveExpired(connection->getKeepAliveReliability()));
|
||||
|
||||
echoRequest("Hello", *connection);
|
||||
}
|
||||
|
||||
|
||||
ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[metrics.created]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.reset]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
|
||||
|
||||
ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count));
|
||||
ASSERT_EQ(1, CurrentMetrics::get(metrics.stored_count));
|
||||
}
|
||||
|
||||
TEST_F(ConnectionPoolTest, ReconnectedWhenConnectionIsNearlyExpired)
|
||||
{
|
||||
auto ka = Poco::Timespan(1, 0); // 1 seconds
|
||||
timeouts.withHTTPKeepAliveTimeout(ka);
|
||||
|
||||
auto pool = getPool();
|
||||
auto metrics = pool->getMetrics();
|
||||
|
||||
{
|
||||
{
|
||||
auto connection = pool->getConnection(timeouts);
|
||||
echoRequest("Hello", *connection);
|
||||
}
|
||||
|
||||
sleepForMilliseconds(900);
|
||||
|
||||
{
|
||||
auto connection = pool->getConnection(timeouts);
|
||||
echoRequest("Hello", *connection);
|
||||
}
|
||||
}
|
||||
|
||||
ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[metrics.created]);
|
||||
ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reused]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.expired]);
|
||||
|
||||
ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count));
|
||||
ASSERT_EQ(1, CurrentMetrics::get(metrics.stored_count));
|
||||
}
|
||||
|
||||
TEST_F(ConnectionPoolTest, ServerOverwriteKeepAlive)
|
||||
{
|
||||
auto ka = Poco::Timespan(30, 0); // 30 seconds
|
||||
timeouts.withHTTPKeepAliveTimeout(ka);
|
||||
|
||||
auto pool = getPool();
|
||||
auto metrics = pool->getMetrics();
|
||||
|
||||
{
|
||||
auto connection = pool->getConnection(timeouts);
|
||||
echoRequest("Hello", *connection);
|
||||
ASSERT_EQ(30, timeouts.http_keep_alive_timeout.totalSeconds());
|
||||
ASSERT_EQ(30, connection->getKeepAliveTimeout().totalSeconds());
|
||||
}
|
||||
|
||||
{
|
||||
setOverWriteKeepAlive(1, 10);
|
||||
auto connection = pool->getConnection(timeouts);
|
||||
echoRequest("Hello", *connection);
|
||||
ASSERT_EQ(30, timeouts.http_keep_alive_timeout.totalSeconds());
|
||||
ASSERT_EQ(1, connection->getKeepAliveTimeout().totalSeconds());
|
||||
}
|
||||
|
||||
{
|
||||
// server do not overwrite it in the following requests but client has to remember last agreed value
|
||||
setOverWriteKeepAlive(0, 0);
|
||||
auto connection = pool->getConnection(timeouts);
|
||||
echoRequest("Hello", *connection);
|
||||
ASSERT_EQ(30, timeouts.http_keep_alive_timeout.totalSeconds());
|
||||
ASSERT_EQ(1, connection->getKeepAliveTimeout().totalSeconds());
|
||||
}
|
||||
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]);
|
||||
ASSERT_EQ(3, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
|
||||
ASSERT_EQ(2, DB::CurrentThread::getProfileEvents()[metrics.reused]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
|
||||
|
||||
ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count));
|
||||
ASSERT_EQ(1, CurrentMetrics::get(metrics.stored_count));
|
||||
}
|
||||
|
||||
TEST_F(ConnectionPoolTest, MaxRequests)
|
||||
{
|
||||
auto ka = Poco::Timespan(30, 0); // 30 seconds
|
||||
timeouts.withHTTPKeepAliveTimeout(ka);
|
||||
auto max_requests = 5;
|
||||
timeouts.http_keep_alive_max_requests = max_requests;
|
||||
|
||||
auto pool = getPool();
|
||||
auto metrics = pool->getMetrics();
|
||||
|
||||
for (int i = 1; i <= max_requests - 1; ++i)
|
||||
{
|
||||
auto connection = pool->getConnection(timeouts);
|
||||
echoRequest("Hello", *connection);
|
||||
ASSERT_EQ(30, connection->getKeepAliveTimeout().totalSeconds());
|
||||
ASSERT_EQ(max_requests, connection->getKeepAliveMaxRequests());
|
||||
ASSERT_EQ(i, connection->getKeepAliveRequest());
|
||||
}
|
||||
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]);
|
||||
ASSERT_EQ(max_requests-1, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
|
||||
ASSERT_EQ(max_requests-2, DB::CurrentThread::getProfileEvents()[metrics.reused]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.expired]);
|
||||
|
||||
ASSERT_EQ(1, CurrentMetrics::get(metrics.active_count));
|
||||
ASSERT_EQ(1, CurrentMetrics::get(metrics.stored_count));
|
||||
|
||||
{
|
||||
auto connection = pool->getConnection(timeouts);
|
||||
echoRequest("Hello", *connection);
|
||||
ASSERT_EQ(30, connection->getKeepAliveTimeout().totalSeconds());
|
||||
ASSERT_EQ(max_requests, connection->getKeepAliveMaxRequests());
|
||||
ASSERT_EQ(max_requests, connection->getKeepAliveRequest());
|
||||
}
|
||||
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.created]);
|
||||
ASSERT_EQ(max_requests-1, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
|
||||
ASSERT_EQ(max_requests-1, DB::CurrentThread::getProfileEvents()[metrics.reused]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
|
||||
ASSERT_EQ(1, DB::CurrentThread::getProfileEvents()[metrics.expired]);
|
||||
|
||||
ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count));
|
||||
ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
|
||||
}
|
||||
|
||||
|
||||
TEST_F(ConnectionPoolTest, ServerOverwriteMaxRequests)
|
||||
{
|
||||
auto ka = Poco::Timespan(30, 0); // 30 seconds
|
||||
timeouts.withHTTPKeepAliveTimeout(ka);
|
||||
|
||||
auto pool = getPool();
|
||||
auto metrics = pool->getMetrics();
|
||||
|
||||
{
|
||||
auto connection = pool->getConnection(timeouts);
|
||||
echoRequest("Hello", *connection);
|
||||
ASSERT_EQ(30, connection->getKeepAliveTimeout().totalSeconds());
|
||||
ASSERT_EQ(1000, connection->getKeepAliveMaxRequests());
|
||||
ASSERT_EQ(1, connection->getKeepAliveRequest());
|
||||
}
|
||||
|
||||
auto max_requests = 3;
|
||||
setOverWriteKeepAlive(5, max_requests);
|
||||
|
||||
for (int i = 2; i <= 10*max_requests; ++i)
|
||||
{
|
||||
auto connection = pool->getConnection(timeouts);
|
||||
echoRequest("Hello", *connection);
|
||||
ASSERT_EQ(5, connection->getKeepAliveTimeout().totalSeconds());
|
||||
ASSERT_EQ(max_requests, connection->getKeepAliveMaxRequests());
|
||||
ASSERT_EQ(((i-1) % max_requests) + 1, connection->getKeepAliveRequest());
|
||||
}
|
||||
|
||||
ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[metrics.created]);
|
||||
ASSERT_EQ(10*max_requests-10, DB::CurrentThread::getProfileEvents()[metrics.preserved]);
|
||||
ASSERT_EQ(10*max_requests-10, DB::CurrentThread::getProfileEvents()[metrics.reused]);
|
||||
ASSERT_EQ(0, DB::CurrentThread::getProfileEvents()[metrics.reset]);
|
||||
ASSERT_EQ(10, DB::CurrentThread::getProfileEvents()[metrics.expired]);
|
||||
|
||||
ASSERT_EQ(0, CurrentMetrics::get(metrics.active_count));
|
||||
ASSERT_EQ(0, CurrentMetrics::get(metrics.stored_count));
|
||||
}
|
||||
|
@ -54,6 +54,7 @@ static constexpr auto DEFAULT_COUNT_OF_HTTP_CONNECTIONS_PER_ENDPOINT = 15;
|
||||
|
||||
static constexpr auto DEFAULT_TCP_KEEP_ALIVE_TIMEOUT = 290;
|
||||
static constexpr auto DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT = 30;
|
||||
static constexpr auto DEFAULT_HTTP_KEEP_ALIVE_MAX_REQUEST = 1000;
|
||||
|
||||
static constexpr auto DBMS_DEFAULT_PATH = "/var/lib/clickhouse/";
|
||||
|
||||
|
@ -128,9 +128,9 @@ namespace DB
|
||||
M(Bool, format_alter_operations_with_parentheses, false, "If enabled, each operation in alter queries will be surrounded with parentheses in formatted queries to make them less ambiguous.", 0) \
|
||||
M(String, default_replica_path, "/clickhouse/tables/{uuid}/{shard}", "The path to the table in ZooKeeper", 0) \
|
||||
M(String, default_replica_name, "{replica}", "The replica name in ZooKeeper", 0) \
|
||||
M(UInt64, disk_connections_soft_limit, 1000, "Connections above this limit have significantly shorter time to live. The limit applies to the disks connections.", 0) \
|
||||
M(UInt64, disk_connections_soft_limit, 5000, "Connections above this limit have significantly shorter time to live. The limit applies to the disks connections.", 0) \
|
||||
M(UInt64, disk_connections_warn_limit, 10000, "Warning massages are written to the logs if number of in-use connections are higher than this limit. The limit applies to the disks connections.", 0) \
|
||||
M(UInt64, disk_connections_store_limit, 12000, "Connections above this limit reset after use. Set to 0 to turn connection cache off. The limit applies to the disks connections.", 0) \
|
||||
M(UInt64, disk_connections_store_limit, 30000, "Connections above this limit reset after use. Set to 0 to turn connection cache off. The limit applies to the disks connections.", 0) \
|
||||
M(UInt64, storage_connections_soft_limit, 100, "Connections above this limit have significantly shorter time to live. The limit applies to the storages connections.", 0) \
|
||||
M(UInt64, storage_connections_warn_limit, 1000, "Warning massages are written to the logs if number of in-use connections are higher than this limit. The limit applies to the storages connections.", 0) \
|
||||
M(UInt64, storage_connections_store_limit, 5000, "Connections above this limit reset after use. Set to 0 to turn connection cache off. The limit applies to the storages connections.", 0) \
|
||||
|
@ -76,6 +76,9 @@ std::unique_ptr<S3::Client> getClient(
|
||||
client_configuration.connectTimeoutMs = config.getUInt(config_prefix + ".connect_timeout_ms", S3::DEFAULT_CONNECT_TIMEOUT_MS);
|
||||
client_configuration.requestTimeoutMs = config.getUInt(config_prefix + ".request_timeout_ms", S3::DEFAULT_REQUEST_TIMEOUT_MS);
|
||||
client_configuration.maxConnections = config.getUInt(config_prefix + ".max_connections", S3::DEFAULT_MAX_CONNECTIONS);
|
||||
client_configuration.http_keep_alive_timeout = config.getUInt(config_prefix + ".http_keep_alive_timeout", S3::DEFAULT_KEEP_ALIVE_TIMEOUT);
|
||||
client_configuration.http_keep_alive_max_requests = config.getUInt(config_prefix + ".http_keep_alive_max_requests", S3::DEFAULT_KEEP_ALIVE_MAX_REQUESTS);
|
||||
|
||||
client_configuration.endpointOverride = uri.endpoint;
|
||||
client_configuration.s3_use_adaptive_timeouts = config.getBool(
|
||||
config_prefix + ".use_adaptive_timeouts", client_configuration.s3_use_adaptive_timeouts);
|
||||
|
@ -56,7 +56,6 @@ UserDefinedSQLObjectsDiskStorage::UserDefinedSQLObjectsDiskStorage(const Context
|
||||
, dir_path{makeDirectoryPathCanonical(dir_path_)}
|
||||
, log{getLogger("UserDefinedSQLObjectsLoaderFromDisk")}
|
||||
{
|
||||
createDirectory();
|
||||
}
|
||||
|
||||
|
||||
@ -122,7 +121,12 @@ void UserDefinedSQLObjectsDiskStorage::reloadObjects()
|
||||
void UserDefinedSQLObjectsDiskStorage::loadObjectsImpl()
|
||||
{
|
||||
LOG_INFO(log, "Loading user defined objects from {}", dir_path);
|
||||
createDirectory();
|
||||
|
||||
if (!std::filesystem::exists(dir_path))
|
||||
{
|
||||
LOG_DEBUG(log, "The directory for user defined objects ({}) does not exist: nothing to load", dir_path);
|
||||
return;
|
||||
}
|
||||
|
||||
std::vector<std::pair<String, ASTPtr>> function_names_and_queries;
|
||||
|
||||
@ -157,7 +161,6 @@ void UserDefinedSQLObjectsDiskStorage::loadObjectsImpl()
|
||||
|
||||
void UserDefinedSQLObjectsDiskStorage::reloadObject(UserDefinedSQLObjectType object_type, const String & object_name)
|
||||
{
|
||||
createDirectory();
|
||||
auto ast = tryLoadObject(object_type, object_name);
|
||||
if (ast)
|
||||
setObject(object_name, *ast);
|
||||
@ -185,6 +188,7 @@ bool UserDefinedSQLObjectsDiskStorage::storeObjectImpl(
|
||||
bool replace_if_exists,
|
||||
const Settings & settings)
|
||||
{
|
||||
createDirectory();
|
||||
String file_path = getFilePath(object_type, object_name);
|
||||
LOG_DEBUG(log, "Storing user-defined object {} to file {}", backQuote(object_name), file_path);
|
||||
|
||||
|
@ -66,13 +66,13 @@ struct DotProduct
|
||||
};
|
||||
|
||||
template <typename Type>
|
||||
static void accumulate(State<Type> & state, Type x, Type y)
|
||||
static NO_SANITIZE_UNDEFINED void accumulate(State<Type> & state, Type x, Type y)
|
||||
{
|
||||
state.sum += x * y;
|
||||
}
|
||||
|
||||
template <typename Type>
|
||||
static void combine(State<Type> & state, const State<Type> & other_state)
|
||||
static NO_SANITIZE_UNDEFINED void combine(State<Type> & state, const State<Type> & other_state)
|
||||
{
|
||||
state.sum += other_state.sum;
|
||||
}
|
||||
|
@ -144,7 +144,12 @@ ConnectionTimeouts ConnectionTimeouts::getAdaptiveTimeouts(const String & method
|
||||
void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts)
|
||||
{
|
||||
session.setTimeout(timeouts.connection_timeout, timeouts.send_timeout, timeouts.receive_timeout);
|
||||
session.setKeepAliveTimeout(timeouts.http_keep_alive_timeout);
|
||||
/// we can not change keep alive timeout for already initiated connections
|
||||
if (!session.connected())
|
||||
{
|
||||
session.setKeepAliveTimeout(timeouts.http_keep_alive_timeout);
|
||||
session.setKeepAliveMaxRequests(int(timeouts.http_keep_alive_max_requests));
|
||||
}
|
||||
}
|
||||
|
||||
ConnectionTimeouts getTimeouts(const Poco::Net::HTTPClientSession & session)
|
||||
|
@ -35,6 +35,8 @@ struct ConnectionTimeouts
|
||||
|
||||
Poco::Timespan tcp_keep_alive_timeout = Poco::Timespan(DEFAULT_TCP_KEEP_ALIVE_TIMEOUT, 0);
|
||||
Poco::Timespan http_keep_alive_timeout = Poco::Timespan(DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT, 0);
|
||||
size_t http_keep_alive_max_requests = DEFAULT_HTTP_KEEP_ALIVE_MAX_REQUEST;
|
||||
|
||||
|
||||
/// Timeouts for HedgedConnections
|
||||
Poco::Timespan hedged_connection_timeout = Poco::Timespan(DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC, 0);
|
||||
@ -69,6 +71,7 @@ APPLY_FOR_ALL_CONNECTION_TIMEOUT_MEMBERS(DECLARE_BUILDER_FOR_MEMBER)
|
||||
|
||||
ConnectionTimeouts & withConnectionTimeout(size_t seconds);
|
||||
ConnectionTimeouts & withConnectionTimeout(Poco::Timespan span);
|
||||
ConnectionTimeouts & withHTTPKeepAliveMaxRequests(size_t requests);
|
||||
};
|
||||
|
||||
/// NOLINTBEGIN(bugprone-macro-parentheses)
|
||||
@ -114,6 +117,12 @@ inline ConnectionTimeouts & ConnectionTimeouts::withConnectionTimeout(Poco::Time
|
||||
return *this;
|
||||
}
|
||||
|
||||
inline ConnectionTimeouts & ConnectionTimeouts::withHTTPKeepAliveMaxRequests(size_t requests)
|
||||
{
|
||||
http_keep_alive_max_requests = requests;
|
||||
return *this;
|
||||
}
|
||||
|
||||
void setTimeouts(Poco::Net::HTTPClientSession & session, const ConnectionTimeouts & timeouts);
|
||||
ConnectionTimeouts getTimeouts(const Poco::Net::HTTPClientSession & session);
|
||||
|
||||
|
@ -345,7 +345,7 @@ void ReadWriteBufferFromHTTP::doWithRetries(std::function<void()> && callable,
|
||||
if (last_attempt || !is_retriable)
|
||||
{
|
||||
if (!mute_logging)
|
||||
LOG_ERROR(log,
|
||||
LOG_DEBUG(log,
|
||||
"Failed to make request to '{}'{}. "
|
||||
"Error: '{}'. "
|
||||
"Failed at try {}/{}.",
|
||||
@ -361,7 +361,7 @@ void ReadWriteBufferFromHTTP::doWithRetries(std::function<void()> && callable,
|
||||
on_retry();
|
||||
|
||||
if (!mute_logging)
|
||||
LOG_INFO(log,
|
||||
LOG_TRACE(log,
|
||||
"Failed to make request to '{}'{}. "
|
||||
"Error: {}. "
|
||||
"Failed at try {}/{}. "
|
||||
|
@ -96,9 +96,9 @@ bool isS3ExpressEndpoint(const std::string & endpoint);
|
||||
|
||||
struct ClientSettings
|
||||
{
|
||||
bool use_virtual_addressing;
|
||||
bool use_virtual_addressing = false;
|
||||
/// Disable checksum to avoid extra read of the input stream
|
||||
bool disable_checksum;
|
||||
bool disable_checksum = false;
|
||||
/// Should client send ComposeObject request after upload to GCS.
|
||||
///
|
||||
/// Previously ComposeObject request was required to make Copy possible,
|
||||
@ -108,8 +108,8 @@ struct ClientSettings
|
||||
///
|
||||
/// Ability to enable it preserved since likely it is required for old
|
||||
/// files.
|
||||
bool gcs_issue_compose_request;
|
||||
bool is_s3express_bucket;
|
||||
bool gcs_issue_compose_request = false;
|
||||
bool is_s3express_bucket = false;
|
||||
};
|
||||
|
||||
/// Client that improves the client from the AWS SDK
|
||||
|
@ -22,6 +22,8 @@ inline static constexpr uint64_t DEFAULT_EXPIRATION_WINDOW_SECONDS = 120;
|
||||
inline static constexpr uint64_t DEFAULT_CONNECT_TIMEOUT_MS = 1000;
|
||||
inline static constexpr uint64_t DEFAULT_REQUEST_TIMEOUT_MS = 30000;
|
||||
inline static constexpr uint64_t DEFAULT_MAX_CONNECTIONS = 100;
|
||||
inline static constexpr uint64_t DEFAULT_KEEP_ALIVE_TIMEOUT = 5;
|
||||
inline static constexpr uint64_t DEFAULT_KEEP_ALIVE_MAX_REQUESTS = 100;
|
||||
|
||||
/// In GCP metadata service can be accessed via DNS regardless of IPv4 or IPv6.
|
||||
static inline constexpr char GCP_METADATA_SERVICE_ENDPOINT[] = "http://metadata.google.internal";
|
||||
|
@ -146,7 +146,9 @@ ConnectionTimeouts getTimeoutsFromConfiguration(const PocoHTTPClientConfiguratio
|
||||
.withSendTimeout(Poco::Timespan(client_configuration.requestTimeoutMs * 1000))
|
||||
.withReceiveTimeout(Poco::Timespan(client_configuration.requestTimeoutMs * 1000))
|
||||
.withTCPKeepAliveTimeout(Poco::Timespan(
|
||||
client_configuration.enableTcpKeepAlive ? client_configuration.tcpKeepAliveIntervalMs * 1000 : 0));
|
||||
client_configuration.enableTcpKeepAlive ? client_configuration.tcpKeepAliveIntervalMs * 1000 : 0))
|
||||
.withHTTPKeepAliveTimeout(Poco::Timespan(client_configuration.http_keep_alive_timeout, 0))
|
||||
.withHTTPKeepAliveMaxRequests(client_configuration.http_keep_alive_max_requests);
|
||||
}
|
||||
|
||||
PocoHTTPClient::PocoHTTPClient(const PocoHTTPClientConfiguration & client_configuration)
|
||||
|
@ -51,6 +51,8 @@ struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration
|
||||
|
||||
/// See PoolBase::BehaviourOnLimit
|
||||
bool s3_use_adaptive_timeouts = true;
|
||||
size_t http_keep_alive_timeout = DEFAULT_HTTP_KEEP_ALIVE_TIMEOUT;
|
||||
size_t http_keep_alive_max_requests = DEFAULT_HTTP_KEEP_ALIVE_MAX_REQUEST;
|
||||
|
||||
std::function<void(const DB::ProxyConfiguration &)> error_report;
|
||||
|
||||
|
@ -159,7 +159,7 @@ void testServerSideEncryption(
|
||||
DB::S3::CredentialsConfiguration
|
||||
{
|
||||
.use_environment_credentials = use_environment_credentials,
|
||||
.use_insecure_imds_request = use_insecure_imds_request
|
||||
.use_insecure_imds_request = use_insecure_imds_request,
|
||||
}
|
||||
);
|
||||
|
||||
|
@ -1187,7 +1187,7 @@ private:
|
||||
else
|
||||
{
|
||||
auto result = std::chrono::system_clock::now() + std::chrono::seconds(calculateDurationWithBackoff(rnd_engine, error_count));
|
||||
LOG_TRACE(log, "Supposed update time for unspecified object is {} (backoff, {} errors.", to_string(result), error_count);
|
||||
LOG_TRACE(log, "Supposed update time for unspecified object is {} (backoff, {} errors)", to_string(result), error_count);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
@ -340,13 +340,10 @@ bool InterpreterInsertQuery::shouldAddSquashingFroStorage(const StoragePtr & tab
|
||||
{
|
||||
auto context_ptr = getContext();
|
||||
const Settings & settings = context_ptr->getSettingsRef();
|
||||
const ASTInsertQuery * query = nullptr;
|
||||
if (query_ptr)
|
||||
query = query_ptr->as<ASTInsertQuery>();
|
||||
|
||||
/// Do not squash blocks if it is a sync INSERT into Distributed, since it lead to double bufferization on client and server side.
|
||||
/// Client-side bufferization might cause excessive timeouts (especially in case of big blocks).
|
||||
return !(settings.distributed_foreground_insert && table->isRemote()) && !async_insert && !no_squash && !(query && query->watch);
|
||||
return !(settings.distributed_foreground_insert && table->isRemote()) && !async_insert && !no_squash;
|
||||
}
|
||||
|
||||
Chain InterpreterInsertQuery::buildPreSinkChain(
|
||||
@ -429,7 +426,7 @@ BlockIO InterpreterInsertQuery::execute()
|
||||
|
||||
std::vector<Chain> presink_chains;
|
||||
std::vector<Chain> sink_chains;
|
||||
if (!distributed_pipeline || query.watch)
|
||||
if (!distributed_pipeline)
|
||||
{
|
||||
/// Number of streams works like this:
|
||||
/// * For the SELECT, use `max_threads`, or `max_insert_threads`, or whatever
|
||||
@ -560,11 +557,6 @@ BlockIO InterpreterInsertQuery::execute()
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (query.watch)
|
||||
{
|
||||
InterpreterWatchQuery interpreter_watch{ query.watch, getContext() };
|
||||
pipeline = interpreter_watch.buildQueryPipeline();
|
||||
}
|
||||
|
||||
ThreadGroupPtr running_group;
|
||||
if (current_thread)
|
||||
@ -591,7 +583,7 @@ BlockIO InterpreterInsertQuery::execute()
|
||||
{
|
||||
res.pipeline = std::move(*distributed_pipeline);
|
||||
}
|
||||
else if (query.select || query.watch)
|
||||
else if (query.select)
|
||||
{
|
||||
const auto & header = presink_chains.at(0).getInputHeader();
|
||||
auto actions_dag = ActionsDAG::makeConvertingActions(
|
||||
|
@ -644,15 +644,6 @@ void logExceptionBeforeStart(
|
||||
}
|
||||
}
|
||||
|
||||
static void setQuerySpecificSettings(ASTPtr & ast, ContextMutablePtr context)
|
||||
{
|
||||
if (auto * ast_insert_into = ast->as<ASTInsertQuery>())
|
||||
{
|
||||
if (ast_insert_into->watch)
|
||||
context->setSetting("output_format_enable_streaming", 1);
|
||||
}
|
||||
}
|
||||
|
||||
void validateAnalyzerSettings(ASTPtr ast, bool context_value)
|
||||
{
|
||||
if (ast->as<ASTSetQuery>())
|
||||
@ -898,8 +889,6 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
|
||||
if (auto * insert_query = ast->as<ASTInsertQuery>())
|
||||
insert_query->tail = istr;
|
||||
|
||||
setQuerySpecificSettings(ast, context);
|
||||
|
||||
/// There is an option of probabilistic logging of queries.
|
||||
/// If it is used - do the random sampling and "collapse" the settings.
|
||||
/// It allows to consistently log queries with all the subqueries in distributed query processing
|
||||
|
@ -123,13 +123,8 @@ void ASTInsertQuery::formatImpl(const FormatSettings & settings, FormatState & s
|
||||
settings.ostr << delim;
|
||||
select->formatImpl(settings, state, frame);
|
||||
}
|
||||
else if (watch)
|
||||
{
|
||||
settings.ostr << delim;
|
||||
watch->formatImpl(settings, state, frame);
|
||||
}
|
||||
|
||||
if (!select && !watch)
|
||||
if (!select)
|
||||
{
|
||||
if (!format.empty())
|
||||
{
|
||||
|
@ -24,7 +24,6 @@ public:
|
||||
ASTPtr settings_ast;
|
||||
|
||||
ASTPtr select;
|
||||
ASTPtr watch;
|
||||
ASTPtr infile;
|
||||
ASTPtr compression;
|
||||
|
||||
@ -63,7 +62,6 @@ public:
|
||||
if (partition_by) { res->partition_by = partition_by->clone(); res->children.push_back(res->partition_by); }
|
||||
if (settings_ast) { res->settings_ast = settings_ast->clone(); res->children.push_back(res->settings_ast); }
|
||||
if (select) { res->select = select->clone(); res->children.push_back(res->select); }
|
||||
if (watch) { res->watch = watch->clone(); res->children.push_back(res->watch); }
|
||||
if (infile) { res->infile = infile->clone(); res->children.push_back(res->infile); }
|
||||
if (compression) { res->compression = compression->clone(); res->children.push_back(res->compression); }
|
||||
|
||||
|
@ -36,7 +36,6 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
ParserKeyword s_format(Keyword::FORMAT);
|
||||
ParserKeyword s_settings(Keyword::SETTINGS);
|
||||
ParserKeyword s_select(Keyword::SELECT);
|
||||
ParserKeyword s_watch(Keyword::WATCH);
|
||||
ParserKeyword s_partition_by(Keyword::PARTITION_BY);
|
||||
ParserKeyword s_with(Keyword::WITH);
|
||||
ParserToken s_lparen(TokenType::OpeningRoundBracket);
|
||||
@ -56,7 +55,6 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
ASTPtr columns;
|
||||
ASTPtr format;
|
||||
ASTPtr select;
|
||||
ASTPtr watch;
|
||||
ASTPtr table_function;
|
||||
ASTPtr settings_ast;
|
||||
ASTPtr partition_by_expr;
|
||||
@ -143,7 +141,7 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
String format_str;
|
||||
Pos before_values = pos;
|
||||
|
||||
/// VALUES or FORMAT or SELECT or WITH or WATCH.
|
||||
/// VALUES or FORMAT or SELECT or WITH.
|
||||
/// After FROM INFILE we expect FORMAT, SELECT, WITH or nothing.
|
||||
if (!infile && s_values.ignore(pos, expected))
|
||||
{
|
||||
@ -175,14 +173,6 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
|
||||
tryGetIdentifierNameInto(format, format_str);
|
||||
}
|
||||
else if (!infile && s_watch.ignore(pos, expected))
|
||||
{
|
||||
/// If WATCH is defined, return to position before WATCH and parse
|
||||
/// rest of query as WATCH query.
|
||||
pos = before_values;
|
||||
ParserWatchQuery watch_p;
|
||||
watch_p.parse(pos, watch, expected);
|
||||
}
|
||||
else if (!infile)
|
||||
{
|
||||
/// If all previous conditions were false and it's not FROM INFILE, query is incorrect
|
||||
@ -286,7 +276,6 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
query->columns = columns;
|
||||
query->format = std::move(format_str);
|
||||
query->select = select;
|
||||
query->watch = watch;
|
||||
query->settings_ast = settings_ast;
|
||||
query->data = data != end ? data : nullptr;
|
||||
query->end = end;
|
||||
@ -295,8 +284,6 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
query->children.push_back(columns);
|
||||
if (select)
|
||||
query->children.push_back(select);
|
||||
if (watch)
|
||||
query->children.push_back(watch);
|
||||
if (settings_ast)
|
||||
query->children.push_back(settings_ast);
|
||||
|
||||
|
@ -235,7 +235,9 @@ public:
|
||||
static bool needChildVisit(const QueryTreeNodePtr &, const QueryTreeNodePtr & child_node)
|
||||
{
|
||||
auto child_node_type = child_node->getNodeType();
|
||||
return !(child_node_type == QueryTreeNodeType::QUERY || child_node_type == QueryTreeNodeType::UNION);
|
||||
return child_node_type != QueryTreeNodeType::QUERY &&
|
||||
child_node_type != QueryTreeNodeType::UNION &&
|
||||
child_node_type != QueryTreeNodeType::LAMBDA;
|
||||
}
|
||||
|
||||
private:
|
||||
|
@ -814,7 +814,8 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
|
||||
bool optimize_move_to_prewhere
|
||||
= settings.optimize_move_to_prewhere && (!is_final || settings.optimize_move_to_prewhere_if_final);
|
||||
|
||||
if (storage->supportsPrewhere() && optimize_move_to_prewhere)
|
||||
auto supported_prewhere_columns = storage->supportedPrewhereColumns();
|
||||
if (storage->canMoveConditionsToPrewhere() && optimize_move_to_prewhere && (!supported_prewhere_columns || supported_prewhere_columns->contains(filter_info.column_name)))
|
||||
{
|
||||
if (!prewhere_info)
|
||||
prewhere_info = std::make_shared<PrewhereInfo>();
|
||||
|
@ -1,4 +1,5 @@
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
#include <memory>
|
||||
#include <numeric>
|
||||
#include <queue>
|
||||
@ -125,14 +126,20 @@ int compareValues(const Values & lhs, const Values & rhs)
|
||||
class IndexAccess
|
||||
{
|
||||
public:
|
||||
explicit IndexAccess(const RangesInDataParts & parts_) : parts(parts_) { }
|
||||
explicit IndexAccess(const RangesInDataParts & parts_) : parts(parts_)
|
||||
{
|
||||
/// Some suffix of index columns might not be loaded (see `primary_key_ratio_of_unique_prefix_values_to_skip_suffix_columns`)
|
||||
/// and we need to use the same set of index columns across all parts.
|
||||
for (const auto & part : parts)
|
||||
loaded_columns = std::min(loaded_columns, part.data_part->getIndex().size());
|
||||
}
|
||||
|
||||
Values getValue(size_t part_idx, size_t mark) const
|
||||
{
|
||||
const auto & index = parts[part_idx].data_part->getIndex();
|
||||
size_t size = index.size();
|
||||
Values values(size);
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
chassert(index.size() >= loaded_columns);
|
||||
Values values(loaded_columns);
|
||||
for (size_t i = 0; i < loaded_columns; ++i)
|
||||
{
|
||||
index[i]->get(mark, values[i]);
|
||||
if (values[i].isNull())
|
||||
@ -199,6 +206,7 @@ public:
|
||||
}
|
||||
private:
|
||||
const RangesInDataParts & parts;
|
||||
size_t loaded_columns = std::numeric_limits<size_t>::max();
|
||||
};
|
||||
|
||||
class RangesInDataPartsBuilder
|
||||
|
@ -1,3 +1,4 @@
|
||||
#include <string_view>
|
||||
#include <Storages/MergeTree/DataPartStorageOnDiskBase.h>
|
||||
#include <Storages/MergeTree/MergeTreeDataPartChecksum.h>
|
||||
#include <Disks/TemporaryFileOnDisk.h>
|
||||
@ -13,6 +14,7 @@
|
||||
#include <Backups/BackupEntryWrappedWith.h>
|
||||
#include <Backups/BackupSettings.h>
|
||||
#include <Disks/SingleDiskVolume.h>
|
||||
#include <Storages/MergeTree/MergeTreeData.h>
|
||||
#include <Storages/MergeTree/IMergeTreeDataPart.h>
|
||||
|
||||
namespace DB
|
||||
@ -64,7 +66,7 @@ std::optional<String> DataPartStorageOnDiskBase::getRelativePathForPrefix(Logger
|
||||
|
||||
auto full_relative_path = fs::path(root_path);
|
||||
if (detached)
|
||||
full_relative_path /= "detached";
|
||||
full_relative_path /= MergeTreeData::DETACHED_DIR_NAME;
|
||||
|
||||
std::optional<String> original_checksums_content;
|
||||
std::optional<Strings> original_files_list;
|
||||
@ -109,7 +111,7 @@ bool DataPartStorageOnDiskBase::looksLikeBrokenDetachedPartHasTheSameContent(con
|
||||
if (!exists("checksums.txt"))
|
||||
return false;
|
||||
|
||||
auto storage_from_detached = create(volume, fs::path(root_path) / "detached", detached_part_path, /*initialize=*/ true);
|
||||
auto storage_from_detached = create(volume, fs::path(root_path) / MergeTreeData::DETACHED_DIR_NAME, detached_part_path, /*initialize=*/ true);
|
||||
if (!storage_from_detached->exists("checksums.txt"))
|
||||
return false;
|
||||
|
||||
@ -490,7 +492,7 @@ MutableDataPartStoragePtr DataPartStorageOnDiskBase::freeze(
|
||||
auto single_disk_volume = std::make_shared<SingleDiskVolume>(disk->getName(), disk, 0);
|
||||
|
||||
/// Do not initialize storage in case of DETACH because part may be broken.
|
||||
bool to_detached = dir_path.starts_with("detached/");
|
||||
bool to_detached = dir_path.starts_with(std::string_view((fs::path(MergeTreeData::DETACHED_DIR_NAME) / "").string()));
|
||||
return create(single_disk_volume, to, dir_path, /*initialize=*/ !to_detached && !params.external_transaction);
|
||||
}
|
||||
|
||||
@ -618,7 +620,7 @@ void DataPartStorageOnDiskBase::remove(
|
||||
if (part_dir_without_slash.has_parent_path())
|
||||
{
|
||||
auto parent_path = part_dir_without_slash.parent_path();
|
||||
if (parent_path == "detached")
|
||||
if (parent_path == MergeTreeData::DETACHED_DIR_NAME)
|
||||
throw Exception(
|
||||
ErrorCodes::LOGICAL_ERROR,
|
||||
"Trying to remove detached part {} with path {} in remove function. It shouldn't happen",
|
||||
|
@ -14,6 +14,7 @@
|
||||
#include <Storages/MergeTree/ReplicatedFetchList.h>
|
||||
#include <Storages/StorageReplicatedMergeTree.h>
|
||||
#include <Storages/MergeTree/DataPartStorageOnDiskFull.h>
|
||||
#include <Storages/MergeTree/MergeTreeData.h>
|
||||
#include <Common/CurrentMetrics.h>
|
||||
#include <Common/NetException.h>
|
||||
#include <Common/randomDelay.h>
|
||||
@ -21,7 +22,6 @@
|
||||
#include <base/scope_guard.h>
|
||||
#include <Poco/Net/HTTPRequest.h>
|
||||
#include <boost/algorithm/string/join.hpp>
|
||||
#include <iterator>
|
||||
#include <base/sort.h>
|
||||
|
||||
|
||||
@ -803,7 +803,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk(
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "`tmp_prefix` and `part_name` cannot be empty or contain '.' or '/' characters.");
|
||||
|
||||
auto part_dir = tmp_prefix + part_name;
|
||||
auto part_relative_path = data.getRelativeDataPath() + String(to_detached ? "detached/" : "");
|
||||
auto part_relative_path = data.getRelativeDataPath() + String(to_detached ? MergeTreeData::DETACHED_DIR_NAME : "");
|
||||
auto volume = std::make_shared<SingleDiskVolume>("volume_" + part_name, disk);
|
||||
|
||||
/// Create temporary part storage to write sent files.
|
||||
|
@ -1844,7 +1844,7 @@ try
|
||||
}
|
||||
catch (...)
|
||||
{
|
||||
if (startsWith(new_relative_path, "detached/"))
|
||||
if (startsWith(new_relative_path, fs::path(MergeTreeData::DETACHED_DIR_NAME) / ""))
|
||||
{
|
||||
// Don't throw when the destination is to the detached folder. It might be able to
|
||||
// recover in some cases, such as fetching parts into multi-disks while some of the
|
||||
@ -1957,7 +1957,7 @@ std::optional<String> IMergeTreeDataPart::getRelativePathForDetachedPart(const S
|
||||
DetachedPartInfo::DETACH_REASONS.end(),
|
||||
prefix) != DetachedPartInfo::DETACH_REASONS.end());
|
||||
if (auto path = getRelativePathForPrefix(prefix, /* detached */ true, broken))
|
||||
return "detached/" + *path;
|
||||
return fs::path(MergeTreeData::DETACHED_DIR_NAME) / *path;
|
||||
return {};
|
||||
}
|
||||
|
||||
|
@ -262,7 +262,7 @@ void MergeTreeData::initializeDirectoriesAndFormatVersion(const std::string & re
|
||||
if (need_create_directories)
|
||||
{
|
||||
disk->createDirectories(relative_data_path);
|
||||
disk->createDirectories(fs::path(relative_data_path) / MergeTreeData::DETACHED_DIR_NAME);
|
||||
disk->createDirectories(fs::path(relative_data_path) / DETACHED_DIR_NAME);
|
||||
}
|
||||
|
||||
if (disk->exists(format_version_path))
|
||||
@ -1713,7 +1713,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks, std::optional<std::un
|
||||
{
|
||||
/// Skip temporary directories, file 'format_version.txt' and directory 'detached'.
|
||||
if (startsWith(it->name(), "tmp") || it->name() == MergeTreeData::FORMAT_VERSION_FILE_NAME
|
||||
|| it->name() == MergeTreeData::DETACHED_DIR_NAME)
|
||||
|| it->name() == DETACHED_DIR_NAME)
|
||||
continue;
|
||||
|
||||
if (auto part_info = MergeTreePartInfo::tryParsePartName(it->name(), format_version))
|
||||
@ -1986,6 +1986,15 @@ void MergeTreeData::waitForOutdatedPartsToBeLoaded() const TSA_NO_THREAD_SAFETY_
|
||||
if (isStaticStorage())
|
||||
return;
|
||||
|
||||
/// If waiting is not required, do NOT log and do NOT enable/disable turbo mode to make `waitForOutdatedPartsToBeLoaded` a lightweight check
|
||||
{
|
||||
std::unique_lock lock(outdated_data_parts_mutex);
|
||||
if (outdated_data_parts_loading_canceled)
|
||||
throw Exception(ErrorCodes::NOT_INITIALIZED, "Loading of outdated data parts was already canceled");
|
||||
if (outdated_data_parts_loading_finished)
|
||||
return;
|
||||
}
|
||||
|
||||
/// We need to load parts as fast as possible
|
||||
getOutdatedPartsLoadingThreadPool().enableTurboMode();
|
||||
SCOPE_EXIT({
|
||||
@ -2796,7 +2805,7 @@ void MergeTreeData::dropAllData()
|
||||
&& settings_ptr->allow_remote_fs_zero_copy_replication;
|
||||
try
|
||||
{
|
||||
bool keep_shared = removeDetachedPart(part.disk, fs::path(relative_data_path) / "detached" / part.dir_name / "", part.dir_name);
|
||||
bool keep_shared = removeDetachedPart(part.disk, fs::path(relative_data_path) / DETACHED_DIR_NAME / part.dir_name / "", part.dir_name);
|
||||
LOG_DEBUG(log, "Dropped detached part {}, keep shared data: {}", part.dir_name, keep_shared);
|
||||
}
|
||||
catch (...)
|
||||
@ -2879,8 +2888,8 @@ void MergeTreeData::dropIfEmpty()
|
||||
if (disk->isBroken())
|
||||
continue;
|
||||
/// Non recursive, exception is thrown if there are more files.
|
||||
disk->removeFileIfExists(fs::path(relative_data_path) / MergeTreeData::FORMAT_VERSION_FILE_NAME);
|
||||
disk->removeDirectory(fs::path(relative_data_path) / MergeTreeData::DETACHED_DIR_NAME);
|
||||
disk->removeFileIfExists(fs::path(relative_data_path) / FORMAT_VERSION_FILE_NAME);
|
||||
disk->removeDirectory(fs::path(relative_data_path) / DETACHED_DIR_NAME);
|
||||
disk->removeDirectory(relative_data_path);
|
||||
}
|
||||
}
|
||||
@ -3443,7 +3452,7 @@ void MergeTreeData::changeSettings(
|
||||
{
|
||||
auto disk = new_storage_policy->getDiskByName(disk_name);
|
||||
disk->createDirectories(relative_data_path);
|
||||
disk->createDirectories(fs::path(relative_data_path) / MergeTreeData::DETACHED_DIR_NAME);
|
||||
disk->createDirectories(fs::path(relative_data_path) / DETACHED_DIR_NAME);
|
||||
}
|
||||
/// FIXME how would that be done while reloading configuration???
|
||||
|
||||
@ -6037,7 +6046,7 @@ DetachedPartsInfo MergeTreeData::getDetachedParts() const
|
||||
|
||||
for (const auto & disk : getDisks())
|
||||
{
|
||||
String detached_path = fs::path(relative_data_path) / MergeTreeData::DETACHED_DIR_NAME;
|
||||
String detached_path = fs::path(relative_data_path) / DETACHED_DIR_NAME;
|
||||
|
||||
/// Note: we don't care about TOCTOU issue here.
|
||||
if (disk->exists(detached_path))
|
||||
@ -6063,7 +6072,7 @@ void MergeTreeData::validateDetachedPartName(const String & name)
|
||||
|
||||
void MergeTreeData::dropDetached(const ASTPtr & partition, bool part, ContextPtr local_context)
|
||||
{
|
||||
PartsTemporaryRename renamed_parts(*this, "detached/");
|
||||
PartsTemporaryRename renamed_parts(*this, DETACHED_DIR_NAME);
|
||||
|
||||
if (part)
|
||||
{
|
||||
@ -6088,7 +6097,7 @@ void MergeTreeData::dropDetached(const ASTPtr & partition, bool part, ContextPtr
|
||||
|
||||
for (auto & [old_name, new_name, disk] : renamed_parts.old_and_new_names)
|
||||
{
|
||||
bool keep_shared = removeDetachedPart(disk, fs::path(relative_data_path) / "detached" / new_name / "", old_name);
|
||||
bool keep_shared = removeDetachedPart(disk, fs::path(relative_data_path) / DETACHED_DIR_NAME / new_name / "", old_name);
|
||||
LOG_DEBUG(log, "Dropped detached part {}, keep shared data: {}", old_name, keep_shared);
|
||||
old_name.clear();
|
||||
}
|
||||
@ -6097,14 +6106,14 @@ void MergeTreeData::dropDetached(const ASTPtr & partition, bool part, ContextPtr
|
||||
MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const ASTPtr & partition, bool attach_part,
|
||||
ContextPtr local_context, PartsTemporaryRename & renamed_parts)
|
||||
{
|
||||
const String source_dir = "detached/";
|
||||
const fs::path source_dir = DETACHED_DIR_NAME;
|
||||
|
||||
/// Let's compose a list of parts that should be added.
|
||||
if (attach_part)
|
||||
{
|
||||
const String part_id = partition->as<ASTLiteral &>().value.safeGet<String>();
|
||||
validateDetachedPartName(part_id);
|
||||
if (temporary_parts.contains(String(DETACHED_DIR_NAME) + "/" + part_id))
|
||||
if (temporary_parts.contains(source_dir / part_id))
|
||||
{
|
||||
LOG_WARNING(log, "Will not try to attach part {} because its directory is temporary, "
|
||||
"probably it's being detached right now", part_id);
|
||||
@ -6181,7 +6190,7 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const
|
||||
LOG_DEBUG(log, "Checking part {}", new_name);
|
||||
|
||||
auto single_disk_volume = std::make_shared<SingleDiskVolume>("volume_" + old_name, disk);
|
||||
auto part = getDataPartBuilder(old_name, single_disk_volume, source_dir + new_name)
|
||||
auto part = getDataPartBuilder(old_name, single_disk_volume, source_dir / new_name)
|
||||
.withPartFormatFromDisk()
|
||||
.build();
|
||||
|
||||
@ -7212,11 +7221,10 @@ String MergeTreeData::getFullPathOnDisk(const DiskPtr & disk) const
|
||||
|
||||
DiskPtr MergeTreeData::tryGetDiskForDetachedPart(const String & part_name) const
|
||||
{
|
||||
String additional_path = "detached/";
|
||||
const auto disks = getStoragePolicy()->getDisks();
|
||||
|
||||
for (const DiskPtr & disk : disks)
|
||||
if (disk->exists(fs::path(relative_data_path) / additional_path / part_name))
|
||||
if (disk->exists(fs::path(relative_data_path) / DETACHED_DIR_NAME / part_name))
|
||||
return disk;
|
||||
|
||||
return nullptr;
|
||||
@ -7791,7 +7799,7 @@ MovePartsOutcome MergeTreeData::moveParts(const CurrentlyMovingPartsTaggerPtr &
|
||||
return result;
|
||||
}
|
||||
|
||||
bool MergeTreeData::partsContainSameProjections(const DataPartPtr & left, const DataPartPtr & right, String & out_reason)
|
||||
bool MergeTreeData::partsContainSameProjections(const DataPartPtr & left, const DataPartPtr & right, PreformattedMessage & out_reason)
|
||||
{
|
||||
auto remove_broken_parts_from_consideration = [](auto & parts)
|
||||
{
|
||||
@ -7813,7 +7821,7 @@ bool MergeTreeData::partsContainSameProjections(const DataPartPtr & left, const
|
||||
|
||||
if (left_projection_parts.size() != right_projection_parts.size())
|
||||
{
|
||||
out_reason = fmt::format(
|
||||
out_reason = PreformattedMessage::create(
|
||||
"Parts have different number of projections: {} in part '{}' and {} in part '{}'",
|
||||
left_projection_parts.size(),
|
||||
left->name,
|
||||
@ -7827,7 +7835,7 @@ bool MergeTreeData::partsContainSameProjections(const DataPartPtr & left, const
|
||||
{
|
||||
if (!right_projection_parts.contains(name))
|
||||
{
|
||||
out_reason = fmt::format(
|
||||
out_reason = PreformattedMessage::create(
|
||||
"The part '{}' doesn't have projection '{}' while part '{}' does", right->name, name, left->name
|
||||
);
|
||||
return false;
|
||||
|
@ -418,7 +418,7 @@ public:
|
||||
static ReservationPtr tryReserveSpace(UInt64 expected_size, const IDataPartStorage & data_part_storage);
|
||||
static ReservationPtr reserveSpace(UInt64 expected_size, const IDataPartStorage & data_part_storage);
|
||||
|
||||
static bool partsContainSameProjections(const DataPartPtr & left, const DataPartPtr & right, String & out_reason);
|
||||
static bool partsContainSameProjections(const DataPartPtr & left, const DataPartPtr & right, PreformattedMessage & out_reason);
|
||||
|
||||
StoragePolicyPtr getStoragePolicy() const override;
|
||||
|
||||
|
@ -136,7 +136,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMerge(
|
||||
const AllowedMergingPredicate & can_merge_callback,
|
||||
bool merge_with_ttl_allowed,
|
||||
const MergeTreeTransactionPtr & txn,
|
||||
String & out_disable_reason,
|
||||
PreformattedMessage & out_disable_reason,
|
||||
const PartitionIdsHint * partitions_hint)
|
||||
{
|
||||
MergeTreeData::DataPartsVector data_parts = getDataPartsToSelectMergeFrom(txn, partitions_hint);
|
||||
@ -145,7 +145,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMerge(
|
||||
|
||||
if (data_parts.empty())
|
||||
{
|
||||
out_disable_reason = "There are no parts in the table";
|
||||
out_disable_reason = PreformattedMessage::create("There are no parts in the table");
|
||||
return SelectPartsDecision::CANNOT_SELECT;
|
||||
}
|
||||
|
||||
@ -153,7 +153,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMerge(
|
||||
|
||||
if (info.parts_selected_precondition == 0)
|
||||
{
|
||||
out_disable_reason = "No parts satisfy preconditions for merge";
|
||||
out_disable_reason = PreformattedMessage::create("No parts satisfy preconditions for merge");
|
||||
return SelectPartsDecision::CANNOT_SELECT;
|
||||
}
|
||||
|
||||
@ -177,9 +177,9 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMerge(
|
||||
/*optimize_skip_merged_partitions=*/true);
|
||||
}
|
||||
|
||||
if (!out_disable_reason.empty())
|
||||
out_disable_reason += ". ";
|
||||
out_disable_reason += "There is no need to merge parts according to merge selector algorithm";
|
||||
if (!out_disable_reason.text.empty())
|
||||
out_disable_reason.text += ". ";
|
||||
out_disable_reason.text += "There is no need to merge parts according to merge selector algorithm";
|
||||
return SelectPartsDecision::CANNOT_SELECT;
|
||||
}
|
||||
|
||||
@ -196,7 +196,7 @@ MergeTreeDataMergerMutator::PartitionIdsHint MergeTreeDataMergerMutator::getPart
|
||||
|
||||
auto metadata_snapshot = data.getInMemoryMetadataPtr();
|
||||
|
||||
String out_reason;
|
||||
PreformattedMessage out_reason;
|
||||
MergeSelectingInfo info = getPossibleMergeRanges(data_parts, can_merge_callback, txn, out_reason);
|
||||
|
||||
if (info.parts_selected_precondition == 0)
|
||||
@ -223,7 +223,7 @@ MergeTreeDataMergerMutator::PartitionIdsHint MergeTreeDataMergerMutator::getPart
|
||||
for (size_t i = 0; i < all_partition_ids.size(); ++i)
|
||||
{
|
||||
auto future_part = std::make_shared<FutureMergedMutatedPart>();
|
||||
String out_disable_reason;
|
||||
PreformattedMessage out_disable_reason;
|
||||
/// This method should have been const, but something went wrong... it's const with dry_run = true
|
||||
auto status = const_cast<MergeTreeDataMergerMutator *>(this)->selectPartsToMergeFromRanges(
|
||||
future_part, /*aggressive*/ false, max_total_size_to_merge, merge_with_ttl_allowed,
|
||||
@ -232,7 +232,7 @@ MergeTreeDataMergerMutator::PartitionIdsHint MergeTreeDataMergerMutator::getPart
|
||||
if (status == SelectPartsDecision::SELECTED)
|
||||
res.insert(all_partition_ids[i]);
|
||||
else
|
||||
LOG_TEST(log, "Nothing to merge in partition {}: {}", all_partition_ids[i], out_disable_reason);
|
||||
LOG_TEST(log, "Nothing to merge in partition {}: {}", all_partition_ids[i], out_disable_reason.text);
|
||||
}
|
||||
|
||||
String best_partition_id_to_optimize = getBestPartitionToOptimizeEntire(info.partitions_info);
|
||||
@ -331,7 +331,7 @@ MergeTreeDataMergerMutator::MergeSelectingInfo MergeTreeDataMergerMutator::getPo
|
||||
const MergeTreeData::DataPartsVector & data_parts,
|
||||
const AllowedMergingPredicate & can_merge_callback,
|
||||
const MergeTreeTransactionPtr & txn,
|
||||
String & out_disable_reason) const
|
||||
PreformattedMessage & out_disable_reason) const
|
||||
{
|
||||
MergeSelectingInfo res;
|
||||
|
||||
@ -444,7 +444,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMergeFromRanges(
|
||||
const StorageMetadataPtr & metadata_snapshot,
|
||||
const IMergeSelector::PartsRanges & parts_ranges,
|
||||
const time_t & current_time,
|
||||
String & out_disable_reason,
|
||||
PreformattedMessage & out_disable_reason,
|
||||
bool dry_run)
|
||||
{
|
||||
const auto data_settings = data.getSettings();
|
||||
@ -515,7 +515,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMergeFromRanges(
|
||||
|
||||
if (parts_to_merge.empty())
|
||||
{
|
||||
out_disable_reason = "Did not find any parts to merge (with usual merge selectors)";
|
||||
out_disable_reason = PreformattedMessage::create("Did not find any parts to merge (with usual merge selectors)");
|
||||
return SelectPartsDecision::CANNOT_SELECT;
|
||||
}
|
||||
}
|
||||
@ -573,20 +573,20 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectAllPartsToMergeWithinParti
|
||||
bool final,
|
||||
const StorageMetadataPtr & metadata_snapshot,
|
||||
const MergeTreeTransactionPtr & txn,
|
||||
String & out_disable_reason,
|
||||
PreformattedMessage & out_disable_reason,
|
||||
bool optimize_skip_merged_partitions)
|
||||
{
|
||||
MergeTreeData::DataPartsVector parts = selectAllPartsFromPartition(partition_id);
|
||||
|
||||
if (parts.empty())
|
||||
{
|
||||
out_disable_reason = "There are no parts inside partition";
|
||||
out_disable_reason = PreformattedMessage::create("There are no parts inside partition");
|
||||
return SelectPartsDecision::CANNOT_SELECT;
|
||||
}
|
||||
|
||||
if (!final && parts.size() == 1)
|
||||
{
|
||||
out_disable_reason = "There is only one part inside partition";
|
||||
out_disable_reason = PreformattedMessage::create("There is only one part inside partition");
|
||||
return SelectPartsDecision::CANNOT_SELECT;
|
||||
}
|
||||
|
||||
@ -595,7 +595,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectAllPartsToMergeWithinParti
|
||||
if (final && optimize_skip_merged_partitions && parts.size() == 1 && parts[0]->info.level > 0 &&
|
||||
(!metadata_snapshot->hasAnyTTL() || parts[0]->checkAllTTLCalculated(metadata_snapshot)))
|
||||
{
|
||||
out_disable_reason = "Partition skipped due to optimize_skip_merged_partitions";
|
||||
out_disable_reason = PreformattedMessage::create("Partition skipped due to optimize_skip_merged_partitions");
|
||||
return SelectPartsDecision::NOTHING_TO_MERGE;
|
||||
}
|
||||
|
||||
@ -636,7 +636,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectAllPartsToMergeWithinParti
|
||||
static_cast<int>((DISK_USAGE_COEFFICIENT_TO_SELECT - 1.0) * 100));
|
||||
}
|
||||
|
||||
out_disable_reason = fmt::format("Insufficient available disk space, required {}", ReadableSize(required_disk_space));
|
||||
out_disable_reason = PreformattedMessage::create("Insufficient available disk space, required {}", ReadableSize(required_disk_space));
|
||||
return SelectPartsDecision::CANNOT_SELECT;
|
||||
}
|
||||
|
||||
|
@ -43,7 +43,7 @@ public:
|
||||
using AllowedMergingPredicate = std::function<bool (const MergeTreeData::DataPartPtr &,
|
||||
const MergeTreeData::DataPartPtr &,
|
||||
const MergeTreeTransaction *,
|
||||
String &)>;
|
||||
PreformattedMessage &)>;
|
||||
|
||||
explicit MergeTreeDataMergerMutator(MergeTreeData & data_);
|
||||
|
||||
@ -92,7 +92,7 @@ public:
|
||||
const MergeTreeData::DataPartsVector & data_parts,
|
||||
const AllowedMergingPredicate & can_merge_callback,
|
||||
const MergeTreeTransactionPtr & txn,
|
||||
String & out_disable_reason) const;
|
||||
PreformattedMessage & out_disable_reason) const;
|
||||
|
||||
/// The third step of selecting parts to merge: takes ranges that we can merge, and selects parts that we want to merge
|
||||
SelectPartsDecision selectPartsToMergeFromRanges(
|
||||
@ -103,7 +103,7 @@ public:
|
||||
const StorageMetadataPtr & metadata_snapshot,
|
||||
const IMergeSelector::PartsRanges & parts_ranges,
|
||||
const time_t & current_time,
|
||||
String & out_disable_reason,
|
||||
PreformattedMessage & out_disable_reason,
|
||||
bool dry_run = false);
|
||||
|
||||
String getBestPartitionToOptimizeEntire(const PartitionsInfo & partitions_info) const;
|
||||
@ -129,7 +129,7 @@ public:
|
||||
const AllowedMergingPredicate & can_merge,
|
||||
bool merge_with_ttl_allowed,
|
||||
const MergeTreeTransactionPtr & txn,
|
||||
String & out_disable_reason,
|
||||
PreformattedMessage & out_disable_reason,
|
||||
const PartitionIdsHint * partitions_hint = nullptr);
|
||||
|
||||
/** Select all the parts in the specified partition for merge, if possible.
|
||||
@ -144,7 +144,7 @@ public:
|
||||
bool final,
|
||||
const StorageMetadataPtr & metadata_snapshot,
|
||||
const MergeTreeTransactionPtr & txn,
|
||||
String & out_disable_reason,
|
||||
PreformattedMessage & out_disable_reason,
|
||||
bool optimize_skip_merged_partitions = false);
|
||||
|
||||
/** Creates a task to merge parts.
|
||||
|
@ -2287,7 +2287,7 @@ bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::operator()(
|
||||
const MergeTreeData::DataPartPtr & left,
|
||||
const MergeTreeData::DataPartPtr & right,
|
||||
const MergeTreeTransaction *,
|
||||
String & out_reason) const
|
||||
PreformattedMessage & out_reason) const
|
||||
{
|
||||
if (left)
|
||||
return canMergeTwoParts(left, right, out_reason);
|
||||
@ -2299,7 +2299,7 @@ template<typename VirtualPartsT, typename MutationsStateT>
|
||||
bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::canMergeTwoParts(
|
||||
const MergeTreeData::DataPartPtr & left,
|
||||
const MergeTreeData::DataPartPtr & right,
|
||||
String & out_reason) const
|
||||
PreformattedMessage & out_reason) const
|
||||
{
|
||||
/// A sketch of a proof of why this method actually works:
|
||||
///
|
||||
@ -2343,19 +2343,19 @@ bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::canMergeTwoParts(
|
||||
{
|
||||
if (pinned_part_uuids_ && pinned_part_uuids_->part_uuids.contains(part->uuid))
|
||||
{
|
||||
out_reason = "Part " + part->name + " has uuid " + toString(part->uuid) + " which is currently pinned";
|
||||
out_reason = PreformattedMessage::create("Part {} has uuid {} which is currently pinned", part->name, part->uuid);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (inprogress_quorum_part_ && part->name == *inprogress_quorum_part_)
|
||||
{
|
||||
out_reason = "Quorum insert for part " + part->name + " is currently in progress";
|
||||
out_reason = PreformattedMessage::create("Quorum insert for part {} is currently in progress", part->name);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (prev_virtual_parts_ && prev_virtual_parts_->getContainingPart(part->info).empty())
|
||||
{
|
||||
out_reason = "Entry for part " + part->name + " hasn't been read from the replication log yet";
|
||||
out_reason = PreformattedMessage::create("Entry for part {} hasn't been read from the replication log yet", part->name);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@ -2369,7 +2369,7 @@ bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::canMergeTwoParts(
|
||||
{
|
||||
if (partition_ids_hint && !partition_ids_hint->contains(left->info.partition_id))
|
||||
{
|
||||
out_reason = fmt::format("Uncommitted block were not loaded for unexpected partition {}", left->info.partition_id);
|
||||
out_reason = PreformattedMessage::create("Uncommitted block were not loaded for unexpected partition {}", left->info.partition_id);
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -2381,8 +2381,7 @@ bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::canMergeTwoParts(
|
||||
auto block_it = block_numbers.upper_bound(left_max_block);
|
||||
if (block_it != block_numbers.end() && *block_it < right_min_block)
|
||||
{
|
||||
out_reason = "Block number " + toString(*block_it) + " is still being inserted between parts "
|
||||
+ left->name + " and " + right->name;
|
||||
out_reason = PreformattedMessage::create("Block number {} is still being inserted between parts {} and {}", *block_it, left->name, right->name);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@ -2401,7 +2400,7 @@ bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::canMergeTwoParts(
|
||||
String containing_part = virtual_parts_->getContainingPart(part->info);
|
||||
if (containing_part != part->name)
|
||||
{
|
||||
out_reason = "Part " + part->name + " has already been assigned a merge into " + containing_part;
|
||||
out_reason = PreformattedMessage::create("Part {} has already been assigned a merge into {}", part->name, containing_part);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@ -2418,9 +2417,9 @@ bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::canMergeTwoParts(
|
||||
Strings covered = virtual_parts_->getPartsCoveredBy(gap_part_info);
|
||||
if (!covered.empty())
|
||||
{
|
||||
out_reason = "There are " + toString(covered.size()) + " parts (from " + covered.front()
|
||||
+ " to " + covered.back() + ") that are still not present or being processed by "
|
||||
+ " other background process on this replica between " + left->name + " and " + right->name;
|
||||
out_reason = PreformattedMessage::create("There are {} parts (from {} to {}) "
|
||||
"that are still not present or being processed by other background process "
|
||||
"on this replica between {} and {}", covered.size(), covered.front(), covered.back(), left->name, right->name);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@ -2436,8 +2435,8 @@ bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::canMergeTwoParts(
|
||||
|
||||
if (left_mutation_ver != right_mutation_ver)
|
||||
{
|
||||
out_reason = "Current mutation versions of parts " + left->name + " and " + right->name + " differ: "
|
||||
+ toString(left_mutation_ver) + " and " + toString(right_mutation_ver) + " respectively";
|
||||
out_reason = PreformattedMessage::create("Current mutation versions of parts {} and {} differ: "
|
||||
"{} and {} respectively", left->name, right->name, left_mutation_ver, right_mutation_ver);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@ -2448,23 +2447,23 @@ bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::canMergeTwoParts(
|
||||
template<typename VirtualPartsT, typename MutationsStateT>
|
||||
bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::canMergeSinglePart(
|
||||
const MergeTreeData::DataPartPtr & part,
|
||||
String & out_reason) const
|
||||
PreformattedMessage & out_reason) const
|
||||
{
|
||||
if (pinned_part_uuids_ && pinned_part_uuids_->part_uuids.contains(part->uuid))
|
||||
{
|
||||
out_reason = fmt::format("Part {} has uuid {} which is currently pinned", part->name, part->uuid);
|
||||
out_reason = PreformattedMessage::create("Part {} has uuid {} which is currently pinned", part->name, part->uuid);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (inprogress_quorum_part_ && part->name == *inprogress_quorum_part_)
|
||||
{
|
||||
out_reason = fmt::format("Quorum insert for part {} is currently in progress", part->name);
|
||||
out_reason = PreformattedMessage::create("Quorum insert for part {} is currently in progress", part->name);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (prev_virtual_parts_ && prev_virtual_parts_->getContainingPart(part->info).empty())
|
||||
{
|
||||
out_reason = fmt::format("Entry for part {} hasn't been read from the replication log yet", part->name);
|
||||
out_reason = PreformattedMessage::create("Entry for part {} hasn't been read from the replication log yet", part->name);
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -2479,7 +2478,7 @@ bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::canMergeSinglePart(
|
||||
String containing_part = virtual_parts_->getContainingPart(part->info);
|
||||
if (containing_part != part->name)
|
||||
{
|
||||
out_reason = fmt::format("Part {} has already been assigned a merge into {}", part->name, containing_part);
|
||||
out_reason = PreformattedMessage::create("Part {} has already been assigned a merge into {}", part->name, containing_part);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@ -2488,7 +2487,7 @@ bool BaseMergePredicate<VirtualPartsT, MutationsStateT>::canMergeSinglePart(
|
||||
}
|
||||
|
||||
|
||||
bool ReplicatedMergeTreeMergePredicate::partParticipatesInReplaceRange(const MergeTreeData::DataPartPtr & part, String & out_reason) const
|
||||
bool ReplicatedMergeTreeMergePredicate::partParticipatesInReplaceRange(const MergeTreeData::DataPartPtr & part, PreformattedMessage & out_reason) const
|
||||
{
|
||||
std::lock_guard lock(queue.state_mutex);
|
||||
for (const auto & entry : queue.queue)
|
||||
@ -2501,7 +2500,7 @@ bool ReplicatedMergeTreeMergePredicate::partParticipatesInReplaceRange(const Mer
|
||||
if (part->info.isDisjoint(MergeTreePartInfo::fromPartName(part_name, queue.format_version)))
|
||||
continue;
|
||||
|
||||
out_reason = fmt::format("Part {} participates in REPLACE_RANGE {} ({})", part_name, entry->new_part_name, entry->znode_name);
|
||||
out_reason = PreformattedMessage::create("Part {} participates in REPLACE_RANGE {} ({})", part_name, entry->new_part_name, entry->znode_name);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
@ -505,19 +505,19 @@ public:
|
||||
bool operator()(const MergeTreeData::DataPartPtr & left,
|
||||
const MergeTreeData::DataPartPtr & right,
|
||||
const MergeTreeTransaction * txn,
|
||||
String & out_reason) const;
|
||||
PreformattedMessage & out_reason) const;
|
||||
|
||||
/// Can we assign a merge with these two parts?
|
||||
/// (assuming that no merge was assigned after the predicate was constructed)
|
||||
/// If we can't and out_reason is not nullptr, set it to the reason why we can't merge.
|
||||
bool canMergeTwoParts(const MergeTreeData::DataPartPtr & left,
|
||||
const MergeTreeData::DataPartPtr & right,
|
||||
String & out_reason) const;
|
||||
PreformattedMessage & out_reason) const;
|
||||
|
||||
/// Can we assign a merge this part and some other part?
|
||||
/// For example a merge of a part and itself is needed for TTL.
|
||||
/// This predicate is checked for the first part of each range.
|
||||
bool canMergeSinglePart(const MergeTreeData::DataPartPtr & part, String & out_reason) const;
|
||||
bool canMergeSinglePart(const MergeTreeData::DataPartPtr & part, PreformattedMessage & out_reason) const;
|
||||
|
||||
CommittingBlocks getCommittingBlocks(zkutil::ZooKeeperPtr & zookeeper, const std::string & zookeeper_path, LoggerPtr log_);
|
||||
|
||||
@ -561,7 +561,7 @@ public:
|
||||
|
||||
/// Returns true if part is needed for some REPLACE_RANGE entry.
|
||||
/// We should not drop part in this case, because replication queue may stuck without that part.
|
||||
bool partParticipatesInReplaceRange(const MergeTreeData::DataPartPtr & part, String & out_reason) const;
|
||||
bool partParticipatesInReplaceRange(const MergeTreeData::DataPartPtr & part, PreformattedMessage & out_reason) const;
|
||||
|
||||
/// Return nonempty optional of desired mutation version and alter version.
|
||||
/// If we have no alter (modify/drop) mutations in mutations queue, than we return biggest possible
|
||||
|
@ -19,6 +19,12 @@ public:
|
||||
bool supportsSampling() const override { return true; }
|
||||
bool supportsFinal() const override { return true; }
|
||||
bool supportsPrewhere() const override { return true; }
|
||||
|
||||
std::optional<NameSet> supportedPrewhereColumns() const override
|
||||
{
|
||||
return original_storage_snapshot ? original_storage_snapshot->storage.supportedPrewhereColumns() : std::nullopt;
|
||||
}
|
||||
|
||||
bool supportsSubcolumns() const override { return true; }
|
||||
bool supportsDynamicSubcolumns() const override { return true; }
|
||||
bool canMoveConditionsToPrewhere() const override
|
||||
|
@ -933,7 +933,7 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge(
|
||||
bool aggressive,
|
||||
const String & partition_id,
|
||||
bool final,
|
||||
String & out_disable_reason,
|
||||
PreformattedMessage & out_disable_reason,
|
||||
TableLockHolder & /* table_lock_holder */,
|
||||
std::unique_lock<std::mutex> & lock,
|
||||
const MergeTreeTransactionPtr & txn,
|
||||
@ -951,7 +951,7 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge(
|
||||
CurrentlyMergingPartsTaggerPtr merging_tagger;
|
||||
MergeList::EntryPtr merge_entry;
|
||||
|
||||
auto can_merge = [this, &lock](const DataPartPtr & left, const DataPartPtr & right, const MergeTreeTransaction * tx, String & disable_reason) -> bool
|
||||
auto can_merge = [this, &lock](const DataPartPtr & left, const DataPartPtr & right, const MergeTreeTransaction * tx, PreformattedMessage & disable_reason) -> bool
|
||||
{
|
||||
if (tx)
|
||||
{
|
||||
@ -960,7 +960,7 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge(
|
||||
if ((left && !left->version.isVisible(tx->getSnapshot(), Tx::EmptyTID))
|
||||
|| (right && !right->version.isVisible(tx->getSnapshot(), Tx::EmptyTID)))
|
||||
{
|
||||
disable_reason = "Some part is not visible in transaction";
|
||||
disable_reason = PreformattedMessage::create("Some part is not visible in transaction");
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -968,7 +968,7 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge(
|
||||
if ((left && left->version.isRemovalTIDLocked())
|
||||
|| (right && right->version.isRemovalTIDLocked()))
|
||||
{
|
||||
disable_reason = "Some part is locked for removal in another cuncurrent transaction";
|
||||
disable_reason = PreformattedMessage::create("Some part is locked for removal in another cuncurrent transaction");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@ -979,7 +979,7 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge(
|
||||
{
|
||||
if (currently_merging_mutating_parts.contains(right))
|
||||
{
|
||||
disable_reason = "Some part currently in a merging or mutating process";
|
||||
disable_reason = PreformattedMessage::create("Some part currently in a merging or mutating process");
|
||||
return false;
|
||||
}
|
||||
else
|
||||
@ -988,13 +988,13 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge(
|
||||
|
||||
if (currently_merging_mutating_parts.contains(left) || currently_merging_mutating_parts.contains(right))
|
||||
{
|
||||
disable_reason = "Some part currently in a merging or mutating process";
|
||||
disable_reason = PreformattedMessage::create("Some part currently in a merging or mutating process");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (getCurrentMutationVersion(left, lock) != getCurrentMutationVersion(right, lock))
|
||||
{
|
||||
disable_reason = "Some parts have different mutation version";
|
||||
disable_reason = PreformattedMessage::create("Some parts have different mutation version");
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -1004,7 +1004,7 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge(
|
||||
auto max_possible_level = getMaxLevelInBetween(left, right);
|
||||
if (max_possible_level > std::max(left->info.level, right->info.level))
|
||||
{
|
||||
disable_reason = fmt::format("There is an outdated part in a gap between two active parts ({}, {}) with merge level {} higher than these active parts have", left->name, right->name, max_possible_level);
|
||||
disable_reason = PreformattedMessage::create("There is an outdated part in a gap between two active parts ({}, {}) with merge level {} higher than these active parts have", left->name, right->name, max_possible_level);
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -1013,11 +1013,11 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge(
|
||||
|
||||
SelectPartsDecision select_decision = SelectPartsDecision::CANNOT_SELECT;
|
||||
|
||||
auto is_background_memory_usage_ok = [](String & disable_reason) -> bool
|
||||
auto is_background_memory_usage_ok = [](PreformattedMessage & disable_reason) -> bool
|
||||
{
|
||||
if (canEnqueueBackgroundTask())
|
||||
return true;
|
||||
disable_reason = fmt::format("Current background tasks memory usage ({}) is more than the limit ({})",
|
||||
disable_reason = PreformattedMessage::create("Current background tasks memory usage ({}) is more than the limit ({})",
|
||||
formatReadableSizeWithBinarySuffix(background_memory_tracker.get()),
|
||||
formatReadableSizeWithBinarySuffix(background_memory_tracker.getSoftLimit()));
|
||||
return false;
|
||||
@ -1045,7 +1045,7 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge(
|
||||
out_disable_reason);
|
||||
}
|
||||
else
|
||||
out_disable_reason = "Current value of max_source_parts_size is zero";
|
||||
out_disable_reason = PreformattedMessage::create("Current value of max_source_parts_size is zero");
|
||||
}
|
||||
}
|
||||
else
|
||||
@ -1086,7 +1086,7 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge(
|
||||
|
||||
if (std::cv_status::timeout == currently_processing_in_background_condition.wait_for(lock, timeout))
|
||||
{
|
||||
out_disable_reason = fmt::format("Timeout ({} ms) while waiting for already running merges before running OPTIMIZE with FINAL", timeout_ms);
|
||||
out_disable_reason = PreformattedMessage::create("Timeout ({} ms) while waiting for already running merges before running OPTIMIZE with FINAL", timeout_ms);
|
||||
break;
|
||||
}
|
||||
}
|
||||
@ -1102,9 +1102,9 @@ MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMerge(
|
||||
|
||||
if (select_decision != SelectPartsDecision::SELECTED)
|
||||
{
|
||||
if (!out_disable_reason.empty())
|
||||
out_disable_reason += ". ";
|
||||
out_disable_reason += "Cannot select parts for optimization";
|
||||
if (!out_disable_reason.text.empty())
|
||||
out_disable_reason.text += ". ";
|
||||
out_disable_reason.text += "Cannot select parts for optimization";
|
||||
|
||||
return {};
|
||||
}
|
||||
@ -1125,7 +1125,7 @@ bool StorageMergeTree::merge(
|
||||
const Names & deduplicate_by_columns,
|
||||
bool cleanup,
|
||||
const MergeTreeTransactionPtr & txn,
|
||||
String & out_disable_reason,
|
||||
PreformattedMessage & out_disable_reason,
|
||||
bool optimize_skip_merged_partitions)
|
||||
{
|
||||
auto table_lock_holder = lockForShare(RWLockImpl::NO_QUERY, getSettings()->lock_acquire_timeout_for_background_operations);
|
||||
@ -1180,7 +1180,7 @@ bool StorageMergeTree::partIsAssignedToBackgroundOperation(const DataPartPtr & p
|
||||
}
|
||||
|
||||
MergeMutateSelectedEntryPtr StorageMergeTree::selectPartsToMutate(
|
||||
const StorageMetadataPtr & metadata_snapshot, String & /* disable_reason */, TableLockHolder & /* table_lock_holder */,
|
||||
const StorageMetadataPtr & metadata_snapshot, PreformattedMessage & /* disable_reason */, TableLockHolder & /* table_lock_holder */,
|
||||
std::unique_lock<std::mutex> & /*currently_processing_in_background_mutex_lock*/)
|
||||
{
|
||||
if (current_mutations_by_version.empty())
|
||||
@ -1396,7 +1396,7 @@ bool StorageMergeTree::scheduleDataProcessingJob(BackgroundJobsAssignee & assign
|
||||
if (merger_mutator.merges_blocker.isCancelled())
|
||||
return false;
|
||||
|
||||
String out_reason;
|
||||
PreformattedMessage out_reason;
|
||||
merge_entry = selectPartsToMerge(metadata_snapshot, false, {}, false, out_reason, shared_lock, lock, txn);
|
||||
|
||||
if (!merge_entry && !current_mutations_by_version.empty())
|
||||
@ -1559,14 +1559,12 @@ bool StorageMergeTree::optimize(
|
||||
|
||||
auto txn = local_context->getCurrentTransaction();
|
||||
|
||||
String disable_reason;
|
||||
PreformattedMessage disable_reason;
|
||||
if (!partition && final)
|
||||
{
|
||||
if (cleanup && this->merging_params.mode != MergingParams::Mode::Replacing)
|
||||
{
|
||||
constexpr const char * message = "Cannot OPTIMIZE with CLEANUP table: {}";
|
||||
disable_reason = "only ReplacingMergeTree can be CLEANUP";
|
||||
throw Exception(ErrorCodes::CANNOT_ASSIGN_OPTIMIZE, message, disable_reason);
|
||||
throw Exception(ErrorCodes::CANNOT_ASSIGN_OPTIMIZE, "Cannot OPTIMIZE with CLEANUP table: only ReplacingMergeTree can be CLEANUP");
|
||||
}
|
||||
|
||||
if (cleanup && !getSettings()->allow_experimental_replacing_merge_with_cleanup)
|
||||
@ -1592,12 +1590,12 @@ bool StorageMergeTree::optimize(
|
||||
local_context->getSettingsRef().optimize_skip_merged_partitions))
|
||||
{
|
||||
constexpr auto message = "Cannot OPTIMIZE table: {}";
|
||||
if (disable_reason.empty())
|
||||
disable_reason = "unknown reason";
|
||||
LOG_INFO(log, message, disable_reason);
|
||||
if (disable_reason.text.empty())
|
||||
disable_reason = PreformattedMessage::create("unknown reason");
|
||||
LOG_INFO(log, message, disable_reason.text);
|
||||
|
||||
if (local_context->getSettingsRef().optimize_throw_if_noop)
|
||||
throw Exception(ErrorCodes::CANNOT_ASSIGN_OPTIMIZE, message, disable_reason);
|
||||
throw Exception(ErrorCodes::CANNOT_ASSIGN_OPTIMIZE, message, disable_reason.text);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@ -1620,12 +1618,12 @@ bool StorageMergeTree::optimize(
|
||||
local_context->getSettingsRef().optimize_skip_merged_partitions))
|
||||
{
|
||||
constexpr auto message = "Cannot OPTIMIZE table: {}";
|
||||
if (disable_reason.empty())
|
||||
disable_reason = "unknown reason";
|
||||
LOG_INFO(log, message, disable_reason);
|
||||
if (disable_reason.text.empty())
|
||||
disable_reason = PreformattedMessage::create("unknown reason");
|
||||
LOG_INFO(log, message, disable_reason.text);
|
||||
|
||||
if (local_context->getSettingsRef().optimize_throw_if_noop)
|
||||
throw Exception(ErrorCodes::CANNOT_ASSIGN_OPTIMIZE, message, disable_reason);
|
||||
throw Exception(ErrorCodes::CANNOT_ASSIGN_OPTIMIZE, message, disable_reason.text);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@ -2024,7 +2022,7 @@ PartitionCommandsResultInfo StorageMergeTree::attachPartition(
|
||||
bool attach_part, ContextPtr local_context)
|
||||
{
|
||||
PartitionCommandsResultInfo results;
|
||||
PartsTemporaryRename renamed_parts(*this, "detached/");
|
||||
PartsTemporaryRename renamed_parts(*this, DETACHED_DIR_NAME);
|
||||
MutableDataPartsVector loaded_parts = tryLoadPartsToAttach(partition, attach_part, local_context, renamed_parts);
|
||||
|
||||
for (size_t i = 0; i < loaded_parts.size(); ++i)
|
||||
|
@ -175,7 +175,7 @@ private:
|
||||
const Names & deduplicate_by_columns,
|
||||
bool cleanup,
|
||||
const MergeTreeTransactionPtr & txn,
|
||||
String & out_disable_reason,
|
||||
PreformattedMessage & out_disable_reason,
|
||||
bool optimize_skip_merged_partitions = false);
|
||||
|
||||
void renameAndCommitEmptyParts(MutableDataPartsVector & new_parts, Transaction & transaction);
|
||||
@ -202,7 +202,7 @@ private:
|
||||
bool aggressive,
|
||||
const String & partition_id,
|
||||
bool final,
|
||||
String & disable_reason,
|
||||
PreformattedMessage & disable_reason,
|
||||
TableLockHolder & table_lock_holder,
|
||||
std::unique_lock<std::mutex> & lock,
|
||||
const MergeTreeTransactionPtr & txn,
|
||||
@ -211,7 +211,7 @@ private:
|
||||
|
||||
|
||||
MergeMutateSelectedEntryPtr selectPartsToMutate(
|
||||
const StorageMetadataPtr & metadata_snapshot, String & disable_reason,
|
||||
const StorageMetadataPtr & metadata_snapshot, PreformattedMessage & disable_reason,
|
||||
TableLockHolder & table_lock_holder, std::unique_lock<std::mutex> & currently_processing_in_background_mutex_lock);
|
||||
|
||||
/// For current mutations queue, returns maximum version of mutation for a part,
|
||||
|
@ -1983,7 +1983,7 @@ MergeTreeData::MutableDataPartPtr StorageReplicatedMergeTree::attachPartHelperFo
|
||||
|
||||
for (const DiskPtr & disk : getStoragePolicy()->getDisks())
|
||||
{
|
||||
for (const auto it = disk->iterateDirectory(fs::path(relative_data_path) / "detached/"); it->isValid(); it->next())
|
||||
for (const auto it = disk->iterateDirectory(fs::path(relative_data_path) / DETACHED_DIR_NAME); it->isValid(); it->next())
|
||||
{
|
||||
const auto part_info = MergeTreePartInfo::tryParsePartName(it->name(), format_version);
|
||||
|
||||
@ -1993,7 +1993,7 @@ MergeTreeData::MutableDataPartPtr StorageReplicatedMergeTree::attachPartHelperFo
|
||||
const auto part_old_name = part_info->getPartNameV1();
|
||||
const auto volume = std::make_shared<SingleDiskVolume>("volume_" + part_old_name, disk);
|
||||
|
||||
auto part = getDataPartBuilder(entry.new_part_name, volume, fs::path("detached") / part_old_name)
|
||||
auto part = getDataPartBuilder(entry.new_part_name, volume, fs::path(DETACHED_DIR_NAME) / part_old_name)
|
||||
.withPartFormatFromDisk()
|
||||
.build();
|
||||
|
||||
@ -2440,7 +2440,7 @@ void StorageReplicatedMergeTree::executeDropRange(const LogEntry & entry)
|
||||
{
|
||||
String part_dir = part_to_detach->getDataPartStorage().getPartDirectory();
|
||||
LOG_INFO(log, "Detaching {}", part_dir);
|
||||
auto holder = getTemporaryPartDirectoryHolder(String(DETACHED_DIR_NAME) + "/" + part_dir);
|
||||
auto holder = getTemporaryPartDirectoryHolder(fs::path(DETACHED_DIR_NAME) / part_dir);
|
||||
part_to_detach->makeCloneInDetached("", metadata_snapshot, /*disk_transaction*/ {});
|
||||
}
|
||||
}
|
||||
@ -2967,7 +2967,7 @@ void StorageReplicatedMergeTree::executeClonePartFromShard(const LogEntry & entr
|
||||
part = get_part();
|
||||
// The fetched part is valuable and should not be cleaned like a temp part.
|
||||
part->is_temp = false;
|
||||
part->renameTo("detached/" + entry.new_part_name, true);
|
||||
part->renameTo(fs::path(DETACHED_DIR_NAME) / entry.new_part_name, true);
|
||||
|
||||
LOG_INFO(log, "Cloned part {} to detached directory", part->name);
|
||||
}
|
||||
@ -3832,7 +3832,7 @@ void StorageReplicatedMergeTree::mergeSelectingTask()
|
||||
merge_pred.emplace(queue.getMergePredicate(zookeeper, partitions_to_merge_in));
|
||||
}
|
||||
|
||||
String out_reason;
|
||||
PreformattedMessage out_reason;
|
||||
if (can_assign_merge &&
|
||||
merger_mutator.selectPartsToMerge(future_merged_part, false, max_source_parts_size_for_merge, *merge_pred,
|
||||
merge_with_ttl_allowed, NO_TRANSACTION_PTR, out_reason, &partitions_to_merge_in) == SelectPartsDecision::SELECTED)
|
||||
@ -4987,7 +4987,7 @@ bool StorageReplicatedMergeTree::fetchPart(
|
||||
{
|
||||
// The fetched part is valuable and should not be cleaned like a temp part.
|
||||
part->is_temp = false;
|
||||
part->renameTo(fs::path("detached") / part_name, true);
|
||||
part->renameTo(fs::path(DETACHED_DIR_NAME) / part_name, true);
|
||||
}
|
||||
}
|
||||
catch (const Exception & e)
|
||||
@ -5814,7 +5814,7 @@ bool StorageReplicatedMergeTree::optimize(
|
||||
future_merged_part->uuid = UUIDHelpers::generateV4();
|
||||
|
||||
constexpr const char * unknown_disable_reason = "unknown reason";
|
||||
String disable_reason = unknown_disable_reason;
|
||||
PreformattedMessage disable_reason = PreformattedMessage::create(unknown_disable_reason);
|
||||
SelectPartsDecision select_decision = SelectPartsDecision::CANNOT_SELECT;
|
||||
|
||||
if (partition_id.empty())
|
||||
@ -5837,10 +5837,10 @@ bool StorageReplicatedMergeTree::optimize(
|
||||
if (select_decision != SelectPartsDecision::SELECTED)
|
||||
{
|
||||
constexpr const char * message_fmt = "Cannot select parts for optimization: {}";
|
||||
assert(disable_reason != unknown_disable_reason);
|
||||
assert(disable_reason.text != unknown_disable_reason);
|
||||
if (!partition_id.empty())
|
||||
disable_reason += fmt::format(" (in partition {})", partition_id);
|
||||
return handle_noop(message_fmt, disable_reason);
|
||||
disable_reason.text += fmt::format(" (in partition {})", partition_id);
|
||||
return handle_noop(message_fmt, disable_reason.text);
|
||||
}
|
||||
|
||||
ReplicatedMergeTreeLogEntryData merge_entry;
|
||||
@ -6547,7 +6547,7 @@ PartitionCommandsResultInfo StorageReplicatedMergeTree::attachPartition(
|
||||
assertNotReadonly();
|
||||
|
||||
PartitionCommandsResultInfo results;
|
||||
PartsTemporaryRename renamed_parts(*this, "detached/");
|
||||
PartsTemporaryRename renamed_parts(*this, DETACHED_DIR_NAME);
|
||||
MutableDataPartsVector loaded_parts = tryLoadPartsToAttach(partition, attach_part, query_context, renamed_parts);
|
||||
|
||||
/// TODO Allow to use quorum here.
|
||||
@ -8509,9 +8509,9 @@ void StorageReplicatedMergeTree::movePartitionToShard(
|
||||
}
|
||||
|
||||
/// canMergeSinglePart is overlapping with dropPart, let's try to use the same code.
|
||||
String out_reason;
|
||||
PreformattedMessage out_reason;
|
||||
if (!merge_pred.canMergeSinglePart(part, out_reason))
|
||||
throw Exception(ErrorCodes::PART_IS_TEMPORARILY_LOCKED, "Part is busy, reason: {}", out_reason);
|
||||
throw Exception(ErrorCodes::PART_IS_TEMPORARILY_LOCKED, "Part is busy, reason: {}", out_reason.text);
|
||||
}
|
||||
|
||||
{
|
||||
@ -8769,18 +8769,18 @@ bool StorageReplicatedMergeTree::dropPartImpl(
|
||||
|
||||
/// There isn't a lot we can do otherwise. Can't cancel merges because it is possible that a replica already
|
||||
/// finished the merge.
|
||||
String out_reason;
|
||||
PreformattedMessage out_reason;
|
||||
if (!merge_pred.canMergeSinglePart(part, out_reason))
|
||||
{
|
||||
if (throw_if_noop)
|
||||
throw Exception::createDeprecated(out_reason, ErrorCodes::PART_IS_TEMPORARILY_LOCKED);
|
||||
throw Exception(out_reason, ErrorCodes::PART_IS_TEMPORARILY_LOCKED);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (merge_pred.partParticipatesInReplaceRange(part, out_reason))
|
||||
{
|
||||
if (throw_if_noop)
|
||||
throw Exception::createDeprecated(out_reason, ErrorCodes::PART_IS_TEMPORARILY_LOCKED);
|
||||
throw Exception(out_reason, ErrorCodes::PART_IS_TEMPORARILY_LOCKED);
|
||||
return false;
|
||||
}
|
||||
|
||||
@ -9986,7 +9986,7 @@ bool StorageReplicatedMergeTree::checkIfDetachedPartExists(const String & part_n
|
||||
{
|
||||
fs::directory_iterator dir_end;
|
||||
for (const std::string & path : getDataPaths())
|
||||
for (fs::directory_iterator dir_it{fs::path(path) / "detached/"}; dir_it != dir_end; ++dir_it)
|
||||
for (fs::directory_iterator dir_it{fs::path(path) / DETACHED_DIR_NAME}; dir_it != dir_end; ++dir_it)
|
||||
if (dir_it->path().filename().string() == part_name)
|
||||
return true;
|
||||
return false;
|
||||
@ -9999,7 +9999,7 @@ bool StorageReplicatedMergeTree::checkIfDetachedPartitionExists(const String & p
|
||||
|
||||
for (const std::string & path : getDataPaths())
|
||||
{
|
||||
for (fs::directory_iterator dir_it{fs::path(path) / "detached/"}; dir_it != dir_end; ++dir_it)
|
||||
for (fs::directory_iterator dir_it{fs::path(path) / DETACHED_DIR_NAME}; dir_it != dir_end; ++dir_it)
|
||||
{
|
||||
const String file_name = dir_it->path().filename().string();
|
||||
auto part_info = MergeTreePartInfo::tryParsePartName(file_name, format_version);
|
||||
|
@ -20,6 +20,7 @@ namespace ErrorCodes
|
||||
{
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
namespace
|
||||
@ -78,6 +79,9 @@ StoragePtr TableFunctionNumbers<multithreaded>::executeImpl(
|
||||
UInt64 length = arguments.size() >= 2 ? evaluateArgument(context, arguments[1]) : evaluateArgument(context, arguments[0]);
|
||||
UInt64 step = arguments.size() == 3 ? evaluateArgument(context, arguments[2]) : 1;
|
||||
|
||||
if (!step)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Table function {} requires step to be a positive number", getName());
|
||||
|
||||
auto res = std::make_shared<StorageSystemNumbers>(
|
||||
StorageID(getDatabaseName(), table_name), multithreaded, std::string{"number"}, length, offset, step);
|
||||
res->startup();
|
||||
|
@ -318,7 +318,7 @@ class CiCache:
|
||||
self.update()
|
||||
|
||||
if self.cache_data_fetched:
|
||||
# there are no record w/o underling data - no need to fetch
|
||||
# there are no records without fetched data - no need to fetch
|
||||
return self
|
||||
|
||||
# clean up
|
||||
@ -773,6 +773,7 @@ class CiOptions:
|
||||
not pr_info.is_pr() and not debug_message
|
||||
): # if commit_message is provided it's test/debug scenario - do not return
|
||||
# CI options can be configured in PRs only
|
||||
# if debug_message is provided - it's a test
|
||||
return res
|
||||
message = debug_message or GitRunner(set_cwd_to_git_root=True).run(
|
||||
f"{GIT_PREFIX} log {pr_info.sha} --format=%B -n 1"
|
||||
@ -790,9 +791,9 @@ class CiOptions:
|
||||
print(f"CI tags from PR body: [{matches_pr}]")
|
||||
matches = list(set(matches + matches_pr))
|
||||
|
||||
if "do not test" in pr_info.labels:
|
||||
# do_not_test could be set in GH labels
|
||||
res.do_not_test = True
|
||||
if "do not test" in pr_info.labels:
|
||||
# do_not_test could be set in GH labels
|
||||
res.do_not_test = True
|
||||
|
||||
for match in matches:
|
||||
if match.startswith("job_"):
|
||||
|
@ -148,6 +148,11 @@ def set_status_comment(commit: Commit, pr_info: PRInfo) -> None:
|
||||
"""It adds or updates the comment status to all Pull Requests but for release
|
||||
one, so the method does nothing for simple pushes and pull requests with
|
||||
`release`/`release-lts` labels"""
|
||||
|
||||
if pr_info.is_merge_queue():
|
||||
# skip report creation for the MQ
|
||||
return
|
||||
|
||||
# to reduce number of parameters, the Github is constructed on the fly
|
||||
gh = Github()
|
||||
gh.__requester = commit._requester # type:ignore #pylint:disable=protected-access
|
||||
@ -441,7 +446,9 @@ def update_mergeable_check(commit: Commit, pr_info: PRInfo, check_name: str) ->
|
||||
or pr_info.release_pr
|
||||
or pr_info.number == 0
|
||||
)
|
||||
if not_run:
|
||||
|
||||
# FIXME: For now, always set mergeable check in the Merge Queue. It's required to pass MQ
|
||||
if not_run and not pr_info.is_merge_queue():
|
||||
# Let's avoid unnecessary work
|
||||
return
|
||||
|
||||
|
@ -26,6 +26,7 @@ NeedsDataType = Dict[str, Dict[str, Union[str, Dict[str, str]]]]
|
||||
DIFF_IN_DOCUMENTATION_EXT = [
|
||||
".html",
|
||||
".md",
|
||||
".mdx",
|
||||
".yml",
|
||||
".txt",
|
||||
".css",
|
||||
|
@ -201,14 +201,17 @@ def main():
|
||||
|
||||
ci_report_url = create_ci_report(pr_info, [])
|
||||
print("::notice ::Can run")
|
||||
post_commit_status(
|
||||
commit,
|
||||
PENDING,
|
||||
ci_report_url,
|
||||
description,
|
||||
CI_STATUS_NAME,
|
||||
pr_info,
|
||||
)
|
||||
|
||||
if not pr_info.is_merge_queue():
|
||||
# we need clean CI status for MQ to merge (no pending statuses)
|
||||
post_commit_status(
|
||||
commit,
|
||||
PENDING,
|
||||
ci_report_url,
|
||||
description,
|
||||
CI_STATUS_NAME,
|
||||
pr_info,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
@ -131,6 +131,11 @@ def main():
|
||||
temp_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
pr_info = PRInfo()
|
||||
|
||||
if pr_info.is_merge_queue() and args.push:
|
||||
print("Auto style fix will be disabled for Merge Queue workflow")
|
||||
args.push = False
|
||||
|
||||
run_cpp_check = True
|
||||
run_shell_check = True
|
||||
run_python_check = True
|
||||
|
@ -21,10 +21,6 @@ INSERT INTO foo FORMAT Values
|
||||
INSERT INTO foo SELECT 1
|
||||
[oneline] insert into foo select 1
|
||||
INSERT INTO foo SELECT 1
|
||||
[multi] insert into foo watch bar
|
||||
INSERT INTO foo WATCH bar
|
||||
[oneline] insert into foo watch bar
|
||||
INSERT INTO foo WATCH bar
|
||||
[multi] insert into foo format tsv
|
||||
INSERT INTO foo FORMAT tsv
|
||||
[oneline] insert into foo format tsv
|
||||
@ -41,12 +37,6 @@ SETTINGS max_threads = 1
|
||||
SELECT 1
|
||||
[oneline] insert into foo settings max_threads=1 select 1
|
||||
INSERT INTO foo SETTINGS max_threads = 1 SELECT 1
|
||||
[multi] insert into foo settings max_threads=1 watch bar
|
||||
INSERT INTO foo
|
||||
SETTINGS max_threads = 1
|
||||
WATCH bar
|
||||
[oneline] insert into foo settings max_threads=1 watch bar
|
||||
INSERT INTO foo SETTINGS max_threads = 1 WATCH bar
|
||||
[multi] insert into foo settings max_threads=1 format tsv
|
||||
INSERT INTO foo
|
||||
SETTINGS max_threads = 1
|
||||
|
@ -40,12 +40,10 @@ $CLICKHOUSE_CLIENT -q 'drop table data_02263'
|
||||
|
||||
run_format_both 'insert into foo values'
|
||||
run_format_both 'insert into foo select 1'
|
||||
run_format_both 'insert into foo watch bar'
|
||||
run_format_both 'insert into foo format tsv'
|
||||
|
||||
run_format_both 'insert into foo settings max_threads=1 values'
|
||||
run_format_both 'insert into foo settings max_threads=1 select 1'
|
||||
run_format_both 'insert into foo settings max_threads=1 watch bar'
|
||||
run_format_both 'insert into foo settings max_threads=1 format tsv'
|
||||
run_format_both 'insert into foo select 1 settings max_threads=1'
|
||||
run_format_both 'insert into foo settings max_threads=1 select 1 settings max_threads=1'
|
||||
|
@ -1,2 +1,3 @@
|
||||
1
|
||||
1
|
||||
1
|
||||
|
@ -34,4 +34,19 @@ FROM
|
||||
) AS t
|
||||
) SETTINGS optimize_uniq_to_count=1;
|
||||
|
||||
-- https://github.com/ClickHouse/ClickHouse/issues/62298
|
||||
DROP TABLE IF EXISTS users;
|
||||
CREATE TABLE users
|
||||
(
|
||||
`id` Int64,
|
||||
`name` String
|
||||
)
|
||||
ENGINE = ReplacingMergeTree
|
||||
ORDER BY (id, name);
|
||||
|
||||
INSERT INTO users VALUES (1, 'pufit'), (1, 'pufit2'), (1, 'pufit3');
|
||||
|
||||
SELECT uniqExact(id) FROM ( SELECT id FROM users WHERE id = 1 GROUP BY id, name );
|
||||
|
||||
DROP TABLE IF EXISTS users;
|
||||
DROP TABLE IF EXISTS tags;
|
||||
|
@ -0,0 +1,3 @@
|
||||
UInt32 1
|
||||
UInt32 2
|
||||
UInt32 3
|
@ -0,0 +1,8 @@
|
||||
set allow_suspicious_low_cardinality_types=1;
|
||||
drop table if exists test;
|
||||
create table test (`x` LowCardinality(Nullable(UInt32)), `y` String) engine = MergeTree order by tuple();
|
||||
insert into test values (1, 'a'), (2, 'bb'), (3, 'ccc'), (4, 'dddd');
|
||||
create table m_table (x UInt32, y String) engine = Merge(currentDatabase(), 'test*');
|
||||
select toTypeName(x), x FROM m_table SETTINGS additional_table_filters = {'m_table':'x != 4'}, optimize_move_to_prewhere=1, allow_experimental_analyzer=1;
|
||||
drop table test;
|
||||
|
@ -4,5 +4,5 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
clickhouse-local --param_rounding 1 --query "SELECT 1 AS x ORDER BY x WITH FILL STEP {rounding:UInt32} SETTINGS allow_experimental_analyzer = 1"
|
||||
clickhouse-local --param_rounding 1 --query "SELECT 1 AS x ORDER BY x WITH FILL STEP {rounding:UInt32} SETTINGS allow_experimental_analyzer = 0"
|
||||
${CLICKHOUSE_LOCAL} --param_rounding 1 --query "SELECT 1 AS x ORDER BY x WITH FILL STEP {rounding:UInt32} SETTINGS allow_experimental_analyzer = 1"
|
||||
${CLICKHOUSE_LOCAL} --param_rounding 1 --query "SELECT 1 AS x ORDER BY x WITH FILL STEP {rounding:UInt32} SETTINGS allow_experimental_analyzer = 0"
|
||||
|
@ -0,0 +1 @@
|
||||
10
|
@ -0,0 +1,27 @@
|
||||
CREATE TABLE vecs_Float32 (v Array(Float32)) ENGINE=Memory;
|
||||
INSERT INTO vecs_Float32
|
||||
SELECT v FROM (
|
||||
SELECT
|
||||
number AS n,
|
||||
[
|
||||
rand(n*10), rand(n*10+1), rand(n*10+2), rand(n*10+3), rand(n*10+4), rand(n*10+5), rand(n*10+6), rand(n*10+7), rand(n*10+8), rand(n*10+9),
|
||||
rand(n*10+10), rand(n*10+11), rand(n*10+12), rand(n*10+13), rand(n*10+14), rand(n*10+15), rand(n*10+16), rand(n*10+17), rand(n*10+18), rand(n*10+19),
|
||||
rand(n*10+20), rand(n*10+21), rand(n*10+22), rand(n*10+23), rand(n*10+24), rand(n*10+25), rand(n*10+26), rand(n*10+27), rand(n*10+28), rand(n*10+29),
|
||||
rand(n*10+30), rand(n*10+31), rand(n*10+32), rand(n*10+33), rand(n*10+34), rand(n*10+35), rand(n*10+36), rand(n*10+37), rand(n*10+38), rand(n*10+39),
|
||||
rand(n*10+40), rand(n*10+41), rand(n*10+42), rand(n*10+43), rand(n*10+44), rand(n*10+45), rand(n*10+46), rand(n*10+47), rand(n*10+48), rand(n*10+49),
|
||||
rand(n*10+50), rand(n*10+51), rand(n*10+52), rand(n*10+53), rand(n*10+54), rand(n*10+55), rand(n*10+56), rand(n*10+57), rand(n*10+58), rand(n*10+59),
|
||||
rand(n*10+60), rand(n*10+61), rand(n*10+62), rand(n*10+63), rand(n*10+64), rand(n*10+65), rand(n*10+66), rand(n*10+67), rand(n*10+68), rand(n*10+69),
|
||||
rand(n*10+70), rand(n*10+71), rand(n*10+72), rand(n*10+73), rand(n*10+74), rand(n*10+75), rand(n*10+76), rand(n*10+77), rand(n*10+78), rand(n*10+79),
|
||||
rand(n*10+80), rand(n*10+81), rand(n*10+82), rand(n*10+83), rand(n*10+84), rand(n*10+85), rand(n*10+86), rand(n*10+87), rand(n*10+88), rand(n*10+89),
|
||||
rand(n*10+90), rand(n*10+91), rand(n*10+92), rand(n*10+93), rand(n*10+94), rand(n*10+95), rand(n*10+96), rand(n*10+97), rand(n*10+98), rand(n*10+99),
|
||||
rand(n*10+100), rand(n*10+101), rand(n*10+102), rand(n*10+103), rand(n*10+104), rand(n*10+105), rand(n*10+106), rand(n*10+107), rand(n*10+108), rand(n*10+109),
|
||||
rand(n*10+110), rand(n*10+111), rand(n*10+112), rand(n*10+113), rand(n*10+114), rand(n*10+115), rand(n*10+116), rand(n*10+117), rand(n*10+118), rand(n*10+119),
|
||||
rand(n*10+120), rand(n*10+121), rand(n*10+122), rand(n*10+123), rand(n*10+124), rand(n*10+125), rand(n*10+126), rand(n*10+127), rand(n*10+128), rand(n*10+129),
|
||||
rand(n*10+130), rand(n*10+131), rand(n*10+132), rand(n*10+133), rand(n*10+134), rand(n*10+135), rand(n*10+136), rand(n*10+137), rand(n*10+138), rand(n*10+139),
|
||||
rand(n*10+140), rand(n*10+141), rand(n*10+142), rand(n*10+143), rand(n*10+144), rand(n*10+145), rand(n*10+146), rand(n*10+147), rand(n*10+148), rand(n*10+149)
|
||||
] AS v
|
||||
FROM system.numbers
|
||||
LIMIT 10
|
||||
);
|
||||
|
||||
WITH (SELECT v FROM vecs_Float32 limit 1) AS a SELECT count(dp) FROM (SELECT dotProduct(a, v) AS dp FROM vecs_Float32);
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user