mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-24 00:22:29 +00:00
Merge branch 'master' into Azure_write_buffer_parallel_upload
This commit is contained in:
Smita Kulkarni
commit
3ace11f92d
@ -39,15 +39,8 @@ Every month we get together with the community (users, contributors, customers,
|
||||
|
||||
## Upcoming Events
|
||||
|
||||
Keep an eye out for upcoming meetups and eventsaround the world. Somewhere else you want us to be? Please feel free to reach out to tyler `<at>` clickhouse `<dot>` com. You can also peruse [ClickHouse Events](https://clickhouse.com/company/news-events) for a list of all upcoming trainings, meetups, speaking engagements, etc.
|
||||
Keep an eye out for upcoming meetups and events around the world. Somewhere else you want us to be? Please feel free to reach out to tyler `<at>` clickhouse `<dot>` com. You can also peruse [ClickHouse Events](https://clickhouse.com/company/news-events) for a list of all upcoming trainings, meetups, speaking engagements, etc.
|
||||
|
||||
* [ClickHouse Meetup in Bellevue](https://www.meetup.com/clickhouse-seattle-user-group/events/298650371/) - Mar 11
|
||||
* [ClickHouse Meetup at Ramp's Offices in NYC](https://www.meetup.com/clickhouse-new-york-user-group/events/298640542/) - Mar 19
|
||||
* [ClickHouse Melbourne Meetup](https://www.meetup.com/clickhouse-australia-user-group/events/299479750/) - Mar 20
|
||||
* [ClickHouse Meetup in Paris](https://www.meetup.com/clickhouse-france-user-group/events/298997115/) - Mar 21
|
||||
* [ClickHouse Meetup in Bengaluru](https://www.meetup.com/clickhouse-bangalore-user-group/events/299479850/) - Mar 23
|
||||
* [ClickHouse Meetup in Zurich](https://www.meetup.com/clickhouse-switzerland-meetup-group/events/299628922/) - Apr 16
|
||||
* [ClickHouse Meetup in Copenhagen](https://www.meetup.com/clickhouse-denmark-meetup-group/events/299629133/) - Apr 23
|
||||
* [ClickHouse Meetup in Dubai](https://www.meetup.com/clickhouse-dubai-meetup-group/events/299629189/) - May 28
|
||||
|
||||
|
||||
|
||||
@ -66,9 +66,11 @@ public:
|
||||
/// The thread and process ids are set.
|
||||
|
||||
Message(
|
||||
const std::string & source, const std::string & text, Priority prio, const char * file, int line, std::string_view fmt_str = {});
|
||||
const std::string & source, const std::string & text, Priority prio, const char * file, int line,
|
||||
std::string_view fmt_str = {}, const std::vector<std::string> & fmt_str_args = {});
|
||||
Message(
|
||||
std::string && source, std::string && text, Priority prio, const char * file, int line, std::string_view fmt_str);
|
||||
std::string && source, std::string && text, Priority prio, const char * file, int line,
|
||||
std::string_view fmt_str, std::vector<std::string> && fmt_str_args);
|
||||
/// Creates a Message with the given source, text, priority,
|
||||
/// source file path and line.
|
||||
///
|
||||
@ -161,6 +163,9 @@ public:
|
||||
std::string_view getFormatString() const;
|
||||
void setFormatString(std::string_view fmt_str);
|
||||
|
||||
const std::vector<std::string> & getFormatStringArgs() const;
|
||||
void setFormatStringArgs(const std::vector<std::string> & fmt_str_args);
|
||||
|
||||
int getSourceLine() const;
|
||||
/// Returns the source file line of the statement
|
||||
/// generating the log message. May be 0
|
||||
@ -210,6 +215,7 @@ private:
|
||||
int _line;
|
||||
StringMap * _pMap;
|
||||
std::string_view _fmt_str;
|
||||
std::vector<std::string> _fmt_str_args;
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -46,7 +46,9 @@ Message::Message(const std::string& source, const std::string& text, Priority pr
|
||||
}
|
||||
|
||||
|
||||
Message::Message(const std::string& source, const std::string& text, Priority prio, const char* file, int line, std::string_view fmt_str):
|
||||
Message::Message(
|
||||
const std::string& source, const std::string& text, Priority prio, const char* file, int line,
|
||||
std::string_view fmt_str, const std::vector<std::string>& fmt_str_args):
|
||||
_source(source),
|
||||
_text(text),
|
||||
_prio(prio),
|
||||
@ -54,13 +56,16 @@ Message::Message(const std::string& source, const std::string& text, Priority pr
|
||||
_file(file),
|
||||
_line(line),
|
||||
_pMap(0),
|
||||
_fmt_str(fmt_str)
|
||||
_fmt_str(fmt_str),
|
||||
_fmt_str_args(fmt_str_args)
|
||||
{
|
||||
init();
|
||||
}
|
||||
|
||||
|
||||
Message::Message(std::string && source, std::string && text, Priority prio, const char * file, int line, std::string_view fmt_str):
|
||||
Message::Message(
|
||||
std::string && source, std::string && text, Priority prio, const char * file, int line,
|
||||
std::string_view fmt_str, std::vector<std::string> && fmt_str_args):
|
||||
_source(std::move(source)),
|
||||
_text(std::move(text)),
|
||||
_prio(prio),
|
||||
@ -68,7 +73,8 @@ Message::Message(std::string && source, std::string && text, Priority prio, cons
|
||||
_file(file),
|
||||
_line(line),
|
||||
_pMap(0),
|
||||
_fmt_str(fmt_str)
|
||||
_fmt_str(fmt_str),
|
||||
_fmt_str_args(std::move(fmt_str_args))
|
||||
{
|
||||
init();
|
||||
}
|
||||
@ -83,7 +89,8 @@ Message::Message(const Message& msg):
|
||||
_pid(msg._pid),
|
||||
_file(msg._file),
|
||||
_line(msg._line),
|
||||
_fmt_str(msg._fmt_str)
|
||||
_fmt_str(msg._fmt_str),
|
||||
_fmt_str_args(msg._fmt_str_args)
|
||||
{
|
||||
if (msg._pMap)
|
||||
_pMap = new StringMap(*msg._pMap);
|
||||
@ -102,7 +109,8 @@ Message::Message(const Message& msg, const std::string& text):
|
||||
_pid(msg._pid),
|
||||
_file(msg._file),
|
||||
_line(msg._line),
|
||||
_fmt_str(msg._fmt_str)
|
||||
_fmt_str(msg._fmt_str),
|
||||
_fmt_str_args(msg._fmt_str_args)
|
||||
{
|
||||
if (msg._pMap)
|
||||
_pMap = new StringMap(*msg._pMap);
|
||||
@ -154,6 +162,7 @@ void Message::swap(Message& msg)
|
||||
swap(_line, msg._line);
|
||||
swap(_pMap, msg._pMap);
|
||||
swap(_fmt_str, msg._fmt_str);
|
||||
swap(_fmt_str_args, msg._fmt_str_args);
|
||||
}
|
||||
|
||||
|
||||
@ -227,6 +236,17 @@ void Message::setFormatString(std::string_view fmt_str)
|
||||
}
|
||||
|
||||
|
||||
const std::vector<std::string>& Message::getFormatStringArgs() const
|
||||
{
|
||||
return _fmt_str_args;
|
||||
}
|
||||
|
||||
void Message::setFormatStringArgs(const std::vector<std::string>& fmt_str_args)
|
||||
{
|
||||
_fmt_str_args = fmt_str_args;
|
||||
}
|
||||
|
||||
|
||||
bool Message::has(const std::string& param) const
|
||||
{
|
||||
return _pMap && (_pMap->find(param) != _pMap->end());
|
||||
|
||||
@ -93,6 +93,7 @@ enable_language(ASM)
|
||||
|
||||
if(COMPILER_CLANG)
|
||||
add_definitions(-Wno-unused-command-line-argument)
|
||||
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld") # only relevant for -DENABLE_OPENSSL_DYNAMIC=1
|
||||
endif()
|
||||
|
||||
if(ARCH_AMD64)
|
||||
@ -146,7 +147,7 @@ elseif(ARCH_AARCH64)
|
||||
else()
|
||||
macro(perl_generate_asm FILE_IN FILE_OUT)
|
||||
add_custom_command(OUTPUT ${FILE_OUT}
|
||||
COMMAND /usr/bin/env perl ${FILE_IN} "linux64" ${FILE_OUT})
|
||||
COMMAND ${CMAKE_COMMAND} -E env "CC=${CMAKE_CXX_COMPILER}" /usr/bin/env perl ${FILE_IN} "linux64" ${FILE_OUT})
|
||||
endmacro()
|
||||
|
||||
perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/aes/asm/aesv8-armx.pl ${OPENSSL_BINARY_DIR}/crypto/aes/aesv8-armx.S)
|
||||
@ -175,7 +176,7 @@ elseif(ARCH_AARCH64)
|
||||
elseif(ARCH_PPC64LE)
|
||||
macro(perl_generate_asm FILE_IN FILE_OUT)
|
||||
add_custom_command(OUTPUT ${FILE_OUT}
|
||||
COMMAND /usr/bin/env perl ${FILE_IN} "linux64v2" ${FILE_OUT})
|
||||
COMMAND ${CMAKE_COMMAND} -E env "CC=${CMAKE_CXX_COMPILER}" /usr/bin/env perl ${FILE_IN} "linux64v2" ${FILE_OUT})
|
||||
endmacro()
|
||||
|
||||
perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/aes/asm/aesp8-ppc.pl ${OPENSSL_BINARY_DIR}/crypto/aes/aesp8-ppc.s)
|
||||
@ -185,7 +186,7 @@ elseif(ARCH_PPC64LE)
|
||||
elseif(ARCH_S390X)
|
||||
macro(perl_generate_asm FILE_IN FILE_OUT)
|
||||
add_custom_command(OUTPUT ${FILE_OUT}
|
||||
COMMAND /usr/bin/env perl ${FILE_IN} "linux64" ${FILE_OUT})
|
||||
COMMAND ${CMAKE_COMMAND} -E env "CC=${CMAKE_CXX_COMPILER}" /usr/bin/env perl ${FILE_IN} "linux64" ${FILE_OUT})
|
||||
endmacro()
|
||||
|
||||
perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/aes/asm/aes-s390x.pl ${OPENSSL_BINARY_DIR}/crypto/aes/aes-s390x.S)
|
||||
@ -193,7 +194,7 @@ elseif(ARCH_S390X)
|
||||
elseif(ARCH_RISCV64)
|
||||
macro(perl_generate_asm FILE_IN FILE_OUT)
|
||||
add_custom_command(OUTPUT ${FILE_OUT}
|
||||
COMMAND /usr/bin/env perl ${FILE_IN} "linux64" ${FILE_OUT})
|
||||
COMMAND ${CMAKE_COMMAND} -E env "CC=${CMAKE_CXX_COMPILER}" /usr/bin/env perl ${FILE_IN} "linux64" ${FILE_OUT})
|
||||
endmacro()
|
||||
|
||||
perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/riscv64cpuid.pl ${OPENSSL_BINARY_DIR}/crypto/riscv64cpuid.S)
|
||||
@ -960,11 +961,6 @@ set(CRYPTO_SRC
|
||||
${OPENSSL_SOURCE_DIR}/crypto/x509/x_req.c
|
||||
${OPENSSL_SOURCE_DIR}/crypto/x509/x_x509.c
|
||||
${OPENSSL_SOURCE_DIR}/crypto/x509/x_x509a.c
|
||||
${OPENSSL_SOURCE_DIR}/engines/e_capi.c
|
||||
${OPENSSL_SOURCE_DIR}/engines/e_dasync.c
|
||||
${OPENSSL_SOURCE_DIR}/engines/e_loader_attic.c
|
||||
${OPENSSL_SOURCE_DIR}/engines/e_ossltest.c
|
||||
${OPENSSL_SOURCE_DIR}/engines/e_padlock.c
|
||||
${OPENSSL_SOURCE_DIR}/providers/baseprov.c
|
||||
${OPENSSL_SOURCE_DIR}/providers/common/bio_prov.c
|
||||
${OPENSSL_SOURCE_DIR}/providers/common/capabilities.c
|
||||
@ -985,8 +981,6 @@ set(CRYPTO_SRC
|
||||
${OPENSSL_SOURCE_DIR}/providers/common/securitycheck.c
|
||||
${OPENSSL_SOURCE_DIR}/providers/common/securitycheck_default.c
|
||||
${OPENSSL_SOURCE_DIR}/providers/defltprov.c
|
||||
${OPENSSL_SOURCE_DIR}/providers/fips/fips_entry.c
|
||||
${OPENSSL_SOURCE_DIR}/providers/fips/fipsprov.c
|
||||
${OPENSSL_SOURCE_DIR}/providers/implementations/asymciphers/rsa_enc.c
|
||||
${OPENSSL_SOURCE_DIR}/providers/implementations/asymciphers/sm2_enc.c
|
||||
${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes.c
|
||||
@ -1145,11 +1139,19 @@ set(CRYPTO_SRC
|
||||
${OPENSSL_SOURCE_DIR}/providers/implementations/signature/sm2_sig.c
|
||||
${OPENSSL_SOURCE_DIR}/providers/implementations/storemgmt/file_store.c
|
||||
${OPENSSL_SOURCE_DIR}/providers/implementations/storemgmt/file_store_any2obj.c
|
||||
${OPENSSL_SOURCE_DIR}/providers/legacyprov.c
|
||||
${OPENSSL_SOURCE_DIR}/providers/nullprov.c
|
||||
${OPENSSL_SOURCE_DIR}/providers/prov_running.c
|
||||
${OPENSSL_SOURCE_DIR}/ssl/record/methods/tls_pad.c
|
||||
${OPENSSL_SOURCE_DIR}/ssl/record/methods/ssl3_cbc.c
|
||||
)
|
||||
|
||||
if(NOT ENABLE_OPENSSL_DYNAMIC)
|
||||
set(CRYPTO_SRC ${CRYPTO_SRC}
|
||||
${OPENSSL_SOURCE_DIR}/providers/fips/fips_entry.c
|
||||
${OPENSSL_SOURCE_DIR}/providers/fips/fipsprov.c
|
||||
)
|
||||
endif()
|
||||
|
||||
if(ARCH_AMD64)
|
||||
if (OS_DARWIN)
|
||||
set(CRYPTO_SRC ${CRYPTO_SRC}
|
||||
@ -1376,8 +1378,6 @@ set(SSL_SRC
|
||||
${OPENSSL_SOURCE_DIR}/ssl/quic/uint_set.c
|
||||
${OPENSSL_SOURCE_DIR}/ssl/record/rec_layer_d1.c
|
||||
${OPENSSL_SOURCE_DIR}/ssl/record/rec_layer_s3.c
|
||||
${OPENSSL_SOURCE_DIR}/ssl/record/methods/tls_pad.c
|
||||
${OPENSSL_SOURCE_DIR}/ssl/record/methods/ssl3_cbc.c
|
||||
${OPENSSL_SOURCE_DIR}/ssl/record/methods/dtls_meth.c
|
||||
${OPENSSL_SOURCE_DIR}/ssl/record/methods/ssl3_meth.c
|
||||
${OPENSSL_SOURCE_DIR}/ssl/record/methods/tls13_meth.c
|
||||
|
||||
@ -44,8 +44,6 @@ source /utils.lib
|
||||
|
||||
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
echo "Azure is disabled"
|
||||
elif [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
|
||||
echo "Azure is disabled"
|
||||
else
|
||||
azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log &
|
||||
fi
|
||||
|
||||
@ -304,10 +304,10 @@ We use the term `MergeTree` to refer to all table engines in the `MergeTree fami
|
||||
|
||||
If you had a `MergeTree` table that was manually replicated, you can convert it to a replicated table. You might need to do this if you have already collected a large amount of data in a `MergeTree` table and now you want to enable replication.
|
||||
|
||||
`MergeTree` table can be automatically converted on server restart if `convert_to_replicated` flag is set at the table's data directory (`/var/lib/clickhouse/store/xxx/xxxyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy/` for `Atomic` database).
|
||||
`MergeTree` table can be automatically converted on server restart if `convert_to_replicated` flag is set at the table's data directory (`/store/xxx/xxxyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy/` for `Atomic` database).
|
||||
Create empty `convert_to_replicated` file and the table will be loaded as replicated on next server restart.
|
||||
|
||||
This query can be used to get the table's data path.
|
||||
This query can be used to get the table's data path. If table has many data paths, you have to use the first one.
|
||||
|
||||
```sql
|
||||
SELECT data_paths FROM system.tables WHERE table = 'table_name' AND database = 'database_name';
|
||||
|
||||
@ -76,7 +76,7 @@ ClickHouse, Inc. does **not** maintain the tools and libraries listed below and
|
||||
- [clickhouse-maxmind-geoip](https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip)
|
||||
- AutoML
|
||||
- [MindsDB](https://mindsdb.com/)
|
||||
- [MindsDB](https://github.com/mindsdb/mindsdb) - Predictive AI layer for ClickHouse database.
|
||||
- [MindsDB](https://github.com/mindsdb/mindsdb) - Integrates with ClickHouse, making data from ClickHouse accessible to a diverse range of AI/ML models.
|
||||
|
||||
## Programming Language Ecosystems {#programming-language-ecosystems}
|
||||
|
||||
|
||||
@ -523,7 +523,7 @@ See settings `cgroups_memory_usage_observer_wait_time` and `cgroup_memory_watche
|
||||
|
||||
Type: Double
|
||||
|
||||
Default: 0.95
|
||||
Default: 0.9
|
||||
|
||||
## max_table_size_to_drop
|
||||
|
||||
|
||||
@ -30,6 +30,16 @@ Columns:
|
||||
- `source_file` (LowCardinality(String)) — Source file from which the logging was done.
|
||||
- `source_line` (UInt64) — Source line from which the logging was done.
|
||||
- `message_format_string` (LowCardinality(String)) — A format string that was used to format the message.
|
||||
- `value1` (String) - Argument 1 that was used to format the message.
|
||||
- `value2` (String) - Argument 2 that was used to format the message.
|
||||
- `value3` (String) - Argument 3 that was used to format the message.
|
||||
- `value4` (String) - Argument 4 that was used to format the message.
|
||||
- `value5` (String) - Argument 5 that was used to format the message.
|
||||
- `value6` (String) - Argument 6 that was used to format the message.
|
||||
- `value7` (String) - Argument 7 that was used to format the message.
|
||||
- `value8` (String) - Argument 8 that was used to format the message.
|
||||
- `value9` (String) - Argument 9 that was used to format the message.
|
||||
- `value10` (String) - Argument 10 that was used to format the message.
|
||||
|
||||
**Example**
|
||||
|
||||
@ -55,4 +65,14 @@ revision: 54440
|
||||
source_file: /ClickHouse/src/Interpreters/DNSCacheUpdater.cpp; void DB::DNSCacheUpdater::start()
|
||||
source_line: 45
|
||||
message_format_string: Update period {} seconds
|
||||
value1: 15
|
||||
value2:
|
||||
value3:
|
||||
value4:
|
||||
value5:
|
||||
value6:
|
||||
value7:
|
||||
value8:
|
||||
value9:
|
||||
value10:
|
||||
```
|
||||
|
||||
@ -1907,6 +1907,12 @@ If the addition results in a value outside the bounds of the data type, the resu
|
||||
date_add(unit, value, date)
|
||||
```
|
||||
|
||||
Alternative syntax:
|
||||
|
||||
``` sql
|
||||
date_add(date, INTERVAL value unit)
|
||||
```
|
||||
|
||||
Aliases: `dateAdd`, `DATE_ADD`.
|
||||
|
||||
**Arguments**
|
||||
@ -1946,6 +1952,20 @@ Result:
|
||||
└───────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT date_add(toDate('2018-01-01'), INTERVAL 3 YEAR);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```text
|
||||
┌─plus(toDate('2018-01-01'), toIntervalYear(3))─┐
|
||||
│ 2021-01-01 │
|
||||
└───────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
|
||||
|
||||
**See Also**
|
||||
|
||||
- [addDate](#addDate)
|
||||
@ -1962,6 +1982,13 @@ If the subtraction results in a value outside the bounds of the data type, the r
|
||||
date_sub(unit, value, date)
|
||||
```
|
||||
|
||||
Alternative syntax:
|
||||
|
||||
``` sql
|
||||
date_sub(date, INTERVAL value unit)
|
||||
```
|
||||
|
||||
|
||||
Aliases: `dateSub`, `DATE_SUB`.
|
||||
|
||||
**Arguments**
|
||||
@ -2002,6 +2029,19 @@ Result:
|
||||
└────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
``` sql
|
||||
SELECT date_sub(toDate('2018-01-01'), INTERVAL 3 YEAR);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─minus(toDate('2018-01-01'), toIntervalYear(3))─┐
|
||||
│ 2015-01-01 │
|
||||
└────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
|
||||
**See Also**
|
||||
|
||||
- [subDate](#subDate)
|
||||
|
||||
@ -8,7 +8,7 @@ sidebar_label: Mathematical
|
||||
|
||||
## e
|
||||
|
||||
Returns e ([Euler's constant](https://en.wikipedia.org/wiki/Euler%27s_constant))
|
||||
Returns e ([Euler's constant](https://en.wikipedia.org/wiki/Euler%27s_constant)).
|
||||
|
||||
**Syntax**
|
||||
|
||||
@ -45,7 +45,7 @@ exp(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -65,7 +65,7 @@ Alias: `ln(x)`
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -83,7 +83,7 @@ exp2(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -111,7 +111,7 @@ log2(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -129,7 +129,7 @@ exp10(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -157,7 +157,7 @@ log10(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -173,7 +173,7 @@ sqrt(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -189,7 +189,7 @@ cbrt(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -207,7 +207,7 @@ erf(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -239,7 +239,7 @@ erfc(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -257,7 +257,7 @@ lgamma(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -275,7 +275,7 @@ gamma(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -293,7 +293,7 @@ sin(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -323,7 +323,7 @@ cos(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -341,7 +341,7 @@ tan(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -359,7 +359,7 @@ asin(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -377,7 +377,7 @@ acos(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -395,7 +395,7 @@ atan(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
|
||||
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -434,7 +434,7 @@ cosh(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` — The angle, in radians. Values from the interval: `-∞ < x < +∞`. [Float64](../../sql-reference/data-types/float.md#float32-float64).
|
||||
- `x` — The angle, in radians. Values from the interval: `-∞ < x < +∞`. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -468,7 +468,7 @@ acosh(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` — Hyperbolic cosine of angle. Values from the interval: `1 <= x < +∞`. [Float64](../../sql-reference/data-types/float.md#float32-float64).
|
||||
- `x` — Hyperbolic cosine of angle. Values from the interval: `1 <= x < +∞`. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -502,7 +502,7 @@ sinh(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` — The angle, in radians. Values from the interval: `-∞ < x < +∞`. [Float64](../../sql-reference/data-types/float.md#float32-float64).
|
||||
- `x` — The angle, in radians. Values from the interval: `-∞ < x < +∞`. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -536,7 +536,7 @@ asinh(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` — Hyperbolic sine of angle. Values from the interval: `-∞ < x < +∞`. [Float64](../../sql-reference/data-types/float.md#float32-float64).
|
||||
- `x` — Hyperbolic sine of angle. Values from the interval: `-∞ < x < +∞`. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -557,6 +557,37 @@ Result:
|
||||
│ 0 │
|
||||
└──────────┘
|
||||
```
|
||||
## tanh
|
||||
|
||||
Returns the [hyperbolic tangent](https://www.mathworks.com/help/matlab/ref/tanh.html).
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
tanh(x)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` — The angle, in radians. Values from the interval: `-∞ < x < +∞`. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Values from the interval: `-1 < tanh(x) < 1`.
|
||||
|
||||
Type: [Float*](../../sql-reference/data-types/float.md#float32-float64).
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
SELECT tanh(0);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```result
|
||||
0
|
||||
```
|
||||
|
||||
## atanh
|
||||
|
||||
@ -570,7 +601,7 @@ atanh(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` — Hyperbolic tangent of angle. Values from the interval: `–1 < x < 1`. [Float64](../../sql-reference/data-types/float.md#float32-float64).
|
||||
- `x` — Hyperbolic tangent of angle. Values from the interval: `–1 < x < 1`. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -604,8 +635,8 @@ atan2(y, x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `y` — y-coordinate of the point through which the ray passes. [Float64](../../sql-reference/data-types/float.md#float32-float64).
|
||||
- `x` — x-coordinate of the point through which the ray passes. [Float64](../../sql-reference/data-types/float.md#float32-float64).
|
||||
- `y` — y-coordinate of the point through which the ray passes. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md).
|
||||
- `x` — x-coordinate of the point through which the ray passes. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -639,8 +670,8 @@ hypot(x, y)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` — The first cathetus of a right-angle triangle. [Float64](../../sql-reference/data-types/float.md#float32-float64).
|
||||
- `y` — The second cathetus of a right-angle triangle. [Float64](../../sql-reference/data-types/float.md#float32-float64).
|
||||
- `x` — The first cathetus of a right-angle triangle. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md).
|
||||
- `y` — The second cathetus of a right-angle triangle. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -674,7 +705,7 @@ log1p(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` — Values from the interval: `-1 < x < +∞`. [Float64](../../sql-reference/data-types/float.md#float32-float64).
|
||||
- `x` — Values from the interval: `-1 < x < +∞`. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -716,6 +747,8 @@ sign(x)
|
||||
- 0 for `x = 0`
|
||||
- 1 for `x > 0`
|
||||
|
||||
Type: [Int8](../../sql-reference/data-types/int-uint.md).
|
||||
|
||||
**Examples**
|
||||
|
||||
Sign for the zero value:
|
||||
@ -772,7 +805,7 @@ degrees(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` — Input in radians. [Float64](../../sql-reference/data-types/float.md#float32-float64).
|
||||
- `x` — Input in radians. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
@ -806,7 +839,7 @@ radians(x)
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `x` — Input in degrees. [Float64](../../sql-reference/data-types/float.md#float32-float64).
|
||||
- `x` — Input in degrees. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
|
||||
@ -88,20 +88,93 @@ Result:
|
||||
|
||||
## length
|
||||
|
||||
Returns the length of a string in bytes (not: in characters or Unicode code points).
|
||||
|
||||
The function also works for arrays.
|
||||
Returns the length of a string in bytes rather than in characters or Unicode code points. The function also works for arrays.
|
||||
|
||||
Alias: `OCTET_LENGTH`
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
length(s)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `s`: An input string or array. [String](../data-types/string)/[Array](../data-types/array).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Length of the string or array `s` in bytes. [UInt64](../data-types/int-uint).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT length('Hello, world!');
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─length('Hello, world!')─┐
|
||||
│ 13 │
|
||||
└─────────────────────────┘
|
||||
```
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT length([1, 2, 3, 4]);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─length([1, 2, 3, 4])─┐
|
||||
│ 4 │
|
||||
└──────────────────────┘
|
||||
```
|
||||
|
||||
|
||||
## lengthUTF8
|
||||
|
||||
Returns the length of a string in Unicode code points (not: in bytes or characters). It assumes that the string contains valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined.
|
||||
Returns the length of a string in Unicode code points rather than in bytes or characters. It assumes that the string contains valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined.
|
||||
|
||||
Alias:
|
||||
Aliases:
|
||||
- `CHAR_LENGTH`
|
||||
- `CHARACTER_LENGTH`
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
lengthUTF8(s)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `s`: String containing valid UTF-8 encoded text. [String](../data-types/string).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Length of the string `s` in Unicode code points. [UInt64](../data-types/int-uint.md).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT lengthUTF8('Здравствуй, мир!');
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─lengthUTF8('Здравствуй, мир!')─┐
|
||||
│ 16 │
|
||||
└────────────────────────────────┘
|
||||
```
|
||||
|
||||
## left
|
||||
|
||||
Returns a substring of string `s` with a specified `offset` starting from the left.
|
||||
@ -1055,6 +1128,34 @@ Result:
|
||||
|
||||
Like `base58Decode` but returns an empty string in case of error.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
tryBase58Decode(encoded)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `encoded`: [String](../../sql-reference/data-types/string.md) column or constant. If the string is not a valid Base58-encoded value, returns an empty string in case of error.
|
||||
|
||||
**Returned value**
|
||||
|
||||
- A string containing the decoded value of the argument.
|
||||
|
||||
**Examples**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT tryBase58Decode('3dc8KtHrwM') as res;
|
||||
```
|
||||
|
||||
```response
|
||||
┌─res─────┐
|
||||
│ Encoded │
|
||||
└─────────┘
|
||||
```
|
||||
|
||||
## base64Encode
|
||||
|
||||
Encodes a String or FixedString as base64.
|
||||
@ -1071,6 +1172,30 @@ Alias: `FROM_BASE64`.
|
||||
|
||||
Like `base64Decode` but returns an empty string in case of error.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
tryBase64Decode(encoded)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `encoded`: [String](../../sql-reference/data-types/string.md) column or constant. If the string is not a valid Base58-encoded value, returns an empty string in case of error.
|
||||
|
||||
**Examples**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT tryBase64Decode('RW5jb2RlZA==') as res;
|
||||
```
|
||||
|
||||
```response
|
||||
┌─res─────┐
|
||||
│ Encoded │
|
||||
└─────────┘
|
||||
```
|
||||
|
||||
## endsWith {#endswith}
|
||||
|
||||
Returns whether string `str` ends with `suffix`.
|
||||
|
||||
@ -1322,9 +1322,9 @@ Result:
|
||||
|
||||
## countSubstrings
|
||||
|
||||
Returns how often substring `needle` occurs in string `haystack`.
|
||||
Returns how often a substring `needle` occurs in a string `haystack`.
|
||||
|
||||
Functions `countSubstringsCaseInsensitive` and `countSubstringsCaseInsensitiveUTF8` provide a case-insensitive and case-insensitive + UTF-8 variants of this function.
|
||||
Functions [`countSubstringsCaseInsensitive`](#countsubstringscaseinsensitive) and [`countSubstringsCaseInsensitiveUTF8`](#countsubstringscaseinsensitiveutf8) provide case-insensitive and case-insensitive + UTF-8 variants of this function respectively.
|
||||
|
||||
**Syntax**
|
||||
|
||||
@ -1371,6 +1371,113 @@ Result:
|
||||
│ 1 │
|
||||
└────────────────────────────────────────┘
|
||||
```
|
||||
## countSubstringsCaseInsensitive
|
||||
|
||||
Returns how often a substring `needle` occurs in a string `haystack`. Ignores case.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
countSubstringsCaseInsensitive(haystack, needle[, start_pos])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `haystack` — String in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal).
|
||||
- `needle` — Substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal).
|
||||
- `start_pos` – Position (1-based) in `haystack` at which the search starts. [UInt](../../sql-reference/data-types/int-uint.md). Optional.
|
||||
|
||||
**Returned values**
|
||||
|
||||
- The number of occurrences.
|
||||
|
||||
Type: [UInt64](../../sql-reference/data-types/int-uint.md).
|
||||
|
||||
**Examples**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT countSubstringsCaseInsensitive('AAAA', 'aa');
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─countSubstringsCaseInsensitive('AAAA', 'aa')─┐
|
||||
│ 2 │
|
||||
└──────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Example with `start_pos` argument:
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT countSubstringsCaseInsensitive('abc___ABC___abc', 'abc', 4);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─countSubstringsCaseInsensitive('abc___ABC___abc', 'abc', 4)─┐
|
||||
│ 2 │
|
||||
└─────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## countSubstringsCaseInsensitiveUTF8
|
||||
|
||||
Returns how often a substring `needle` occurs in a string `haystack`. Ignores case and assumes that `haystack` is a UTF8 string.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
countSubstringsCaseInsensitiveUTF8(haystack, needle[, start_pos])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `haystack` — UTF-8 string in which the search is performed. [String](../../sql-reference/syntax.md#syntax-string-literal).
|
||||
- `needle` — Substring to be searched. [String](../../sql-reference/syntax.md#syntax-string-literal).
|
||||
- `start_pos` – Position (1-based) in `haystack` at which the search starts. [UInt](../../sql-reference/data-types/int-uint.md). Optional.
|
||||
|
||||
**Returned values**
|
||||
|
||||
- The number of occurrences.
|
||||
|
||||
Type: [UInt64](../../sql-reference/data-types/int-uint.md).
|
||||
|
||||
**Examples**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT countSubstringsCaseInsensitiveUTF8('ложка, кошка, картошка', 'КА');
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─countSubstringsCaseInsensitiveUTF8('ложка, кошка, картошка', 'КА')─┐
|
||||
│ 4 │
|
||||
└────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Example with `start_pos` argument:
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SELECT countSubstringsCaseInsensitiveUTF8('ложка, кошка, картошка', 'КА', 13);
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─countSubstringsCaseInsensitiveUTF8('ложка, кошка, картошка', 'КА', 13)─┐
|
||||
│ 2 │
|
||||
└────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## countMatches
|
||||
|
||||
@ -1421,7 +1528,40 @@ Result:
|
||||
|
||||
## countMatchesCaseInsensitive
|
||||
|
||||
Like `countMatches(haystack, pattern)` but matching ignores the case.
|
||||
Returns the number of regular expression matches for a pattern in a haystack like [`countMatches`](#countmatches) but matching ignores the case.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
countMatchesCaseInsensitive(haystack, pattern)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `haystack` — The string to search in. [String](../../sql-reference/syntax.md#syntax-string-literal).
|
||||
- `pattern` — The regular expression with [re2 syntax](https://github.com/google/re2/wiki/Syntax). [String](../../sql-reference/data-types/string.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The number of matches.
|
||||
|
||||
Type: [UInt64](../../sql-reference/data-types/int-uint.md).
|
||||
|
||||
**Examples**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT countMatchesCaseInsensitive('AAAA', 'aa');
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─countMatchesCaseInsensitive('AAAA', 'aa')────┐
|
||||
│ 2 │
|
||||
└──────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## regexpExtract
|
||||
|
||||
|
||||
@ -817,6 +817,42 @@ Result:
|
||||
└─────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## flattenTuple
|
||||
|
||||
Returns a flattened `output` tuple from a nested named `input` tuple. Elements of the `output` tuple are the paths from the original `input` tuple. For instance: `Tuple(a Int, Tuple(b Int, c Int)) -> Tuple(a Int, b Int, c Int)`. `flattenTuple` can be used to select all paths from type `Object` as separate columns.
|
||||
|
||||
**Syntax**
|
||||
|
||||
```sql
|
||||
flattenTuple(input)
|
||||
```
|
||||
|
||||
**Parameters**
|
||||
|
||||
- `input`: Nested named tuple to flatten. [Tuple](../data-types/tuple).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- `output` tuple whose elements are paths from the original `input`. [Tuple](../data-types/tuple).
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
CREATE TABLE t_flatten_tuple(t Tuple(t1 Nested(a UInt32, s String), b UInt32, t2 Tuple(k String, v UInt32))) ENGINE = Memory;
|
||||
INSERT INTO t_flatten_tuple VALUES (([(1, 'a'), (2, 'b')], 3, ('c', 4)));
|
||||
SELECT flattenTuple(t) FROM t_flatten_tuple;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─flattenTuple(t)───────────┐
|
||||
│ ([1,2],['a','b'],3,'c',4) │
|
||||
└───────────────────────────┘
|
||||
```
|
||||
|
||||
## Distance functions
|
||||
|
||||
All supported functions are described in [distance functions documentation](../../sql-reference/functions/distance-functions.md).
|
||||
|
||||
@ -21,6 +21,8 @@ SELECT [DISTINCT [ON (column1, column2, ...)]] expr_list
|
||||
[WHERE expr]
|
||||
[GROUP BY expr_list] [WITH ROLLUP|WITH CUBE] [WITH TOTALS]
|
||||
[HAVING expr]
|
||||
[WINDOW window_expr_list]
|
||||
[QUALIFY expr]
|
||||
[ORDER BY expr_list] [WITH FILL] [FROM expr] [TO expr] [STEP expr] [INTERPOLATE [(expr_list)]]
|
||||
[LIMIT [offset_value, ]n BY columns]
|
||||
[LIMIT [n, ]m] [WITH TIES]
|
||||
@ -45,6 +47,7 @@ Specifics of each optional clause are covered in separate sections, which are li
|
||||
- [GROUP BY clause](../../../sql-reference/statements/select/group-by.md)
|
||||
- [LIMIT BY clause](../../../sql-reference/statements/select/limit-by.md)
|
||||
- [HAVING clause](../../../sql-reference/statements/select/having.md)
|
||||
- [QUALIFY clause](../../../sql-reference/statements/select/qualify.md)
|
||||
- [LIMIT clause](../../../sql-reference/statements/select/limit.md)
|
||||
- [OFFSET clause](../../../sql-reference/statements/select/offset.md)
|
||||
- [UNION clause](../../../sql-reference/statements/select/union.md)
|
||||
|
||||
34
docs/en/sql-reference/statements/select/qualify.md
Normal file
34
docs/en/sql-reference/statements/select/qualify.md
Normal file
@ -0,0 +1,34 @@
|
||||
---
|
||||
slug: /en/sql-reference/statements/select/qualify
|
||||
sidebar_label: QUALIFY
|
||||
---
|
||||
|
||||
# QUALIFY Clause
|
||||
|
||||
Allows filtering window functions results. It is similar to the [WHERE](../../../sql-reference/statements/select/where.md) clause, but the difference is that `WHERE` is performed before window functions evaluation, while `QUALIFY` is performed after it.
|
||||
|
||||
It is possible to reference window functions results from `SELECT` clause in `QUALIFY` clause by their alias. Alternatively, `QUALIFY` clause can filter on results of additional window functions that are not returned in query results.
|
||||
|
||||
## Limitations
|
||||
|
||||
`QUALIFY` can’t be used if there are no window functions to evaluate. Use `WHERE` instead.
|
||||
|
||||
## Examples
|
||||
|
||||
Example:
|
||||
|
||||
``` sql
|
||||
SELECT number, COUNT() OVER (PARTITION BY number % 3) AS partition_count
|
||||
FROM numbers(10)
|
||||
QUALIFY partition_count = 4
|
||||
ORDER BY number;
|
||||
```
|
||||
|
||||
``` text
|
||||
┌─number─┬─partition_count─┐
|
||||
│ 0 │ 4 │
|
||||
│ 3 │ 4 │
|
||||
│ 6 │ 4 │
|
||||
│ 9 │ 4 │
|
||||
└────────┴─────────────────┘
|
||||
```
|
||||
@ -29,6 +29,16 @@ slug: /ru/operations/system-tables/text_log
|
||||
- `source_file` (LowCardinality(String)) — исходный файл, из которого была сделана запись.
|
||||
- `source_line` (UInt64) — исходная строка, из которой была сделана запись.
|
||||
- `message_format_string` (LowCardinality(String)) — форматная строка, с помощью которой было отформатировано сообщение.
|
||||
- `value1` (String) - аргумент 1, который использовался для форматирования сообщения.
|
||||
- `value2` (String) - аргумент 2, который использовался для форматирования сообщения.
|
||||
- `value3` (String) - аргумент 3, который использовался для форматирования сообщения.
|
||||
- `value4` (String) - аргумент 4, который использовался для форматирования сообщения.
|
||||
- `value5` (String) - аргумент 5, который использовался для форматирования сообщения.
|
||||
- `value6` (String) - аргумент 6, который использовался для форматирования сообщения.
|
||||
- `value7` (String) - аргумент 7, который использовался для форматирования сообщения.
|
||||
- `value8` (String) - аргумент 8, который использовался для форматирования сообщения.
|
||||
- `value9` (String) - аргумент 9, который использовался для форматирования сообщения.
|
||||
- `value10` (String) - аргумент 10, который использовался для форматирования сообщения.
|
||||
|
||||
**Пример**
|
||||
|
||||
@ -53,4 +63,14 @@ revision: 54440
|
||||
source_file: /ClickHouse/src/Interpreters/DNSCacheUpdater.cpp; void DB::DNSCacheUpdater::start()
|
||||
source_line: 45
|
||||
message_format_string: Update period {} seconds
|
||||
value1: 15
|
||||
value2:
|
||||
value3:
|
||||
value4:
|
||||
value5:
|
||||
value6:
|
||||
value7:
|
||||
value8:
|
||||
value9:
|
||||
value10:
|
||||
```
|
||||
|
||||
@ -729,6 +729,15 @@ int mainEntryClickHouseInstall(int argc, char ** argv)
|
||||
}
|
||||
}
|
||||
|
||||
/// Don't allow relative paths because install script may cd to / when installing
|
||||
/// And having path=./ may break the system
|
||||
if (log_path.is_relative())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Log path is relative: {}", log_path.string());
|
||||
if (data_path.is_relative())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Data path is relative: {}", data_path.string());
|
||||
if (pid_path.is_relative())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Pid path is relative: {}", pid_path.string());
|
||||
|
||||
/// Create directories for data and log.
|
||||
|
||||
if (fs::exists(log_path))
|
||||
|
||||
@ -742,9 +742,9 @@
|
||||
It also enables 'changeable_in_readonly' constraint type -->
|
||||
<settings_constraints_replace_previous>true</settings_constraints_replace_previous>
|
||||
|
||||
<!-- By default, for backward compatibility create table with a specific table engine ignores grant,
|
||||
<!-- By default, for backward compatibility creating table with a specific table engine ignores grant,
|
||||
however you can change this behaviour by setting this to true -->
|
||||
<table_engines_require_grant>true</table_engines_require_grant>
|
||||
<table_engines_require_grant>false</table_engines_require_grant>
|
||||
|
||||
<!-- Number of seconds since last access a role is stored in the Role Cache -->
|
||||
<role_cache_expiration_time_seconds>600</role_cache_expiration_time_seconds>
|
||||
|
||||
@ -7,6 +7,10 @@ max-branches=50
|
||||
max-nested-blocks=10
|
||||
max-statements=200
|
||||
|
||||
[tool.isort]
|
||||
profile = "black"
|
||||
src_paths = ["src", "tests/ci", "tests/sqllogic"]
|
||||
|
||||
[tool.pylint.FORMAT]
|
||||
#ignore-long-lines = (# )?<?https?://\S+>?$
|
||||
|
||||
|
||||
@ -607,7 +607,8 @@ AuthResult AccessControl::authenticate(const Credentials & credentials, const Po
|
||||
/// We use the same message for all authentication failures because we don't want to give away any unnecessary information for security reasons,
|
||||
/// only the log will show the exact reason.
|
||||
throw Exception(PreformattedMessage{message.str(),
|
||||
"{}: Authentication failed: password is incorrect, or there is no user with such name.{}"},
|
||||
"{}: Authentication failed: password is incorrect, or there is no user with such name.{}",
|
||||
std::vector<std::string>{credentials.getUserName()}},
|
||||
ErrorCodes::AUTHENTICATION_FAILED);
|
||||
}
|
||||
}
|
||||
|
||||
@ -56,7 +56,9 @@ void IdentifierNode::updateTreeHashImpl(HashState & state, CompareOptions) const
|
||||
|
||||
QueryTreeNodePtr IdentifierNode::cloneImpl() const
|
||||
{
|
||||
return std::make_shared<IdentifierNode>(identifier);
|
||||
auto clone_identifier_node = std::make_shared<IdentifierNode>(identifier);
|
||||
clone_identifier_node->table_expression_modifiers = table_expression_modifiers;
|
||||
return clone_identifier_node;
|
||||
}
|
||||
|
||||
ASTPtr IdentifierNode::toASTImpl(const ConvertToASTOptions & /* options */) const
|
||||
|
||||
@ -7895,6 +7895,9 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier
|
||||
if (query_node_typed.hasHaving() && query_node_typed.isGroupByWithTotals() && is_rollup_or_cube)
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "WITH TOTALS and WITH ROLLUP or CUBE are not supported together in presence of HAVING");
|
||||
|
||||
if (query_node_typed.hasQualify() && query_node_typed.isGroupByWithTotals() && is_rollup_or_cube)
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "WITH TOTALS and WITH ROLLUP or CUBE are not supported together in presence of QUALIFY");
|
||||
|
||||
/// Initialize aliases in query node scope
|
||||
QueryExpressionsAliasVisitor visitor(scope);
|
||||
|
||||
@ -7919,6 +7922,9 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier
|
||||
if (query_node_typed.hasWindow())
|
||||
visitor.visit(query_node_typed.getWindowNode());
|
||||
|
||||
if (query_node_typed.hasQualify())
|
||||
visitor.visit(query_node_typed.getQualify());
|
||||
|
||||
if (query_node_typed.hasOrderBy())
|
||||
visitor.visit(query_node_typed.getOrderByNode());
|
||||
|
||||
@ -8067,6 +8073,9 @@ void QueryAnalyzer::resolveQuery(const QueryTreeNodePtr & query_node, Identifier
|
||||
if (query_node_typed.hasWindow())
|
||||
resolveWindowNodeList(query_node_typed.getWindowNode(), scope);
|
||||
|
||||
if (query_node_typed.hasQualify())
|
||||
resolveExpressionNode(query_node_typed.getQualify(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/);
|
||||
|
||||
if (query_node_typed.hasOrderBy())
|
||||
{
|
||||
replaceNodesWithPositionalArguments(query_node_typed.getOrderByNode(), query_node_typed.getProjection().getNodes(), scope);
|
||||
|
||||
@ -51,10 +51,15 @@ public:
|
||||
{
|
||||
const auto & second_const_value = second_const_node->getValue();
|
||||
if (second_const_value.isNull()
|
||||
|| (lower_name == "sum" && isInt64OrUInt64FieldType(second_const_value.getType()) && second_const_value.get<UInt64>() == 0))
|
||||
|| (lower_name == "sum" && isInt64OrUInt64FieldType(second_const_value.getType()) && second_const_value.get<UInt64>() == 0
|
||||
&& !function_node->getResultType()->isNullable()))
|
||||
{
|
||||
/// avg(if(cond, a, null)) -> avgIf(a, cond)
|
||||
/// avg(if(cond, nullable_a, null)) -> avgIfOrNull(a, cond)
|
||||
|
||||
/// sum(if(cond, a, 0)) -> sumIf(a, cond)
|
||||
/// sum(if(cond, nullable_a, 0)) **is not** equivalent to sumIfOrNull(cond, nullable_a) as
|
||||
/// it changes the output when no rows pass the condition (from 0 to NULL)
|
||||
function_arguments_nodes.resize(2);
|
||||
function_arguments_nodes[0] = std::move(if_arguments_nodes[1]);
|
||||
function_arguments_nodes[1] = std::move(if_arguments_nodes[0]);
|
||||
@ -66,10 +71,13 @@ public:
|
||||
{
|
||||
const auto & first_const_value = first_const_node->getValue();
|
||||
if (first_const_value.isNull()
|
||||
|| (lower_name == "sum" && isInt64OrUInt64FieldType(first_const_value.getType()) && first_const_value.get<UInt64>() == 0))
|
||||
|| (lower_name == "sum" && isInt64OrUInt64FieldType(first_const_value.getType()) && first_const_value.get<UInt64>() == 0
|
||||
&& !function_node->getResultType()->isNullable()))
|
||||
{
|
||||
/// avg(if(cond, null, a) -> avgIf(a, !cond))
|
||||
/// avg(if(cond, null, a) -> avgIfOrNullable(a, !cond))
|
||||
|
||||
/// sum(if(cond, 0, a) -> sumIf(a, !cond))
|
||||
/// sum(if(cond, 0, nullable_a) **is not** sumIf(a, !cond)) -> Same as above
|
||||
auto not_function = std::make_shared<FunctionNode>("not");
|
||||
auto & not_function_arguments = not_function->getArguments().getNodes();
|
||||
not_function_arguments.push_back(std::move(if_arguments_nodes[0]));
|
||||
|
||||
@ -197,6 +197,12 @@ void QueryNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, s
|
||||
getWindow().dumpTreeImpl(buffer, format_state, indent + 4);
|
||||
}
|
||||
|
||||
if (hasQualify())
|
||||
{
|
||||
buffer << '\n' << std::string(indent + 2, ' ') << "QUALIFY\n";
|
||||
getQualify()->dumpTreeImpl(buffer, format_state, indent + 4);
|
||||
}
|
||||
|
||||
if (hasOrderBy())
|
||||
{
|
||||
buffer << '\n' << std::string(indent + 2, ' ') << "ORDER BY\n";
|
||||
@ -381,6 +387,9 @@ ASTPtr QueryNode::toASTImpl(const ConvertToASTOptions & options) const
|
||||
if (hasWindow())
|
||||
select_query->setExpression(ASTSelectQuery::Expression::WINDOW, getWindow().toAST(options));
|
||||
|
||||
if (hasQualify())
|
||||
select_query->setExpression(ASTSelectQuery::Expression::QUALIFY, getQualify()->toAST(options));
|
||||
|
||||
if (hasOrderBy())
|
||||
select_query->setExpression(ASTSelectQuery::Expression::ORDER_BY, getOrderBy().toAST(options));
|
||||
|
||||
|
||||
@ -416,6 +416,24 @@ public:
|
||||
return children[window_child_index];
|
||||
}
|
||||
|
||||
/// Returns true if query node QUALIFY section is not empty, false otherwise
|
||||
bool hasQualify() const
|
||||
{
|
||||
return getQualify() != nullptr;
|
||||
}
|
||||
|
||||
/// Get QUALIFY section node
|
||||
const QueryTreeNodePtr & getQualify() const
|
||||
{
|
||||
return children[qualify_child_index];
|
||||
}
|
||||
|
||||
/// Get QUALIFY section node
|
||||
QueryTreeNodePtr & getQualify()
|
||||
{
|
||||
return children[qualify_child_index];
|
||||
}
|
||||
|
||||
/// Returns true if query node ORDER BY section is not empty, false otherwise
|
||||
bool hasOrderBy() const
|
||||
{
|
||||
@ -622,13 +640,14 @@ private:
|
||||
static constexpr size_t group_by_child_index = 5;
|
||||
static constexpr size_t having_child_index = 6;
|
||||
static constexpr size_t window_child_index = 7;
|
||||
static constexpr size_t order_by_child_index = 8;
|
||||
static constexpr size_t interpolate_child_index = 9;
|
||||
static constexpr size_t limit_by_limit_child_index = 10;
|
||||
static constexpr size_t limit_by_offset_child_index = 11;
|
||||
static constexpr size_t limit_by_child_index = 12;
|
||||
static constexpr size_t limit_child_index = 13;
|
||||
static constexpr size_t offset_child_index = 14;
|
||||
static constexpr size_t qualify_child_index = 8;
|
||||
static constexpr size_t order_by_child_index = 9;
|
||||
static constexpr size_t interpolate_child_index = 10;
|
||||
static constexpr size_t limit_by_limit_child_index = 11;
|
||||
static constexpr size_t limit_by_offset_child_index = 12;
|
||||
static constexpr size_t limit_by_child_index = 13;
|
||||
static constexpr size_t limit_child_index = 14;
|
||||
static constexpr size_t offset_child_index = 15;
|
||||
static constexpr size_t children_size = offset_child_index + 1;
|
||||
};
|
||||
|
||||
|
||||
@ -330,6 +330,10 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression(const ASTPtr & select_q
|
||||
if (window_list)
|
||||
current_query_tree->getWindowNode() = buildWindowList(window_list, current_context);
|
||||
|
||||
auto qualify_expression = select_query_typed.qualify();
|
||||
if (qualify_expression)
|
||||
current_query_tree->getQualify() = buildExpression(qualify_expression, current_context);
|
||||
|
||||
auto select_order_by_list = select_query_typed.orderBy();
|
||||
if (select_order_by_list)
|
||||
current_query_tree->getOrderByNode() = buildSortList(select_order_by_list, current_context);
|
||||
|
||||
@ -56,6 +56,9 @@ void validateFilters(const QueryTreeNodePtr & query_node)
|
||||
|
||||
if (query_node_typed.hasHaving())
|
||||
validateFilter(query_node_typed.getHaving(), "HAVING", query_node);
|
||||
|
||||
if (query_node_typed.hasQualify())
|
||||
validateFilter(query_node_typed.getQualify(), "QUALIFY", query_node);
|
||||
}
|
||||
|
||||
namespace
|
||||
|
||||
@ -47,7 +47,7 @@ Suggest::Suggest()
|
||||
"GRANT", "REVOKE", "OPTION", "ADMIN", "EXCEPT", "REPLACE", "IDENTIFIED", "HOST",
|
||||
"NAME", "READONLY", "WRITABLE", "PERMISSIVE", "FOR", "RESTRICTIVE", "RANDOMIZED", "INTERVAL",
|
||||
"LIMITS", "ONLY", "TRACKING", "IP", "REGEXP", "ILIKE", "CLEANUP", "APPEND",
|
||||
"IGNORE NULLS", "RESPECT NULLS", "OVER", "PASTE"});
|
||||
"IGNORE NULLS", "RESPECT NULLS", "OVER", "PASTE", "WINDOW", "QUALIFY"});
|
||||
}
|
||||
|
||||
static String getLoadSuggestionQuery(Int32 suggestion_limit, bool basic_suggestion)
|
||||
|
||||
@ -154,7 +154,7 @@ public:
|
||||
void updatePermutation(PermutationSortDirection, PermutationSortStability,
|
||||
size_t, int, Permutation &, EqualRanges &) const override
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getPermutation is not supported for ColumnUnique.");
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method updatePermutation is not supported for ColumnUnique.");
|
||||
}
|
||||
|
||||
std::vector<MutableColumnPtr> scatter(IColumn::ColumnIndex, const IColumn::Selector &) const override
|
||||
|
||||
@ -77,7 +77,7 @@ void CgroupsMemoryUsageObserver::setMemoryUsageLimits(uint64_t hard_limit_, uint
|
||||
{
|
||||
if (up)
|
||||
{
|
||||
LOG_WARNING(log, "Exceeded sort memory limit ({})", ReadableSize(soft_limit_));
|
||||
LOG_WARNING(log, "Exceeded soft memory limit ({})", ReadableSize(soft_limit_));
|
||||
|
||||
#if USE_JEMALLOC
|
||||
LOG_INFO(log, "Purging jemalloc arenas");
|
||||
|
||||
@ -391,6 +391,7 @@ PreformattedMessage getCurrentExceptionMessageAndPattern(bool with_stacktrace, b
|
||||
{
|
||||
WriteBufferFromOwnString stream;
|
||||
std::string_view message_format_string;
|
||||
std::vector<std::string> message_format_string_args;
|
||||
|
||||
try
|
||||
{
|
||||
@ -402,6 +403,7 @@ PreformattedMessage getCurrentExceptionMessageAndPattern(bool with_stacktrace, b
|
||||
<< (with_extra_info ? getExtraExceptionInfo(e) : "")
|
||||
<< " (version " << VERSION_STRING << VERSION_OFFICIAL << ")";
|
||||
message_format_string = e.tryGetMessageFormatString();
|
||||
message_format_string_args = e.getMessageFormatStringArgs();
|
||||
}
|
||||
catch (const Poco::Exception & e)
|
||||
{
|
||||
@ -462,7 +464,7 @@ PreformattedMessage getCurrentExceptionMessageAndPattern(bool with_stacktrace, b
|
||||
catch (...) {} // NOLINT(bugprone-empty-catch)
|
||||
}
|
||||
|
||||
return PreformattedMessage{stream.str(), message_format_string};
|
||||
return PreformattedMessage{stream.str(), message_format_string, message_format_string_args};
|
||||
}
|
||||
|
||||
|
||||
@ -581,7 +583,7 @@ PreformattedMessage getExceptionMessageAndPattern(const Exception & e, bool with
|
||||
}
|
||||
catch (...) {} // NOLINT(bugprone-empty-catch)
|
||||
|
||||
return PreformattedMessage{stream.str(), e.tryGetMessageFormatString()};
|
||||
return PreformattedMessage{stream.str(), e.tryGetMessageFormatString(), e.getMessageFormatStringArgs()};
|
||||
}
|
||||
|
||||
std::string getExceptionMessage(std::exception_ptr e, bool with_stacktrace)
|
||||
|
||||
@ -13,6 +13,7 @@
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include <fmt/core.h>
|
||||
#include <fmt/format.h>
|
||||
#include <Poco/Exception.h>
|
||||
|
||||
@ -59,6 +60,7 @@ public:
|
||||
std::terminate();
|
||||
capture_thread_frame_pointers = thread_frame_pointers;
|
||||
message_format_string = msg.format_string;
|
||||
message_format_string_args = msg.format_string_args;
|
||||
}
|
||||
|
||||
Exception(PreformattedMessage && msg, int code): Exception(std::move(msg.text), code)
|
||||
@ -67,6 +69,7 @@ public:
|
||||
std::terminate();
|
||||
capture_thread_frame_pointers = thread_frame_pointers;
|
||||
message_format_string = msg.format_string;
|
||||
message_format_string_args = msg.format_string_args;
|
||||
}
|
||||
|
||||
/// Collect call stacks of all previous jobs' schedulings leading to this thread job's execution
|
||||
@ -107,12 +110,7 @@ public:
|
||||
|
||||
// Format message with fmt::format, like the logging functions.
|
||||
template <typename... Args>
|
||||
Exception(int code, FormatStringHelper<Args...> fmt, Args &&... args)
|
||||
: Exception(fmt::format(fmt.fmt_str, std::forward<Args>(args)...), code)
|
||||
{
|
||||
capture_thread_frame_pointers = thread_frame_pointers;
|
||||
message_format_string = fmt.message_format_string;
|
||||
}
|
||||
Exception(int code, FormatStringHelper<Args...> fmt, Args &&... args) : Exception(fmt.format(std::forward<Args>(args)...), code) {}
|
||||
|
||||
struct CreateFromPocoTag {};
|
||||
struct CreateFromSTDTag {};
|
||||
@ -152,6 +150,8 @@ public:
|
||||
|
||||
std::string_view tryGetMessageFormatString() const { return message_format_string; }
|
||||
|
||||
std::vector<std::string> getMessageFormatStringArgs() const { return message_format_string_args; }
|
||||
|
||||
private:
|
||||
#ifndef STD_EXCEPTION_HAS_STACK_TRACE
|
||||
StackTrace trace;
|
||||
@ -162,6 +162,7 @@ private:
|
||||
|
||||
protected:
|
||||
std::string_view message_format_string;
|
||||
std::vector<std::string> message_format_string_args;
|
||||
/// Local copy of static per-thread thread_frame_pointers, should be mutable to be unpoisoned on printout
|
||||
mutable std::vector<StackTrace::FramePointers> capture_thread_frame_pointers;
|
||||
};
|
||||
@ -193,26 +194,29 @@ public:
|
||||
// Format message with fmt::format, like the logging functions.
|
||||
template <typename... Args>
|
||||
ErrnoException(int code, FormatStringHelper<Args...> fmt, Args &&... args)
|
||||
: Exception(fmt::format(fmt.fmt_str, std::forward<Args>(args)...), code), saved_errno(errno)
|
||||
: Exception(fmt.format(std::forward<Args>(args)...), code), saved_errno(errno)
|
||||
{
|
||||
addMessage(", {}", errnoToString(saved_errno));
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
ErrnoException(int code, int with_errno, FormatStringHelper<Args...> fmt, Args &&... args)
|
||||
: Exception(fmt.format(std::forward<Args>(args)...), code), saved_errno(with_errno)
|
||||
{
|
||||
capture_thread_frame_pointers = thread_frame_pointers;
|
||||
message_format_string = fmt.message_format_string;
|
||||
addMessage(", {}", errnoToString(saved_errno));
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
[[noreturn]] static void throwWithErrno(int code, int with_errno, FormatStringHelper<Args...> fmt, Args &&... args)
|
||||
{
|
||||
auto e = ErrnoException(fmt::format(fmt.fmt_str, std::forward<Args>(args)...), code, with_errno);
|
||||
e.message_format_string = fmt.message_format_string;
|
||||
auto e = ErrnoException(code, with_errno, std::move(fmt), std::forward<Args>(args)...);
|
||||
throw e; /// NOLINT
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
[[noreturn]] static void throwFromPath(int code, const std::string & path, FormatStringHelper<Args...> fmt, Args &&... args)
|
||||
{
|
||||
auto e = ErrnoException(fmt::format(fmt.fmt_str, std::forward<Args>(args)...), code, errno);
|
||||
e.message_format_string = fmt.message_format_string;
|
||||
auto e = ErrnoException(code, errno, std::move(fmt), std::forward<Args>(args)...);
|
||||
e.path = path;
|
||||
throw e; /// NOLINT
|
||||
}
|
||||
@ -221,8 +225,7 @@ public:
|
||||
[[noreturn]] static void
|
||||
throwFromPathWithErrno(int code, const std::string & path, int with_errno, FormatStringHelper<Args...> fmt, Args &&... args)
|
||||
{
|
||||
auto e = ErrnoException(fmt::format(fmt.fmt_str, std::forward<Args>(args)...), code, with_errno);
|
||||
e.message_format_string = fmt.message_format_string;
|
||||
auto e = ErrnoException(code, with_errno, std::move(fmt), std::forward<Args>(args)...);
|
||||
e.path = path;
|
||||
throw e; /// NOLINT
|
||||
}
|
||||
|
||||
@ -2,8 +2,11 @@
|
||||
|
||||
#include <base/defines.h>
|
||||
#include <base/types.h>
|
||||
#include <fmt/args.h>
|
||||
#include <fmt/core.h>
|
||||
#include <fmt/format.h>
|
||||
#include <mutex>
|
||||
#include <type_traits>
|
||||
#include <unordered_map>
|
||||
#include <Poco/Logger.h>
|
||||
#include <Poco/Message.h>
|
||||
@ -14,6 +17,10 @@ struct PreformattedMessage;
|
||||
consteval void formatStringCheckArgsNumImpl(std::string_view str, size_t nargs);
|
||||
template <typename T> constexpr std::string_view tryGetStaticFormatString(T && x);
|
||||
|
||||
[[maybe_unused]] inline void tryGetFormattedArgs(std::vector<std::string>&) {};
|
||||
template <typename T, typename... Ts> [[maybe_unused]] inline void tryGetFormattedArgs(std::vector<std::string>&, T &&, Ts && ...);
|
||||
template <typename... Args> inline std::string tryGetArgsAndFormat(std::vector<std::string>&, fmt::format_string<Args...>, Args && ...);
|
||||
|
||||
/// Extract format string from a string literal and constructs consteval fmt::format_string
|
||||
template <typename... Args>
|
||||
struct FormatStringHelperImpl
|
||||
@ -39,6 +46,7 @@ struct PreformattedMessage
|
||||
{
|
||||
std::string text;
|
||||
std::string_view format_string;
|
||||
std::vector<std::string> format_string_args;
|
||||
|
||||
template <typename... Args>
|
||||
static PreformattedMessage create(FormatStringHelper<Args...> fmt, Args &&... args);
|
||||
@ -47,22 +55,26 @@ struct PreformattedMessage
|
||||
operator std::string () && { return std::move(text); } /// NOLINT
|
||||
operator fmt::format_string<> () const { UNREACHABLE(); } /// NOLINT
|
||||
|
||||
void apply(std::string & out_text, std::string_view & out_format_string) const &
|
||||
void apply(std::string & out_text, std::string_view & out_format_string, std::vector<std::string> & out_format_string_args) const &
|
||||
{
|
||||
out_text = text;
|
||||
out_format_string = format_string;
|
||||
out_format_string_args = format_string_args;
|
||||
}
|
||||
void apply(std::string & out_text, std::string_view & out_format_string) &&
|
||||
void apply(std::string & out_text, std::string_view & out_format_string, std::vector<std::string> & out_format_string_args) &&
|
||||
{
|
||||
out_text = std::move(text);
|
||||
out_format_string = format_string;
|
||||
out_format_string_args = std::move(format_string_args);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename... Args>
|
||||
PreformattedMessage FormatStringHelperImpl<Args...>::format(Args && ...args) const
|
||||
{
|
||||
return PreformattedMessage{fmt::format(fmt_str, std::forward<Args>(args)...), message_format_string};
|
||||
std::vector<std::string> out_format_string_args;
|
||||
std::string msg_text = tryGetArgsAndFormat(out_format_string_args, fmt_str, std::forward<Args>(args)...);
|
||||
return PreformattedMessage{msg_text, message_format_string, out_format_string_args};
|
||||
}
|
||||
|
||||
template <typename... Args>
|
||||
@ -113,12 +125,23 @@ template <typename T> constexpr std::string_view tryGetStaticFormatString(T && x
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename... Ts> void tryGetFormattedArgs(std::vector<std::string>& out, T && x, Ts && ...rest)
|
||||
{
|
||||
if constexpr (std::is_base_of_v<fmt::detail::view, std::decay_t<T>>)
|
||||
out.push_back(fmt::format("{}", std::remove_reference_t<T>(x)));
|
||||
else
|
||||
out.push_back(fmt::format("{}", std::forward<T>(x)));
|
||||
|
||||
tryGetFormattedArgs(out, std::forward<Ts>(rest)...);
|
||||
}
|
||||
|
||||
/// Constexpr ifs are not like ifdefs, and compiler still checks that unneeded code can be compiled
|
||||
/// This template is useful to avoid compilation failures when condition of some "constexpr if" is false
|
||||
template<bool enable> struct ConstexprIfsAreNotIfdefs
|
||||
{
|
||||
template <typename T> constexpr static std::string_view getStaticFormatString(T &&) { return {}; }
|
||||
template <typename T> static PreformattedMessage getPreformatted(T &&) { return {}; }
|
||||
template <typename... Args> static std::string getArgsAndFormat(std::vector<std::string>&, fmt::format_string<Args...>, Args &&...) { return {}; }
|
||||
};
|
||||
|
||||
template<> struct ConstexprIfsAreNotIfdefs<true>
|
||||
@ -133,8 +156,19 @@ template<> struct ConstexprIfsAreNotIfdefs<true>
|
||||
}
|
||||
|
||||
template <typename T> static T && getPreformatted(T && x) { return std::forward<T>(x); }
|
||||
|
||||
template <typename... Args> static std::string getArgsAndFormat(std::vector<std::string>& out, fmt::format_string<Args...> fmt_str, Args && ...args)
|
||||
{
|
||||
return tryGetArgsAndFormat(out, std::move(fmt_str), std::forward<Args>(args)...);
|
||||
}
|
||||
};
|
||||
|
||||
template <typename... Args> inline std::string tryGetArgsAndFormat(std::vector<std::string>& out, fmt::format_string<Args...> fmt_str, Args && ...args)
|
||||
{
|
||||
tryGetFormattedArgs(out, args...);
|
||||
return fmt::format(fmt_str, std::forward<Args>(args)...);
|
||||
}
|
||||
|
||||
template <typename... Ts> constexpr size_t numArgs(Ts &&...) { return sizeof...(Ts); }
|
||||
template <typename T, typename... Ts> constexpr auto firstArg(T && x, Ts &&...) { return std::forward<T>(x); }
|
||||
/// For implicit conversion of fmt::basic_runtime<> to char* for std::string ctor
|
||||
|
||||
@ -518,7 +518,8 @@ bool ZooKeeper::existsWatch(const std::string & path, Coordination::Stat * stat,
|
||||
return code != Coordination::Error::ZNONODE;
|
||||
}
|
||||
|
||||
Coordination::Error ZooKeeper::getImpl(const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback)
|
||||
Coordination::Error ZooKeeper::getImpl(
|
||||
const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallbackPtr watch_callback)
|
||||
{
|
||||
auto future_result = asyncTryGetNoThrow(path, watch_callback);
|
||||
|
||||
@ -541,6 +542,11 @@ Coordination::Error ZooKeeper::getImpl(const std::string & path, std::string & r
|
||||
}
|
||||
}
|
||||
|
||||
Coordination::Error ZooKeeper::getImpl(const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback)
|
||||
{
|
||||
return getImpl(path, res, stat, watch_callback ? std::make_shared<Coordination::WatchCallback>(watch_callback) : Coordination::WatchCallbackPtr{});
|
||||
}
|
||||
|
||||
std::string ZooKeeper::get(const std::string & path, Coordination::Stat * stat, const EventPtr & watch)
|
||||
{
|
||||
Coordination::Error code = Coordination::Error::ZOK;
|
||||
@ -561,6 +567,17 @@ std::string ZooKeeper::getWatch(const std::string & path, Coordination::Stat * s
|
||||
throw KeeperException(code, "Can't get data for node '{}': node doesn't exist", path);
|
||||
}
|
||||
|
||||
|
||||
std::string ZooKeeper::getWatch(const std::string & path, Coordination::Stat * stat, Coordination::WatchCallbackPtr watch_callback)
|
||||
{
|
||||
Coordination::Error code = Coordination::Error::ZOK;
|
||||
std::string res;
|
||||
if (tryGetWatch(path, res, stat, watch_callback, &code))
|
||||
return res;
|
||||
else
|
||||
throw KeeperException(code, "Can't get data for node '{}': node doesn't exist", path);
|
||||
}
|
||||
|
||||
bool ZooKeeper::tryGet(
|
||||
const std::string & path,
|
||||
std::string & res,
|
||||
@ -571,6 +588,25 @@ bool ZooKeeper::tryGet(
|
||||
return tryGetWatch(path, res, stat, callbackForEvent(watch), return_code);
|
||||
}
|
||||
|
||||
bool ZooKeeper::tryGetWatch(
|
||||
const std::string & path,
|
||||
std::string & res,
|
||||
Coordination::Stat * stat,
|
||||
Coordination::WatchCallbackPtr watch_callback,
|
||||
Coordination::Error * return_code)
|
||||
{
|
||||
Coordination::Error code = getImpl(path, res, stat, watch_callback);
|
||||
|
||||
if (!(code == Coordination::Error::ZOK || code == Coordination::Error::ZNONODE))
|
||||
throw KeeperException::fromPath(code, path);
|
||||
|
||||
if (return_code)
|
||||
*return_code = code;
|
||||
|
||||
return code == Coordination::Error::ZOK;
|
||||
|
||||
}
|
||||
|
||||
bool ZooKeeper::tryGetWatch(
|
||||
const std::string & path,
|
||||
std::string & res,
|
||||
@ -589,6 +625,7 @@ bool ZooKeeper::tryGetWatch(
|
||||
return code == Coordination::Error::ZOK;
|
||||
}
|
||||
|
||||
|
||||
Coordination::Error ZooKeeper::setImpl(const std::string & path, const std::string & data,
|
||||
int32_t version, Coordination::Stat * stat)
|
||||
{
|
||||
@ -1062,6 +1099,11 @@ std::future<Coordination::GetResponse> ZooKeeper::asyncGet(const std::string & p
|
||||
}
|
||||
|
||||
std::future<Coordination::GetResponse> ZooKeeper::asyncTryGetNoThrow(const std::string & path, Coordination::WatchCallback watch_callback)
|
||||
{
|
||||
return asyncTryGetNoThrow(path, watch_callback ? std::make_shared<Coordination::WatchCallback>(watch_callback) : Coordination::WatchCallbackPtr{});
|
||||
}
|
||||
|
||||
std::future<Coordination::GetResponse> ZooKeeper::asyncTryGetNoThrow(const std::string & path, Coordination::WatchCallbackPtr watch_callback)
|
||||
{
|
||||
auto promise = std::make_shared<std::promise<Coordination::GetResponse>>();
|
||||
auto future = promise->get_future();
|
||||
@ -1071,8 +1113,7 @@ std::future<Coordination::GetResponse> ZooKeeper::asyncTryGetNoThrow(const std::
|
||||
promise->set_value(response);
|
||||
};
|
||||
|
||||
impl->get(path, std::move(callback),
|
||||
watch_callback ? std::make_shared<Coordination::WatchCallback>(watch_callback) : Coordination::WatchCallbackPtr{});
|
||||
impl->get(path, std::move(callback), watch_callback);
|
||||
return future;
|
||||
}
|
||||
|
||||
|
||||
@ -306,6 +306,7 @@ public:
|
||||
|
||||
std::string get(const std::string & path, Coordination::Stat * stat = nullptr, const EventPtr & watch = nullptr);
|
||||
std::string getWatch(const std::string & path, Coordination::Stat * stat, Coordination::WatchCallback watch_callback);
|
||||
std::string getWatch(const std::string & path, Coordination::Stat * stat, Coordination::WatchCallbackPtr watch_callback);
|
||||
|
||||
using MultiGetResponse = MultiReadResponses<Coordination::GetResponse, false>;
|
||||
using MultiTryGetResponse = MultiReadResponses<Coordination::GetResponse, true>;
|
||||
@ -338,6 +339,13 @@ public:
|
||||
Coordination::WatchCallback watch_callback,
|
||||
Coordination::Error * code = nullptr);
|
||||
|
||||
bool tryGetWatch(
|
||||
const std::string & path,
|
||||
std::string & res,
|
||||
Coordination::Stat * stat,
|
||||
Coordination::WatchCallbackPtr watch_callback,
|
||||
Coordination::Error * code = nullptr);
|
||||
|
||||
template <typename TIter>
|
||||
MultiTryGetResponse tryGet(TIter start, TIter end)
|
||||
{
|
||||
@ -520,6 +528,8 @@ public:
|
||||
/// Like the previous one but don't throw any exceptions on future.get()
|
||||
FutureGet asyncTryGetNoThrow(const std::string & path, Coordination::WatchCallback watch_callback = {});
|
||||
|
||||
FutureGet asyncTryGetNoThrow(const std::string & path, Coordination::WatchCallbackPtr watch_callback = {});
|
||||
|
||||
using FutureExists = std::future<Coordination::ExistsResponse>;
|
||||
FutureExists asyncExists(const std::string & path, Coordination::WatchCallback watch_callback = {});
|
||||
/// Like the previous one but don't throw any exceptions on future.get()
|
||||
@ -625,6 +635,8 @@ private:
|
||||
Coordination::Error removeImpl(const std::string & path, int32_t version);
|
||||
Coordination::Error getImpl(
|
||||
const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback);
|
||||
Coordination::Error getImpl(
|
||||
const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallbackPtr watch_callback);
|
||||
Coordination::Error setImpl(const std::string & path, const std::string & data, int32_t version, Coordination::Stat * stat);
|
||||
Coordination::Error getChildrenImpl(
|
||||
const std::string & path,
|
||||
|
||||
@ -22,13 +22,16 @@ ZooKeeperLock::ZooKeeperLock(
|
||||
const ZooKeeperPtr & zookeeper_,
|
||||
const std::string & lock_prefix_,
|
||||
const std::string & lock_name_,
|
||||
const std::string & lock_message_)
|
||||
const std::string & lock_message_,
|
||||
bool throw_if_lost_)
|
||||
: zookeeper(zookeeper_)
|
||||
, lock_path(fs::path(lock_prefix_) / lock_name_)
|
||||
, lock_message(lock_message_)
|
||||
, throw_if_lost(throw_if_lost_)
|
||||
, log(getLogger("zkutil::Lock"))
|
||||
{
|
||||
zookeeper->createIfNotExists(lock_prefix_, "");
|
||||
LOG_TRACE(log, "Trying to create zookeeper lock on path {} for session {}", lock_path, zookeeper->getClientID());
|
||||
}
|
||||
|
||||
ZooKeeperLock::~ZooKeeperLock()
|
||||
@ -45,7 +48,7 @@ ZooKeeperLock::~ZooKeeperLock()
|
||||
|
||||
bool ZooKeeperLock::isLocked() const
|
||||
{
|
||||
return locked;
|
||||
return locked && !zookeeper->expired();
|
||||
}
|
||||
|
||||
const std::string & ZooKeeperLock::getLockPath() const
|
||||
@ -56,7 +59,10 @@ const std::string & ZooKeeperLock::getLockPath() const
|
||||
void ZooKeeperLock::unlock()
|
||||
{
|
||||
if (!locked)
|
||||
{
|
||||
LOG_TRACE(log, "Lock on path {} for session {} is not locked, exiting", lock_path, zookeeper->getClientID());
|
||||
return;
|
||||
}
|
||||
|
||||
locked = false;
|
||||
|
||||
@ -71,12 +77,19 @@ void ZooKeeperLock::unlock()
|
||||
bool result = zookeeper->exists(lock_path, &stat);
|
||||
|
||||
if (result && stat.ephemeralOwner == zookeeper->getClientID())
|
||||
{
|
||||
zookeeper->remove(lock_path, -1);
|
||||
LOG_TRACE(log, "Lock on path {} for session {} is unlocked", lock_path, zookeeper->getClientID());
|
||||
}
|
||||
else if (result)
|
||||
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Lock is lost, it has another owner. Path: {}, message: {}, owner: {}, our id: {}",
|
||||
lock_path, lock_message, stat.ephemeralOwner, zookeeper->getClientID());
|
||||
else if (throw_if_lost)
|
||||
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Lock is lost, node does not exist. Path: {}, message: {}, our id: {}",
|
||||
lock_path, lock_message, zookeeper->getClientID());
|
||||
else
|
||||
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Lock is lost, node does not exist. Path: {}, message: {}", lock_path, lock_message);
|
||||
LOG_INFO(log, "Lock is lost, node does not exist. Path: {}, message: {}, our id: {}",
|
||||
lock_path, lock_message, zookeeper->getClientID());
|
||||
}
|
||||
|
||||
bool ZooKeeperLock::tryLock()
|
||||
@ -96,9 +109,9 @@ bool ZooKeeperLock::tryLock()
|
||||
}
|
||||
|
||||
std::unique_ptr<ZooKeeperLock> createSimpleZooKeeperLock(
|
||||
const ZooKeeperPtr & zookeeper, const String & lock_prefix, const String & lock_name, const String & lock_message)
|
||||
const ZooKeeperPtr & zookeeper, const String & lock_prefix, const String & lock_name, const String & lock_message, bool throw_if_lost)
|
||||
{
|
||||
return std::make_unique<ZooKeeperLock>(zookeeper, lock_prefix, lock_name, lock_message);
|
||||
return std::make_unique<ZooKeeperLock>(zookeeper, lock_prefix, lock_name, lock_message, throw_if_lost);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -32,7 +32,8 @@ public:
|
||||
const ZooKeeperPtr & zookeeper_,
|
||||
const std::string & lock_prefix_,
|
||||
const std::string & lock_name_,
|
||||
const std::string & lock_message_ = "");
|
||||
const std::string & lock_message_ = "",
|
||||
bool throw_if_lost_ = true);
|
||||
|
||||
~ZooKeeperLock();
|
||||
|
||||
@ -46,12 +47,13 @@ private:
|
||||
|
||||
std::string lock_path;
|
||||
std::string lock_message;
|
||||
bool throw_if_lost{true};
|
||||
LoggerPtr log;
|
||||
bool locked = false;
|
||||
|
||||
};
|
||||
|
||||
std::unique_ptr<ZooKeeperLock> createSimpleZooKeeperLock(
|
||||
const ZooKeeperPtr & zookeeper, const String & lock_prefix, const String & lock_name, const String & lock_message);
|
||||
const ZooKeeperPtr & zookeeper, const String & lock_prefix, const String & lock_name, const String & lock_message, bool throw_if_lost = true);
|
||||
|
||||
}
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#include <Common/COW.h>
|
||||
#include <iostream>
|
||||
#include <base/defines.h>
|
||||
|
||||
|
||||
class IColumn : public COW<IColumn>
|
||||
@ -15,8 +16,6 @@ public:
|
||||
|
||||
virtual int get() const = 0;
|
||||
virtual void set(int value) = 0;
|
||||
|
||||
virtual MutablePtr test() const = 0;
|
||||
};
|
||||
|
||||
using ColumnPtr = IColumn::Ptr;
|
||||
@ -31,58 +30,63 @@ private:
|
||||
explicit ConcreteColumn(int data_) : data(data_) {}
|
||||
ConcreteColumn(const ConcreteColumn &) = default;
|
||||
|
||||
MutableColumnPtr test() const override
|
||||
{
|
||||
MutableColumnPtr res = create(123);
|
||||
return res;
|
||||
}
|
||||
|
||||
public:
|
||||
int get() const override { return data; }
|
||||
void set(int value) override { data = value; }
|
||||
};
|
||||
|
||||
template <typename ColPtr>
|
||||
void print(const ColumnPtr & x, const ColPtr & y)
|
||||
{
|
||||
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
|
||||
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
|
||||
std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n";
|
||||
}
|
||||
|
||||
int main(int, char **)
|
||||
{
|
||||
ColumnPtr x = ConcreteColumn::create(1);
|
||||
ColumnPtr y = x;//x->test();
|
||||
|
||||
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
|
||||
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
|
||||
std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n";
|
||||
ColumnPtr y = x;
|
||||
print(x, y);
|
||||
chassert(x->get() == 1 && y->get() == 1);
|
||||
chassert(x->use_count() == 2 && y->use_count() == 2);
|
||||
chassert(x.get() == y.get());
|
||||
|
||||
{
|
||||
MutableColumnPtr mut = IColumn::mutate(std::move(y));
|
||||
mut->set(2);
|
||||
print(x, mut);
|
||||
chassert(x->get() == 1 && mut->get() == 2);
|
||||
chassert(x->use_count() == 1 && mut->use_count() == 1);
|
||||
chassert(x.get() != mut.get());
|
||||
|
||||
std::cerr << "refcounts: " << x->use_count() << ", " << mut->use_count() << "\n";
|
||||
std::cerr << "addresses: " << x.get() << ", " << mut.get() << "\n";
|
||||
y = std::move(mut);
|
||||
}
|
||||
|
||||
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
|
||||
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
|
||||
std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n";
|
||||
print(x, y);
|
||||
chassert(x->get() == 1 && y->get() == 2);
|
||||
chassert(x->use_count() == 1 && y->use_count() == 1);
|
||||
chassert(x.get() != y.get());
|
||||
|
||||
x = ConcreteColumn::create(0);
|
||||
|
||||
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
|
||||
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
|
||||
std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n";
|
||||
print(x, y);
|
||||
chassert(x->get() == 0 && y->get() == 2);
|
||||
chassert(x->use_count() == 1 && y->use_count() == 1);
|
||||
chassert(x.get() != y.get());
|
||||
|
||||
{
|
||||
MutableColumnPtr mut = IColumn::mutate(std::move(y));
|
||||
mut->set(3);
|
||||
print(x, mut);
|
||||
chassert(x->get() == 0 && mut->get() == 3);
|
||||
chassert(x->use_count() == 1 && mut->use_count() == 1);
|
||||
chassert(x.get() != mut.get());
|
||||
|
||||
std::cerr << "refcounts: " << x->use_count() << ", " << mut->use_count() << "\n";
|
||||
std::cerr << "addresses: " << x.get() << ", " << mut.get() << "\n";
|
||||
y = std::move(mut);
|
||||
}
|
||||
|
||||
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
|
||||
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
|
||||
print(x, y);
|
||||
chassert(x->get() == 0 && y->get() == 3);
|
||||
chassert(x->use_count() == 1 && y->use_count() == 1);
|
||||
chassert(x.get() != y.get());
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#include <Common/COW.h>
|
||||
#include <iostream>
|
||||
#include <base/defines.h>
|
||||
|
||||
|
||||
class IColumn : public COW<IColumn>
|
||||
@ -61,47 +62,58 @@ public:
|
||||
void set(int value) override { wrapped->set(value); }
|
||||
};
|
||||
|
||||
template <typename ColPtr>
|
||||
void print(const ColumnPtr & x, const ColPtr & y)
|
||||
{
|
||||
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
|
||||
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
|
||||
std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n";
|
||||
}
|
||||
|
||||
int main(int, char **)
|
||||
{
|
||||
ColumnPtr x = ColumnComposition::create(1);
|
||||
ColumnPtr y = x;
|
||||
|
||||
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
|
||||
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
|
||||
std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n";
|
||||
print(x, y);
|
||||
chassert(x->get() == 1 && y->get() == 1);
|
||||
chassert(x->use_count() == 2 && y->use_count() == 2);
|
||||
chassert(x.get() == y.get());
|
||||
|
||||
{
|
||||
MutableColumnPtr mut = IColumn::mutate(std::move(y));
|
||||
mut->set(2);
|
||||
print(x, mut);
|
||||
chassert(x->get() == 1 && mut->get() == 2);
|
||||
chassert(x->use_count() == 1 && mut->use_count() == 1);
|
||||
chassert(x.get() != mut.get());
|
||||
|
||||
std::cerr << "refcounts: " << x->use_count() << ", " << mut->use_count() << "\n";
|
||||
std::cerr << "addresses: " << x.get() << ", " << mut.get() << "\n";
|
||||
y = std::move(mut);
|
||||
}
|
||||
|
||||
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
|
||||
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
|
||||
std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n";
|
||||
print(x, y);
|
||||
chassert(x->get() == 1 && y->get() == 2);
|
||||
chassert(x->use_count() == 1 && y->use_count() == 1);
|
||||
chassert(x.get() != y.get());
|
||||
|
||||
x = ColumnComposition::create(0);
|
||||
|
||||
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
|
||||
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
|
||||
std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n";
|
||||
print(x, y);
|
||||
chassert(x->get() == 0 && y->get() == 2);
|
||||
chassert(x->use_count() == 1 && y->use_count() == 1);
|
||||
chassert(x.get() != y.get());
|
||||
|
||||
{
|
||||
MutableColumnPtr mut = IColumn::mutate(std::move(y));
|
||||
mut->set(3);
|
||||
print(x, mut);
|
||||
chassert(x->get() == 0 && mut->get() == 3);
|
||||
chassert(x->use_count() == 1 && mut->use_count() == 1);
|
||||
chassert(x.get() != mut.get());
|
||||
|
||||
std::cerr << "refcounts: " << x->use_count() << ", " << mut->use_count() << "\n";
|
||||
std::cerr << "addresses: " << x.get() << ", " << mut.get() << "\n";
|
||||
y = std::move(mut);
|
||||
}
|
||||
|
||||
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
|
||||
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
|
||||
print(x, y);
|
||||
chassert(x->get() == 0 && y->get() == 3);
|
||||
chassert(x->use_count() == 1 && y->use_count() == 1);
|
||||
chassert(x.get() != y.get());
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
|
||||
/// Macros for convenient usage of Poco logger.
|
||||
#include <unistd.h>
|
||||
#include <fmt/args.h>
|
||||
#include <fmt/format.h>
|
||||
#include <Poco/Logger.h>
|
||||
#include <Poco/Message.h>
|
||||
@ -80,6 +81,7 @@ namespace impl
|
||||
\
|
||||
std::string_view _format_string; \
|
||||
std::string _formatted_message; \
|
||||
std::vector<std::string> _format_string_args; \
|
||||
\
|
||||
if constexpr (LogTypeInfo::is_static) \
|
||||
{ \
|
||||
@ -91,17 +93,17 @@ namespace impl
|
||||
if constexpr (is_preformatted_message) \
|
||||
{ \
|
||||
static_assert(_nargs == 1 || !is_preformatted_message); \
|
||||
ConstexprIfsAreNotIfdefs<is_preformatted_message>::getPreformatted(LOG_IMPL_FIRST_ARG(__VA_ARGS__)).apply(_formatted_message, _format_string); \
|
||||
ConstexprIfsAreNotIfdefs<is_preformatted_message>::getPreformatted(LOG_IMPL_FIRST_ARG(__VA_ARGS__)).apply(_formatted_message, _format_string, _format_string_args); \
|
||||
} \
|
||||
else \
|
||||
{ \
|
||||
_formatted_message = _nargs == 1 ? firstArg(__VA_ARGS__) : fmt::format(__VA_ARGS__); \
|
||||
_formatted_message = _nargs == 1 ? firstArg(__VA_ARGS__) : ConstexprIfsAreNotIfdefs<!is_preformatted_message>::getArgsAndFormat(_format_string_args, __VA_ARGS__); \
|
||||
} \
|
||||
\
|
||||
std::string _file_function = __FILE__ "; "; \
|
||||
_file_function += __PRETTY_FUNCTION__; \
|
||||
Poco::Message _poco_message(_logger->name(), std::move(_formatted_message), \
|
||||
(PRIORITY), _file_function.c_str(), __LINE__, _format_string); \
|
||||
(PRIORITY), _file_function.c_str(), __LINE__, _format_string, _format_string_args); \
|
||||
_channel->log(_poco_message); \
|
||||
} \
|
||||
catch (const Poco::Exception & logger_exception) \
|
||||
|
||||
@ -132,7 +132,9 @@ static PollPidResult pollPid(pid_t pid, int timeout_in_ms)
|
||||
if (kq == -1)
|
||||
return PollPidResult::FAILED;
|
||||
|
||||
struct kevent change = {.ident = 0};
|
||||
struct kevent change;
|
||||
change.ident = 0;
|
||||
|
||||
EV_SET(&change, pid, EVFILT_PROC, EV_ADD, NOTE_EXIT, 0, NULL);
|
||||
|
||||
int event_add_result = HANDLE_EINTR(kevent(kq, &change, 1, NULL, 0, NULL));
|
||||
@ -144,7 +146,9 @@ static PollPidResult pollPid(pid_t pid, int timeout_in_ms)
|
||||
return PollPidResult::FAILED;
|
||||
}
|
||||
|
||||
struct kevent event = {.ident = 0};
|
||||
struct kevent event;
|
||||
event.ident = 0;
|
||||
|
||||
struct timespec remaining_timespec = {.tv_sec = timeout_in_ms / 1000, .tv_nsec = (timeout_in_ms % 1000) * 1000000};
|
||||
int ready = HANDLE_EINTR(kevent(kq, nullptr, 0, &event, 1, &remaining_timespec));
|
||||
PollPidResult result = ready < 0 ? PollPidResult::FAILED : PollPidResult::RESTART;
|
||||
|
||||
@ -95,16 +95,21 @@ static void setReplicatedEngine(ASTCreateQuery * create_query, ContextPtr contex
|
||||
create_query->storage->set(create_query->storage->engine, engine->clone());
|
||||
}
|
||||
|
||||
String DatabaseOrdinary::getConvertToReplicatedFlagPath(const String & name, bool tableStarted)
|
||||
String DatabaseOrdinary::getConvertToReplicatedFlagPath(const String & name, const StoragePolicyPtr storage_policy, bool tableStarted)
|
||||
{
|
||||
fs::path data_path;
|
||||
if (storage_policy->getDisks().empty())
|
||||
data_path = getContext()->getPath();
|
||||
else
|
||||
data_path = storage_policy->getDisks()[0]->getPath();
|
||||
|
||||
if (!tableStarted)
|
||||
{
|
||||
auto create_query = tryGetCreateTableQuery(name, getContext());
|
||||
data_path = fs::path(getContext()->getPath()) / getTableDataPath(create_query->as<ASTCreateQuery &>());
|
||||
data_path = data_path / getTableDataPath(create_query->as<ASTCreateQuery &>());
|
||||
}
|
||||
else
|
||||
data_path = fs::path(getContext()->getPath()) / getTableDataPath(name);
|
||||
data_path = data_path / getTableDataPath(name);
|
||||
|
||||
return (data_path / CONVERT_TO_REPLICATED_FLAG_NAME).string();
|
||||
}
|
||||
@ -120,7 +125,14 @@ void DatabaseOrdinary::convertMergeTreeToReplicatedIfNeeded(ASTPtr ast, const Qu
|
||||
if (!create_query->storage || !create_query->storage->engine->name.ends_with("MergeTree") || create_query->storage->engine->name.starts_with("Replicated") || create_query->storage->engine->name.starts_with("Shared"))
|
||||
return;
|
||||
|
||||
auto convert_to_replicated_flag_path = getConvertToReplicatedFlagPath(qualified_name.table, false);
|
||||
/// Get table's storage policy
|
||||
MergeTreeSettings default_settings = getContext()->getMergeTreeSettings();
|
||||
auto policy = getContext()->getStoragePolicy(default_settings.storage_policy);
|
||||
if (auto * query_settings = create_query->storage->settings)
|
||||
if (Field * policy_setting = query_settings->changes.tryGet("storage_policy"))
|
||||
policy = getContext()->getStoragePolicy(policy_setting->safeGet<String>());
|
||||
|
||||
auto convert_to_replicated_flag_path = getConvertToReplicatedFlagPath(qualified_name.table, policy, false);
|
||||
|
||||
if (!fs::exists(convert_to_replicated_flag_path))
|
||||
return;
|
||||
@ -288,7 +300,7 @@ void DatabaseOrdinary::restoreMetadataAfterConvertingToReplicated(StoragePtr tab
|
||||
if (!rmt)
|
||||
return;
|
||||
|
||||
auto convert_to_replicated_flag_path = getConvertToReplicatedFlagPath(name.table, true);
|
||||
auto convert_to_replicated_flag_path = getConvertToReplicatedFlagPath(name.table, table->getStoragePolicy(), true);
|
||||
if (!fs::exists(convert_to_replicated_flag_path))
|
||||
return;
|
||||
|
||||
|
||||
@ -86,7 +86,7 @@ protected:
|
||||
private:
|
||||
void convertMergeTreeToReplicatedIfNeeded(ASTPtr ast, const QualifiedTableName & qualified_name, const String & file_name);
|
||||
void restoreMetadataAfterConvertingToReplicated(StoragePtr table, const QualifiedTableName & name);
|
||||
String getConvertToReplicatedFlagPath(const String & name, bool tableStarted);
|
||||
String getConvertToReplicatedFlagPath(const String & name, StoragePolicyPtr storage_policy, bool tableStarted);
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@ -1708,9 +1708,18 @@ void registerDatabaseReplicated(DatabaseFactory & factory)
|
||||
String shard_name = safeGetLiteralValue<String>(arguments[1], "Replicated");
|
||||
String replica_name = safeGetLiteralValue<String>(arguments[2], "Replicated");
|
||||
|
||||
zookeeper_path = args.context->getMacros()->expand(zookeeper_path);
|
||||
shard_name = args.context->getMacros()->expand(shard_name);
|
||||
replica_name = args.context->getMacros()->expand(replica_name);
|
||||
/// Expand macros.
|
||||
Macros::MacroExpansionInfo info;
|
||||
info.table_id.database_name = args.database_name;
|
||||
info.table_id.uuid = args.uuid;
|
||||
zookeeper_path = args.context->getMacros()->expand(zookeeper_path, info);
|
||||
|
||||
info.level = 0;
|
||||
info.table_id.uuid = UUIDHelpers::Nil;
|
||||
shard_name = args.context->getMacros()->expand(shard_name, info);
|
||||
|
||||
info.level = 0;
|
||||
replica_name = args.context->getMacros()->expand(replica_name, info);
|
||||
|
||||
DatabaseReplicatedSettings database_replicated_settings{};
|
||||
if (engine_define->settings)
|
||||
|
||||
@ -188,7 +188,8 @@ Azure::Storage::Blobs::BlobClientOptions getAzureBlobClientOptions(const Poco::U
|
||||
retry_options.MaxRetryDelay = std::chrono::milliseconds(config.getUInt(config_prefix + ".retry_max_backoff_ms", 1000));
|
||||
|
||||
using CurlOptions = Azure::Core::Http::CurlTransportOptions;
|
||||
CurlOptions curl_options{.NoSignal = true};
|
||||
CurlOptions curl_options;
|
||||
curl_options.NoSignal = true;
|
||||
|
||||
if (config.has(config_prefix + ".curl_ip_resolve"))
|
||||
{
|
||||
|
||||
@ -21,6 +21,8 @@ struct AzureBlobStorageEndpoint
|
||||
String getEndpoint()
|
||||
{
|
||||
String url = storage_account_url;
|
||||
if (url.ends_with('/'))
|
||||
url.pop_back();
|
||||
|
||||
if (!account_name.empty())
|
||||
url += "/" + account_name;
|
||||
|
||||
@ -3736,6 +3736,7 @@ namespace
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "ClickHouse doesn't support type recursion ({})", field_descriptor->full_name());
|
||||
}
|
||||
pending_resolution.emplace(field_descriptor);
|
||||
SCOPE_EXIT({ pending_resolution.erase(field_descriptor); });
|
||||
|
||||
if (allow_repeat && field_descriptor->is_map())
|
||||
{
|
||||
|
||||
@ -794,7 +794,7 @@ inline bool tryParseImpl<DataTypeIPv6>(DataTypeIPv6::FieldType & x, ReadBuffer &
|
||||
if (isNativeNumber(result_type) && !(result_type.getName() == "IPv4" || result_type.getName() == "IPv6"))
|
||||
message_buf << ". Note: there are to" << result_type.getName() << "OrZero and to" << result_type.getName() << "OrNull functions, which returns zero/NULL instead of throwing exception.";
|
||||
|
||||
throw Exception(PreformattedMessage{message_buf.str(), "Cannot parse string {} as {}: syntax error {}"}, ErrorCodes::CANNOT_PARSE_TEXT);
|
||||
throw Exception(PreformattedMessage{message_buf.str(), "Cannot parse string {} as {}: syntax error {}", {String(read_buffer.buffer().begin(), read_buffer.buffer().size()), result_type.getName()}}, ErrorCodes::CANNOT_PARSE_TEXT);
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -38,8 +38,14 @@ public:
|
||||
: format_name(std::move(format_name_))
|
||||
, arguments_column_names(std::move(arguments_column_names_))
|
||||
, context(std::move(context_))
|
||||
, format_settings(getFormatSettings(context))
|
||||
{
|
||||
FormatFactory::instance().checkFormatName(format_name);
|
||||
|
||||
/// We don't need handling exceptions while formatting as a row.
|
||||
/// But it can be enabled in query sent via http interface.
|
||||
format_settings.json.valid_output_on_exception = false;
|
||||
format_settings.xml.valid_output_on_exception = false;
|
||||
}
|
||||
|
||||
String getName() const override { return name; }
|
||||
@ -68,7 +74,6 @@ public:
|
||||
}
|
||||
|
||||
materializeBlockInplace(arg_columns);
|
||||
auto format_settings = getFormatSettings(context);
|
||||
auto out = FormatFactory::instance().getOutputFormat(format_name, buffer, arg_columns, context, format_settings);
|
||||
|
||||
/// This function make sense only for row output formats.
|
||||
@ -104,6 +109,7 @@ private:
|
||||
String format_name;
|
||||
Names arguments_column_names;
|
||||
ContextPtr context;
|
||||
FormatSettings format_settings;
|
||||
};
|
||||
|
||||
template <bool no_newline>
|
||||
|
||||
@ -82,6 +82,7 @@ void EvictionCandidates::removeQueueEntries(const CachePriorityGuard::Lock & loc
|
||||
auto queue_iterator = candidate->getQueueIterator();
|
||||
queue_iterator->invalidate();
|
||||
|
||||
chassert(candidate->releasable());
|
||||
candidate->file_segment->resetQueueIterator();
|
||||
/// We need to set removed flag in file segment metadata,
|
||||
/// because in dynamic cache resize we first remove queue entries,
|
||||
@ -122,7 +123,13 @@ void EvictionCandidates::evict()
|
||||
while (!key_candidates.candidates.empty())
|
||||
{
|
||||
auto & candidate = key_candidates.candidates.back();
|
||||
chassert(candidate->releasable());
|
||||
if (!candidate->releasable())
|
||||
{
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR,
|
||||
"Eviction candidate is not releasable: {} (evicting or removed flag: {})",
|
||||
candidate->file_segment->getInfoForLog(), candidate->isEvictingOrRemoved(*locked_key));
|
||||
}
|
||||
|
||||
const auto segment = candidate->file_segment;
|
||||
|
||||
IFileCachePriority::IteratorPtr iterator;
|
||||
|
||||
@ -128,6 +128,11 @@ const FileCache::UserInfo & FileCache::getInternalUser()
|
||||
return user;
|
||||
}
|
||||
|
||||
bool FileCache::isInitialized() const
|
||||
{
|
||||
return is_initialized.load(std::memory_order_seq_cst);
|
||||
}
|
||||
|
||||
const String & FileCache::getBasePath() const
|
||||
{
|
||||
return metadata.getBaseDirectory();
|
||||
|
||||
@ -80,6 +80,8 @@ public:
|
||||
|
||||
void initialize();
|
||||
|
||||
bool isInitialized() const;
|
||||
|
||||
const String & getBasePath() const;
|
||||
|
||||
static Key createKeyForPath(const String & path);
|
||||
|
||||
@ -790,6 +790,9 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti
|
||||
auto as_storage_metadata = as_storage->getInMemoryMetadataPtr();
|
||||
properties.columns = as_storage_metadata->getColumns();
|
||||
|
||||
if (!create.comment && !as_storage_metadata->comment.empty())
|
||||
create.set(create.comment, std::make_shared<ASTLiteral>(as_storage_metadata->comment));
|
||||
|
||||
/// Secondary indices and projections make sense only for MergeTree family of storage engines.
|
||||
/// We should not copy them for other storages.
|
||||
if (create.storage && endsWith(create.storage->engine->name, "MergeTree"))
|
||||
|
||||
@ -397,22 +397,31 @@ BlockIO InterpreterSystemQuery::execute()
|
||||
{
|
||||
auto caches = FileCacheFactory::instance().getAll();
|
||||
for (const auto & [_, cache_data] : caches)
|
||||
{
|
||||
if (!cache_data->cache->isInitialized())
|
||||
continue;
|
||||
|
||||
cache_data->cache->removeAllReleasable(user_id);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
auto cache = FileCacheFactory::instance().getByName(query.filesystem_cache_name)->cache;
|
||||
if (query.key_to_drop.empty())
|
||||
|
||||
if (cache->isInitialized())
|
||||
{
|
||||
cache->removeAllReleasable(user_id);
|
||||
}
|
||||
else
|
||||
{
|
||||
auto key = FileCacheKey::fromKeyString(query.key_to_drop);
|
||||
if (query.offset_to_drop.has_value())
|
||||
cache->removeFileSegment(key, query.offset_to_drop.value(), user_id);
|
||||
if (query.key_to_drop.empty())
|
||||
{
|
||||
cache->removeAllReleasable(user_id);
|
||||
}
|
||||
else
|
||||
cache->removeKey(key, user_id);
|
||||
{
|
||||
auto key = FileCacheKey::fromKeyString(query.key_to_drop);
|
||||
if (query.offset_to_drop.has_value())
|
||||
cache->removeFileSegment(key, query.offset_to_drop.value(), user_id);
|
||||
else
|
||||
cache->removeKey(key, user_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
@ -212,6 +212,7 @@ scope_guard MergeTreeTransaction::beforeCommit()
|
||||
|
||||
void MergeTreeTransaction::afterCommit(CSN assigned_csn) noexcept
|
||||
{
|
||||
auto blocker = CannotAllocateThreadFaultInjector::blockFaultInjections();
|
||||
LockMemoryExceptionInThread memory_tracker_lock(VariableContext::Global);
|
||||
/// Write allocated CSN into version metadata, so we will know CSN without reading it from transaction log
|
||||
/// and we will be able to remove old entries from transaction log in ZK.
|
||||
@ -248,6 +249,7 @@ void MergeTreeTransaction::afterCommit(CSN assigned_csn) noexcept
|
||||
|
||||
bool MergeTreeTransaction::rollback() noexcept
|
||||
{
|
||||
auto blocker = CannotAllocateThreadFaultInjector::blockFaultInjections();
|
||||
LockMemoryExceptionInThread memory_tracker_lock(VariableContext::Global);
|
||||
CSN expected = Tx::UnknownCSN;
|
||||
bool need_rollback = csn.compare_exchange_strong(expected, Tx::RolledBackCSN);
|
||||
|
||||
@ -86,6 +86,7 @@ struct QueryLogElement
|
||||
String exception;
|
||||
String stack_trace;
|
||||
std::string_view exception_format_string{};
|
||||
std::vector<std::string> exception_format_string_args{};
|
||||
|
||||
ClientInfo client_info;
|
||||
|
||||
|
||||
@ -53,6 +53,16 @@ ColumnsDescription TextLogElement::getColumnsDescription()
|
||||
{"source_line", std::make_shared<DataTypeUInt64>(), "Source line from which the logging was done."},
|
||||
|
||||
{"message_format_string", std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>()), "A format string that was used to format the message."},
|
||||
{"value1", std::make_shared<DataTypeString>(), "Argument 1 that was used to format the message."},
|
||||
{"value2", std::make_shared<DataTypeString>(), "Argument 2 that was used to format the message."},
|
||||
{"value3", std::make_shared<DataTypeString>(), "Argument 3 that was used to format the message."},
|
||||
{"value4", std::make_shared<DataTypeString>(), "Argument 4 that was used to format the message."},
|
||||
{"value5", std::make_shared<DataTypeString>(), "Argument 5 that was used to format the message."},
|
||||
{"value6", std::make_shared<DataTypeString>(), "Argument 6 that was used to format the message."},
|
||||
{"value7", std::make_shared<DataTypeString>(), "Argument 7 that was used to format the message."},
|
||||
{"value8", std::make_shared<DataTypeString>(), "Argument 8 that was used to format the message."},
|
||||
{"value9", std::make_shared<DataTypeString>(), "Argument 9 that was used to format the message."},
|
||||
{"value10", std::make_shared<DataTypeString>(), "Argument 10 that was used to format the message."},
|
||||
};
|
||||
}
|
||||
|
||||
@ -79,6 +89,16 @@ void TextLogElement::appendToBlock(MutableColumns & columns) const
|
||||
columns[i++]->insert(source_line);
|
||||
|
||||
columns[i++]->insert(message_format_string);
|
||||
columns[i++]->insert(value1);
|
||||
columns[i++]->insert(value2);
|
||||
columns[i++]->insert(value3);
|
||||
columns[i++]->insert(value4);
|
||||
columns[i++]->insert(value5);
|
||||
columns[i++]->insert(value6);
|
||||
columns[i++]->insert(value7);
|
||||
columns[i++]->insert(value8);
|
||||
columns[i++]->insert(value9);
|
||||
columns[i++]->insert(value10);
|
||||
}
|
||||
|
||||
TextLog::TextLog(ContextPtr context_,
|
||||
|
||||
@ -29,6 +29,16 @@ struct TextLogElement
|
||||
UInt64 source_line{};
|
||||
|
||||
std::string_view message_format_string;
|
||||
String value1;
|
||||
String value2;
|
||||
String value3;
|
||||
String value4;
|
||||
String value5;
|
||||
String value6;
|
||||
String value7;
|
||||
String value8;
|
||||
String value9;
|
||||
String value10;
|
||||
|
||||
static std::string name() { return "TextLog"; }
|
||||
static ColumnsDescription getColumnsDescription();
|
||||
|
||||
@ -1249,7 +1249,7 @@ bool TreeRewriterResult::collectUsedColumns(const ASTPtr & query, bool is_select
|
||||
|
||||
if (no_throw)
|
||||
return false;
|
||||
throw Exception(PreformattedMessage{ss.str(), format_string}, ErrorCodes::UNKNOWN_IDENTIFIER);
|
||||
throw Exception(PreformattedMessage{ss.str(), format_string, std::vector<std::string>{}}, ErrorCodes::UNKNOWN_IDENTIFIER);
|
||||
}
|
||||
|
||||
required_source_columns.swap(source_columns);
|
||||
|
||||
@ -200,6 +200,7 @@ static void logException(ContextPtr context, QueryLogElement & elem, bool log_er
|
||||
/// so we pass elem.exception_format_string as format string instead.
|
||||
PreformattedMessage message;
|
||||
message.format_string = elem.exception_format_string;
|
||||
message.format_string_args = elem.exception_format_string_args;
|
||||
|
||||
if (elem.stack_trace.empty() || !log_error)
|
||||
message.text = fmt::format("{} (from {}){} (in query: {})", elem.exception,
|
||||
@ -504,6 +505,7 @@ void logQueryException(
|
||||
auto exception_message = getCurrentExceptionMessageAndPattern(/* with_stacktrace */ false);
|
||||
elem.exception = std::move(exception_message.text);
|
||||
elem.exception_format_string = exception_message.format_string;
|
||||
elem.exception_format_string_args = exception_message.format_string_args;
|
||||
|
||||
QueryStatusPtr process_list_elem = context->getProcessListElement();
|
||||
|
||||
@ -597,6 +599,7 @@ void logExceptionBeforeStart(
|
||||
auto exception_message = getCurrentExceptionMessageAndPattern(/* with_stacktrace */ false);
|
||||
elem.exception = std::move(exception_message.text);
|
||||
elem.exception_format_string = exception_message.format_string;
|
||||
elem.exception_format_string_args = exception_message.format_string_args;
|
||||
|
||||
elem.client_info = context->getClientInfo();
|
||||
|
||||
|
||||
@ -131,6 +131,21 @@ void OwnSplitChannel::logSplit(const Poco::Message & msg)
|
||||
elem.source_line = msg.getSourceLine();
|
||||
elem.message_format_string = msg.getFormatString();
|
||||
|
||||
#define SET_VALUE_IF_EXISTS(INDEX) if ((INDEX) <= msg.getFormatStringArgs().size()) (elem.value##INDEX) = msg.getFormatStringArgs()[(INDEX) - 1]
|
||||
|
||||
SET_VALUE_IF_EXISTS(1);
|
||||
SET_VALUE_IF_EXISTS(2);
|
||||
SET_VALUE_IF_EXISTS(3);
|
||||
SET_VALUE_IF_EXISTS(4);
|
||||
SET_VALUE_IF_EXISTS(5);
|
||||
SET_VALUE_IF_EXISTS(6);
|
||||
SET_VALUE_IF_EXISTS(7);
|
||||
SET_VALUE_IF_EXISTS(8);
|
||||
SET_VALUE_IF_EXISTS(9);
|
||||
SET_VALUE_IF_EXISTS(10);
|
||||
|
||||
#undef SET_VALUE_IF_EXISTS
|
||||
|
||||
std::shared_ptr<SystemLogQueue<TextLogElement>> text_log_locked{};
|
||||
text_log_locked = text_log.lock();
|
||||
if (text_log_locked)
|
||||
|
||||
@ -144,6 +144,12 @@ void ASTSelectQuery::formatImpl(const FormatSettings & s, FormatState & state, F
|
||||
window()->as<ASTExpressionList &>().formatImplMultiline(s, state, frame);
|
||||
}
|
||||
|
||||
if (qualify())
|
||||
{
|
||||
s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "QUALIFY " << (s.hilite ? hilite_none : "");
|
||||
qualify()->formatImpl(s, state, frame);
|
||||
}
|
||||
|
||||
if (!order_by_all && orderBy())
|
||||
{
|
||||
s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "ORDER BY" << (s.hilite ? hilite_none : "");
|
||||
|
||||
@ -25,6 +25,7 @@ public:
|
||||
GROUP_BY,
|
||||
HAVING,
|
||||
WINDOW,
|
||||
QUALIFY,
|
||||
ORDER_BY,
|
||||
LIMIT_BY_OFFSET,
|
||||
LIMIT_BY_LENGTH,
|
||||
@ -55,6 +56,8 @@ public:
|
||||
return "HAVING";
|
||||
case Expression::WINDOW:
|
||||
return "WINDOW";
|
||||
case Expression::QUALIFY:
|
||||
return "QUALIFY";
|
||||
case Expression::ORDER_BY:
|
||||
return "ORDER BY";
|
||||
case Expression::LIMIT_BY_OFFSET:
|
||||
@ -95,6 +98,7 @@ public:
|
||||
ASTPtr & refPrewhere() { return getExpression(Expression::PREWHERE); }
|
||||
ASTPtr & refWhere() { return getExpression(Expression::WHERE); }
|
||||
ASTPtr & refHaving() { return getExpression(Expression::HAVING); }
|
||||
ASTPtr & refQualify() { return getExpression(Expression::QUALIFY); }
|
||||
|
||||
ASTPtr with() const { return getExpression(Expression::WITH); }
|
||||
ASTPtr select() const { return getExpression(Expression::SELECT); }
|
||||
@ -104,6 +108,7 @@ public:
|
||||
ASTPtr groupBy() const { return getExpression(Expression::GROUP_BY); }
|
||||
ASTPtr having() const { return getExpression(Expression::HAVING); }
|
||||
ASTPtr window() const { return getExpression(Expression::WINDOW); }
|
||||
ASTPtr qualify() const { return getExpression(Expression::QUALIFY); }
|
||||
ASTPtr orderBy() const { return getExpression(Expression::ORDER_BY); }
|
||||
ASTPtr limitByOffset() const { return getExpression(Expression::LIMIT_BY_OFFSET); }
|
||||
ASTPtr limitByLength() const { return getExpression(Expression::LIMIT_BY_LENGTH); }
|
||||
@ -113,7 +118,7 @@ public:
|
||||
ASTPtr settings() const { return getExpression(Expression::SETTINGS); }
|
||||
ASTPtr interpolate() const { return getExpression(Expression::INTERPOLATE); }
|
||||
|
||||
bool hasFiltration() const { return where() || prewhere() || having(); }
|
||||
bool hasFiltration() const { return where() || prewhere() || having() || qualify(); }
|
||||
|
||||
/// Set/Reset/Remove expression.
|
||||
void setExpression(Expression expr, ASTPtr && ast);
|
||||
|
||||
@ -507,6 +507,7 @@ namespace DB
|
||||
MR_MACROS(WHEN, "WHEN") \
|
||||
MR_MACROS(WHERE, "WHERE") \
|
||||
MR_MACROS(WINDOW, "WINDOW") \
|
||||
MR_MACROS(QUALIFY, "QUALIFY") \
|
||||
MR_MACROS(WITH_ADMIN_OPTION, "WITH ADMIN OPTION") \
|
||||
MR_MACROS(WITH_CHECK, "WITH CHECK") \
|
||||
MR_MACROS(WITH_FILL, "WITH FILL") \
|
||||
|
||||
@ -1481,6 +1481,7 @@ const char * ParserAlias::restricted_keywords[] =
|
||||
"USING",
|
||||
"WHERE",
|
||||
"WINDOW",
|
||||
"QUALIFY",
|
||||
"WITH",
|
||||
"INTERSECT",
|
||||
"EXCEPT",
|
||||
|
||||
@ -49,6 +49,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
ParserKeyword s_totals(Keyword::TOTALS);
|
||||
ParserKeyword s_having(Keyword::HAVING);
|
||||
ParserKeyword s_window(Keyword::WINDOW);
|
||||
ParserKeyword s_qualify(Keyword::QUALIFY);
|
||||
ParserKeyword s_order_by(Keyword::ORDER_BY);
|
||||
ParserKeyword s_limit(Keyword::LIMIT);
|
||||
ParserKeyword s_settings(Keyword::SETTINGS);
|
||||
@ -86,6 +87,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
ASTPtr group_expression_list;
|
||||
ASTPtr having_expression;
|
||||
ASTPtr window_list;
|
||||
ASTPtr qualify_expression;
|
||||
ASTPtr order_expression_list;
|
||||
ASTPtr interpolate_expression_list;
|
||||
ASTPtr limit_by_length;
|
||||
@ -266,6 +268,13 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
}
|
||||
}
|
||||
|
||||
/// QUALIFY expr
|
||||
if (s_qualify.ignore(pos, expected))
|
||||
{
|
||||
if (!exp_elem.parse(pos, qualify_expression, expected))
|
||||
return false;
|
||||
}
|
||||
|
||||
/// ORDER BY expr ASC|DESC COLLATE 'locale' list
|
||||
if (s_order_by.ignore(pos, expected))
|
||||
{
|
||||
@ -489,6 +498,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
select_query->setExpression(ASTSelectQuery::Expression::GROUP_BY, std::move(group_expression_list));
|
||||
select_query->setExpression(ASTSelectQuery::Expression::HAVING, std::move(having_expression));
|
||||
select_query->setExpression(ASTSelectQuery::Expression::WINDOW, std::move(window_list));
|
||||
select_query->setExpression(ASTSelectQuery::Expression::QUALIFY, std::move(qualify_expression));
|
||||
select_query->setExpression(ASTSelectQuery::Expression::ORDER_BY, std::move(order_expression_list));
|
||||
select_query->setExpression(ASTSelectQuery::Expression::LIMIT_BY_OFFSET, std::move(limit_by_offset));
|
||||
select_query->setExpression(ASTSelectQuery::Expression::LIMIT_BY_LENGTH, std::move(limit_by_length));
|
||||
|
||||
@ -1367,6 +1367,16 @@ void Planner::buildPlanForQueryNode()
|
||||
select_query_info.has_aggregates = hasAggregateFunctionNodes(query_tree);
|
||||
select_query_info.need_aggregate = query_node.hasGroupBy() || select_query_info.has_aggregates;
|
||||
|
||||
if (!select_query_info.has_window && query_node.hasQualify())
|
||||
{
|
||||
if (query_node.hasHaving())
|
||||
query_node.getHaving() = mergeConditionNodes({query_node.getHaving(), query_node.getQualify()}, query_context);
|
||||
else
|
||||
query_node.getHaving() = query_node.getQualify();
|
||||
|
||||
query_node.getQualify() = {};
|
||||
}
|
||||
|
||||
if (!select_query_info.need_aggregate && query_node.hasHaving())
|
||||
{
|
||||
if (query_node.hasWhere())
|
||||
@ -1636,6 +1646,9 @@ void Planner::buildPlanForQueryNode()
|
||||
addWindowSteps(query_plan, planner_context, window_analysis_result);
|
||||
}
|
||||
|
||||
if (expression_analysis_result.hasQualify())
|
||||
addFilterStep(query_plan, expression_analysis_result.getQualify(), "QUALIFY", result_actions_to_execute);
|
||||
|
||||
const auto & projection_analysis_result = expression_analysis_result.getProjection();
|
||||
addExpressionStep(query_plan, projection_analysis_result.projection_actions, "Projection", result_actions_to_execute);
|
||||
|
||||
|
||||
@ -513,6 +513,16 @@ PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(const QueryTreeNo
|
||||
if (window_analysis_result_optional)
|
||||
current_output_columns = actions_chain.getLastStepAvailableOutputColumns();
|
||||
|
||||
std::optional<FilterAnalysisResult> qualify_analysis_result_optional;
|
||||
std::optional<size_t> qualify_action_step_index_optional;
|
||||
|
||||
if (query_node.hasQualify())
|
||||
{
|
||||
qualify_analysis_result_optional = analyzeFilter(query_node.getQualify(), current_output_columns, planner_context, actions_chain);
|
||||
qualify_action_step_index_optional = actions_chain.getLastStepIndex();
|
||||
current_output_columns = actions_chain.getLastStepAvailableOutputColumns();
|
||||
}
|
||||
|
||||
auto projection_analysis_result = analyzeProjection(query_node, current_output_columns, planner_context, actions_chain);
|
||||
current_output_columns = actions_chain.getLastStepAvailableOutputColumns();
|
||||
|
||||
@ -604,7 +614,7 @@ PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(const QueryTreeNo
|
||||
|
||||
PlannerExpressionsAnalysisResult expressions_analysis_result(std::move(projection_analysis_result));
|
||||
|
||||
if (where_action_step_index_optional && where_analysis_result_optional)
|
||||
if (where_analysis_result_optional && where_action_step_index_optional)
|
||||
{
|
||||
auto & where_analysis_result = *where_analysis_result_optional;
|
||||
auto & where_actions_chain_node = actions_chain.at(*where_action_step_index_optional);
|
||||
@ -615,7 +625,7 @@ PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(const QueryTreeNo
|
||||
if (aggregation_analysis_result_optional)
|
||||
expressions_analysis_result.addAggregation(std::move(*aggregation_analysis_result_optional));
|
||||
|
||||
if (having_action_step_index_optional && having_analysis_result_optional)
|
||||
if (having_analysis_result_optional && having_action_step_index_optional)
|
||||
{
|
||||
auto & having_analysis_result = *having_analysis_result_optional;
|
||||
auto & having_actions_chain_node = actions_chain.at(*having_action_step_index_optional);
|
||||
@ -626,6 +636,14 @@ PlannerExpressionsAnalysisResult buildExpressionAnalysisResult(const QueryTreeNo
|
||||
if (window_analysis_result_optional)
|
||||
expressions_analysis_result.addWindow(std::move(*window_analysis_result_optional));
|
||||
|
||||
if (qualify_analysis_result_optional && qualify_action_step_index_optional)
|
||||
{
|
||||
auto & qualify_analysis_result = *qualify_analysis_result_optional;
|
||||
auto & qualify_actions_chain_node = actions_chain.at(*qualify_action_step_index_optional);
|
||||
qualify_analysis_result.remove_filter_column = !qualify_actions_chain_node->getChildRequiredOutputColumnsNames().contains(qualify_analysis_result.filter_column_name);
|
||||
expressions_analysis_result.addQualify(std::move(qualify_analysis_result));
|
||||
}
|
||||
|
||||
if (sort_analysis_result_optional)
|
||||
expressions_analysis_result.addSort(std::move(*sort_analysis_result_optional));
|
||||
|
||||
|
||||
@ -129,6 +129,21 @@ public:
|
||||
window_analysis_result = std::move(window_analysis_result_);
|
||||
}
|
||||
|
||||
bool hasQualify() const
|
||||
{
|
||||
return qualify_analysis_result.filter_actions != nullptr;
|
||||
}
|
||||
|
||||
const FilterAnalysisResult & getQualify() const
|
||||
{
|
||||
return qualify_analysis_result;
|
||||
}
|
||||
|
||||
void addQualify(FilterAnalysisResult qualify_analysis_result_)
|
||||
{
|
||||
qualify_analysis_result = std::move(qualify_analysis_result_);
|
||||
}
|
||||
|
||||
bool hasSort() const
|
||||
{
|
||||
return sort_analysis_result.before_order_by_actions != nullptr;
|
||||
@ -165,6 +180,7 @@ private:
|
||||
AggregationAnalysisResult aggregation_analysis_result;
|
||||
FilterAnalysisResult having_analysis_result;
|
||||
WindowAnalysisResult window_analysis_result;
|
||||
FilterAnalysisResult qualify_analysis_result;
|
||||
SortAnalysisResult sort_analysis_result;
|
||||
LimitByAnalysisResult limit_by_analysis_result;
|
||||
};
|
||||
|
||||
@ -125,9 +125,8 @@ TableExpressionSet extractTableExpressionsSet(const QueryTreeNodePtr & node)
|
||||
return res;
|
||||
}
|
||||
|
||||
std::optional<JoinTableSide> extractJoinTableSideFromExpression(//const ActionsDAG::Node * expression_root_node,
|
||||
std::optional<JoinTableSide> extractJoinTableSideFromExpression(
|
||||
const IQueryTreeNode * expression_root_node,
|
||||
//const std::unordered_set<const ActionsDAG::Node *> & join_expression_dag_input_nodes,
|
||||
const TableExpressionSet & left_table_expressions,
|
||||
const TableExpressionSet & right_table_expressions,
|
||||
const JoinNode & join_node)
|
||||
@ -136,30 +135,11 @@ std::optional<JoinTableSide> extractJoinTableSideFromExpression(//const ActionsD
|
||||
std::vector<const IQueryTreeNode *> nodes_to_process;
|
||||
nodes_to_process.push_back(expression_root_node);
|
||||
|
||||
// std::cerr << "==== extractJoinTableSideFromExpression\n";
|
||||
// std::cerr << "inp nodes" << std::endl;
|
||||
// for (const auto * node : join_expression_dag_input_nodes)
|
||||
// std::cerr << reinterpret_cast<const void *>(node) << ' ' << node->result_name << std::endl;
|
||||
|
||||
|
||||
// std::cerr << "l names" << std::endl;
|
||||
// for (const auto & l : left_table_expression_columns_names)
|
||||
// std::cerr << l << std::endl;
|
||||
|
||||
// std::cerr << "r names" << std::endl;
|
||||
// for (const auto & r : right_table_expression_columns_names)
|
||||
// std::cerr << r << std::endl;
|
||||
|
||||
// const auto * left_table_expr = join_node.getLeftTableExpression().get();
|
||||
// const auto * right_table_expr = join_node.getRightTableExpression().get();
|
||||
|
||||
while (!nodes_to_process.empty())
|
||||
{
|
||||
const auto * node_to_process = nodes_to_process.back();
|
||||
nodes_to_process.pop_back();
|
||||
|
||||
//std::cerr << "... " << reinterpret_cast<const void *>(node_to_process) << ' ' << node_to_process->result_name << std::endl;
|
||||
|
||||
if (const auto * function_node = node_to_process->as<FunctionNode>())
|
||||
{
|
||||
for (const auto & child : function_node->getArguments())
|
||||
@ -172,22 +152,7 @@ std::optional<JoinTableSide> extractJoinTableSideFromExpression(//const ActionsD
|
||||
if (!column_node)
|
||||
continue;
|
||||
|
||||
// if (!join_expression_dag_input_nodes.contains(node_to_process))
|
||||
// continue;
|
||||
|
||||
const auto & input_name = column_node->getColumnName();
|
||||
|
||||
// bool left_table_expression_contains_input = left_table_expression_columns_names.contains(input_name);
|
||||
// bool right_table_expression_contains_input = right_table_expression_columns_names.contains(input_name);
|
||||
|
||||
// if (!left_table_expression_contains_input && !right_table_expression_contains_input)
|
||||
// throw Exception(ErrorCodes::INVALID_JOIN_ON_EXPRESSION,
|
||||
// "JOIN {} actions has column {} that do not exist in left {} or right {} table expression columns",
|
||||
// join_node.formatASTForErrorMessage(),
|
||||
// input_name,
|
||||
// boost::join(left_table_expression_columns_names, ", "),
|
||||
// boost::join(right_table_expression_columns_names, ", "));
|
||||
|
||||
const auto * column_source = column_node->getColumnSource().get();
|
||||
if (!column_source)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "No source for column {} in JOIN {}", input_name, join_node.formatASTForErrorMessage());
|
||||
@ -235,9 +200,6 @@ void buildJoinClause(
|
||||
ActionsDAGPtr & left_dag,
|
||||
ActionsDAGPtr & right_dag,
|
||||
const PlannerContextPtr & planner_context,
|
||||
//ActionsDAGPtr join_expression_dag,
|
||||
//const std::unordered_set<const ActionsDAG::Node *> & join_expression_dag_input_nodes,
|
||||
//const ActionsDAG::Node * join_expressions_actions_node,
|
||||
const QueryTreeNodePtr & join_expression,
|
||||
const TableExpressionSet & left_table_expressions,
|
||||
const TableExpressionSet & right_table_expressions,
|
||||
@ -245,22 +207,16 @@ void buildJoinClause(
|
||||
JoinClause & join_clause)
|
||||
{
|
||||
std::string function_name;
|
||||
|
||||
//std::cerr << join_expression_dag->dumpDAG() << std::endl;
|
||||
auto * function_node = join_expression->as<FunctionNode>();
|
||||
if (function_node)
|
||||
function_name = function_node->getFunction()->getName();
|
||||
|
||||
// if (join_expressions_actions_node->function)
|
||||
// function_name = join_expressions_actions_node->function->getName();
|
||||
|
||||
/// For 'and' function go into children
|
||||
if (function_name == "and")
|
||||
{
|
||||
for (const auto & child : function_node->getArguments())
|
||||
{
|
||||
buildJoinClause(//join_expression_dag,
|
||||
//join_expression_dag_input_nodes,
|
||||
buildJoinClause(
|
||||
left_dag,
|
||||
right_dag,
|
||||
planner_context,
|
||||
@ -279,17 +235,15 @@ void buildJoinClause(
|
||||
|
||||
if (function_name == "equals" || function_name == "isNotDistinctFrom" || is_asof_join_inequality)
|
||||
{
|
||||
const auto left_child = function_node->getArguments().getNodes().at(0);//join_expressions_actions_node->children.at(0);
|
||||
const auto right_child = function_node->getArguments().getNodes().at(1); //join_expressions_actions_node->children.at(1);
|
||||
const auto left_child = function_node->getArguments().getNodes().at(0);
|
||||
const auto right_child = function_node->getArguments().getNodes().at(1);
|
||||
|
||||
auto left_expression_side_optional = extractJoinTableSideFromExpression(left_child.get(),
|
||||
//join_expression_dag_input_nodes,
|
||||
left_table_expressions,
|
||||
right_table_expressions,
|
||||
join_node);
|
||||
|
||||
auto right_expression_side_optional = extractJoinTableSideFromExpression(right_child.get(),
|
||||
//join_expression_dag_input_nodes,
|
||||
left_table_expressions,
|
||||
right_table_expressions,
|
||||
join_node);
|
||||
@ -314,7 +268,6 @@ void buildJoinClause(
|
||||
}
|
||||
else
|
||||
{
|
||||
// std::cerr << "===============\n";
|
||||
auto left_expression_side = *left_expression_side_optional;
|
||||
auto right_expression_side = *right_expression_side_optional;
|
||||
|
||||
@ -361,8 +314,7 @@ void buildJoinClause(
|
||||
return;
|
||||
}
|
||||
|
||||
auto expression_side_optional = extractJoinTableSideFromExpression(//join_expressions_actions_node,
|
||||
//join_expression_dag_input_nodes,
|
||||
auto expression_side_optional = extractJoinTableSideFromExpression(
|
||||
join_expression.get(),
|
||||
left_table_expressions,
|
||||
right_table_expressions,
|
||||
@ -377,32 +329,15 @@ void buildJoinClause(
|
||||
join_clause.addCondition(expression_side, node);
|
||||
}
|
||||
|
||||
JoinClausesAndActions buildJoinClausesAndActions(//const ColumnsWithTypeAndName & join_expression_input_columns,
|
||||
JoinClausesAndActions buildJoinClausesAndActions(
|
||||
const ColumnsWithTypeAndName & left_table_expression_columns,
|
||||
const ColumnsWithTypeAndName & right_table_expression_columns,
|
||||
const JoinNode & join_node,
|
||||
const PlannerContextPtr & planner_context)
|
||||
{
|
||||
//ActionsDAGPtr join_expression_actions = std::make_shared<ActionsDAG>(join_expression_input_columns);
|
||||
|
||||
ActionsDAGPtr left_join_actions = std::make_shared<ActionsDAG>(left_table_expression_columns);
|
||||
ActionsDAGPtr right_join_actions = std::make_shared<ActionsDAG>(right_table_expression_columns);
|
||||
|
||||
// LOG_TRACE(getLogger("Planner"), "buildJoinClausesAndActions cols {} ", left_join_actions->dumpDAG());
|
||||
// LOG_TRACE(getLogger("Planner"), "buildJoinClausesAndActions cols {} ", right_join_actions->dumpDAG());
|
||||
|
||||
/** In ActionsDAG if input node has constant representation additional constant column is added.
|
||||
* That way we cannot simply check that node has INPUT type during resolution of expression join table side.
|
||||
* Put all nodes after actions dag initialization in set.
|
||||
* To check if actions dag node is input column, we check if set contains it.
|
||||
*/
|
||||
// const auto & join_expression_actions_nodes = join_expression_actions->getNodes();
|
||||
|
||||
// std::unordered_set<const ActionsDAG::Node *> join_expression_dag_input_nodes;
|
||||
// join_expression_dag_input_nodes.reserve(join_expression_actions_nodes.size());
|
||||
// for (const auto & node : join_expression_actions_nodes)
|
||||
// join_expression_dag_input_nodes.insert(&node);
|
||||
|
||||
/** It is possible to have constant value in JOIN ON section, that we need to ignore during DAG construction.
|
||||
* If we do not ignore it, this function will be replaced by underlying constant.
|
||||
* For example ASOF JOIN does not support JOIN with constants, and we should process it like ordinary JOIN.
|
||||
@ -411,9 +346,6 @@ JoinClausesAndActions buildJoinClausesAndActions(//const ColumnsWithTypeAndName
|
||||
* ON (t1.id = t2.id) AND 1 != 1 AND (t1.value >= t1.value);
|
||||
*/
|
||||
auto join_expression = join_node.getJoinExpression();
|
||||
// LOG_TRACE(getLogger("Planner"), "buildJoinClausesAndActions expr {} ", join_expression->formatConvertedASTForErrorMessage());
|
||||
// LOG_TRACE(getLogger("Planner"), "buildJoinClausesAndActions expr {} ", join_expression->dumpTree());
|
||||
|
||||
auto * constant_join_expression = join_expression->as<ConstantNode>();
|
||||
|
||||
if (constant_join_expression && constant_join_expression->hasSourceExpression())
|
||||
@ -425,19 +357,6 @@ JoinClausesAndActions buildJoinClausesAndActions(//const ColumnsWithTypeAndName
|
||||
"JOIN {} join expression expected function",
|
||||
join_node.formatASTForErrorMessage());
|
||||
|
||||
// PlannerActionsVisitor join_expression_visitor(planner_context);
|
||||
// auto join_expression_dag_node_raw_pointers = join_expression_visitor.visit(join_expression_actions, join_expression);
|
||||
// if (join_expression_dag_node_raw_pointers.size() != 1)
|
||||
// throw Exception(ErrorCodes::LOGICAL_ERROR,
|
||||
// "JOIN {} ON clause contains multiple expressions",
|
||||
// join_node.formatASTForErrorMessage());
|
||||
|
||||
// const auto * join_expressions_actions_root_node = join_expression_dag_node_raw_pointers[0];
|
||||
// if (!join_expressions_actions_root_node->function)
|
||||
// throw Exception(ErrorCodes::INVALID_JOIN_ON_EXPRESSION,
|
||||
// "JOIN {} join expression expected function",
|
||||
// join_node.formatASTForErrorMessage());
|
||||
|
||||
size_t left_table_expression_columns_size = left_table_expression_columns.size();
|
||||
|
||||
Names join_left_actions_names;
|
||||
@ -470,7 +389,6 @@ JoinClausesAndActions buildJoinClausesAndActions(//const ColumnsWithTypeAndName
|
||||
auto join_right_table_expressions = extractTableExpressionsSet(join_node.getRightTableExpression());
|
||||
|
||||
JoinClausesAndActions result;
|
||||
//result.join_expression_actions = join_expression_actions;
|
||||
|
||||
const auto & function_name = function_node->getFunction()->getName();
|
||||
if (function_name == "or")
|
||||
@ -479,8 +397,7 @@ JoinClausesAndActions buildJoinClausesAndActions(//const ColumnsWithTypeAndName
|
||||
{
|
||||
result.join_clauses.emplace_back();
|
||||
|
||||
buildJoinClause(//join_expression_actions,
|
||||
//join_expression_dag_input_nodes,
|
||||
buildJoinClause(
|
||||
left_join_actions,
|
||||
right_join_actions,
|
||||
planner_context,
|
||||
@ -499,9 +416,7 @@ JoinClausesAndActions buildJoinClausesAndActions(//const ColumnsWithTypeAndName
|
||||
left_join_actions,
|
||||
right_join_actions,
|
||||
planner_context,
|
||||
//join_expression_actions,
|
||||
//join_expression_dag_input_nodes,
|
||||
join_expression, //join_expressions_actions_root_node,
|
||||
join_expression,
|
||||
join_left_table_expressions,
|
||||
join_right_table_expressions,
|
||||
join_node,
|
||||
@ -621,12 +536,6 @@ JoinClausesAndActions buildJoinClausesAndActions(//const ColumnsWithTypeAndName
|
||||
result.left_join_expressions_actions = left_join_actions->clone();
|
||||
result.left_join_tmp_expression_actions = std::move(left_join_actions);
|
||||
result.left_join_expressions_actions->removeUnusedActions(join_left_actions_names);
|
||||
|
||||
// for (const auto & name : join_right_actions_names)
|
||||
// std::cerr << ".. " << name << std::endl;
|
||||
|
||||
// std::cerr << right_join_actions->dumpDAG() << std::endl;
|
||||
|
||||
result.right_join_expressions_actions = right_join_actions->clone();
|
||||
result.right_join_tmp_expression_actions = std::move(right_join_actions);
|
||||
result.right_join_expressions_actions->removeUnusedActions(join_right_actions_names);
|
||||
@ -648,10 +557,7 @@ JoinClausesAndActions buildJoinClausesAndActions(
|
||||
"JOIN {} join does not have ON section",
|
||||
join_node_typed.formatASTForErrorMessage());
|
||||
|
||||
// auto join_expression_input_columns = left_table_expression_columns;
|
||||
// join_expression_input_columns.insert(join_expression_input_columns.end(), right_table_expression_columns.begin(), right_table_expression_columns.end());
|
||||
|
||||
return buildJoinClausesAndActions(/*join_expression_input_columns,*/ left_table_expression_columns, right_table_expression_columns, join_node_typed, planner_context);
|
||||
return buildJoinClausesAndActions(left_table_expression_columns, right_table_expression_columns, join_node_typed, planner_context);
|
||||
}
|
||||
|
||||
std::optional<bool> tryExtractConstantFromJoinNode(const QueryTreeNodePtr & join_node)
|
||||
|
||||
@ -987,7 +987,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder(
|
||||
|
||||
/// We take full part if it contains enough marks or
|
||||
/// if we know limit and part contains less than 'limit' rows.
|
||||
bool take_full_part = marks_in_part <= need_marks || (input_order_info->limit && input_order_info->limit < part.getRowsCount());
|
||||
bool take_full_part = marks_in_part <= need_marks || (input_order_info->limit && input_order_info->limit > part.getRowsCount());
|
||||
|
||||
/// We take the whole part if it is small enough.
|
||||
if (take_full_part)
|
||||
@ -1792,6 +1792,11 @@ bool ReadFromMergeTree::requestOutputEachPartitionThroughSeparatePort()
|
||||
if (isQueryWithFinal())
|
||||
return false;
|
||||
|
||||
/// With parallel replicas we have to have only a single instance of `MergeTreeReadPoolParallelReplicas` per replica.
|
||||
/// With aggregation-by-partitions optimisation we might create a separate pool for each partition.
|
||||
if (is_parallel_reading_from_replicas)
|
||||
return false;
|
||||
|
||||
const auto & settings = context->getSettingsRef();
|
||||
|
||||
const auto partitions_cnt = countPartitions(prepared_parts);
|
||||
|
||||
@ -915,11 +915,22 @@ void StorageKafka::updateGlobalConfiguration(cppkafka::Configuration & kafka_con
|
||||
#endif // USE_KRB5
|
||||
|
||||
// No need to add any prefix, messages can be distinguished
|
||||
kafka_config.set_log_callback([this](cppkafka::KafkaHandleBase &, int level, const std::string & facility, const std::string & message)
|
||||
{
|
||||
auto [poco_level, client_logs_level] = parseSyslogLevel(level);
|
||||
LOG_IMPL(log, client_logs_level, poco_level, "[rdk:{}] {}", facility, message);
|
||||
});
|
||||
kafka_config.set_log_callback(
|
||||
[this](cppkafka::KafkaHandleBase & handle, int level, const std::string & facility, const std::string & message)
|
||||
{
|
||||
auto [poco_level, client_logs_level] = parseSyslogLevel(level);
|
||||
const auto & kafka_object_config = handle.get_configuration();
|
||||
const std::string client_id_key{"client.id"};
|
||||
chassert(kafka_object_config.has_property(client_id_key) && "Kafka configuration doesn't have expected client.id set");
|
||||
LOG_IMPL(
|
||||
log,
|
||||
client_logs_level,
|
||||
poco_level,
|
||||
"[client.id:{}] [rdk:{}] {}",
|
||||
kafka_object_config.get(client_id_key),
|
||||
facility,
|
||||
message);
|
||||
});
|
||||
|
||||
/// NOTE: statistics should be consumed, otherwise it creates too much
|
||||
/// entries in the queue, that leads to memory leak and slow shutdown.
|
||||
|
||||
@ -2250,9 +2250,11 @@ static BoolMask forAnyHyperrectangle(
|
||||
if (left_bounded && right_bounded)
|
||||
hyperrectangle[prefix_size] = Range(left_keys[prefix_size], true, right_keys[prefix_size], true);
|
||||
else if (left_bounded)
|
||||
hyperrectangle[prefix_size] = Range::createLeftBounded(left_keys[prefix_size], true, data_types[prefix_size]->isNullable());
|
||||
hyperrectangle[prefix_size]
|
||||
= Range::createLeftBounded(left_keys[prefix_size], true, isNullableOrLowCardinalityNullable(data_types[prefix_size]));
|
||||
else if (right_bounded)
|
||||
hyperrectangle[prefix_size] = Range::createRightBounded(right_keys[prefix_size], true, data_types[prefix_size]->isNullable());
|
||||
hyperrectangle[prefix_size]
|
||||
= Range::createRightBounded(right_keys[prefix_size], true, isNullableOrLowCardinalityNullable(data_types[prefix_size]));
|
||||
|
||||
return callback(hyperrectangle);
|
||||
}
|
||||
@ -2262,13 +2264,15 @@ static BoolMask forAnyHyperrectangle(
|
||||
if (left_bounded && right_bounded)
|
||||
hyperrectangle[prefix_size] = Range(left_keys[prefix_size], false, right_keys[prefix_size], false);
|
||||
else if (left_bounded)
|
||||
hyperrectangle[prefix_size] = Range::createLeftBounded(left_keys[prefix_size], false, data_types[prefix_size]->isNullable());
|
||||
hyperrectangle[prefix_size]
|
||||
= Range::createLeftBounded(left_keys[prefix_size], false, isNullableOrLowCardinalityNullable(data_types[prefix_size]));
|
||||
else if (right_bounded)
|
||||
hyperrectangle[prefix_size] = Range::createRightBounded(right_keys[prefix_size], false, data_types[prefix_size]->isNullable());
|
||||
hyperrectangle[prefix_size]
|
||||
= Range::createRightBounded(right_keys[prefix_size], false, isNullableOrLowCardinalityNullable(data_types[prefix_size]));
|
||||
|
||||
for (size_t i = prefix_size + 1; i < key_size; ++i)
|
||||
{
|
||||
if (data_types[i]->isNullable())
|
||||
if (isNullableOrLowCardinalityNullable(data_types[i]))
|
||||
hyperrectangle[i] = Range::createWholeUniverse();
|
||||
else
|
||||
hyperrectangle[i] = Range::createWholeUniverseWithoutNull();
|
||||
@ -2324,7 +2328,7 @@ BoolMask KeyCondition::checkInRange(
|
||||
key_ranges.reserve(used_key_size);
|
||||
for (size_t i = 0; i < used_key_size; ++i)
|
||||
{
|
||||
if (data_types[i]->isNullable())
|
||||
if (isNullableOrLowCardinalityNullable(data_types[i]))
|
||||
key_ranges.push_back(Range::createWholeUniverse());
|
||||
else
|
||||
key_ranges.push_back(Range::createWholeUniverseWithoutNull());
|
||||
|
||||
@ -383,7 +383,8 @@ MergeTreeDataSelectSamplingData MergeTreeDataSelectExecutor::getSampling(
|
||||
if (has_lower_limit)
|
||||
{
|
||||
if (!key_condition.addCondition(
|
||||
sampling_key.column_names[0], Range::createLeftBounded(lower, true, sampling_key.data_types[0]->isNullable())))
|
||||
sampling_key.column_names[0],
|
||||
Range::createLeftBounded(lower, true, isNullableOrLowCardinalityNullable(sampling_key.data_types[0]))))
|
||||
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Sampling column not in primary key");
|
||||
|
||||
ASTPtr args = std::make_shared<ASTExpressionList>();
|
||||
@ -401,7 +402,8 @@ MergeTreeDataSelectSamplingData MergeTreeDataSelectExecutor::getSampling(
|
||||
if (has_upper_limit)
|
||||
{
|
||||
if (!key_condition.addCondition(
|
||||
sampling_key.column_names[0], Range::createRightBounded(upper, false, sampling_key.data_types[0]->isNullable())))
|
||||
sampling_key.column_names[0],
|
||||
Range::createRightBounded(upper, false, isNullableOrLowCardinalityNullable(sampling_key.data_types[0]))))
|
||||
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Sampling column not in primary key");
|
||||
|
||||
ASTPtr args = std::make_shared<ASTExpressionList>();
|
||||
|
||||
@ -299,8 +299,35 @@ void StorageBuffer::read(
|
||||
}
|
||||
else
|
||||
{
|
||||
auto src_table_query_info = query_info;
|
||||
if (src_table_query_info.prewhere_info)
|
||||
{
|
||||
auto actions_dag = ActionsDAG::makeConvertingActions(
|
||||
header_after_adding_defaults.getColumnsWithTypeAndName(),
|
||||
header.getColumnsWithTypeAndName(),
|
||||
ActionsDAG::MatchColumnsMode::Name);
|
||||
|
||||
if (src_table_query_info.prewhere_info->row_level_filter)
|
||||
{
|
||||
src_table_query_info.prewhere_info->row_level_filter = ActionsDAG::merge(
|
||||
std::move(*actions_dag->clone()),
|
||||
std::move(*src_table_query_info.prewhere_info->row_level_filter));
|
||||
|
||||
src_table_query_info.prewhere_info->row_level_filter->removeUnusedActions();
|
||||
}
|
||||
|
||||
if (src_table_query_info.prewhere_info->prewhere_actions)
|
||||
{
|
||||
src_table_query_info.prewhere_info->prewhere_actions = ActionsDAG::merge(
|
||||
std::move(*actions_dag->clone()),
|
||||
std::move(*src_table_query_info.prewhere_info->prewhere_actions));
|
||||
|
||||
src_table_query_info.prewhere_info->prewhere_actions->removeUnusedActions();
|
||||
}
|
||||
}
|
||||
|
||||
destination->read(
|
||||
query_plan, columns_intersection, destination_snapshot, query_info,
|
||||
query_plan, columns_intersection, destination_snapshot, src_table_query_info,
|
||||
local_context, processed_stage, max_block_size, num_streams);
|
||||
|
||||
if (query_plan.isInitialized())
|
||||
|
||||
@ -401,6 +401,11 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree(
|
||||
{
|
||||
try
|
||||
{
|
||||
if (LoadingStrictnessLevel::ATTACH == mode && current_zookeeper && current_zookeeper->exists(replica_path + "/is_active"))
|
||||
{
|
||||
throw Exception(ErrorCodes::REPLICA_ALREADY_EXISTS, "There already is an active replica with this replica path {}", replica_path);
|
||||
}
|
||||
|
||||
if (current_zookeeper && current_zookeeper->exists(replica_path + "/host"))
|
||||
{
|
||||
/// Check it earlier if we can (we don't want incompatible version to start).
|
||||
@ -5836,11 +5841,24 @@ bool StorageReplicatedMergeTree::optimize(
|
||||
|
||||
if (select_decision != SelectPartsDecision::SELECTED)
|
||||
{
|
||||
constexpr const char * message_fmt = "Cannot select parts for optimization: {}";
|
||||
assert(disable_reason.text != unknown_disable_reason);
|
||||
if (!partition_id.empty())
|
||||
disable_reason.text += fmt::format(" (in partition {})", partition_id);
|
||||
return handle_noop(message_fmt, disable_reason.text);
|
||||
if (try_no + 1 < max_retries)
|
||||
{
|
||||
/// Here we trying to have a similar behaviour to ordinary MergeTree: if some merges are already in progress - let's wait for them to finish.
|
||||
/// This way `optimize final` won't just silently be a noop (if also `optimize_throw_if_noop=false`), but will wait for the active merges and repeat an attempt to schedule final merge.
|
||||
/// This guarantees are enough for tests, because there we have full control over insertions.
|
||||
const auto wait_timeout = query_context->getSettingsRef().receive_timeout.totalMilliseconds() / max_retries;
|
||||
/// DEFAULT (and not LIGHTWEIGHT) because merges are not condidered lightweight; empty `source_replicas` means "all replicas"
|
||||
waitForProcessingQueue(wait_timeout, SyncReplicaMode::DEFAULT, {});
|
||||
continue;
|
||||
}
|
||||
else
|
||||
{
|
||||
constexpr const char * message_fmt = "Cannot select parts for optimization: {}";
|
||||
assert(disable_reason.text != unknown_disable_reason);
|
||||
if (!partition_id.empty())
|
||||
disable_reason.text += fmt::format(" (in partition {})", partition_id);
|
||||
return handle_noop(message_fmt, disable_reason.text);
|
||||
}
|
||||
}
|
||||
|
||||
ReplicatedMergeTreeLogEntryData merge_entry;
|
||||
|
||||
@ -622,6 +622,20 @@ Chunk SystemZooKeeperSource::generate()
|
||||
ZooKeeperRetriesControl("", nullptr, retries_seetings, query_status).retryLoop(
|
||||
[&]() { get_responses = get_zookeeper()->tryGet(paths_to_get); });
|
||||
|
||||
/// Add children count to query total rows. We can not get total rows in advance,
|
||||
/// because it is too heavy to get row count for non exact paths.
|
||||
/// Please be aware that there might be minor setbacks in the query progress,
|
||||
/// but overall it should reflect the advancement of the query.
|
||||
size_t children_count = 0;
|
||||
for (size_t i = 0, size = get_tasks.size(); i < size; ++i)
|
||||
{
|
||||
auto & res = get_responses[i];
|
||||
if (res.error == Coordination::Error::ZNONODE)
|
||||
continue; /// Node was deleted meanwhile.
|
||||
children_count += res.stat.numChildren;
|
||||
}
|
||||
addTotalRowsApprox(children_count);
|
||||
|
||||
for (size_t i = 0, size = get_tasks.size(); i < size; ++i)
|
||||
{
|
||||
auto & res = get_responses[i];
|
||||
|
||||
@ -10,11 +10,8 @@ from pathlib import Path
|
||||
from shutil import copy2
|
||||
from typing import List, Optional, Union
|
||||
|
||||
# isort: off
|
||||
from github.Commit import Commit
|
||||
|
||||
# isort: on
|
||||
|
||||
from build_download_helper import download_build_with_progress
|
||||
from commit_status_helper import post_commit_status
|
||||
from compress_files import SUFFIX, compress_fast, decompress_fast
|
||||
|
||||
@ -1,31 +1,27 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from typing import Tuple
|
||||
import subprocess
|
||||
import logging
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Tuple
|
||||
|
||||
from ci_config import CI_CONFIG, BuildConfig
|
||||
|
||||
from env_helper import (
|
||||
REPO_COPY,
|
||||
S3_BUILDS_BUCKET,
|
||||
TEMP_PATH,
|
||||
)
|
||||
from git_helper import Git
|
||||
from pr_info import PRInfo
|
||||
from report import FAILURE, JobReport, StatusType, SUCCESS
|
||||
from tee_popen import TeePopen
|
||||
import docker_images_helper
|
||||
from ci_config import CI_CONFIG, BuildConfig
|
||||
from env_helper import REPO_COPY, S3_BUILDS_BUCKET, TEMP_PATH
|
||||
from git_helper import Git
|
||||
from lambda_shared_package.lambda_shared.pr import Labels
|
||||
from pr_info import PRInfo
|
||||
from report import FAILURE, SUCCESS, JobReport, StatusType
|
||||
from stopwatch import Stopwatch
|
||||
from tee_popen import TeePopen
|
||||
from version_helper import (
|
||||
ClickHouseVersion,
|
||||
get_version_from_repo,
|
||||
update_version_local,
|
||||
)
|
||||
from stopwatch import Stopwatch
|
||||
|
||||
IMAGE_NAME = "clickhouse/binary-builder"
|
||||
BUILD_LOG_NAME = "build_log.log"
|
||||
@ -111,6 +107,10 @@ def build_clickhouse(
|
||||
return build_log_path, SUCCESS if success else FAILURE
|
||||
|
||||
|
||||
def is_release_pr(pr_info: PRInfo) -> bool:
|
||||
return Labels.RELEASE in pr_info.labels or Labels.RELEASE_LTS in pr_info.labels
|
||||
|
||||
|
||||
def get_release_or_pr(pr_info: PRInfo, version: ClickHouseVersion) -> Tuple[str, str]:
|
||||
"Return prefixes for S3 artifacts paths"
|
||||
# FIXME performance
|
||||
@ -119,7 +119,7 @@ def get_release_or_pr(pr_info: PRInfo, version: ClickHouseVersion) -> Tuple[str,
|
||||
# It should be fixed in performance-comparison image eventually
|
||||
# For performance tests we always set PRs prefix
|
||||
performance_pr = "PRs/0"
|
||||
if "release" in pr_info.labels or "release-lts" in pr_info.labels:
|
||||
if is_release_pr(pr_info):
|
||||
# for release pull requests we use branch names prefixes, not pr numbers
|
||||
return pr_info.head_ref, performance_pr
|
||||
if pr_info.number == 0:
|
||||
@ -163,7 +163,7 @@ def main():
|
||||
official_flag = pr_info.number == 0
|
||||
|
||||
version_type = "testing"
|
||||
if "release" in pr_info.labels or "release-lts" in pr_info.labels:
|
||||
if is_release_pr(pr_info):
|
||||
version_type = "stable"
|
||||
official_flag = True
|
||||
|
||||
|
||||
@ -8,11 +8,8 @@ import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, List, Optional, Union
|
||||
|
||||
# isort: off
|
||||
import requests
|
||||
|
||||
# isort: on
|
||||
|
||||
import get_robot_token as grt # we need an updated ROBOT_TOKEN
|
||||
from ci_config import CI_CONFIG
|
||||
|
||||
|
||||
@ -139,7 +139,8 @@ def main():
|
||||
additional_files=[report_path],
|
||||
).dump()
|
||||
|
||||
if summary_status == ERROR:
|
||||
# We should fail the report job to rerun it in the following attempts
|
||||
if summary_status != SUCCESS:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
|
||||
@ -9,7 +9,7 @@ from threading import Thread
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import requests
|
||||
from lambda_shared.pr import check_pr_description
|
||||
from lambda_shared.pr import Labels, check_pr_description
|
||||
from lambda_shared.token import get_cached_access_token
|
||||
|
||||
NEED_RERUN_OR_CANCELL_WORKFLOWS = {
|
||||
@ -261,7 +261,7 @@ def main(event):
|
||||
print("Freshly opened PR, nothing to do")
|
||||
return
|
||||
|
||||
if action == "closed" or label == "do not test":
|
||||
if action == "closed" or label == Labels.DO_NOT_TEST:
|
||||
print("PR merged/closed or manually labeled 'do not test', will kill workflows")
|
||||
workflow_descriptions = get_workflows_description_for_pull_request(
|
||||
pull_request, token
|
||||
@ -281,7 +281,7 @@ def main(event):
|
||||
exec_workflow_url(urls_to_cancel, token)
|
||||
return
|
||||
|
||||
if label == "can be tested":
|
||||
if label == Labels.CAN_BE_TESTED:
|
||||
print("PR marked with can be tested label, rerun workflow")
|
||||
workflow_descriptions = get_workflows_description_for_pull_request(
|
||||
pull_request, token
|
||||
|
||||
@ -37,19 +37,10 @@ from env_helper import TEMP_PATH
|
||||
from get_robot_token import get_best_robot_token
|
||||
from git_helper import git_runner, is_shallow
|
||||
from github_helper import GitHub, PullRequest, PullRequests, Repository
|
||||
from lambda_shared_package.lambda_shared.pr import Labels
|
||||
from ssh import SSHKey
|
||||
|
||||
|
||||
class Labels:
|
||||
MUST_BACKPORT = "pr-must-backport"
|
||||
MUST_BACKPORT_CLOUD = "pr-must-backport-cloud"
|
||||
BACKPORT = "pr-backport"
|
||||
BACKPORTS_CREATED = "pr-backports-created"
|
||||
BACKPORTS_CREATED_CLOUD = "pr-backports-created-cloud"
|
||||
CHERRYPICK = "pr-cherrypick"
|
||||
DO_NOT_TEST = "do not test"
|
||||
|
||||
|
||||
class ReleaseBranch:
|
||||
CHERRYPICK_DESCRIPTION = """Original pull-request #{pr_number}
|
||||
|
||||
@ -99,7 +90,7 @@ close it.
|
||||
name: str,
|
||||
pr: PullRequest,
|
||||
repo: Repository,
|
||||
backport_created_label: str = Labels.BACKPORTS_CREATED,
|
||||
backport_created_label: str = Labels.PR_BACKPORTS_CREATED,
|
||||
):
|
||||
self.name = name
|
||||
self.pr = pr
|
||||
@ -247,12 +238,12 @@ close it.
|
||||
pr_number=self.pr.number,
|
||||
pr_url=self.pr.html_url,
|
||||
backport_created_label=self.backport_created_label,
|
||||
label_cherrypick=Labels.CHERRYPICK,
|
||||
label_cherrypick=Labels.PR_CHERRYPICK,
|
||||
),
|
||||
base=self.backport_branch,
|
||||
head=self.cherrypick_branch,
|
||||
)
|
||||
self.cherrypick_pr.add_to_labels(Labels.CHERRYPICK)
|
||||
self.cherrypick_pr.add_to_labels(Labels.PR_CHERRYPICK)
|
||||
self.cherrypick_pr.add_to_labels(Labels.DO_NOT_TEST)
|
||||
self._assign_new_pr(self.cherrypick_pr)
|
||||
# update cherrypick PR to get the state for PR.mergable
|
||||
@ -288,7 +279,7 @@ close it.
|
||||
base=self.name,
|
||||
head=self.backport_branch,
|
||||
)
|
||||
self.backport_pr.add_to_labels(Labels.BACKPORT)
|
||||
self.backport_pr.add_to_labels(Labels.PR_BACKPORT)
|
||||
self._assign_new_pr(self.backport_pr)
|
||||
|
||||
def ping_cherry_pick_assignees(self, dry_run: bool) -> None:
|
||||
@ -518,7 +509,7 @@ class Backport:
|
||||
)
|
||||
bp_cp_prs = self.gh.get_pulls_from_search(
|
||||
query=f"type:pr repo:{self._repo_name} {query_suffix}",
|
||||
label=f"{Labels.BACKPORT},{Labels.CHERRYPICK}",
|
||||
label=f"{Labels.PR_BACKPORT},{Labels.PR_CHERRYPICK}",
|
||||
)
|
||||
for br in branches:
|
||||
br.pop_prs(bp_cp_prs)
|
||||
@ -588,8 +579,8 @@ def parse_args():
|
||||
)
|
||||
parser.add_argument(
|
||||
"--backport-created-label",
|
||||
default=Labels.BACKPORTS_CREATED,
|
||||
choices=(Labels.BACKPORTS_CREATED, Labels.BACKPORTS_CREATED_CLOUD),
|
||||
default=Labels.PR_BACKPORTS_CREATED,
|
||||
choices=(Labels.PR_BACKPORTS_CREATED, Labels.PR_BACKPORTS_CREATED_CLOUD),
|
||||
help="label to mark PRs as backported",
|
||||
)
|
||||
parser.add_argument(
|
||||
|
||||
@ -17,7 +17,7 @@ from typing import Any, Dict, List, Optional, Sequence, Set, Tuple, Union
|
||||
import docker_images_helper
|
||||
import upload_result_helper
|
||||
from build_check import get_release_or_pr
|
||||
from ci_config import CI_CONFIG, Build, CIStages, JobNames, Labels
|
||||
from ci_config import CI_CONFIG, Build, CILabels, CIStages, JobNames
|
||||
from ci_utils import GHActions, is_hex, normalize_string
|
||||
from clickhouse_helper import (
|
||||
CiLogsCredentials,
|
||||
@ -800,7 +800,7 @@ class CiOptions:
|
||||
if not res.ci_jobs:
|
||||
res.ci_jobs = []
|
||||
res.ci_jobs.append(match.removeprefix("job_"))
|
||||
elif match.startswith("ci_set_") and match in Labels:
|
||||
elif match.startswith("ci_set_") and match in CILabels:
|
||||
if not res.ci_sets:
|
||||
res.ci_sets = []
|
||||
res.ci_sets.append(match)
|
||||
@ -816,12 +816,12 @@ class CiOptions:
|
||||
res.exclude_keywords.append(
|
||||
normalize_check_name(match.removeprefix("ci_exclude_"))
|
||||
)
|
||||
elif match == Labels.NO_CI_CACHE:
|
||||
elif match == CILabels.NO_CI_CACHE:
|
||||
res.no_ci_cache = True
|
||||
print("NOTE: CI Cache will be disabled")
|
||||
elif match == Labels.DO_NOT_TEST_LABEL:
|
||||
elif match == CILabels.DO_NOT_TEST_LABEL:
|
||||
res.do_not_test = True
|
||||
elif match == Labels.NO_MERGE_COMMIT:
|
||||
elif match == CILabels.NO_MERGE_COMMIT:
|
||||
res.no_merge_commit = True
|
||||
print("NOTE: Merge Commit will be disabled")
|
||||
elif match.startswith("batch_"):
|
||||
@ -920,7 +920,7 @@ class CiOptions:
|
||||
|
||||
# 3. Handle "do not test"
|
||||
if self.do_not_test:
|
||||
label_config = CI_CONFIG.get_label_config(Labels.DO_NOT_TEST_LABEL)
|
||||
label_config = CI_CONFIG.get_label_config(CILabels.DO_NOT_TEST_LABEL)
|
||||
assert label_config
|
||||
print(
|
||||
f"NOTE: CI 'do not test' setting applied, set jobs: [{label_config.run_jobs}]"
|
||||
@ -1547,7 +1547,7 @@ def _fetch_commit_tokens(message: str, pr_info: PRInfo) -> List[str]:
|
||||
res = [
|
||||
match
|
||||
for match in matches
|
||||
if match in Labels or match.startswith("job_") or match.startswith("batch_")
|
||||
if match in CILabels or match.startswith("job_") or match.startswith("batch_")
|
||||
]
|
||||
print(f"CI modifyers from commit message: [{res}]")
|
||||
res_2 = []
|
||||
@ -1556,7 +1556,9 @@ def _fetch_commit_tokens(message: str, pr_info: PRInfo) -> List[str]:
|
||||
res_2 = [
|
||||
match
|
||||
for match in matches
|
||||
if match in Labels or match.startswith("job_") or match.startswith("batch_")
|
||||
if match in CILabels
|
||||
or match.startswith("job_")
|
||||
or match.startswith("batch_")
|
||||
]
|
||||
print(f"CI modifyers from PR body: [{res_2}]")
|
||||
return list(set(res + res_2))
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from copy import deepcopy
|
||||
import logging
|
||||
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Callable, Dict, Iterable, List, Literal, Optional, Union
|
||||
@ -37,7 +37,7 @@ class Runners(metaclass=WithIter):
|
||||
FUZZER_UNIT_TESTER = "fuzzer-unit-tester"
|
||||
|
||||
|
||||
class Labels(metaclass=WithIter):
|
||||
class CILabels(metaclass=WithIter):
|
||||
"""
|
||||
Label names or commit tokens in normalized form
|
||||
"""
|
||||
@ -181,6 +181,13 @@ class JobNames(metaclass=WithIter):
|
||||
BUGFIX_VALIDATE = "Bugfix validation"
|
||||
|
||||
|
||||
class StatusNames(metaclass=WithIter):
|
||||
"Class with statuses that aren't related to particular jobs"
|
||||
CI = "CI running"
|
||||
MERGEABLE = "Mergeable Check"
|
||||
SYNC = "A Sync"
|
||||
|
||||
|
||||
# dynamically update JobName with Build jobs
|
||||
for attr_name in dir(Build):
|
||||
if not attr_name.startswith("__") and not callable(getattr(Build, attr_name)):
|
||||
@ -275,7 +282,7 @@ builds_job_config = JobConfig(
|
||||
run_command="build_check.py $BUILD_NAME",
|
||||
)
|
||||
fuzzer_build_job_config = deepcopy(builds_job_config)
|
||||
fuzzer_build_job_config.run_by_label = Labels.libFuzzer
|
||||
fuzzer_build_job_config.run_by_label = CILabels.libFuzzer
|
||||
|
||||
|
||||
@dataclass
|
||||
@ -818,28 +825,28 @@ class CIConfig:
|
||||
|
||||
CI_CONFIG = CIConfig(
|
||||
label_configs={
|
||||
Labels.DO_NOT_TEST_LABEL: LabelConfig(run_jobs=[JobNames.STYLE_CHECK]),
|
||||
Labels.CI_SET_FAST: LabelConfig(
|
||||
CILabels.DO_NOT_TEST_LABEL: LabelConfig(run_jobs=[JobNames.STYLE_CHECK]),
|
||||
CILabels.CI_SET_FAST: LabelConfig(
|
||||
run_jobs=[
|
||||
JobNames.STYLE_CHECK,
|
||||
JobNames.FAST_TEST,
|
||||
]
|
||||
),
|
||||
Labels.CI_SET_ARM: LabelConfig(
|
||||
CILabels.CI_SET_ARM: LabelConfig(
|
||||
run_jobs=[
|
||||
JobNames.STYLE_CHECK,
|
||||
Build.PACKAGE_AARCH64,
|
||||
JobNames.INTEGRATION_TEST_ARM,
|
||||
]
|
||||
),
|
||||
Labels.CI_SET_INTEGRATION: LabelConfig(
|
||||
CILabels.CI_SET_INTEGRATION: LabelConfig(
|
||||
run_jobs=[
|
||||
JobNames.STYLE_CHECK,
|
||||
Build.PACKAGE_RELEASE,
|
||||
JobNames.INTEGRATION_TEST,
|
||||
]
|
||||
),
|
||||
Labels.CI_SET_ANALYZER: LabelConfig(
|
||||
CILabels.CI_SET_ANALYZER: LabelConfig(
|
||||
run_jobs=[
|
||||
JobNames.STYLE_CHECK,
|
||||
JobNames.FAST_TEST,
|
||||
@ -849,7 +856,7 @@ CI_CONFIG = CIConfig(
|
||||
JobNames.INTEGRATION_TEST_ASAN_ANALYZER,
|
||||
]
|
||||
),
|
||||
Labels.CI_SET_STATLESS: LabelConfig(
|
||||
CILabels.CI_SET_STATLESS: LabelConfig(
|
||||
run_jobs=[
|
||||
JobNames.STYLE_CHECK,
|
||||
JobNames.FAST_TEST,
|
||||
@ -857,7 +864,7 @@ CI_CONFIG = CIConfig(
|
||||
JobNames.STATELESS_TEST_RELEASE,
|
||||
]
|
||||
),
|
||||
Labels.CI_SET_STATLESS_ASAN: LabelConfig(
|
||||
CILabels.CI_SET_STATLESS_ASAN: LabelConfig(
|
||||
run_jobs=[
|
||||
JobNames.STYLE_CHECK,
|
||||
JobNames.FAST_TEST,
|
||||
@ -865,7 +872,7 @@ CI_CONFIG = CIConfig(
|
||||
JobNames.STATELESS_TEST_ASAN,
|
||||
]
|
||||
),
|
||||
Labels.CI_SET_STATEFUL: LabelConfig(
|
||||
CILabels.CI_SET_STATEFUL: LabelConfig(
|
||||
run_jobs=[
|
||||
JobNames.STYLE_CHECK,
|
||||
JobNames.FAST_TEST,
|
||||
@ -873,7 +880,7 @@ CI_CONFIG = CIConfig(
|
||||
JobNames.STATEFUL_TEST_RELEASE,
|
||||
]
|
||||
),
|
||||
Labels.CI_SET_STATEFUL_ASAN: LabelConfig(
|
||||
CILabels.CI_SET_STATEFUL_ASAN: LabelConfig(
|
||||
run_jobs=[
|
||||
JobNames.STYLE_CHECK,
|
||||
JobNames.FAST_TEST,
|
||||
@ -881,7 +888,7 @@ CI_CONFIG = CIConfig(
|
||||
JobNames.STATEFUL_TEST_ASAN,
|
||||
]
|
||||
),
|
||||
Labels.CI_SET_REDUCED: LabelConfig(
|
||||
CILabels.CI_SET_REDUCED: LabelConfig(
|
||||
run_jobs=[
|
||||
job
|
||||
for job in JobNames
|
||||
@ -1335,7 +1342,7 @@ CI_CONFIG = CIConfig(
|
||||
JobNames.LIBFUZZER_TEST: TestConfig(
|
||||
Build.FUZZERS,
|
||||
job_config=JobConfig(
|
||||
run_by_label=Labels.libFuzzer,
|
||||
run_by_label=CILabels.libFuzzer,
|
||||
timeout=10800,
|
||||
run_command='libfuzzer_test_check.py "$CHECK_NAME" 10800',
|
||||
),
|
||||
@ -1348,7 +1355,7 @@ CI_CONFIG.validate()
|
||||
# checks required by Mergeable Check
|
||||
REQUIRED_CHECKS = [
|
||||
"PR Check",
|
||||
"A Sync", # Cloud sync
|
||||
StatusNames.SYNC,
|
||||
JobNames.BUILD_CHECK,
|
||||
JobNames.BUILD_CHECK_SPECIAL,
|
||||
JobNames.DOCS_CHECK,
|
||||
@ -1461,9 +1468,9 @@ CHECK_DESCRIPTIONS = [
|
||||
lambda x: x.startswith("Integration tests ("),
|
||||
),
|
||||
CheckDescription(
|
||||
"Mergeable Check",
|
||||
StatusNames.MERGEABLE,
|
||||
"Checks if all other necessary checks are successful",
|
||||
lambda x: x == "Mergeable Check",
|
||||
lambda x: x == StatusNames.MERGEABLE,
|
||||
),
|
||||
CheckDescription(
|
||||
"Performance Comparison",
|
||||
|
||||
@ -9,7 +9,6 @@ from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Union
|
||||
|
||||
# isort: off
|
||||
from github import Github
|
||||
from github.Commit import Commit
|
||||
from github.CommitStatus import CommitStatus
|
||||
@ -18,11 +17,10 @@ from github.GithubObject import NotSet
|
||||
from github.IssueComment import IssueComment
|
||||
from github.Repository import Repository
|
||||
|
||||
# isort: on
|
||||
|
||||
from ci_config import CHECK_DESCRIPTIONS, REQUIRED_CHECKS, CheckDescription
|
||||
from ci_config import CHECK_DESCRIPTIONS, REQUIRED_CHECKS, CheckDescription, StatusNames
|
||||
from env_helper import GITHUB_REPOSITORY, GITHUB_RUN_URL, TEMP_PATH
|
||||
from pr_info import SKIP_MERGEABLE_CHECK_LABEL, PRInfo
|
||||
from lambda_shared_package.lambda_shared.pr import Labels
|
||||
from pr_info import PRInfo
|
||||
from report import (
|
||||
ERROR,
|
||||
FAILURE,
|
||||
@ -38,9 +36,7 @@ from upload_result_helper import upload_results
|
||||
|
||||
RETRY = 5
|
||||
CommitStatuses = List[CommitStatus]
|
||||
MERGEABLE_NAME = "Mergeable Check"
|
||||
GH_REPO = None # type: Optional[Repository]
|
||||
CI_STATUS_NAME = "CI running"
|
||||
STATUS_FILE_PATH = Path(TEMP_PATH) / "status.json"
|
||||
|
||||
|
||||
@ -161,16 +157,16 @@ def set_status_comment(commit: Commit, pr_info: PRInfo) -> None:
|
||||
if not statuses:
|
||||
return
|
||||
|
||||
if not [status for status in statuses if status.context == CI_STATUS_NAME]:
|
||||
if not [status for status in statuses if status.context == StatusNames.CI]:
|
||||
# This is the case, when some statuses already exist for the check,
|
||||
# but not the CI_STATUS_NAME. We should create it as pending.
|
||||
# but not the StatusNames.CI. We should create it as pending.
|
||||
# W/o pr_info to avoid recursion, and yes, one extra create_ci_report
|
||||
post_commit_status(
|
||||
commit,
|
||||
PENDING,
|
||||
create_ci_report(pr_info, statuses),
|
||||
"The report for running CI",
|
||||
CI_STATUS_NAME,
|
||||
StatusNames.CI,
|
||||
)
|
||||
|
||||
# We update the report in generate_status_comment function, so do it each
|
||||
@ -302,7 +298,7 @@ def create_ci_report(pr_info: PRInfo, statuses: CommitStatuses) -> str:
|
||||
)
|
||||
)
|
||||
return upload_results(
|
||||
S3Helper(), pr_info.number, pr_info.sha, test_results, [], CI_STATUS_NAME
|
||||
S3Helper(), pr_info.number, pr_info.sha, test_results, [], StatusNames.CI
|
||||
)
|
||||
|
||||
|
||||
@ -431,7 +427,7 @@ def set_mergeable_check(
|
||||
state: StatusType = SUCCESS,
|
||||
) -> None:
|
||||
commit.create_status(
|
||||
context=MERGEABLE_NAME,
|
||||
context=StatusNames.MERGEABLE,
|
||||
description=format_description(description),
|
||||
state=state,
|
||||
target_url=GITHUB_RUN_URL,
|
||||
@ -441,7 +437,7 @@ def set_mergeable_check(
|
||||
def update_mergeable_check(commit: Commit, pr_info: PRInfo, check_name: str) -> None:
|
||||
"check if the check_name in REQUIRED_CHECKS and then trigger update"
|
||||
not_run = (
|
||||
pr_info.labels.intersection({SKIP_MERGEABLE_CHECK_LABEL, "release"})
|
||||
pr_info.labels.intersection({Labels.SKIP_MERGEABLE_CHECK, Labels.RELEASE})
|
||||
or check_name not in REQUIRED_CHECKS
|
||||
or pr_info.release_pr
|
||||
or pr_info.number == 0
|
||||
@ -459,14 +455,14 @@ def update_mergeable_check(commit: Commit, pr_info: PRInfo, check_name: str) ->
|
||||
|
||||
|
||||
def trigger_mergeable_check(commit: Commit, statuses: CommitStatuses) -> None:
|
||||
"""calculate and update MERGEABLE_NAME"""
|
||||
"""calculate and update StatusNames.MERGEABLE"""
|
||||
required_checks = [
|
||||
status for status in statuses if status.context in REQUIRED_CHECKS
|
||||
]
|
||||
|
||||
mergeable_status = None
|
||||
for status in statuses:
|
||||
if status.context == MERGEABLE_NAME:
|
||||
if status.context == StatusNames.MERGEABLE:
|
||||
mergeable_status = status
|
||||
break
|
||||
|
||||
|
||||
@ -8,11 +8,8 @@ import sys
|
||||
from pathlib import Path
|
||||
from typing import List, Tuple
|
||||
|
||||
# isort: off
|
||||
from pip._vendor.packaging.version import Version
|
||||
|
||||
# isort: on
|
||||
|
||||
from build_download_helper import download_builds_filter
|
||||
from docker_images_helper import DockerImage, get_docker_image, pull_image
|
||||
from env_helper import REPORT_PATH, TEMP_PATH
|
||||
|
||||
@ -8,11 +8,8 @@ import time
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
# isort: off
|
||||
from github import Github
|
||||
|
||||
# isort: on
|
||||
|
||||
from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse
|
||||
from commit_status_helper import format_description, get_commit, post_commit_status
|
||||
from docker_images_helper import DockerImageData, docker_login, get_images_oredered_list
|
||||
@ -225,9 +222,11 @@ def main():
|
||||
parent_version = (
|
||||
None
|
||||
if not image.parent
|
||||
else image_tags[image.parent]
|
||||
if not args.suffix
|
||||
else f"{image_tags[image.parent]}-{args.suffix}"
|
||||
else (
|
||||
image_tags[image.parent]
|
||||
if not args.suffix
|
||||
else f"{image_tags[image.parent]}-{args.suffix}"
|
||||
)
|
||||
)
|
||||
|
||||
res = process_single_image(
|
||||
|
||||
@ -8,11 +8,8 @@ import subprocess
|
||||
import sys
|
||||
from typing import List, Tuple
|
||||
|
||||
# isort: off
|
||||
from github import Github
|
||||
|
||||
# isort: on
|
||||
|
||||
from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse
|
||||
from commit_status_helper import format_description, get_commit, post_commit_status
|
||||
from docker_images_helper import docker_login, get_images_oredered_list
|
||||
|
||||
@ -1,13 +1,10 @@
|
||||
#!/usr/bin/env python3
|
||||
import logging
|
||||
|
||||
# isort: off
|
||||
from github import Github
|
||||
|
||||
# isort: on
|
||||
|
||||
from ci_config import StatusNames
|
||||
from commit_status_helper import (
|
||||
CI_STATUS_NAME,
|
||||
get_commit,
|
||||
get_commit_filtered_statuses,
|
||||
post_commit_status,
|
||||
@ -29,7 +26,7 @@ def main():
|
||||
trigger_mergeable_check(commit, statuses)
|
||||
|
||||
if not pr_info.is_merge_queue:
|
||||
statuses = [s for s in statuses if s.context == CI_STATUS_NAME]
|
||||
statuses = [s for s in statuses if s.context == StatusNames.CI]
|
||||
if not statuses:
|
||||
return
|
||||
# Take the latest status
|
||||
@ -40,7 +37,7 @@ def main():
|
||||
SUCCESS,
|
||||
status.target_url,
|
||||
"All checks finished",
|
||||
CI_STATUS_NAME,
|
||||
StatusNames.CI,
|
||||
pr_info,
|
||||
dump_to_file=True,
|
||||
)
|
||||
|
||||
@ -33,7 +33,7 @@ set -e
|
||||
trap "bash -ex /packages/preserve_logs.sh" ERR
|
||||
test_env='TEST_THE_DEFAULT_PARAMETER=15'
|
||||
echo "$test_env" >> /etc/default/clickhouse
|
||||
systemctl start clickhouse-server
|
||||
systemctl restart clickhouse-server
|
||||
clickhouse-client -q 'SELECT version()'
|
||||
grep "$test_env" /proc/$(cat /var/run/clickhouse-server/clickhouse-server.pid)/environ"""
|
||||
initd_test = r"""#!/bin/bash
|
||||
|
||||
@ -48,11 +48,35 @@ TRUSTED_CONTRIBUTORS = {
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
class Labels:
|
||||
CAN_BE_TESTED = "can be tested"
|
||||
DO_NOT_TEST = "do not test"
|
||||
MUST_BACKPORT = "pr-must-backport"
|
||||
MUST_BACKPORT_CLOUD = "pr-must-backport-cloud"
|
||||
JEPSEN_TEST = "jepsen-test"
|
||||
SKIP_MERGEABLE_CHECK = "skip mergeable check"
|
||||
PR_BACKPORT = "pr-backport"
|
||||
PR_BACKPORTS_CREATED = "pr-backports-created"
|
||||
PR_BACKPORTS_CREATED_CLOUD = "pr-backports-created-cloud"
|
||||
PR_CHERRYPICK = "pr-cherrypick"
|
||||
PR_CI = "pr-ci"
|
||||
PR_FEATURE = "pr-feature"
|
||||
PR_SYNCED_TO_CLOUD = "pr-synced-to-cloud"
|
||||
PR_SYNC_UPSTREAM = "pr-sync-upstream"
|
||||
RELEASE = "release"
|
||||
RELEASE_LTS = "release-lts"
|
||||
SUBMODULE_CHANGED = "submodule changed"
|
||||
|
||||
# pr-bugfix autoport can lead to issues in releases, let's do ci fixes only
|
||||
AUTO_BACKPORT = {"pr-ci"}
|
||||
|
||||
|
||||
# Descriptions are used in .github/PULL_REQUEST_TEMPLATE.md, keep comments there
|
||||
# updated accordingly
|
||||
# The following lists are append only, try to avoid editing them
|
||||
# They still could be cleaned out after the decent time though.
|
||||
LABELS = {
|
||||
LABEL_CATEGORIES = {
|
||||
"pr-backward-incompatible": ["Backward Incompatible Change"],
|
||||
"pr-bugfix": [
|
||||
"Bug Fix",
|
||||
@ -81,7 +105,9 @@ LABELS = {
|
||||
"pr-ci": ["CI Fix or Improvement (changelog entry is not required)"],
|
||||
}
|
||||
|
||||
CATEGORY_TO_LABEL = {c: lb for lb, categories in LABELS.items() for c in categories}
|
||||
CATEGORY_TO_LABEL = {
|
||||
c: lb for lb, categories in LABEL_CATEGORIES.items() for c in categories
|
||||
}
|
||||
|
||||
|
||||
def check_pr_description(pr_body: str, repo_name: str) -> Tuple[str, str]:
|
||||
@ -93,7 +119,7 @@ def check_pr_description(pr_body: str, repo_name: str) -> Tuple[str, str]:
|
||||
|
||||
# Check if body contains "Reverts ClickHouse/ClickHouse#36337"
|
||||
if [True for line in lines if re.match(rf"\AReverts {repo_name}#[\d]+\Z", line)]:
|
||||
return "", LABELS["pr-not-for-changelog"][0]
|
||||
return "", LABEL_CATEGORIES["pr-not-for-changelog"][0]
|
||||
|
||||
category = ""
|
||||
entry = ""
|
||||
|
||||
@ -9,13 +9,10 @@ from os import getenv
|
||||
from pprint import pformat
|
||||
from typing import Dict, List
|
||||
|
||||
# isort: off
|
||||
from github.PaginatedList import PaginatedList
|
||||
from github.PullRequestReview import PullRequestReview
|
||||
from github.WorkflowRun import WorkflowRun
|
||||
|
||||
# isort: on
|
||||
|
||||
from commit_status_helper import get_commit_filtered_statuses
|
||||
from get_robot_token import get_best_robot_token
|
||||
from github_helper import GitHub, NamedUser, PullRequest, Repository
|
||||
|
||||
@ -9,11 +9,8 @@ import sys
|
||||
import traceback
|
||||
from pathlib import Path
|
||||
|
||||
# isort: off
|
||||
from github import Github
|
||||
|
||||
# isort: on
|
||||
|
||||
from build_download_helper import download_builds_filter
|
||||
from ci_config import CI_CONFIG
|
||||
from clickhouse_helper import get_instance_id, get_instance_type
|
||||
|
||||
@ -6,12 +6,8 @@ import re
|
||||
from typing import Dict, List, Set, Union
|
||||
from urllib.parse import quote
|
||||
|
||||
# isort: off
|
||||
# for some reason this line moves to the end
|
||||
from unidiff import PatchSet # type: ignore
|
||||
|
||||
# isort: on
|
||||
|
||||
from build_download_helper import get_gh_api
|
||||
from env_helper import (
|
||||
GITHUB_EVENT_PATH,
|
||||
@ -19,8 +15,8 @@ from env_helper import (
|
||||
GITHUB_RUN_URL,
|
||||
GITHUB_SERVER_URL,
|
||||
)
|
||||
from lambda_shared_package.lambda_shared.pr import Labels
|
||||
|
||||
SKIP_MERGEABLE_CHECK_LABEL = "skip mergeable check"
|
||||
NeedsDataType = Dict[str, Dict[str, Union[str, Dict[str, str]]]]
|
||||
|
||||
DIFF_IN_DOCUMENTATION_EXT = [
|
||||
@ -256,7 +252,7 @@ class PRInfo:
|
||||
self.head_ref = pull_request["head"]["ref"]
|
||||
self.head_name = pull_request["head"]["repo"]["full_name"]
|
||||
self.pr_html_url = pull_request["html_url"]
|
||||
if "pr-backport" in self.labels:
|
||||
if Labels.PR_BACKPORT in self.labels:
|
||||
# head1...head2 gives changes in head2 since merge base
|
||||
# Thag's why we need {self.head_ref}...master to get
|
||||
# files changed in upstream AND master...{self.head_ref}
|
||||
@ -279,7 +275,7 @@ class PRInfo:
|
||||
]
|
||||
else:
|
||||
self.diff_urls.append(self.compare_pr_url(pull_request))
|
||||
if "release" in self.labels:
|
||||
if Labels.RELEASE in self.labels:
|
||||
# For release PRs we must get not only files changed in the PR
|
||||
# itself, but as well files changed since we branched out
|
||||
self.diff_urls.append(
|
||||
|
||||
@ -25,6 +25,7 @@ from contextlib import contextmanager
|
||||
from typing import Any, Final, Iterator, List, Optional, Tuple
|
||||
|
||||
from git_helper import Git, commit, release_branch
|
||||
from lambda_shared_package.lambda_shared.pr import Labels
|
||||
from report import SUCCESS
|
||||
from version_helper import (
|
||||
FILE_WITH_VERSION_PATH,
|
||||
@ -407,9 +408,9 @@ class Release:
|
||||
self._git.update()
|
||||
new_version = self.version.patch_update()
|
||||
version_type = self.get_stable_release_type()
|
||||
pr_labels = "--label release"
|
||||
pr_labels = f"--label {Labels.RELEASE}"
|
||||
if version_type == VersionType.LTS:
|
||||
pr_labels += " --label release-lts"
|
||||
pr_labels += f" --label {Labels.RELEASE_LTS}"
|
||||
new_version.with_description(version_type)
|
||||
self._update_cmake_contributors(new_version)
|
||||
self._commit_cmake_contributors(new_version)
|
||||
|
||||
@ -3,14 +3,10 @@ import logging
|
||||
import sys
|
||||
from typing import Tuple
|
||||
|
||||
# isort: off
|
||||
from github import Github
|
||||
|
||||
# isort: on
|
||||
|
||||
from cherry_pick import Labels
|
||||
from ci_config import StatusNames
|
||||
from commit_status_helper import (
|
||||
CI_STATUS_NAME,
|
||||
create_ci_report,
|
||||
format_description,
|
||||
get_commit,
|
||||
@ -23,6 +19,7 @@ from get_robot_token import get_best_robot_token
|
||||
from lambda_shared_package.lambda_shared.pr import (
|
||||
CATEGORY_TO_LABEL,
|
||||
TRUSTED_CONTRIBUTORS,
|
||||
Labels,
|
||||
check_pr_description,
|
||||
)
|
||||
from pr_info import PRInfo
|
||||
@ -32,13 +29,8 @@ TRUSTED_ORG_IDS = {
|
||||
54801242, # clickhouse
|
||||
}
|
||||
|
||||
OK_SKIP_LABELS = {"release", "pr-backport", "pr-cherrypick"}
|
||||
CAN_BE_TESTED_LABEL = "can be tested"
|
||||
FEATURE_LABEL = "pr-feature"
|
||||
SUBMODULE_CHANGED_LABEL = "submodule changed"
|
||||
OK_SKIP_LABELS = {Labels.RELEASE, Labels.PR_BACKPORT, Labels.PR_CHERRYPICK}
|
||||
PR_CHECK = "PR Check"
|
||||
# pr-bugfix autoport can lead to issues in releases, let's do ci fixes only
|
||||
AUTO_BACKPORT_LABELS = ["pr-ci"]
|
||||
|
||||
|
||||
def pr_is_by_trusted_user(pr_user_login, pr_user_orgs):
|
||||
@ -71,11 +63,12 @@ def should_run_ci_for_pr(pr_info: PRInfo) -> Tuple[bool, str]:
|
||||
if OK_SKIP_LABELS.intersection(pr_info.labels):
|
||||
return True, "Don't try new checks for release/backports/cherry-picks"
|
||||
|
||||
if CAN_BE_TESTED_LABEL not in pr_info.labels and not pr_is_by_trusted_user(
|
||||
if Labels.CAN_BE_TESTED not in pr_info.labels and not pr_is_by_trusted_user(
|
||||
pr_info.user_login, pr_info.user_orgs
|
||||
):
|
||||
print(
|
||||
f"PRs by untrusted users need the '{CAN_BE_TESTED_LABEL}' label - please contact a member of the core team"
|
||||
f"PRs by untrusted users need the '{Labels.CAN_BE_TESTED}' label - "
|
||||
"please contact a member of the core team"
|
||||
)
|
||||
return False, "Needs 'can be tested' label"
|
||||
|
||||
@ -119,11 +112,11 @@ def main():
|
||||
pr_labels_to_remove.append(label)
|
||||
|
||||
if pr_info.has_changes_in_submodules():
|
||||
pr_labels_to_add.append(SUBMODULE_CHANGED_LABEL)
|
||||
elif SUBMODULE_CHANGED_LABEL in pr_info.labels:
|
||||
pr_labels_to_remove.append(SUBMODULE_CHANGED_LABEL)
|
||||
pr_labels_to_add.append(Labels.SUBMODULE_CHANGED)
|
||||
elif Labels.SUBMODULE_CHANGED in pr_info.labels:
|
||||
pr_labels_to_remove.append(Labels.SUBMODULE_CHANGED)
|
||||
|
||||
if any(label in AUTO_BACKPORT_LABELS for label in pr_labels_to_add):
|
||||
if any(label in Labels.AUTO_BACKPORT for label in pr_labels_to_add):
|
||||
backport_labels = [Labels.MUST_BACKPORT, Labels.MUST_BACKPORT_CLOUD]
|
||||
pr_labels_to_add += [
|
||||
label for label in backport_labels if label not in pr_info.labels
|
||||
@ -163,16 +156,19 @@ def main():
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
if FEATURE_LABEL in pr_info.labels and not pr_info.has_changes_in_documentation():
|
||||
if (
|
||||
Labels.PR_FEATURE in pr_info.labels
|
||||
and not pr_info.has_changes_in_documentation()
|
||||
):
|
||||
print(
|
||||
f"The '{FEATURE_LABEL}' in the labels, "
|
||||
f"The '{Labels.PR_FEATURE}' in the labels, "
|
||||
"but there's no changed documentation"
|
||||
)
|
||||
post_commit_status(
|
||||
commit,
|
||||
FAILURE,
|
||||
"",
|
||||
f"expect adding docs for {FEATURE_LABEL}",
|
||||
f"expect adding docs for {Labels.PR_FEATURE}",
|
||||
PR_CHECK,
|
||||
pr_info,
|
||||
)
|
||||
@ -209,7 +205,7 @@ def main():
|
||||
PENDING,
|
||||
ci_report_url,
|
||||
description,
|
||||
CI_STATUS_NAME,
|
||||
StatusNames.CI,
|
||||
pr_info,
|
||||
)
|
||||
|
||||
|
||||
@ -77,23 +77,35 @@ def commit_push_staged(pr_info: PRInfo) -> None:
|
||||
return
|
||||
git_staged = git_runner("git diff --cached --name-only")
|
||||
if not git_staged:
|
||||
logging.info("No fixes are staged")
|
||||
return
|
||||
remote_url = pr_info.event["pull_request"]["base"]["repo"]["ssh_url"]
|
||||
head = git_runner("git rev-parse HEAD^{}")
|
||||
git_runner(f"{GIT_PREFIX} commit -m 'Automatic style fix'")
|
||||
# The fetch to avoid issue 'pushed branch tip is behind its remote'
|
||||
fetch_cmd = (
|
||||
f"{GIT_PREFIX} fetch {remote_url} --no-recurse-submodules --depth=2 {head}"
|
||||
)
|
||||
push_cmd = f"{GIT_PREFIX} push {remote_url} HEAD:{pr_info.head_ref}"
|
||||
|
||||
def push_fix() -> None:
|
||||
"""
|
||||
Stash staged changes to commit them on the top of the PR's head.
|
||||
`pull_request` event runs on top of a temporary merge_commit, we need to avoid
|
||||
including it in the autofix
|
||||
"""
|
||||
remote_url = pr_info.event["pull_request"]["base"]["repo"]["ssh_url"]
|
||||
head = pr_info.sha
|
||||
git_runner(f"{GIT_PREFIX} commit -m 'Automatic style fix'")
|
||||
fix_commit = git_runner("git rev-parse HEAD")
|
||||
logging.info(
|
||||
"Fetching PR's head, check it out and cherry-pick autofix: %s", head
|
||||
)
|
||||
git_runner(
|
||||
f"{GIT_PREFIX} fetch {remote_url} --no-recurse-submodules --depth=1 {head}"
|
||||
)
|
||||
git_runner(f"git reset --hard {head}")
|
||||
git_runner(f"{GIT_PREFIX} cherry-pick {fix_commit}")
|
||||
git_runner(f"{GIT_PREFIX} push {remote_url} HEAD:{pr_info.head_ref}")
|
||||
|
||||
if os.getenv("ROBOT_CLICKHOUSE_SSH_KEY", ""):
|
||||
with SSHKey("ROBOT_CLICKHOUSE_SSH_KEY"):
|
||||
git_runner(fetch_cmd)
|
||||
git_runner(push_cmd)
|
||||
push_fix()
|
||||
return
|
||||
|
||||
git_runner(fetch_cmd)
|
||||
git_runner(push_cmd)
|
||||
push_fix()
|
||||
|
||||
|
||||
def _check_mime(file: Union[Path, str], mime: str) -> bool:
|
||||
@ -179,8 +191,14 @@ def main():
|
||||
future = executor.submit(subprocess.run, cmd_shell, shell=True)
|
||||
_ = future.result()
|
||||
|
||||
autofix_description = ""
|
||||
if args.push:
|
||||
commit_push_staged(pr_info)
|
||||
try:
|
||||
commit_push_staged(pr_info)
|
||||
except subprocess.SubprocessError:
|
||||
# do not fail the whole script if the autofix didn't work out
|
||||
logging.error("Unable to push the autofix. Continue.")
|
||||
autofix_description = "Failed to push autofix to the PR. "
|
||||
|
||||
subprocess.check_call(
|
||||
f"python3 ../../utils/check-style/process_style_check_result.py --in-results-dir {temp_path} "
|
||||
@ -192,7 +210,7 @@ def main():
|
||||
state, description, test_results, additional_files = process_result(temp_path)
|
||||
|
||||
JobReport(
|
||||
description=description,
|
||||
description=f"{autofix_description}{description}",
|
||||
test_results=test_results,
|
||||
status=state,
|
||||
start_time=stopwatch.start_time_str,
|
||||
|
||||
@ -5,10 +5,11 @@
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
from get_robot_token import get_best_robot_token
|
||||
from pr_info import PRInfo
|
||||
from github_helper import GitHub
|
||||
from ci_config import StatusNames
|
||||
from commit_status_helper import get_commit, post_commit_status
|
||||
from get_robot_token import get_best_robot_token
|
||||
from github_helper import GitHub
|
||||
from pr_info import PRInfo
|
||||
from report import SUCCESS
|
||||
|
||||
|
||||
@ -56,7 +57,7 @@ def set_sync_status(gh, pr_info, sync_pr):
|
||||
# FIXME: uncomment posting red Sync status to prohibit merge in MQ if PR state fetching works good
|
||||
if not sync_pr:
|
||||
# post_commit_status(
|
||||
# get_commit(gh, pr_info.sha), FAILURE, "", "Sync PR not found", "A Sync"
|
||||
# get_commit(gh, pr_info.sha), FAILURE, "", "Sync PR not found", StatusNames.SYNC
|
||||
# )
|
||||
return
|
||||
|
||||
@ -73,7 +74,9 @@ def set_sync_status(gh, pr_info, sync_pr):
|
||||
|
||||
if sync_pr.mergeable_state == "clean":
|
||||
print(f"Sync PR [{sync_pr.number}] is clean")
|
||||
post_commit_status(get_commit(gh, pr_info.sha), SUCCESS, "", "", "A Sync")
|
||||
post_commit_status(
|
||||
get_commit(gh, pr_info.sha), SUCCESS, "", "", StatusNames.SYNC
|
||||
)
|
||||
else:
|
||||
print(
|
||||
f"Sync PR [{sync_pr}] is not mergeable, state [{sync_pr.mergeable_state}]"
|
||||
@ -83,7 +86,7 @@ def set_sync_status(gh, pr_info, sync_pr):
|
||||
# FAILURE,
|
||||
# "",
|
||||
# f"state: {sync_pr.mergeable_state}",
|
||||
# "A Sync",
|
||||
# StatusNames.SYNC,
|
||||
# )
|
||||
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user