thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-23 16:12:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'rschu1ze/master' into revert-revert-62511

Browse Source
This commit is contained in:
Robert Schulze 2024-04-25 14:28:39 +00:00
commit 9e511023c5
No known key found for this signature in database
GPG Key ID: 26703B55FB13728A
64 changed files with 904 additions and 318 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
README.md
View File

@ -39,15 +39,8 @@ Every month we get together with the community (users, contributors, customers,
## Upcoming Events
Keep an eye out for upcoming meetups and eventsaround the world. Somewhere else you want us to be? Please feel free to reach out to tyler `<at>` clickhouse `<dot>` com. You can also peruse [ClickHouse Events](https://clickhouse.com/company/news-events) for a list of all upcoming trainings, meetups, speaking engagements, etc.
Keep an eye out for upcoming meetups and events around the world. Somewhere else you want us to be? Please feel free to reach out to tyler `<at>` clickhouse `<dot>` com. You can also peruse [ClickHouse Events](https://clickhouse.com/company/news-events) for a list of all upcoming trainings, meetups, speaking engagements, etc.
* [ClickHouse Meetup in Bellevue](https://www.meetup.com/clickhouse-seattle-user-group/events/298650371/) - Mar 11
* [ClickHouse Meetup at Ramp's Offices in NYC](https://www.meetup.com/clickhouse-new-york-user-group/events/298640542/) - Mar 19
* [ClickHouse Melbourne Meetup](https://www.meetup.com/clickhouse-australia-user-group/events/299479750/) - Mar 20
* [ClickHouse Meetup in Paris](https://www.meetup.com/clickhouse-france-user-group/events/298997115/) - Mar 21
* [ClickHouse Meetup in Bengaluru](https://www.meetup.com/clickhouse-bangalore-user-group/events/299479850/) - Mar 23
* [ClickHouse Meetup in Zurich](https://www.meetup.com/clickhouse-switzerland-meetup-group/events/299628922/) - Apr 16
* [ClickHouse Meetup in Copenhagen](https://www.meetup.com/clickhouse-denmark-meetup-group/events/299629133/) - Apr 23
* [ClickHouse Meetup in Dubai](https://www.meetup.com/clickhouse-dubai-meetup-group/events/299629189/) - May 28

10
base/poco/Foundation/include/Poco/Message.h
View File

@ -66,9 +66,11 @@ public:
/// The thread and process ids are set.
Message(
const std::string & source, const std::string & text, Priority prio, const char * file, int line, std::string_view fmt_str = {});
const std::string & source, const std::string & text, Priority prio, const char * file, int line,
std::string_view fmt_str = {}, const std::vector<std::string> & fmt_str_args = {});
Message(
std::string && source, std::string && text, Priority prio, const char * file, int line, std::string_view fmt_str);
std::string && source, std::string && text, Priority prio, const char * file, int line,
std::string_view fmt_str, std::vector<std::string> && fmt_str_args);
/// Creates a Message with the given source, text, priority,
/// source file path and line.
///
@ -161,6 +163,9 @@ public:
std::string_view getFormatString() const;
void setFormatString(std::string_view fmt_str);
const std::vector<std::string> & getFormatStringArgs() const;
void setFormatStringArgs(const std::vector<std::string> & fmt_str_args);
int getSourceLine() const;
/// Returns the source file line of the statement
/// generating the log message. May be 0
@ -210,6 +215,7 @@ private:
int _line;
StringMap * _pMap;
std::string_view _fmt_str;
std::vector<std::string> _fmt_str_args;
};

32
base/poco/Foundation/src/Message.cpp
View File

@ -46,7 +46,9 @@ Message::Message(const std::string& source, const std::string& text, Priority pr
}
Message::Message(const std::string& source, const std::string& text, Priority prio, const char* file, int line, std::string_view fmt_str):
Message::Message(
const std::string& source, const std::string& text, Priority prio, const char* file, int line,
std::string_view fmt_str, const std::vector<std::string>& fmt_str_args):
_source(source),
_text(text),
_prio(prio),
@ -54,13 +56,16 @@ Message::Message(const std::string& source, const std::string& text, Priority pr
_file(file),
_line(line),
_pMap(0),
_fmt_str(fmt_str)
_fmt_str(fmt_str),
_fmt_str_args(fmt_str_args)
{
init();
}
Message::Message(std::string && source, std::string && text, Priority prio, const char * file, int line, std::string_view fmt_str):
Message::Message(
std::string && source, std::string && text, Priority prio, const char * file, int line,
std::string_view fmt_str, std::vector<std::string> && fmt_str_args):
_source(std::move(source)),
_text(std::move(text)),
_prio(prio),
@ -68,7 +73,8 @@ Message::Message(std::string && source, std::string && text, Priority prio, cons
_file(file),
_line(line),
_pMap(0),
_fmt_str(fmt_str)
_fmt_str(fmt_str),
_fmt_str_args(std::move(fmt_str_args))
{
init();
}
@ -83,7 +89,8 @@ Message::Message(const Message& msg):
_pid(msg._pid),
_file(msg._file),
_line(msg._line),
_fmt_str(msg._fmt_str)
_fmt_str(msg._fmt_str),
_fmt_str_args(msg._fmt_str_args)
{
if (msg._pMap)
_pMap = new StringMap(*msg._pMap);
@ -102,7 +109,8 @@ Message::Message(const Message& msg, const std::string& text):
_pid(msg._pid),
_file(msg._file),
_line(msg._line),
_fmt_str(msg._fmt_str)
_fmt_str(msg._fmt_str),
_fmt_str_args(msg._fmt_str_args)
{
if (msg._pMap)
_pMap = new StringMap(*msg._pMap);
@ -154,6 +162,7 @@ void Message::swap(Message& msg)
swap(_line, msg._line);
swap(_pMap, msg._pMap);
swap(_fmt_str, msg._fmt_str);
swap(_fmt_str_args, msg._fmt_str_args);
}
@ -227,6 +236,17 @@ void Message::setFormatString(std::string_view fmt_str)
}
const std::vector<std::string>& Message::getFormatStringArgs() const
{
return _fmt_str_args;
}
void Message::setFormatStringArgs(const std::vector<std::string>& fmt_str_args)
{
_fmt_str_args = fmt_str_args;
}
bool Message::has(const std::string& param) const
{
return _pMap && (_pMap->find(param) != _pMap->end());

20
contrib/openssl-cmake/CMakeLists.txt
View File

@ -93,6 +93,7 @@ enable_language(ASM)
if(COMPILER_CLANG)
add_definitions(-Wno-unused-command-line-argument)
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld") # only relevant for -DENABLE_OPENSSL_DYNAMIC=1
endif()
if(ARCH_AMD64)
@ -960,11 +961,6 @@ set(CRYPTO_SRC
${OPENSSL_SOURCE_DIR}/crypto/x509/x_req.c
${OPENSSL_SOURCE_DIR}/crypto/x509/x_x509.c
${OPENSSL_SOURCE_DIR}/crypto/x509/x_x509a.c
${OPENSSL_SOURCE_DIR}/engines/e_capi.c
${OPENSSL_SOURCE_DIR}/engines/e_dasync.c
${OPENSSL_SOURCE_DIR}/engines/e_loader_attic.c
${OPENSSL_SOURCE_DIR}/engines/e_ossltest.c
${OPENSSL_SOURCE_DIR}/engines/e_padlock.c
${OPENSSL_SOURCE_DIR}/providers/baseprov.c
${OPENSSL_SOURCE_DIR}/providers/common/bio_prov.c
${OPENSSL_SOURCE_DIR}/providers/common/capabilities.c
@ -985,8 +981,6 @@ set(CRYPTO_SRC
${OPENSSL_SOURCE_DIR}/providers/common/securitycheck.c
${OPENSSL_SOURCE_DIR}/providers/common/securitycheck_default.c
${OPENSSL_SOURCE_DIR}/providers/defltprov.c
${OPENSSL_SOURCE_DIR}/providers/fips/fips_entry.c
${OPENSSL_SOURCE_DIR}/providers/fips/fipsprov.c
${OPENSSL_SOURCE_DIR}/providers/implementations/asymciphers/rsa_enc.c
${OPENSSL_SOURCE_DIR}/providers/implementations/asymciphers/sm2_enc.c
${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes.c
@ -1145,11 +1139,19 @@ set(CRYPTO_SRC
${OPENSSL_SOURCE_DIR}/providers/implementations/signature/sm2_sig.c
${OPENSSL_SOURCE_DIR}/providers/implementations/storemgmt/file_store.c
${OPENSSL_SOURCE_DIR}/providers/implementations/storemgmt/file_store_any2obj.c
${OPENSSL_SOURCE_DIR}/providers/legacyprov.c
${OPENSSL_SOURCE_DIR}/providers/nullprov.c
${OPENSSL_SOURCE_DIR}/providers/prov_running.c
${OPENSSL_SOURCE_DIR}/ssl/record/methods/tls_pad.c
${OPENSSL_SOURCE_DIR}/ssl/record/methods/ssl3_cbc.c
)
if(NOT ENABLE_OPENSSL_DYNAMIC)
set(CRYPTO_SRC ${CRYPTO_SRC}
${OPENSSL_SOURCE_DIR}/providers/fips/fips_entry.c
${OPENSSL_SOURCE_DIR}/providers/fips/fipsprov.c
)
endif()
if(ARCH_AMD64)
if (OS_DARWIN)
set(CRYPTO_SRC ${CRYPTO_SRC}
@ -1376,8 +1378,6 @@ set(SSL_SRC
${OPENSSL_SOURCE_DIR}/ssl/quic/uint_set.c
${OPENSSL_SOURCE_DIR}/ssl/record/rec_layer_d1.c
${OPENSSL_SOURCE_DIR}/ssl/record/rec_layer_s3.c
${OPENSSL_SOURCE_DIR}/ssl/record/methods/tls_pad.c
${OPENSSL_SOURCE_DIR}/ssl/record/methods/ssl3_cbc.c
${OPENSSL_SOURCE_DIR}/ssl/record/methods/dtls_meth.c
${OPENSSL_SOURCE_DIR}/ssl/record/methods/ssl3_meth.c
${OPENSSL_SOURCE_DIR}/ssl/record/methods/tls13_meth.c

2
docker/test/stateless/run.sh
View File

@ -44,8 +44,6 @@ source /utils.lib
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
echo "Azure is disabled"
elif [[ -n "$USE_SHARED_CATALOG" ]] && [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
echo "Azure is disabled"
else
azurite-blob --blobHost 0.0.0.0 --blobPort 10000 --debug /azurite_log &
fi

4
docs/en/engines/table-engines/mergetree-family/replication.md
View File

@ -304,10 +304,10 @@ We use the term `MergeTree` to refer to all table engines in the `MergeTree fami
If you had a `MergeTree` table that was manually replicated, you can convert it to a replicated table. You might need to do this if you have already collected a large amount of data in a `MergeTree` table and now you want to enable replication.
`MergeTree` table can be automatically converted on server restart if `convert_to_replicated` flag is set at the table's data directory (`/var/lib/clickhouse/store/xxx/xxxyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy/` for `Atomic` database).
`MergeTree` table can be automatically converted on server restart if `convert_to_replicated` flag is set at the table's data directory (`/store/xxx/xxxyyyyy-yyyy-yyyy-yyyy-yyyyyyyyyyyy/` for `Atomic` database).
Create empty `convert_to_replicated` file and the table will be loaded as replicated on next server restart.
This query can be used to get the table's data path.
This query can be used to get the table's data path. If table has many data paths, you have to use the first one.
```sql
SELECT data_paths FROM system.tables WHERE table = 'table_name' AND database = 'database_name';

2
docs/en/interfaces/third-party/integrations.md vendored
View File

@ -76,7 +76,7 @@ ClickHouse, Inc. does **not** maintain the tools and libraries listed below and
- [clickhouse-maxmind-geoip](https://github.com/AlexeyKupershtokh/clickhouse-maxmind-geoip)
- AutoML
- [MindsDB](https://mindsdb.com/)
- [MindsDB](https://github.com/mindsdb/mindsdb) - Predictive AI layer for ClickHouse database.
- [MindsDB](https://github.com/mindsdb/mindsdb) - Integrates with ClickHouse, making data from ClickHouse accessible to a diverse range of AI/ML models.
## Programming Language Ecosystems {#programming-language-ecosystems}

2
docs/en/operations/server-configuration-parameters/settings.md
View File

@ -523,7 +523,7 @@ See settings `cgroups_memory_usage_observer_wait_time` and `cgroup_memory_watche
Type: Double
Default: 0.95
Default: 0.9
## max_table_size_to_drop

20
docs/en/operations/system-tables/text_log.md
View File

@ -30,6 +30,16 @@ Columns:
- `source_file` (LowCardinality(String)) — Source file from which the logging was done.
- `source_line` (UInt64) — Source line from which the logging was done.
- `message_format_string` (LowCardinality(String)) — A format string that was used to format the message.
- `value1` (String) - Argument 1 that was used to format the message.
- `value2` (String) - Argument 2 that was used to format the message.
- `value3` (String) - Argument 3 that was used to format the message.
- `value4` (String) - Argument 4 that was used to format the message.
- `value5` (String) - Argument 5 that was used to format the message.
- `value6` (String) - Argument 6 that was used to format the message.
- `value7` (String) - Argument 7 that was used to format the message.
- `value8` (String) - Argument 8 that was used to format the message.
- `value9` (String) - Argument 9 that was used to format the message.
- `value10` (String) - Argument 10 that was used to format the message.
**Example**
@ -55,4 +65,14 @@ revision: 54440
source_file: /ClickHouse/src/Interpreters/DNSCacheUpdater.cpp; void DB::DNSCacheUpdater::start()
source_line: 45
message_format_string: Update period {} seconds
value1: 15
value2:
value3:
value4:
value5:
value6:
value7:
value8:
value9:
value10:
```

68
docs/en/sql-reference/functions/math-functions.md
View File

@ -8,7 +8,7 @@ sidebar_label: Mathematical
## e
Returns e ([Euler's constant](https://en.wikipedia.org/wiki/Euler%27s_constant))
Returns e ([Euler's constant](https://en.wikipedia.org/wiki/Euler%27s_constant)).
**Syntax**
@ -45,7 +45,7 @@ exp(x)
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -65,7 +65,7 @@ Alias: `ln(x)`
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -83,7 +83,7 @@ exp2(x)
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -111,7 +111,7 @@ log2(x)
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -129,7 +129,7 @@ exp10(x)
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -157,7 +157,7 @@ log10(x)
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -173,7 +173,7 @@ sqrt(x)
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -189,7 +189,7 @@ cbrt(x)
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -207,7 +207,7 @@ erf(x)
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -239,7 +239,7 @@ erfc(x)
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -257,7 +257,7 @@ lgamma(x)
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -275,7 +275,7 @@ gamma(x)
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -293,7 +293,7 @@ sin(x)
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -323,7 +323,7 @@ cos(x)
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -341,7 +341,7 @@ tan(x)
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -359,7 +359,7 @@ asin(x)
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -377,7 +377,7 @@ acos(x)
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -395,7 +395,7 @@ atan(x)
**Arguments**
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md)
- `x` - [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -434,7 +434,7 @@ cosh(x)
**Arguments**
- `x` — The angle, in radians. Values from the interval: `-∞ < x < +∞`. [Float64](../../sql-reference/data-types/float.md#float32-float64).
- `x` — The angle, in radians. Values from the interval: `-∞ < x < +∞`. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -468,7 +468,7 @@ acosh(x)
**Arguments**
- `x` — Hyperbolic cosine of angle. Values from the interval: `1 <= x < +∞`. [Float64](../../sql-reference/data-types/float.md#float32-float64).
- `x` — Hyperbolic cosine of angle. Values from the interval: `1 <= x < +∞`. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -502,7 +502,7 @@ sinh(x)
**Arguments**
- `x` — The angle, in radians. Values from the interval: `-∞ < x < +∞`. [Float64](../../sql-reference/data-types/float.md#float32-float64).
- `x` — The angle, in radians. Values from the interval: `-∞ < x < +∞`. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -536,7 +536,7 @@ asinh(x)
**Arguments**
- `x` — Hyperbolic sine of angle. Values from the interval: `-∞ < x < +∞`. [Float64](../../sql-reference/data-types/float.md#float32-float64).
- `x` — Hyperbolic sine of angle. Values from the interval: `-∞ < x < +∞`. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -569,13 +569,13 @@ tanh(x)
**Arguments**
- `x` — The angle, in radians. Values from the interval: `-∞ < x < +∞`. [Float64](../../sql-reference/data-types/float.md#float32-float64).
- `x` — The angle, in radians. Values from the interval: `-∞ < x < +∞`. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
- Values from the interval: `-1 < tanh(x) < 1`.
Type: [Float64](../../sql-reference/data-types/float.md#float32-float64).
Type: [Float*](../../sql-reference/data-types/float.md#float32-float64).
**Example**
@ -601,7 +601,7 @@ atanh(x)
**Arguments**
- `x` — Hyperbolic tangent of angle. Values from the interval: `–1 < x < 1`. [Float64](../../sql-reference/data-types/float.md#float32-float64).
- `x` — Hyperbolic tangent of angle. Values from the interval: `–1 < x < 1`. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -635,8 +635,8 @@ atan2(y, x)
**Arguments**
- `y` — y-coordinate of the point through which the ray passes. [Float64](../../sql-reference/data-types/float.md#float32-float64).
- `x` — x-coordinate of the point through which the ray passes. [Float64](../../sql-reference/data-types/float.md#float32-float64).
- `y` — y-coordinate of the point through which the ray passes. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md).
- `x` — x-coordinate of the point through which the ray passes. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md).
**Returned value**
@ -670,8 +670,8 @@ hypot(x, y)
**Arguments**
- `x` — The first cathetus of a right-angle triangle. [Float64](../../sql-reference/data-types/float.md#float32-float64).
- `y` — The second cathetus of a right-angle triangle. [Float64](../../sql-reference/data-types/float.md#float32-float64).
- `x` — The first cathetus of a right-angle triangle. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md).
- `y` — The second cathetus of a right-angle triangle. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md).
**Returned value**
@ -705,7 +705,7 @@ log1p(x)
**Arguments**
- `x` — Values from the interval: `-1 < x < +∞`. [Float64](../../sql-reference/data-types/float.md#float32-float64).
- `x` — Values from the interval: `-1 < x < +∞`. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -747,6 +747,8 @@ sign(x)
- 0 for `x = 0`
- 1 for `x > 0`
Type: [Int8](../../sql-reference/data-types/int-uint.md).
**Examples**
Sign for the zero value:
@ -803,7 +805,7 @@ degrees(x)
**Arguments**
- `x` — Input in radians. [Float64](../../sql-reference/data-types/float.md#float32-float64).
- `x` — Input in radians. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**
@ -837,7 +839,7 @@ radians(x)
**Arguments**
- `x` — Input in degrees. [Float64](../../sql-reference/data-types/float.md#float32-float64).
- `x` — Input in degrees. [(U)Int*](../../sql-reference/data-types/int-uint.md), [Float*](../../sql-reference/data-types/float.md) or [Decimal*](../../sql-reference/data-types/decimal.md).
**Returned value**

135
docs/en/sql-reference/functions/string-functions.md
View File

@ -88,20 +88,93 @@ Result:
## length
Returns the length of a string in bytes (not: in characters or Unicode code points).
The function also works for arrays.
Returns the length of a string in bytes rather than in characters or Unicode code points. The function also works for arrays.
Alias: `OCTET_LENGTH`
**Syntax**
```sql
length(s)
```
**Parameters**
- `s`: An input string or array. [String](../data-types/string)/[Array](../data-types/array).
**Returned value**
- Length of the string or array `s` in bytes. [UInt64](../data-types/int-uint).
**Example**
Query:
```sql
SELECT length('Hello, world!');
```
Result:
```response
┌─length('Hello, world!')─┐
│ 13 │
└─────────────────────────┘
```
Query:
```sql
SELECT length([1, 2, 3, 4]);
```
Result:
```response
┌─length([1, 2, 3, 4])─┐
│ 4 │
└──────────────────────┘
```
## lengthUTF8
Returns the length of a string in Unicode code points (not: in bytes or characters). It assumes that the string contains valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined.
Returns the length of a string in Unicode code points rather than in bytes or characters. It assumes that the string contains valid UTF-8 encoded text. If this assumption is violated, no exception is thrown and the result is undefined.
Alias:
Aliases:
- `CHAR_LENGTH`
- `CHARACTER_LENGTH`
**Syntax**
```sql
lengthUTF8(s)
```
**Parameters**
- `s`: String containing valid UTF-8 encoded text. [String](../data-types/string).
**Returned value**
- Length of the string `s` in Unicode code points. [UInt64](../data-types/int-uint.md).
**Example**
Query:
```sql
SELECT lengthUTF8('Здравствуй, мир!');
```
Result:
```response
┌─lengthUTF8('Здравствуй, мир!')─┐
│ 16 │
└────────────────────────────────┘
```
## left
Returns a substring of string `s` with a specified `offset` starting from the left.
@ -1055,6 +1128,34 @@ Result:
Like `base58Decode` but returns an empty string in case of error.
**Syntax**
```sql
tryBase58Decode(encoded)
```
**Parameters**
- `encoded`: [String](../../sql-reference/data-types/string.md) column or constant. If the string is not a valid Base58-encoded value, returns an empty string in case of error.
**Returned value**
- A string containing the decoded value of the argument.
**Examples**
Query:
```sql
SELECT tryBase58Decode('3dc8KtHrwM') as res;
```
```response
┌─res─────┐
│ Encoded │
└─────────┘
```
## base64Encode
Encodes a String or FixedString as base64.
@ -1071,6 +1172,30 @@ Alias: `FROM_BASE64`.
Like `base64Decode` but returns an empty string in case of error.
**Syntax**
```sql
tryBase64Decode(encoded)
```
**Parameters**
- `encoded`: [String](../../sql-reference/data-types/string.md) column or constant. If the string is not a valid Base58-encoded value, returns an empty string in case of error.
**Examples**
Query:
```sql
SELECT tryBase64Decode('RW5jb2RlZA==') as res;
```
```response
┌─res─────┐
│ Encoded │
└─────────┘
```
## endsWith {#endswith}
Returns whether string `str` ends with `suffix`.

20
docs/ru/operations/system-tables/text_log.md
View File

@ -29,6 +29,16 @@ slug: /ru/operations/system-tables/text_log
- `source_file` (LowCardinality(String)) — исходный файл, из которого была сделана запись.
- `source_line` (UInt64) — исходная строка, из которой была сделана запись.
- `message_format_string` (LowCardinality(String)) — форматная строка, с помощью которой было отформатировано сообщение.
- `value1` (String) - аргумент 1, который использовался для форматирования сообщения.
- `value2` (String) - аргумент 2, который использовался для форматирования сообщения.
- `value3` (String) - аргумент 3, который использовался для форматирования сообщения.
- `value4` (String) - аргумент 4, который использовался для форматирования сообщения.
- `value5` (String) - аргумент 5, который использовался для форматирования сообщения.
- `value6` (String) - аргумент 6, который использовался для форматирования сообщения.
- `value7` (String) - аргумент 7, который использовался для форматирования сообщения.
- `value8` (String) - аргумент 8, который использовался для форматирования сообщения.
- `value9` (String) - аргумент 9, который использовался для форматирования сообщения.
- `value10` (String) - аргумент 10, который использовался для форматирования сообщения.
**Пример**
@ -53,4 +63,14 @@ revision: 54440
source_file: /ClickHouse/src/Interpreters/DNSCacheUpdater.cpp; void DB::DNSCacheUpdater::start()
source_line: 45
message_format_string: Update period {} seconds
value1: 15
value2:
value3:
value4:
value5:
value6:
value7:
value8:
value9:
value10:
```

9
programs/install/Install.cpp
View File

@ -729,6 +729,15 @@ int mainEntryClickHouseInstall(int argc, char ** argv)
}
}
/// Don't allow relative paths because install script may cd to / when installing
/// And having path=./ may break the system
if (log_path.is_relative())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Log path is relative: {}", log_path.string());
if (data_path.is_relative())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Data path is relative: {}", data_path.string());
if (pid_path.is_relative())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Pid path is relative: {}", pid_path.string());
/// Create directories for data and log.
if (fs::exists(log_path))

3
src/Access/AccessControl.cpp
View File

@ -607,7 +607,8 @@ AuthResult AccessControl::authenticate(const Credentials & credentials, const Po
/// We use the same message for all authentication failures because we don't want to give away any unnecessary information for security reasons,
/// only the log will show the exact reason.
throw Exception(PreformattedMessage{message.str(),
"{}: Authentication failed: password is incorrect, or there is no user with such name.{}"},
"{}: Authentication failed: password is incorrect, or there is no user with such name.{}",
std::vector<std::string>{credentials.getUserName()}},
ErrorCodes::AUTHENTICATION_FAILED);
}
}

14
src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.cpp
View File

@ -51,10 +51,15 @@ public:
{
const auto & second_const_value = second_const_node->getValue();
if (second_const_value.isNull()
|| (lower_name == "sum" && isInt64OrUInt64FieldType(second_const_value.getType()) && second_const_value.get<UInt64>() == 0))
|| (lower_name == "sum" && isInt64OrUInt64FieldType(second_const_value.getType()) && second_const_value.get<UInt64>() == 0
&& !function_node->getResultType()->isNullable()))
{
/// avg(if(cond, a, null)) -> avgIf(a, cond)
/// avg(if(cond, nullable_a, null)) -> avgIfOrNull(a, cond)
/// sum(if(cond, a, 0)) -> sumIf(a, cond)
/// sum(if(cond, nullable_a, 0)) **is not** equivalent to sumIfOrNull(cond, nullable_a) as
/// it changes the output when no rows pass the condition (from 0 to NULL)
function_arguments_nodes.resize(2);
function_arguments_nodes[0] = std::move(if_arguments_nodes[1]);
function_arguments_nodes[1] = std::move(if_arguments_nodes[0]);
@ -66,10 +71,13 @@ public:
{
const auto & first_const_value = first_const_node->getValue();
if (first_const_value.isNull()
|| (lower_name == "sum" && isInt64OrUInt64FieldType(first_const_value.getType()) && first_const_value.get<UInt64>() == 0))
|| (lower_name == "sum" && isInt64OrUInt64FieldType(first_const_value.getType()) && first_const_value.get<UInt64>() == 0
&& !function_node->getResultType()->isNullable()))
{
/// avg(if(cond, null, a) -> avgIf(a, !cond))
/// avg(if(cond, null, a) -> avgIfOrNullable(a, !cond))
/// sum(if(cond, 0, a) -> sumIf(a, !cond))
/// sum(if(cond, 0, nullable_a) **is not** sumIf(a, !cond)) -> Same as above
auto not_function = std::make_shared<FunctionNode>("not");
auto & not_function_arguments = not_function->getArguments().getNodes();
not_function_arguments.push_back(std::move(if_arguments_nodes[0]));

6
src/Common/Exception.cpp
View File

@ -391,6 +391,7 @@ PreformattedMessage getCurrentExceptionMessageAndPattern(bool with_stacktrace, b
{
WriteBufferFromOwnString stream;
std::string_view message_format_string;
std::vector<std::string> message_format_string_args;
try
{
@ -402,6 +403,7 @@ PreformattedMessage getCurrentExceptionMessageAndPattern(bool with_stacktrace, b
<< (with_extra_info ? getExtraExceptionInfo(e) : "")
<< " (version " << VERSION_STRING << VERSION_OFFICIAL << ")";
message_format_string = e.tryGetMessageFormatString();
message_format_string_args = e.getMessageFormatStringArgs();
}
catch (const Poco::Exception & e)
{
@ -462,7 +464,7 @@ PreformattedMessage getCurrentExceptionMessageAndPattern(bool with_stacktrace, b
catch (...) {} // NOLINT(bugprone-empty-catch)
}
return PreformattedMessage{stream.str(), message_format_string};
return PreformattedMessage{stream.str(), message_format_string, message_format_string_args};
}
@ -581,7 +583,7 @@ PreformattedMessage getExceptionMessageAndPattern(const Exception & e, bool with
}
catch (...) {} // NOLINT(bugprone-empty-catch)
return PreformattedMessage{stream.str(), e.tryGetMessageFormatString()};
return PreformattedMessage{stream.str(), e.tryGetMessageFormatString(), e.getMessageFormatStringArgs()};
}
std::string getExceptionMessage(std::exception_ptr e, bool with_stacktrace)

33
src/Common/Exception.h
View File

@ -13,6 +13,7 @@
#include <memory>
#include <vector>
#include <fmt/core.h>
#include <fmt/format.h>
#include <Poco/Exception.h>
@ -59,6 +60,7 @@ public:
std::terminate();
capture_thread_frame_pointers = thread_frame_pointers;
message_format_string = msg.format_string;
message_format_string_args = msg.format_string_args;
}
Exception(PreformattedMessage && msg, int code): Exception(std::move(msg.text), code)
@ -67,6 +69,7 @@ public:
std::terminate();
capture_thread_frame_pointers = thread_frame_pointers;
message_format_string = msg.format_string;
message_format_string_args = msg.format_string_args;
}
/// Collect call stacks of all previous jobs' schedulings leading to this thread job's execution
@ -107,12 +110,7 @@ public:
// Format message with fmt::format, like the logging functions.
template <typename... Args>
Exception(int code, FormatStringHelper<Args...> fmt, Args &&... args)
: Exception(fmt::format(fmt.fmt_str, std::forward<Args>(args)...), code)
{
capture_thread_frame_pointers = thread_frame_pointers;
message_format_string = fmt.message_format_string;
}
Exception(int code, FormatStringHelper<Args...> fmt, Args &&... args) : Exception(fmt.format(std::forward<Args>(args)...), code) {}
struct CreateFromPocoTag {};
struct CreateFromSTDTag {};
@ -152,6 +150,8 @@ public:
std::string_view tryGetMessageFormatString() const { return message_format_string; }
std::vector<std::string> getMessageFormatStringArgs() const { return message_format_string_args; }
private:
#ifndef STD_EXCEPTION_HAS_STACK_TRACE
StackTrace trace;
@ -162,6 +162,7 @@ private:
protected:
std::string_view message_format_string;
std::vector<std::string> message_format_string_args;
/// Local copy of static per-thread thread_frame_pointers, should be mutable to be unpoisoned on printout
mutable std::vector<StackTrace::FramePointers> capture_thread_frame_pointers;
};
@ -193,26 +194,29 @@ public:
// Format message with fmt::format, like the logging functions.
template <typename... Args>
ErrnoException(int code, FormatStringHelper<Args...> fmt, Args &&... args)
: Exception(fmt::format(fmt.fmt_str, std::forward<Args>(args)...), code), saved_errno(errno)
: Exception(fmt.format(std::forward<Args>(args)...), code), saved_errno(errno)
{
addMessage(", {}", errnoToString(saved_errno));
}
template <typename... Args>
ErrnoException(int code, int with_errno, FormatStringHelper<Args...> fmt, Args &&... args)
: Exception(fmt.format(std::forward<Args>(args)...), code), saved_errno(with_errno)
{
capture_thread_frame_pointers = thread_frame_pointers;
message_format_string = fmt.message_format_string;
addMessage(", {}", errnoToString(saved_errno));
}
template <typename... Args>
[[noreturn]] static void throwWithErrno(int code, int with_errno, FormatStringHelper<Args...> fmt, Args &&... args)
{
auto e = ErrnoException(fmt::format(fmt.fmt_str, std::forward<Args>(args)...), code, with_errno);
e.message_format_string = fmt.message_format_string;
auto e = ErrnoException(code, with_errno, std::move(fmt), std::forward<Args>(args)...);
throw e; /// NOLINT
}
template <typename... Args>
[[noreturn]] static void throwFromPath(int code, const std::string & path, FormatStringHelper<Args...> fmt, Args &&... args)
{
auto e = ErrnoException(fmt::format(fmt.fmt_str, std::forward<Args>(args)...), code, errno);
e.message_format_string = fmt.message_format_string;
auto e = ErrnoException(code, errno, std::move(fmt), std::forward<Args>(args)...);
e.path = path;
throw e; /// NOLINT
}
@ -221,8 +225,7 @@ public:
[[noreturn]] static void
throwFromPathWithErrno(int code, const std::string & path, int with_errno, FormatStringHelper<Args...> fmt, Args &&... args)
{
auto e = ErrnoException(fmt::format(fmt.fmt_str, std::forward<Args>(args)...), code, with_errno);
e.message_format_string = fmt.message_format_string;
auto e = ErrnoException(code, with_errno, std::move(fmt), std::forward<Args>(args)...);
e.path = path;
throw e; /// NOLINT
}

40
src/Common/LoggingFormatStringHelpers.h
View File

@ -2,8 +2,11 @@
#include <base/defines.h>
#include <base/types.h>
#include <fmt/args.h>
#include <fmt/core.h>
#include <fmt/format.h>
#include <mutex>
#include <type_traits>
#include <unordered_map>
#include <Poco/Logger.h>
#include <Poco/Message.h>
@ -14,6 +17,10 @@ struct PreformattedMessage;
consteval void formatStringCheckArgsNumImpl(std::string_view str, size_t nargs);
template <typename T> constexpr std::string_view tryGetStaticFormatString(T && x);
[[maybe_unused]] inline void tryGetFormattedArgs(std::vector<std::string>&) {};
template <typename T, typename... Ts> [[maybe_unused]] inline void tryGetFormattedArgs(std::vector<std::string>&, T &&, Ts && ...);
template <typename... Args> inline std::string tryGetArgsAndFormat(std::vector<std::string>&, fmt::format_string<Args...>, Args && ...);
/// Extract format string from a string literal and constructs consteval fmt::format_string
template <typename... Args>
struct FormatStringHelperImpl
@ -39,6 +46,7 @@ struct PreformattedMessage
{
std::string text;
std::string_view format_string;
std::vector<std::string> format_string_args;
template <typename... Args>
static PreformattedMessage create(FormatStringHelper<Args...> fmt, Args &&... args);
@ -47,22 +55,26 @@ struct PreformattedMessage
operator std::string () && { return std::move(text); } /// NOLINT
operator fmt::format_string<> () const { UNREACHABLE(); } /// NOLINT
void apply(std::string & out_text, std::string_view & out_format_string) const &
void apply(std::string & out_text, std::string_view & out_format_string, std::vector<std::string> & out_format_string_args) const &
{
out_text = text;
out_format_string = format_string;
out_format_string_args = format_string_args;
}
void apply(std::string & out_text, std::string_view & out_format_string) &&
void apply(std::string & out_text, std::string_view & out_format_string, std::vector<std::string> & out_format_string_args) &&
{
out_text = std::move(text);
out_format_string = format_string;
out_format_string_args = std::move(format_string_args);
}
};
template <typename... Args>
PreformattedMessage FormatStringHelperImpl<Args...>::format(Args && ...args) const
{
return PreformattedMessage{fmt::format(fmt_str, std::forward<Args>(args)...), message_format_string};
std::vector<std::string> out_format_string_args;
std::string msg_text = tryGetArgsAndFormat(out_format_string_args, fmt_str, std::forward<Args>(args)...);
return PreformattedMessage{msg_text, message_format_string, out_format_string_args};
}
template <typename... Args>
@ -113,12 +125,23 @@ template <typename T> constexpr std::string_view tryGetStaticFormatString(T && x
}
}
template <typename T, typename... Ts> void tryGetFormattedArgs(std::vector<std::string>& out, T && x, Ts && ...rest)
{
if constexpr (std::is_base_of_v<fmt::detail::view, std::decay_t<T>>)
out.push_back(fmt::format("{}", std::remove_reference_t<T>(x)));
else
out.push_back(fmt::format("{}", std::forward<T>(x)));
tryGetFormattedArgs(out, std::forward<Ts>(rest)...);
}
/// Constexpr ifs are not like ifdefs, and compiler still checks that unneeded code can be compiled
/// This template is useful to avoid compilation failures when condition of some "constexpr if" is false
template<bool enable> struct ConstexprIfsAreNotIfdefs
{
template <typename T> constexpr static std::string_view getStaticFormatString(T &&) { return {}; }
template <typename T> static PreformattedMessage getPreformatted(T &&) { return {}; }
template <typename... Args> static std::string getArgsAndFormat(std::vector<std::string>&, fmt::format_string<Args...>, Args &&...) { return {}; }
};
template<> struct ConstexprIfsAreNotIfdefs<true>
@ -133,8 +156,19 @@ template<> struct ConstexprIfsAreNotIfdefs<true>
}
template <typename T> static T && getPreformatted(T && x) { return std::forward<T>(x); }
template <typename... Args> static std::string getArgsAndFormat(std::vector<std::string>& out, fmt::format_string<Args...> fmt_str, Args && ...args)
{
return tryGetArgsAndFormat(out, std::move(fmt_str), std::forward<Args>(args)...);
}
};
template <typename... Args> inline std::string tryGetArgsAndFormat(std::vector<std::string>& out, fmt::format_string<Args...> fmt_str, Args && ...args)
{
tryGetFormattedArgs(out, args...);
return fmt::format(fmt_str, std::forward<Args>(args)...);
}
template <typename... Ts> constexpr size_t numArgs(Ts &&...) { return sizeof...(Ts); }
template <typename T, typename... Ts> constexpr auto firstArg(T && x, Ts &&...) { return std::forward<T>(x); }
/// For implicit conversion of fmt::basic_runtime<> to char* for std::string ctor

47
src/Common/ZooKeeper/ZooKeeper.cpp
View File

@ -518,7 +518,8 @@ bool ZooKeeper::existsWatch(const std::string & path, Coordination::Stat * stat,
return code != Coordination::Error::ZNONODE;
}
Coordination::Error ZooKeeper::getImpl(const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback)
Coordination::Error ZooKeeper::getImpl(
const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallbackPtr watch_callback)
{
auto future_result = asyncTryGetNoThrow(path, watch_callback);
@ -541,6 +542,11 @@ Coordination::Error ZooKeeper::getImpl(const std::string & path, std::string & r
}
}
Coordination::Error ZooKeeper::getImpl(const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback)
{
return getImpl(path, res, stat, watch_callback ? std::make_shared<Coordination::WatchCallback>(watch_callback) : Coordination::WatchCallbackPtr{});
}
std::string ZooKeeper::get(const std::string & path, Coordination::Stat * stat, const EventPtr & watch)
{
Coordination::Error code = Coordination::Error::ZOK;
@ -561,6 +567,17 @@ std::string ZooKeeper::getWatch(const std::string & path, Coordination::Stat * s
throw KeeperException(code, "Can't get data for node '{}': node doesn't exist", path);
}
std::string ZooKeeper::getWatch(const std::string & path, Coordination::Stat * stat, Coordination::WatchCallbackPtr watch_callback)
{
Coordination::Error code = Coordination::Error::ZOK;
std::string res;
if (tryGetWatch(path, res, stat, watch_callback, &code))
return res;
else
throw KeeperException(code, "Can't get data for node '{}': node doesn't exist", path);
}
bool ZooKeeper::tryGet(
const std::string & path,
std::string & res,
@ -571,6 +588,25 @@ bool ZooKeeper::tryGet(
return tryGetWatch(path, res, stat, callbackForEvent(watch), return_code);
}
bool ZooKeeper::tryGetWatch(
const std::string & path,
std::string & res,
Coordination::Stat * stat,
Coordination::WatchCallbackPtr watch_callback,
Coordination::Error * return_code)
{
Coordination::Error code = getImpl(path, res, stat, watch_callback);
if (!(code == Coordination::Error::ZOK || code == Coordination::Error::ZNONODE))
throw KeeperException::fromPath(code, path);
if (return_code)
*return_code = code;
return code == Coordination::Error::ZOK;
}
bool ZooKeeper::tryGetWatch(
const std::string & path,
std::string & res,
@ -589,6 +625,7 @@ bool ZooKeeper::tryGetWatch(
return code == Coordination::Error::ZOK;
}
Coordination::Error ZooKeeper::setImpl(const std::string & path, const std::string & data,
int32_t version, Coordination::Stat * stat)
{
@ -1062,6 +1099,11 @@ std::future<Coordination::GetResponse> ZooKeeper::asyncGet(const std::string & p
}
std::future<Coordination::GetResponse> ZooKeeper::asyncTryGetNoThrow(const std::string & path, Coordination::WatchCallback watch_callback)
{
return asyncTryGetNoThrow(path, watch_callback ? std::make_shared<Coordination::WatchCallback>(watch_callback) : Coordination::WatchCallbackPtr{});
}
std::future<Coordination::GetResponse> ZooKeeper::asyncTryGetNoThrow(const std::string & path, Coordination::WatchCallbackPtr watch_callback)
{
auto promise = std::make_shared<std::promise<Coordination::GetResponse>>();
auto future = promise->get_future();
@ -1071,8 +1113,7 @@ std::future<Coordination::GetResponse> ZooKeeper::asyncTryGetNoThrow(const std::
promise->set_value(response);
};
impl->get(path, std::move(callback),
watch_callback ? std::make_shared<Coordination::WatchCallback>(watch_callback) : Coordination::WatchCallbackPtr{});
impl->get(path, std::move(callback), watch_callback);
return future;
}

12
src/Common/ZooKeeper/ZooKeeper.h
View File

@ -306,6 +306,7 @@ public:
std::string get(const std::string & path, Coordination::Stat * stat = nullptr, const EventPtr & watch = nullptr);
std::string getWatch(const std::string & path, Coordination::Stat * stat, Coordination::WatchCallback watch_callback);
std::string getWatch(const std::string & path, Coordination::Stat * stat, Coordination::WatchCallbackPtr watch_callback);
using MultiGetResponse = MultiReadResponses<Coordination::GetResponse, false>;
using MultiTryGetResponse = MultiReadResponses<Coordination::GetResponse, true>;
@ -338,6 +339,13 @@ public:
Coordination::WatchCallback watch_callback,
Coordination::Error * code = nullptr);
bool tryGetWatch(
const std::string & path,
std::string & res,
Coordination::Stat * stat,
Coordination::WatchCallbackPtr watch_callback,
Coordination::Error * code = nullptr);
template <typename TIter>
MultiTryGetResponse tryGet(TIter start, TIter end)
{
@ -520,6 +528,8 @@ public:
/// Like the previous one but don't throw any exceptions on future.get()
FutureGet asyncTryGetNoThrow(const std::string & path, Coordination::WatchCallback watch_callback = {});
FutureGet asyncTryGetNoThrow(const std::string & path, Coordination::WatchCallbackPtr watch_callback = {});
using FutureExists = std::future<Coordination::ExistsResponse>;
FutureExists asyncExists(const std::string & path, Coordination::WatchCallback watch_callback = {});
/// Like the previous one but don't throw any exceptions on future.get()
@ -625,6 +635,8 @@ private:
Coordination::Error removeImpl(const std::string & path, int32_t version);
Coordination::Error getImpl(
const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallback watch_callback);
Coordination::Error getImpl(
const std::string & path, std::string & res, Coordination::Stat * stat, Coordination::WatchCallbackPtr watch_callback);
Coordination::Error setImpl(const std::string & path, const std::string & data, int32_t version, Coordination::Stat * stat);
Coordination::Error getChildrenImpl(
const std::string & path,

23
src/Common/ZooKeeper/ZooKeeperLock.cpp
View File

@ -22,13 +22,16 @@ ZooKeeperLock::ZooKeeperLock(
const ZooKeeperPtr & zookeeper_,
const std::string & lock_prefix_,
const std::string & lock_name_,
const std::string & lock_message_)
const std::string & lock_message_,
bool throw_if_lost_)
: zookeeper(zookeeper_)
, lock_path(fs::path(lock_prefix_) / lock_name_)
, lock_message(lock_message_)
, throw_if_lost(throw_if_lost_)
, log(getLogger("zkutil::Lock"))
{
zookeeper->createIfNotExists(lock_prefix_, "");
LOG_TRACE(log, "Trying to create zookeeper lock on path {} for session {}", lock_path, zookeeper->getClientID());
}
ZooKeeperLock::~ZooKeeperLock()
@ -45,7 +48,7 @@ ZooKeeperLock::~ZooKeeperLock()
bool ZooKeeperLock::isLocked() const
{
return locked;
return locked && !zookeeper->expired();
}
const std::string & ZooKeeperLock::getLockPath() const
@ -56,7 +59,10 @@ const std::string & ZooKeeperLock::getLockPath() const
void ZooKeeperLock::unlock()
{
if (!locked)
{
LOG_TRACE(log, "Lock on path {} for session {} is not locked, exiting", lock_path, zookeeper->getClientID());
return;
}
locked = false;
@ -71,12 +77,19 @@ void ZooKeeperLock::unlock()
bool result = zookeeper->exists(lock_path, &stat);
if (result && stat.ephemeralOwner == zookeeper->getClientID())
{
zookeeper->remove(lock_path, -1);
LOG_TRACE(log, "Lock on path {} for session {} is unlocked", lock_path, zookeeper->getClientID());
}
else if (result)
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Lock is lost, it has another owner. Path: {}, message: {}, owner: {}, our id: {}",
lock_path, lock_message, stat.ephemeralOwner, zookeeper->getClientID());
else if (throw_if_lost)
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Lock is lost, node does not exist. Path: {}, message: {}, our id: {}",
lock_path, lock_message, zookeeper->getClientID());
else
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Lock is lost, node does not exist. Path: {}, message: {}", lock_path, lock_message);
LOG_INFO(log, "Lock is lost, node does not exist. Path: {}, message: {}, our id: {}",
lock_path, lock_message, zookeeper->getClientID());
}
bool ZooKeeperLock::tryLock()
@ -96,9 +109,9 @@ bool ZooKeeperLock::tryLock()
}
std::unique_ptr<ZooKeeperLock> createSimpleZooKeeperLock(
const ZooKeeperPtr & zookeeper, const String & lock_prefix, const String & lock_name, const String & lock_message)
const ZooKeeperPtr & zookeeper, const String & lock_prefix, const String & lock_name, const String & lock_message, bool throw_if_lost)
{
return std::make_unique<ZooKeeperLock>(zookeeper, lock_prefix, lock_name, lock_message);
return std::make_unique<ZooKeeperLock>(zookeeper, lock_prefix, lock_name, lock_message, throw_if_lost);
}

6
src/Common/ZooKeeper/ZooKeeperLock.h
View File

@ -32,7 +32,8 @@ public:
const ZooKeeperPtr & zookeeper_,
const std::string & lock_prefix_,
const std::string & lock_name_,
const std::string & lock_message_ = "");
const std::string & lock_message_ = "",
bool throw_if_lost_ = true);
~ZooKeeperLock();
@ -46,12 +47,13 @@ private:
std::string lock_path;
std::string lock_message;
bool throw_if_lost{true};
LoggerPtr log;
bool locked = false;
};
std::unique_ptr<ZooKeeperLock> createSimpleZooKeeperLock(
const ZooKeeperPtr & zookeeper, const String & lock_prefix, const String & lock_name, const String & lock_message);
const ZooKeeperPtr & zookeeper, const String & lock_prefix, const String & lock_name, const String & lock_message, bool throw_if_lost = true);
}

62
src/Common/examples/cow_columns.cpp
View File

@ -1,5 +1,6 @@
#include <Common/COW.h>
#include <iostream>
#include <base/defines.h>
class IColumn : public COW<IColumn>
@ -15,8 +16,6 @@ public:
virtual int get() const = 0;
virtual void set(int value) = 0;
virtual MutablePtr test() const = 0;
};
using ColumnPtr = IColumn::Ptr;
@ -31,58 +30,63 @@ private:
explicit ConcreteColumn(int data_) : data(data_) {}
ConcreteColumn(const ConcreteColumn &) = default;
MutableColumnPtr test() const override
{
MutableColumnPtr res = create(123);
return res;
}
public:
int get() const override { return data; }
void set(int value) override { data = value; }
};
template <typename ColPtr>
void print(const ColumnPtr & x, const ColPtr & y)
{
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n";
}
int main(int, char **)
{
ColumnPtr x = ConcreteColumn::create(1);
ColumnPtr y = x;//x->test();
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n";
ColumnPtr y = x;
print(x, y);
chassert(x->get() == 1 && y->get() == 1);
chassert(x->use_count() == 2 && y->use_count() == 2);
chassert(x.get() == y.get());
{
MutableColumnPtr mut = IColumn::mutate(std::move(y));
mut->set(2);
print(x, mut);
chassert(x->get() == 1 && mut->get() == 2);
chassert(x->use_count() == 1 && mut->use_count() == 1);
chassert(x.get() != mut.get());
std::cerr << "refcounts: " << x->use_count() << ", " << mut->use_count() << "\n";
std::cerr << "addresses: " << x.get() << ", " << mut.get() << "\n";
y = std::move(mut);
}
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n";
print(x, y);
chassert(x->get() == 1 && y->get() == 2);
chassert(x->use_count() == 1 && y->use_count() == 1);
chassert(x.get() != y.get());
x = ConcreteColumn::create(0);
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n";
print(x, y);
chassert(x->get() == 0 && y->get() == 2);
chassert(x->use_count() == 1 && y->use_count() == 1);
chassert(x.get() != y.get());
{
MutableColumnPtr mut = IColumn::mutate(std::move(y));
mut->set(3);
print(x, mut);
chassert(x->get() == 0 && mut->get() == 3);
chassert(x->use_count() == 1 && mut->use_count() == 1);
chassert(x.get() != mut.get());
std::cerr << "refcounts: " << x->use_count() << ", " << mut->use_count() << "\n";
std::cerr << "addresses: " << x.get() << ", " << mut.get() << "\n";
y = std::move(mut);
}
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
print(x, y);
chassert(x->get() == 0 && y->get() == 3);
chassert(x->use_count() == 1 && y->use_count() == 1);
chassert(x.get() != y.get());
return 0;
}

52
src/Common/examples/cow_compositions.cpp
View File

@ -1,5 +1,6 @@
#include <Common/COW.h>
#include <iostream>
#include <base/defines.h>
class IColumn : public COW<IColumn>
@ -61,47 +62,58 @@ public:
void set(int value) override { wrapped->set(value); }
};
template <typename ColPtr>
void print(const ColumnPtr & x, const ColPtr & y)
{
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n";
}
int main(int, char **)
{
ColumnPtr x = ColumnComposition::create(1);
ColumnPtr y = x;
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n";
print(x, y);
chassert(x->get() == 1 && y->get() == 1);
chassert(x->use_count() == 2 && y->use_count() == 2);
chassert(x.get() == y.get());
{
MutableColumnPtr mut = IColumn::mutate(std::move(y));
mut->set(2);
print(x, mut);
chassert(x->get() == 1 && mut->get() == 2);
chassert(x->use_count() == 1 && mut->use_count() == 1);
chassert(x.get() != mut.get());
std::cerr << "refcounts: " << x->use_count() << ", " << mut->use_count() << "\n";
std::cerr << "addresses: " << x.get() << ", " << mut.get() << "\n";
y = std::move(mut);
}
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n";
print(x, y);
chassert(x->get() == 1 && y->get() == 2);
chassert(x->use_count() == 1 && y->use_count() == 1);
chassert(x.get() != y.get());
x = ColumnComposition::create(0);
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
std::cerr << "addresses: " << x.get() << ", " << y.get() << "\n";
print(x, y);
chassert(x->get() == 0 && y->get() == 2);
chassert(x->use_count() == 1 && y->use_count() == 1);
chassert(x.get() != y.get());
{
MutableColumnPtr mut = IColumn::mutate(std::move(y));
mut->set(3);
print(x, mut);
chassert(x->get() == 0 && mut->get() == 3);
chassert(x->use_count() == 1 && mut->use_count() == 1);
chassert(x.get() != mut.get());
std::cerr << "refcounts: " << x->use_count() << ", " << mut->use_count() << "\n";
std::cerr << "addresses: " << x.get() << ", " << mut.get() << "\n";
y = std::move(mut);
}
std::cerr << "values: " << x->get() << ", " << y->get() << "\n";
std::cerr << "refcounts: " << x->use_count() << ", " << y->use_count() << "\n";
print(x, y);
chassert(x->get() == 0 && y->get() == 3);
chassert(x->use_count() == 1 && y->use_count() == 1);
chassert(x.get() != y.get());
return 0;
}

8
src/Common/logger_useful.h
View File

@ -2,6 +2,7 @@
/// Macros for convenient usage of Poco logger.
#include <unistd.h>
#include <fmt/args.h>
#include <fmt/format.h>
#include <Poco/Logger.h>
#include <Poco/Message.h>
@ -80,6 +81,7 @@ namespace impl
\
std::string_view _format_string; \
std::string _formatted_message; \
std::vector<std::string> _format_string_args; \
\
if constexpr (LogTypeInfo::is_static) \
{ \
@ -91,17 +93,17 @@ namespace impl
if constexpr (is_preformatted_message) \
{ \
static_assert(_nargs == 1 || !is_preformatted_message); \
ConstexprIfsAreNotIfdefs<is_preformatted_message>::getPreformatted(LOG_IMPL_FIRST_ARG(__VA_ARGS__)).apply(_formatted_message, _format_string); \
ConstexprIfsAreNotIfdefs<is_preformatted_message>::getPreformatted(LOG_IMPL_FIRST_ARG(__VA_ARGS__)).apply(_formatted_message, _format_string, _format_string_args); \
} \
else \
{ \
_formatted_message = _nargs == 1 ? firstArg(__VA_ARGS__) : fmt::format(__VA_ARGS__); \
_formatted_message = _nargs == 1 ? firstArg(__VA_ARGS__) : ConstexprIfsAreNotIfdefs<!is_preformatted_message>::getArgsAndFormat(_format_string_args, __VA_ARGS__); \
} \
\
std::string _file_function = __FILE__ "; "; \
_file_function += __PRETTY_FUNCTION__; \
Poco::Message _poco_message(_logger->name(), std::move(_formatted_message), \
(PRIORITY), _file_function.c_str(), __LINE__, _format_string); \
(PRIORITY), _file_function.c_str(), __LINE__, _format_string, _format_string_args); \
_channel->log(_poco_message); \
} \
catch (const Poco::Exception & logger_exception) \

8
src/Common/waitForPid.cpp
View File

@ -132,7 +132,9 @@ static PollPidResult pollPid(pid_t pid, int timeout_in_ms)
if (kq == -1)
return PollPidResult::FAILED;
struct kevent change = {.ident = 0};
struct kevent change;
change.ident = 0;
EV_SET(&change, pid, EVFILT_PROC, EV_ADD, NOTE_EXIT, 0, NULL);
int event_add_result = HANDLE_EINTR(kevent(kq, &change, 1, NULL, 0, NULL));
@ -144,7 +146,9 @@ static PollPidResult pollPid(pid_t pid, int timeout_in_ms)
return PollPidResult::FAILED;
}
struct kevent event = {.ident = 0};
struct kevent event;
event.ident = 0;
struct timespec remaining_timespec = {.tv_sec = timeout_in_ms / 1000, .tv_nsec = (timeout_in_ms % 1000) * 1000000};
int ready = HANDLE_EINTR(kevent(kq, nullptr, 0, &event, 1, &remaining_timespec));
PollPidResult result = ready < 0 ? PollPidResult::FAILED : PollPidResult::RESTART;

22
src/Databases/DatabaseOrdinary.cpp
View File

@ -95,16 +95,21 @@ static void setReplicatedEngine(ASTCreateQuery * create_query, ContextPtr contex
create_query->storage->set(create_query->storage->engine, engine->clone());
}
String DatabaseOrdinary::getConvertToReplicatedFlagPath(const String & name, bool tableStarted)
String DatabaseOrdinary::getConvertToReplicatedFlagPath(const String & name, const StoragePolicyPtr storage_policy, bool tableStarted)
{
fs::path data_path;
if (storage_policy->getDisks().empty())
data_path = getContext()->getPath();
else
data_path = storage_policy->getDisks()[0]->getPath();
if (!tableStarted)
{
auto create_query = tryGetCreateTableQuery(name, getContext());
data_path = fs::path(getContext()->getPath()) / getTableDataPath(create_query->as<ASTCreateQuery &>());
data_path = data_path / getTableDataPath(create_query->as<ASTCreateQuery &>());
}
else
data_path = fs::path(getContext()->getPath()) / getTableDataPath(name);
data_path = data_path / getTableDataPath(name);
return (data_path / CONVERT_TO_REPLICATED_FLAG_NAME).string();
}
@ -120,7 +125,14 @@ void DatabaseOrdinary::convertMergeTreeToReplicatedIfNeeded(ASTPtr ast, const Qu
if (!create_query->storage || !create_query->storage->engine->name.ends_with("MergeTree") || create_query->storage->engine->name.starts_with("Replicated") || create_query->storage->engine->name.starts_with("Shared"))
return;
auto convert_to_replicated_flag_path = getConvertToReplicatedFlagPath(qualified_name.table, false);
/// Get table's storage policy
MergeTreeSettings default_settings = getContext()->getMergeTreeSettings();
auto policy = getContext()->getStoragePolicy(default_settings.storage_policy);
if (auto * query_settings = create_query->storage->settings)
if (Field * policy_setting = query_settings->changes.tryGet("storage_policy"))
policy = getContext()->getStoragePolicy(policy_setting->safeGet<String>());
auto convert_to_replicated_flag_path = getConvertToReplicatedFlagPath(qualified_name.table, policy, false);
if (!fs::exists(convert_to_replicated_flag_path))
return;
@ -288,7 +300,7 @@ void DatabaseOrdinary::restoreMetadataAfterConvertingToReplicated(StoragePtr tab
if (!rmt)
return;
auto convert_to_replicated_flag_path = getConvertToReplicatedFlagPath(name.table, true);
auto convert_to_replicated_flag_path = getConvertToReplicatedFlagPath(name.table, table->getStoragePolicy(), true);
if (!fs::exists(convert_to_replicated_flag_path))
return;

2
src/Databases/DatabaseOrdinary.h
View File

@ -86,7 +86,7 @@ protected:
private:
void convertMergeTreeToReplicatedIfNeeded(ASTPtr ast, const QualifiedTableName & qualified_name, const String & file_name);
void restoreMetadataAfterConvertingToReplicated(StoragePtr table, const QualifiedTableName & name);
String getConvertToReplicatedFlagPath(const String & name, bool tableStarted);
String getConvertToReplicatedFlagPath(const String & name, StoragePolicyPtr storage_policy, bool tableStarted);
};
}

3
src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.cpp
View File

@ -188,7 +188,8 @@ Azure::Storage::Blobs::BlobClientOptions getAzureBlobClientOptions(const Poco::U
retry_options.MaxRetryDelay = std::chrono::milliseconds(config.getUInt(config_prefix + ".retry_max_backoff_ms", 1000));
using CurlOptions = Azure::Core::Http::CurlTransportOptions;
CurlOptions curl_options{.NoSignal = true};
CurlOptions curl_options;
curl_options.NoSignal = true;
if (config.has(config_prefix + ".curl_ip_resolve"))
{

2
src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageAuth.h
View File

@ -21,6 +21,8 @@ struct AzureBlobStorageEndpoint
String getEndpoint()
{
String url = storage_account_url;
if (url.ends_with('/'))
url.pop_back();
if (!account_name.empty())
url += "/" + account_name;

1
src/Formats/ProtobufSerializer.cpp
View File

@ -3736,6 +3736,7 @@ namespace
throw Exception(ErrorCodes::BAD_ARGUMENTS, "ClickHouse doesn't support type recursion ({})", field_descriptor->full_name());
}
pending_resolution.emplace(field_descriptor);
SCOPE_EXIT({ pending_resolution.erase(field_descriptor); });
if (allow_repeat && field_descriptor->is_map())
{

2
src/Functions/FunctionsConversion.cpp
View File

@ -794,7 +794,7 @@ inline bool tryParseImpl<DataTypeIPv6>(DataTypeIPv6::FieldType & x, ReadBuffer &
if (isNativeNumber(result_type) && !(result_type.getName() == "IPv4" || result_type.getName() == "IPv6"))
message_buf << ". Note: there are to" << result_type.getName() << "OrZero and to" << result_type.getName() << "OrNull functions, which returns zero/NULL instead of throwing exception.";
throw Exception(PreformattedMessage{message_buf.str(), "Cannot parse string {} as {}: syntax error {}"}, ErrorCodes::CANNOT_PARSE_TEXT);
throw Exception(PreformattedMessage{message_buf.str(), "Cannot parse string {} as {}: syntax error {}", {String(read_buffer.buffer().begin(), read_buffer.buffer().size()), result_type.getName()}}, ErrorCodes::CANNOT_PARSE_TEXT);
}

5
src/Interpreters/Cache/FileCache.cpp
View File

@ -128,6 +128,11 @@ const FileCache::UserInfo & FileCache::getInternalUser()
return user;
}
bool FileCache::isInitialized() const
{
return is_initialized.load(std::memory_order_seq_cst);
}
const String & FileCache::getBasePath() const
{
return metadata.getBaseDirectory();

2
src/Interpreters/Cache/FileCache.h
View File

@ -80,6 +80,8 @@ public:
void initialize();
bool isInitialized() const;
const String & getBasePath() const;
static Key createKeyForPath(const String & path);

27
src/Interpreters/InterpreterSystemQuery.cpp
View File

@ -397,22 +397,31 @@ BlockIO InterpreterSystemQuery::execute()
{
auto caches = FileCacheFactory::instance().getAll();
for (const auto & [_, cache_data] : caches)
{
if (!cache_data->cache->isInitialized())
continue;
cache_data->cache->removeAllReleasable(user_id);
}
}
else
{
auto cache = FileCacheFactory::instance().getByName(query.filesystem_cache_name)->cache;
if (query.key_to_drop.empty())
if (cache->isInitialized())
{
cache->removeAllReleasable(user_id);
}
else
{
auto key = FileCacheKey::fromKeyString(query.key_to_drop);
if (query.offset_to_drop.has_value())
cache->removeFileSegment(key, query.offset_to_drop.value(), user_id);
if (query.key_to_drop.empty())
{
cache->removeAllReleasable(user_id);
}
else
cache->removeKey(key, user_id);
{
auto key = FileCacheKey::fromKeyString(query.key_to_drop);
if (query.offset_to_drop.has_value())
cache->removeFileSegment(key, query.offset_to_drop.value(), user_id);
else
cache->removeKey(key, user_id);
}
}
}
break;

1
src/Interpreters/QueryLog.h
View File

@ -86,6 +86,7 @@ struct QueryLogElement
String exception;
String stack_trace;
std::string_view exception_format_string{};
std::vector<std::string> exception_format_string_args{};
ClientInfo client_info;

20
src/Interpreters/TextLog.cpp
View File

@ -53,6 +53,16 @@ ColumnsDescription TextLogElement::getColumnsDescription()
{"source_line", std::make_shared<DataTypeUInt64>(), "Source line from which the logging was done."},
{"message_format_string", std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>()), "A format string that was used to format the message."},
{"value1", std::make_shared<DataTypeString>(), "Argument 1 that was used to format the message."},
{"value2", std::make_shared<DataTypeString>(), "Argument 2 that was used to format the message."},
{"value3", std::make_shared<DataTypeString>(), "Argument 3 that was used to format the message."},
{"value4", std::make_shared<DataTypeString>(), "Argument 4 that was used to format the message."},
{"value5", std::make_shared<DataTypeString>(), "Argument 5 that was used to format the message."},
{"value6", std::make_shared<DataTypeString>(), "Argument 6 that was used to format the message."},
{"value7", std::make_shared<DataTypeString>(), "Argument 7 that was used to format the message."},
{"value8", std::make_shared<DataTypeString>(), "Argument 8 that was used to format the message."},
{"value9", std::make_shared<DataTypeString>(), "Argument 9 that was used to format the message."},
{"value10", std::make_shared<DataTypeString>(), "Argument 10 that was used to format the message."},
};
}
@ -79,6 +89,16 @@ void TextLogElement::appendToBlock(MutableColumns & columns) const
columns[i++]->insert(source_line);
columns[i++]->insert(message_format_string);
columns[i++]->insert(value1);
columns[i++]->insert(value2);
columns[i++]->insert(value3);
columns[i++]->insert(value4);
columns[i++]->insert(value5);
columns[i++]->insert(value6);
columns[i++]->insert(value7);
columns[i++]->insert(value8);
columns[i++]->insert(value9);
columns[i++]->insert(value10);
}
TextLog::TextLog(ContextPtr context_,

10
src/Interpreters/TextLog.h
View File

@ -29,6 +29,16 @@ struct TextLogElement
UInt64 source_line{};
std::string_view message_format_string;
String value1;
String value2;
String value3;
String value4;
String value5;
String value6;
String value7;
String value8;
String value9;
String value10;
static std::string name() { return "TextLog"; }
static ColumnsDescription getColumnsDescription();

2
src/Interpreters/TreeRewriter.cpp
View File

@ -1249,7 +1249,7 @@ bool TreeRewriterResult::collectUsedColumns(const ASTPtr & query, bool is_select
if (no_throw)
return false;
throw Exception(PreformattedMessage{ss.str(), format_string}, ErrorCodes::UNKNOWN_IDENTIFIER);
throw Exception(PreformattedMessage{ss.str(), format_string, std::vector<std::string>{}}, ErrorCodes::UNKNOWN_IDENTIFIER);
}
required_source_columns.swap(source_columns);

3
src/Interpreters/executeQuery.cpp
View File

@ -200,6 +200,7 @@ static void logException(ContextPtr context, QueryLogElement & elem, bool log_er
/// so we pass elem.exception_format_string as format string instead.
PreformattedMessage message;
message.format_string = elem.exception_format_string;
message.format_string_args = elem.exception_format_string_args;
if (elem.stack_trace.empty() || !log_error)
message.text = fmt::format("{} (from {}){} (in query: {})", elem.exception,
@ -504,6 +505,7 @@ void logQueryException(
auto exception_message = getCurrentExceptionMessageAndPattern(/* with_stacktrace */ false);
elem.exception = std::move(exception_message.text);
elem.exception_format_string = exception_message.format_string;
elem.exception_format_string_args = exception_message.format_string_args;
QueryStatusPtr process_list_elem = context->getProcessListElement();
@ -597,6 +599,7 @@ void logExceptionBeforeStart(
auto exception_message = getCurrentExceptionMessageAndPattern(/* with_stacktrace */ false);
elem.exception = std::move(exception_message.text);
elem.exception_format_string = exception_message.format_string;
elem.exception_format_string_args = exception_message.format_string_args;
elem.client_info = context->getClientInfo();

15
src/Loggers/OwnSplitChannel.cpp
View File

@ -131,6 +131,21 @@ void OwnSplitChannel::logSplit(const Poco::Message & msg)
elem.source_line = msg.getSourceLine();
elem.message_format_string = msg.getFormatString();
#define SET_VALUE_IF_EXISTS(INDEX) if ((INDEX) <= msg.getFormatStringArgs().size()) (elem.value##INDEX) = msg.getFormatStringArgs()[(INDEX) - 1]
SET_VALUE_IF_EXISTS(1);
SET_VALUE_IF_EXISTS(2);
SET_VALUE_IF_EXISTS(3);
SET_VALUE_IF_EXISTS(4);
SET_VALUE_IF_EXISTS(5);
SET_VALUE_IF_EXISTS(6);
SET_VALUE_IF_EXISTS(7);
SET_VALUE_IF_EXISTS(8);
SET_VALUE_IF_EXISTS(9);
SET_VALUE_IF_EXISTS(10);
#undef SET_VALUE_IF_EXISTS
std::shared_ptr<SystemLogQueue<TextLogElement>> text_log_locked{};
text_log_locked = text_log.lock();
if (text_log_locked)

112
src/Planner/PlannerJoins.cpp
View File

@ -125,9 +125,8 @@ TableExpressionSet extractTableExpressionsSet(const QueryTreeNodePtr & node)
return res;
}
std::optional<JoinTableSide> extractJoinTableSideFromExpression(//const ActionsDAG::Node * expression_root_node,
std::optional<JoinTableSide> extractJoinTableSideFromExpression(
const IQueryTreeNode * expression_root_node,
//const std::unordered_set<const ActionsDAG::Node *> & join_expression_dag_input_nodes,
const TableExpressionSet & left_table_expressions,
const TableExpressionSet & right_table_expressions,
const JoinNode & join_node)
@ -136,30 +135,11 @@ std::optional<JoinTableSide> extractJoinTableSideFromExpression(//const ActionsD
std::vector<const IQueryTreeNode *> nodes_to_process;
nodes_to_process.push_back(expression_root_node);
// std::cerr << "==== extractJoinTableSideFromExpression\n";
// std::cerr << "inp nodes" << std::endl;
// for (const auto * node : join_expression_dag_input_nodes)
// std::cerr << reinterpret_cast<const void *>(node) << ' ' << node->result_name << std::endl;
// std::cerr << "l names" << std::endl;
// for (const auto & l : left_table_expression_columns_names)
// std::cerr << l << std::endl;
// std::cerr << "r names" << std::endl;
// for (const auto & r : right_table_expression_columns_names)
// std::cerr << r << std::endl;
// const auto * left_table_expr = join_node.getLeftTableExpression().get();
// const auto * right_table_expr = join_node.getRightTableExpression().get();
while (!nodes_to_process.empty())
{
const auto * node_to_process = nodes_to_process.back();
nodes_to_process.pop_back();
//std::cerr << "... " << reinterpret_cast<const void *>(node_to_process) << ' ' << node_to_process->result_name << std::endl;
if (const auto * function_node = node_to_process->as<FunctionNode>())
{
for (const auto & child : function_node->getArguments())
@ -172,22 +152,7 @@ std::optional<JoinTableSide> extractJoinTableSideFromExpression(//const ActionsD
if (!column_node)
continue;
// if (!join_expression_dag_input_nodes.contains(node_to_process))
// continue;
const auto & input_name = column_node->getColumnName();
// bool left_table_expression_contains_input = left_table_expression_columns_names.contains(input_name);
// bool right_table_expression_contains_input = right_table_expression_columns_names.contains(input_name);
// if (!left_table_expression_contains_input && !right_table_expression_contains_input)
// throw Exception(ErrorCodes::INVALID_JOIN_ON_EXPRESSION,
// "JOIN {} actions has column {} that do not exist in left {} or right {} table expression columns",
// join_node.formatASTForErrorMessage(),
// input_name,
// boost::join(left_table_expression_columns_names, ", "),
// boost::join(right_table_expression_columns_names, ", "));
const auto * column_source = column_node->getColumnSource().get();
if (!column_source)
throw Exception(ErrorCodes::LOGICAL_ERROR, "No source for column {} in JOIN {}", input_name, join_node.formatASTForErrorMessage());
@ -235,9 +200,6 @@ void buildJoinClause(
ActionsDAGPtr & left_dag,
ActionsDAGPtr & right_dag,
const PlannerContextPtr & planner_context,
//ActionsDAGPtr join_expression_dag,
//const std::unordered_set<const ActionsDAG::Node *> & join_expression_dag_input_nodes,
//const ActionsDAG::Node * join_expressions_actions_node,
const QueryTreeNodePtr & join_expression,
const TableExpressionSet & left_table_expressions,
const TableExpressionSet & right_table_expressions,
@ -245,22 +207,16 @@ void buildJoinClause(
JoinClause & join_clause)
{
std::string function_name;
//std::cerr << join_expression_dag->dumpDAG() << std::endl;
auto * function_node = join_expression->as<FunctionNode>();
if (function_node)
function_name = function_node->getFunction()->getName();
// if (join_expressions_actions_node->function)
// function_name = join_expressions_actions_node->function->getName();
/// For 'and' function go into children
if (function_name == "and")
{
for (const auto & child : function_node->getArguments())
{
buildJoinClause(//join_expression_dag,
//join_expression_dag_input_nodes,
buildJoinClause(
left_dag,
right_dag,
planner_context,
@ -279,17 +235,15 @@ void buildJoinClause(
if (function_name == "equals" || function_name == "isNotDistinctFrom" || is_asof_join_inequality)
{
const auto left_child = function_node->getArguments().getNodes().at(0);//join_expressions_actions_node->children.at(0);
const auto right_child = function_node->getArguments().getNodes().at(1); //join_expressions_actions_node->children.at(1);
const auto left_child = function_node->getArguments().getNodes().at(0);
const auto right_child = function_node->getArguments().getNodes().at(1);
auto left_expression_side_optional = extractJoinTableSideFromExpression(left_child.get(),
//join_expression_dag_input_nodes,
left_table_expressions,
right_table_expressions,
join_node);
auto right_expression_side_optional = extractJoinTableSideFromExpression(right_child.get(),
//join_expression_dag_input_nodes,
left_table_expressions,
right_table_expressions,
join_node);
@ -314,7 +268,6 @@ void buildJoinClause(
}
else
{
// std::cerr << "===============\n";
auto left_expression_side = *left_expression_side_optional;
auto right_expression_side = *right_expression_side_optional;
@ -361,8 +314,7 @@ void buildJoinClause(
return;
}
auto expression_side_optional = extractJoinTableSideFromExpression(//join_expressions_actions_node,
//join_expression_dag_input_nodes,
auto expression_side_optional = extractJoinTableSideFromExpression(
join_expression.get(),
left_table_expressions,
right_table_expressions,
@ -377,32 +329,15 @@ void buildJoinClause(
join_clause.addCondition(expression_side, node);
}
JoinClausesAndActions buildJoinClausesAndActions(//const ColumnsWithTypeAndName & join_expression_input_columns,
JoinClausesAndActions buildJoinClausesAndActions(
const ColumnsWithTypeAndName & left_table_expression_columns,
const ColumnsWithTypeAndName & right_table_expression_columns,
const JoinNode & join_node,
const PlannerContextPtr & planner_context)
{
//ActionsDAGPtr join_expression_actions = std::make_shared<ActionsDAG>(join_expression_input_columns);
ActionsDAGPtr left_join_actions = std::make_shared<ActionsDAG>(left_table_expression_columns);
ActionsDAGPtr right_join_actions = std::make_shared<ActionsDAG>(right_table_expression_columns);
// LOG_TRACE(getLogger("Planner"), "buildJoinClausesAndActions cols {} ", left_join_actions->dumpDAG());
// LOG_TRACE(getLogger("Planner"), "buildJoinClausesAndActions cols {} ", right_join_actions->dumpDAG());
/** In ActionsDAG if input node has constant representation additional constant column is added.
* That way we cannot simply check that node has INPUT type during resolution of expression join table side.
* Put all nodes after actions dag initialization in set.
* To check if actions dag node is input column, we check if set contains it.
*/
// const auto & join_expression_actions_nodes = join_expression_actions->getNodes();
// std::unordered_set<const ActionsDAG::Node *> join_expression_dag_input_nodes;
// join_expression_dag_input_nodes.reserve(join_expression_actions_nodes.size());
// for (const auto & node : join_expression_actions_nodes)
// join_expression_dag_input_nodes.insert(&node);
/** It is possible to have constant value in JOIN ON section, that we need to ignore during DAG construction.
* If we do not ignore it, this function will be replaced by underlying constant.
* For example ASOF JOIN does not support JOIN with constants, and we should process it like ordinary JOIN.
@ -411,9 +346,6 @@ JoinClausesAndActions buildJoinClausesAndActions(//const ColumnsWithTypeAndName
* ON (t1.id = t2.id) AND 1 != 1 AND (t1.value >= t1.value);
*/
auto join_expression = join_node.getJoinExpression();
// LOG_TRACE(getLogger("Planner"), "buildJoinClausesAndActions expr {} ", join_expression->formatConvertedASTForErrorMessage());
// LOG_TRACE(getLogger("Planner"), "buildJoinClausesAndActions expr {} ", join_expression->dumpTree());
auto * constant_join_expression = join_expression->as<ConstantNode>();
if (constant_join_expression && constant_join_expression->hasSourceExpression())
@ -425,19 +357,6 @@ JoinClausesAndActions buildJoinClausesAndActions(//const ColumnsWithTypeAndName
"JOIN {} join expression expected function",
join_node.formatASTForErrorMessage());
// PlannerActionsVisitor join_expression_visitor(planner_context);
// auto join_expression_dag_node_raw_pointers = join_expression_visitor.visit(join_expression_actions, join_expression);
// if (join_expression_dag_node_raw_pointers.size() != 1)
// throw Exception(ErrorCodes::LOGICAL_ERROR,
// "JOIN {} ON clause contains multiple expressions",
// join_node.formatASTForErrorMessage());
// const auto * join_expressions_actions_root_node = join_expression_dag_node_raw_pointers[0];
// if (!join_expressions_actions_root_node->function)
// throw Exception(ErrorCodes::INVALID_JOIN_ON_EXPRESSION,
// "JOIN {} join expression expected function",
// join_node.formatASTForErrorMessage());
size_t left_table_expression_columns_size = left_table_expression_columns.size();
Names join_left_actions_names;
@ -470,7 +389,6 @@ JoinClausesAndActions buildJoinClausesAndActions(//const ColumnsWithTypeAndName
auto join_right_table_expressions = extractTableExpressionsSet(join_node.getRightTableExpression());
JoinClausesAndActions result;
//result.join_expression_actions = join_expression_actions;
const auto & function_name = function_node->getFunction()->getName();
if (function_name == "or")
@ -479,8 +397,7 @@ JoinClausesAndActions buildJoinClausesAndActions(//const ColumnsWithTypeAndName
{
result.join_clauses.emplace_back();
buildJoinClause(//join_expression_actions,
//join_expression_dag_input_nodes,
buildJoinClause(
left_join_actions,
right_join_actions,
planner_context,
@ -499,9 +416,7 @@ JoinClausesAndActions buildJoinClausesAndActions(//const ColumnsWithTypeAndName
left_join_actions,
right_join_actions,
planner_context,
//join_expression_actions,
//join_expression_dag_input_nodes,
join_expression, //join_expressions_actions_root_node,
join_expression,
join_left_table_expressions,
join_right_table_expressions,
join_node,
@ -621,12 +536,6 @@ JoinClausesAndActions buildJoinClausesAndActions(//const ColumnsWithTypeAndName
result.left_join_expressions_actions = left_join_actions->clone();
result.left_join_tmp_expression_actions = std::move(left_join_actions);
result.left_join_expressions_actions->removeUnusedActions(join_left_actions_names);
// for (const auto & name : join_right_actions_names)
// std::cerr << ".. " << name << std::endl;
// std::cerr << right_join_actions->dumpDAG() << std::endl;
result.right_join_expressions_actions = right_join_actions->clone();
result.right_join_tmp_expression_actions = std::move(right_join_actions);
result.right_join_expressions_actions->removeUnusedActions(join_right_actions_names);
@ -648,10 +557,7 @@ JoinClausesAndActions buildJoinClausesAndActions(
"JOIN {} join does not have ON section",
join_node_typed.formatASTForErrorMessage());
// auto join_expression_input_columns = left_table_expression_columns;
// join_expression_input_columns.insert(join_expression_input_columns.end(), right_table_expression_columns.begin(), right_table_expression_columns.end());
return buildJoinClausesAndActions(/*join_expression_input_columns,*/ left_table_expression_columns, right_table_expression_columns, join_node_typed, planner_context);
return buildJoinClausesAndActions(left_table_expression_columns, right_table_expression_columns, join_node_typed, planner_context);
}
std::optional<bool> tryExtractConstantFromJoinNode(const QueryTreeNodePtr & join_node)

2
src/Processors/QueryPlan/ReadFromMergeTree.cpp
View File

@ -987,7 +987,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder(
/// We take full part if it contains enough marks or
/// if we know limit and part contains less than 'limit' rows.
bool take_full_part = marks_in_part <= need_marks || (input_order_info->limit && input_order_info->limit < part.getRowsCount());
bool take_full_part = marks_in_part <= need_marks || (input_order_info->limit && input_order_info->limit > part.getRowsCount());
/// We take the whole part if it is small enough.
if (take_full_part)

16
src/Storages/MergeTree/KeyCondition.cpp
View File

@ -2250,9 +2250,11 @@ static BoolMask forAnyHyperrectangle(
if (left_bounded && right_bounded)
hyperrectangle[prefix_size] = Range(left_keys[prefix_size], true, right_keys[prefix_size], true);
else if (left_bounded)
hyperrectangle[prefix_size] = Range::createLeftBounded(left_keys[prefix_size], true, data_types[prefix_size]->isNullable());
hyperrectangle[prefix_size]
= Range::createLeftBounded(left_keys[prefix_size], true, isNullableOrLowCardinalityNullable(data_types[prefix_size]));
else if (right_bounded)
hyperrectangle[prefix_size] = Range::createRightBounded(right_keys[prefix_size], true, data_types[prefix_size]->isNullable());
hyperrectangle[prefix_size]
= Range::createRightBounded(right_keys[prefix_size], true, isNullableOrLowCardinalityNullable(data_types[prefix_size]));
return callback(hyperrectangle);
}
@ -2262,13 +2264,15 @@ static BoolMask forAnyHyperrectangle(
if (left_bounded && right_bounded)
hyperrectangle[prefix_size] = Range(left_keys[prefix_size], false, right_keys[prefix_size], false);
else if (left_bounded)
hyperrectangle[prefix_size] = Range::createLeftBounded(left_keys[prefix_size], false, data_types[prefix_size]->isNullable());
hyperrectangle[prefix_size]
= Range::createLeftBounded(left_keys[prefix_size], false, isNullableOrLowCardinalityNullable(data_types[prefix_size]));
else if (right_bounded)
hyperrectangle[prefix_size] = Range::createRightBounded(right_keys[prefix_size], false, data_types[prefix_size]->isNullable());
hyperrectangle[prefix_size]
= Range::createRightBounded(right_keys[prefix_size], false, isNullableOrLowCardinalityNullable(data_types[prefix_size]));
for (size_t i = prefix_size + 1; i < key_size; ++i)
{
if (data_types[i]->isNullable())
if (isNullableOrLowCardinalityNullable(data_types[i]))
hyperrectangle[i] = Range::createWholeUniverse();
else
hyperrectangle[i] = Range::createWholeUniverseWithoutNull();
@ -2324,7 +2328,7 @@ BoolMask KeyCondition::checkInRange(
key_ranges.reserve(used_key_size);
for (size_t i = 0; i < used_key_size; ++i)
{
if (data_types[i]->isNullable())
if (isNullableOrLowCardinalityNullable(data_types[i]))
key_ranges.push_back(Range::createWholeUniverse());
else
key_ranges.push_back(Range::createWholeUniverseWithoutNull());

6
src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
View File

@ -383,7 +383,8 @@ MergeTreeDataSelectSamplingData MergeTreeDataSelectExecutor::getSampling(
if (has_lower_limit)
{
if (!key_condition.addCondition(
sampling_key.column_names[0], Range::createLeftBounded(lower, true, sampling_key.data_types[0]->isNullable())))
sampling_key.column_names[0],
Range::createLeftBounded(lower, true, isNullableOrLowCardinalityNullable(sampling_key.data_types[0]))))
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Sampling column not in primary key");
ASTPtr args = std::make_shared<ASTExpressionList>();
@ -401,7 +402,8 @@ MergeTreeDataSelectSamplingData MergeTreeDataSelectExecutor::getSampling(
if (has_upper_limit)
{
if (!key_condition.addCondition(
sampling_key.column_names[0], Range::createRightBounded(upper, false, sampling_key.data_types[0]->isNullable())))
sampling_key.column_names[0],
Range::createRightBounded(upper, false, isNullableOrLowCardinalityNullable(sampling_key.data_types[0]))))
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Sampling column not in primary key");
ASTPtr args = std::make_shared<ASTExpressionList>();

2
tests/ci/install_check.py
View File

@ -33,7 +33,7 @@ set -e
trap "bash -ex /packages/preserve_logs.sh" ERR
test_env='TEST_THE_DEFAULT_PARAMETER=15'
echo "$test_env" >> /etc/default/clickhouse
systemctl start clickhouse-server
systemctl restart clickhouse-server
clickhouse-client -q 'SELECT version()'
grep "$test_env" /proc/$(cat /var/run/clickhouse-server/clickhouse-server.pid)/environ"""
initd_test = r"""#!/bin/bash

10
tests/integration/test_modify_engine_on_restart/common.py
View File

@ -2,9 +2,13 @@ from helpers.cluster import ClickHouseCluster
def get_table_path(node, table, database):
return node.query(
sql=f"SELECT data_paths FROM system.tables WHERE table = '{table}' and database = '{database}'"
).strip("'[]\n")
return (
node.query(
sql=f"SELECT data_paths FROM system.tables WHERE table = '{table}' and database = '{database}' LIMIT 1"
)
.split(",")[0]
.strip("'[]\n")
)
def check_flags_deleted(node, database_name, tables):

42
tests/integration/test_modify_engine_on_restart/configs/config.d/storage_policies.xml Normal file
View File

@ -0,0 +1,42 @@
<clickhouse>
<storage_configuration>
<disks>
<default>
<keep_free_space_bytes>1024</keep_free_space_bytes>
</default>
<jbod1>
<path>/jbod1/</path>
</jbod1>
<jbod2>
<path>/jbod2/</path>
</jbod2>
<s3>
<type>s3</type>
<endpoint>http://minio1:9001/root/data/</endpoint>
<access_key_id>minio</access_key_id>
<secret_access_key>minio123</secret_access_key>
</s3>
</disks>
<policies>
<jbod>
<volumes>
<jbod_volume>
<disk>jbod1</disk>
<disk>jbod2</disk>
</jbod_volume>
</volumes>
</jbod>
<s3>
<volumes>
<s3_volume>
<disk>s3</disk>
</s3_volume>
</volumes>
</s3>
</policies>
</storage_configuration>
<merge_tree>
<storage_policy>jbod</storage_policy>
</merge_tree>
</clickhouse>

102
tests/integration/test_modify_engine_on_restart/test_storage_policies.py Normal file
View File

@ -0,0 +1,102 @@
import pytest
from test_modify_engine_on_restart.common import check_flags_deleted, set_convert_flags
from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__)
ch1 = cluster.add_instance(
"ch1",
main_configs=[
"configs/config.d/clusters.xml",
"configs/config.d/distributed_ddl.xml",
"configs/config.d/storage_policies.xml",
],
with_zookeeper=True,
with_minio=True,
macros={"replica": "node1"},
stay_alive=True,
)
database_name = "modify_engine_storage_policies"
@pytest.fixture(scope="module")
def started_cluster():
try:
cluster.start()
yield cluster
finally:
cluster.shutdown()
def q(node, query):
return node.query(database=database_name, sql=query)
def create_tables():
# Implicit jbod (set default in config)
q(
ch1,
"CREATE TABLE jbod_imp ( A Int64, D Date, S String ) ENGINE MergeTree() PARTITION BY toYYYYMM(D) ORDER BY A;",
)
# Explicit jbod
q(
ch1,
"""
CREATE TABLE jbod_exp ( A Int64, D Date, S String ) ENGINE MergeTree() PARTITION BY toYYYYMM(D) ORDER BY A
SETTINGS storage_policy='jbod';
""",
)
# s3
q(
ch1,
"""
CREATE TABLE s3 ( A Int64, D Date, S String ) ENGINE MergeTree() PARTITION BY toYYYYMM(D) ORDER BY A
SETTINGS storage_policy='s3';
""",
)
# Default
q(
ch1,
"""
CREATE TABLE default ( A Int64, D Date, S String ) ENGINE MergeTree() PARTITION BY toYYYYMM(D) ORDER BY A
SETTINGS storage_policy='default';
""",
)
def check_tables(converted):
engine_prefix = ""
if converted:
engine_prefix = "Replicated"
assert (
q(
ch1,
f"SELECT name, engine FROM system.tables WHERE database = '{database_name}'",
).strip()
== f"default\t{engine_prefix}MergeTree\njbod_exp\t{engine_prefix}MergeTree\njbod_imp\t{engine_prefix}MergeTree\ns3\t{engine_prefix}MergeTree"
)
def test_modify_engine_on_restart(started_cluster):
ch1.query("CREATE DATABASE " + database_name)
create_tables()
check_tables(False)
ch1.restart_clickhouse()
check_tables(False)
set_convert_flags(ch1, database_name, ["default", "jbod_exp", "jbod_imp", "s3"])
ch1.restart_clickhouse()
check_flags_deleted(ch1, database_name, ["default", "jbod_exp", "jbod_imp", "s3"])
check_tables(True)

2
tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh
View File

@ -37,7 +37,7 @@ function thread4()
{
while true; do
REPLICA=$(($RANDOM % 10))
$CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE alter_table_$REPLICA FINAL";
$CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE alter_table_$REPLICA FINAL SETTINGS receive_timeout=1";
sleep 0.$RANDOM;
done
}

3
tests/queries/0_stateless/02499_monotonicity_toUnixTimestamp64.sh
View File

@ -13,8 +13,7 @@ $CLICKHOUSE_CLIENT -q "create table t(ts DateTime64) engine=MergeTree order by t
max_block_size=8192
query_id="${CLICKHOUSE_DATABASE}_02499_$RANDOM$RANDOM"
$CLICKHOUSE_CLIENT --query_id="$query_id" -q "select ts from t order by toUnixTimestamp64Nano(ts) limit 10 format Null settings max_block_size = $max_block_size, optimize_read_in_order = 1;"
$CLICKHOUSE_CLIENT --query_id="$query_id" -q "select ts from t order by toUnixTimestamp64Nano(ts) limit 10 format Null settings max_block_size = $max_block_size, optimize_read_in_order = 1, max_threads = 1;"
$CLICKHOUSE_CLIENT -q "system flush logs;"
$CLICKHOUSE_CLIENT --param_query_id="$query_id" -q "select read_rows <= $max_block_size from system.query_log where event_date >= yesterday() and current_database = '$CLICKHOUSE_DATABASE' and query_id = {query_id:String} and type = 'QueryFinish';"

9
tests/queries/0_stateless/02813_func_now_and_alias.sql
View File

@ -1,3 +1,6 @@
SELECT now() = current_timestamp();
SELECT now() = CURRENT_TIMESTAMP();
SELECT now() = current_TIMESTAMP();
-- "Tests" current_timestamp() which is an alias of now().
-- Since the function is non-deterministic, only check that no bad things happen (don't check the returned value).
SELECT count() FROM (SELECT current_timestamp());
SELECT count() FROM (SELECT CURRENT_TIMESTAMP());
SELECT count() FROM (SELECT current_TIMESTAMP());

42
tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.reference
View File

@ -43,30 +43,34 @@ QUERY id: 0
FUNCTION id: 2, function_name: tuple, function_type: ordinary, result_type: Tuple(Nullable(UInt64), Nullable(Nothing))
ARGUMENTS
LIST id: 3, nodes: 2
FUNCTION id: 4, function_name: sumOrNullIf, function_type: aggregate, result_type: Nullable(UInt64)
FUNCTION id: 4, function_name: sum, function_type: aggregate, result_type: Nullable(UInt64)
ARGUMENTS
LIST id: 5, nodes: 2
CONSTANT id: 6, constant_value: UInt64_1, constant_value_type: Nullable(UInt8)
EXPRESSION
FUNCTION id: 7, function_name: toNullable, function_type: ordinary, result_type: Nullable(UInt8)
ARGUMENTS
LIST id: 8, nodes: 1
CONSTANT id: 9, constant_value: UInt64_1, constant_value_type: UInt8
FUNCTION id: 10, function_name: equals, function_type: ordinary, result_type: UInt8
LIST id: 5, nodes: 1
FUNCTION id: 6, function_name: if, function_type: ordinary, result_type: Nullable(UInt8)
ARGUMENTS
LIST id: 11, nodes: 2
FUNCTION id: 12, function_name: modulo, function_type: ordinary, result_type: UInt8
LIST id: 7, nodes: 3
FUNCTION id: 8, function_name: equals, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 13, nodes: 2
COLUMN id: 14, column_name: number, result_type: UInt64, source_id: 15
CONSTANT id: 16, constant_value: UInt64_2, constant_value_type: UInt8
CONSTANT id: 17, constant_value: UInt64_0, constant_value_type: UInt8
CONSTANT id: 18, constant_value: NULL, constant_value_type: Nullable(Nothing)
LIST id: 9, nodes: 2
FUNCTION id: 10, function_name: modulo, function_type: ordinary, result_type: UInt8
ARGUMENTS
LIST id: 11, nodes: 2
COLUMN id: 12, column_name: number, result_type: UInt64, source_id: 13
CONSTANT id: 14, constant_value: UInt64_2, constant_value_type: UInt8
CONSTANT id: 15, constant_value: UInt64_0, constant_value_type: UInt8
CONSTANT id: 16, constant_value: UInt64_1, constant_value_type: Nullable(UInt8)
EXPRESSION
FUNCTION id: 17, function_name: toNullable, function_type: ordinary, result_type: Nullable(UInt8)
ARGUMENTS
LIST id: 18, nodes: 1
CONSTANT id: 19, constant_value: UInt64_1, constant_value_type: UInt8
CONSTANT id: 20, constant_value: UInt64_0, constant_value_type: UInt8
CONSTANT id: 21, constant_value: NULL, constant_value_type: Nullable(Nothing)
JOIN TREE
TABLE_FUNCTION id: 15, alias: __table1, table_function_name: numbers
TABLE_FUNCTION id: 13, alias: __table1, table_function_name: numbers
ARGUMENTS
LIST id: 19, nodes: 1
CONSTANT id: 20, constant_value: UInt64_10, constant_value_type: UInt8
LIST id: 22, nodes: 1
CONSTANT id: 23, constant_value: UInt64_10, constant_value_type: UInt8
((6150),3)
QUERY id: 0
PROJECTION COLUMNS

1
tests/queries/0_stateless/03013_forbid_attach_table_if_active_replica_already_exists.sh
View File

@ -1,4 +1,5 @@
#!/usr/bin/env bash
# Tags: no-shared-merge-tree
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh

2
tests/queries/0_stateless/03015_optimize_final_rmt.sh
View File

@ -1,5 +1,5 @@
#!/usr/bin/env bash
# Tags: long, no-random-settings, no-random-merge-tree-settings
# Tags: long, no-random-settings, no-random-merge-tree-settings, no-shared-merge-tree
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh

3
tests/queries/0_stateless/03096_text_log_format_string_args_not_empty.reference Normal file
View File

@ -0,0 +1,3 @@
1
1
1

13
tests/queries/0_stateless/03096_text_log_format_string_args_not_empty.sql Normal file
View File

@ -0,0 +1,13 @@
set allow_experimental_analyzer = true;
select count; -- { serverError 47 }
select conut(); -- { serverError 46 }
system flush logs;
select count() > 0 from system.text_log where message_format_string = 'Peak memory usage{}: {}.' and value1 is not null and value2 like '% MiB';
select count() > 0 from system.text_log where level = 'Error' and message_format_string = 'Unknown {}{} identifier \'{}\' in scope {}{}' and value1 = 'expression' and value3 = 'count' and value4 = 'SELECT count';
select count() > 0 from system.text_log where level = 'Error' and message_format_string = 'Function with name \'{}\' does not exists. In scope {}{}' and value1 = 'conut' and value2 = 'SELECT conut()' and value3 ilike '%\'count\'%';

1
tests/queries/0_stateless/03129_low_cardinality_nullable_non_first_primary_key.reference Normal file
View File

@ -0,0 +1 @@
1650

10
tests/queries/0_stateless/03129_low_cardinality_nullable_non_first_primary_key.sql Normal file
View File

@ -0,0 +1,10 @@
DROP TABLE IF EXISTS small;
CREATE TABLE small (`dt` DateTime, `user_email` LowCardinality(Nullable(String)))
ENGINE = MergeTree order by (dt, user_email) settings allow_nullable_key = 1, min_bytes_for_wide_part=0, min_rows_for_wide_part=0;
INSERT INTO small (dt, user_email) SELECT number, if(number % 3 = 2, NULL, number) FROM numbers(1e2);
SELECT SUM(dt::int) FROM small WHERE user_email IS NULL;
DROP TABLE small;

1
tests/queries/0_stateless/03130_nested_type.reference Normal file
View File

@ -0,0 +1 @@
inner Tuple(\n a Tuple(\n seconds Int64,\n nanos Int32),\n b Tuple(\n seconds Int64,\n nanos Int32),\n c Tuple(\n seconds Int64,\n nanos Int32))

9
tests/queries/0_stateless/03130_nested_type.sh Executable file
View File

@ -0,0 +1,9 @@
#!/usr/bin/env bash
# Tags: no-fasttest
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
SCHEMADIR="$CUR_DIR/format_schemas"
$CLICKHOUSE_LOCAL -q "DESCRIBE TABLE file('nonexist', 'Protobuf') SETTINGS format_schema='$SCHEMADIR/03130_nested_schema.proto:Outer'"

34
tests/queries/0_stateless/03131_rewrite_sum_if_nullable.reference Normal file
View File

@ -0,0 +1,34 @@
-- https://github.com/ClickHouse/ClickHouse/issues/62890
-- { echoOn }
SELECT sum(if(materialize(0), toNullable(1), 0));
0
SELECT sum(if(materialize(0), toNullable(1), materialize(0)));
0
SELECT sum(if(materialize(0), materialize(toNullable(1)), materialize(0)));
0
SELECT sum(if(materialize(0), materialize(1), materialize(0)));
0
SELECT sum(if(dummy, 0, toNullable(0)));
0
SELECT sum(if(dummy, materialize(0), toNullable(0)));
0
SELECT sum(if(dummy, materialize(0), materialize(toNullable(0))));
0
SELECT sum(if(s == '', v, 0)) b from VALUES ('v Nullable(Int64), s String',(1, 'x'));
0
SELECT sumOrNull(if(materialize(0), toNullable(1), 0));
0
SELECT sumOrNull(if(materialize(0), 1, 0));
0
SELECT sum(if(materialize(0), toNullable(1), 0)) settings aggregate_functions_null_for_empty=1;
0
SELECT sum(if(materialize(0), 1, 0)) settings aggregate_functions_null_for_empty=1;
0
SELECT sumOrNull(if(materialize(1), toNullable(1), 10));
1
SELECT sumOrNull(if(materialize(1), 1, 10));
1
SELECT sum(if(materialize(1), toNullable(1), 10)) settings aggregate_functions_null_for_empty=1;
1
SELECT sum(if(materialize( 1), 1, 10)) settings aggregate_functions_null_for_empty=1;
1

20
tests/queries/0_stateless/03131_rewrite_sum_if_nullable.sql Normal file
View File

@ -0,0 +1,20 @@
-- https://github.com/ClickHouse/ClickHouse/issues/62890
-- { echoOn }
SELECT sum(if(materialize(0), toNullable(1), 0));
SELECT sum(if(materialize(0), toNullable(1), materialize(0)));
SELECT sum(if(materialize(0), materialize(toNullable(1)), materialize(0)));
SELECT sum(if(materialize(0), materialize(1), materialize(0)));
SELECT sum(if(dummy, 0, toNullable(0)));
SELECT sum(if(dummy, materialize(0), toNullable(0)));
SELECT sum(if(dummy, materialize(0), materialize(toNullable(0))));
SELECT sum(if(s == '', v, 0)) b from VALUES ('v Nullable(Int64), s String',(1, 'x'));
SELECT sumOrNull(if(materialize(0), toNullable(1), 0));
SELECT sumOrNull(if(materialize(0), 1, 0));
SELECT sum(if(materialize(0), toNullable(1), 0)) settings aggregate_functions_null_for_empty=1;
SELECT sum(if(materialize(0), 1, 0)) settings aggregate_functions_null_for_empty=1;
SELECT sumOrNull(if(materialize(1), toNullable(1), 10));
SELECT sumOrNull(if(materialize(1), 1, 10));
SELECT sum(if(materialize(1), toNullable(1), 10)) settings aggregate_functions_null_for_empty=1;
SELECT sum(if(materialize( 1), 1, 10)) settings aggregate_functions_null_for_empty=1;

16
tests/queries/0_stateless/format_schemas/03130_nested_schema.proto Normal file
View File

@ -0,0 +1,16 @@
syntax = "proto3";
message Duration {
int64 seconds = 1;
int32 nanos = 2;
}
message Inner {
Duration a = 7;
Duration b = 8;
Duration c = 9;
}
message Outer {
Inner inner = 6;
}