mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-19 16:20:50 +00:00
Merge branch 'master' into tighten-limits-functional-tests
This commit is contained in:
Alexey Milovidov
commit
27a06f9eec
@ -19,6 +19,7 @@
|
||||
|
||||
|
||||
#include "Poco/Foundation.h"
|
||||
#include <Poco/Types.h>
|
||||
|
||||
|
||||
namespace Poco
|
||||
@ -135,6 +136,12 @@ public:
|
||||
static const UUID & x500();
|
||||
/// Returns the namespace identifier for the X500 namespace.
|
||||
|
||||
UInt32 getTimeLow() const { return _timeLow; }
|
||||
UInt16 getTimeMid() const { return _timeMid; }
|
||||
UInt16 getTimeHiAndVersion() const { return _timeHiAndVersion; }
|
||||
UInt16 getClockSeq() const { return _clockSeq; }
|
||||
std::array<UInt8, 6> getNode() const { return std::array<UInt8, 6>{_node[0], _node[1], _node[2], _node[3], _node[4], _node[5]}; }
|
||||
|
||||
protected:
|
||||
UUID(UInt32 timeLow, UInt32 timeMid, UInt32 timeHiAndVersion, UInt16 clockSeq, UInt8 node[]);
|
||||
UUID(const char * bytes, Version version);
|
||||
|
||||
@ -76,13 +76,13 @@ std::string Binary::toString(int indent) const
|
||||
|
||||
UUID Binary::uuid() const
|
||||
{
|
||||
if (_subtype == 0x04 && _buffer.size() == 16)
|
||||
if ((_subtype == 0x04 || _subtype == 0x03) && _buffer.size() == 16)
|
||||
{
|
||||
UUID uuid;
|
||||
uuid.copyFrom((const char*) _buffer.begin());
|
||||
return uuid;
|
||||
}
|
||||
throw BadCastException("Invalid subtype");
|
||||
throw BadCastException("Invalid subtype: " + std::to_string(_subtype) + ", size: " + std::to_string(_buffer.size()));
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -35,7 +35,9 @@ ENV UBSAN_OPTIONS='print_stacktrace=1 max_allocation_size_mb=32768'
|
||||
ENV MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1 max_allocation_size_mb=32768'
|
||||
ENV LSAN_OPTIONS='max_allocation_size_mb=32768'
|
||||
|
||||
# for external_symbolizer_path
|
||||
# for external_symbolizer_path, and also ensure that llvm-symbolizer really
|
||||
# exists (since you don't want to fallback to addr2line, it is very slow)
|
||||
RUN test -f /usr/bin/llvm-symbolizer-${LLVM_VERSION}
|
||||
RUN ln -s /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer
|
||||
|
||||
RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen en_US.UTF-8
|
||||
|
||||
@ -3,6 +3,12 @@
|
||||
# shellcheck disable=SC1091
|
||||
source /setup_export_logs.sh
|
||||
|
||||
# shellcheck source=../stateless/stress_tests.lib
|
||||
source /stress_tests.lib
|
||||
|
||||
# Avoid overlaps with previous runs
|
||||
dmesg --clear
|
||||
|
||||
# fail on errors, verbose and export all env variables
|
||||
set -e -x -a
|
||||
|
||||
@ -72,8 +78,12 @@ if [[ -n "$BUGFIX_VALIDATE_CHECK" ]] && [[ "$BUGFIX_VALIDATE_CHECK" -eq 1 ]]; th
|
||||
remove_keeper_config "latest_logs_cache_size_threshold" "[[:digit:]]\+"
|
||||
fi
|
||||
|
||||
export IS_FLAKY_CHECK=0
|
||||
|
||||
# For flaky check we also enable thread fuzzer
|
||||
if [ "$NUM_TRIES" -gt "1" ]; then
|
||||
export IS_FLAKY_CHECK=1
|
||||
|
||||
export THREAD_FUZZER_CPU_TIME_PERIOD_US=1000
|
||||
export THREAD_FUZZER_SLEEP_PROBABILITY=0.1
|
||||
export THREAD_FUZZER_SLEEP_TIME_US_MAX=100000
|
||||
@ -260,7 +270,7 @@ function run_tests()
|
||||
| ts '%Y-%m-%d %H:%M:%S' \
|
||||
| tee -a test_output/test_result.txt
|
||||
set -e
|
||||
DURATION=$((START_TIME - SECONDS))
|
||||
DURATION=$((SECONDS - START_TIME))
|
||||
|
||||
echo "Elapsed ${DURATION} seconds."
|
||||
if [[ $DURATION -ge $TIMEOUT ]]
|
||||
@ -299,22 +309,22 @@ stop_logs_replication
|
||||
failed_to_save_logs=0
|
||||
for table in query_log zookeeper_log trace_log transactions_info_log metric_log blob_storage_log error_log
|
||||
do
|
||||
err=$(clickhouse-client -q "select * from system.$table into outfile '/test_output/$table.tsv.gz' format TSVWithNamesAndTypes")
|
||||
echo "$err"
|
||||
[[ "0" != "${#err}" ]] && failed_to_save_logs=1
|
||||
if ! clickhouse-client -q "select * from system.$table into outfile '/test_output/$table.tsv.zst' format TSVWithNamesAndTypes"; then
|
||||
failed_to_save_logs=1
|
||||
fi
|
||||
if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
err=$( { clickhouse-client --port 19000 -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst; } 2>&1 )
|
||||
echo "$err"
|
||||
[[ "0" != "${#err}" ]] && failed_to_save_logs=1
|
||||
err=$( { clickhouse-client --port 29000 -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.2.tsv.zst; } 2>&1 )
|
||||
echo "$err"
|
||||
[[ "0" != "${#err}" ]] && failed_to_save_logs=1
|
||||
if ! clickhouse-client --port 19000 -q "select * from system.$table into outfile '/test_output/$table.1.tsv.zst' format TSVWithNamesAndTypes"; then
|
||||
failed_to_save_logs=1
|
||||
fi
|
||||
if ! clickhouse-client --port 29000 -q "select * from system.$table into outfile '/test_output/$table.2.tsv.zst' format TSVWithNamesAndTypes"; then
|
||||
failed_to_save_logs=1
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
|
||||
err=$( { clickhouse-client --port 19000 -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst; } 2>&1 )
|
||||
echo "$err"
|
||||
[[ "0" != "${#err}" ]] && failed_to_save_logs=1
|
||||
if ! clickhouse-client --port 29000 -q "select * from system.$table into outfile '/test_output/$table.2.tsv.zst' format TSVWithNamesAndTypes"; then
|
||||
failed_to_save_logs=1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
@ -387,6 +397,8 @@ do
|
||||
| zstd --threads=0 > "/test_output/trace-log-$trace_type-flamegraph.tsv.zst" ||:
|
||||
done
|
||||
|
||||
# Grep logs for sanitizer asserts, crashes and other critical errors
|
||||
check_logs_for_critical_errors
|
||||
|
||||
# Compressed (FIXME: remove once only github actions will be left)
|
||||
rm /var/log/clickhouse-server/clickhouse-server.log
|
||||
|
||||
@ -242,7 +242,7 @@ function check_server_start()
|
||||
function check_logs_for_critical_errors()
|
||||
{
|
||||
# Sanitizer asserts
|
||||
sed -n '/WARNING:.*anitizer/,/^$/p' /var/log/clickhouse-server/stderr.log >> /test_output/tmp
|
||||
sed -n '/WARNING:.*anitizer/,/^$/p' /var/log/clickhouse-server/stderr*.log >> /test_output/tmp
|
||||
rg -Fav -e "ASan doesn't fully support makecontext/swapcontext functions" -e "DB::Exception" /test_output/tmp > /dev/null \
|
||||
&& echo -e "Sanitizer assert (in stderr.log)$FAIL$(head_escaped /test_output/tmp)" >> /test_output/test_results.tsv \
|
||||
|| echo -e "No sanitizer asserts$OK" >> /test_output/test_results.tsv
|
||||
|
||||
@ -5,14 +5,6 @@ FROM ubuntu:22.04
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
# FIXME: rebuild for clang 18.1.3, that contains a workaround [1] for
|
||||
# sanitizers issue [2]:
|
||||
#
|
||||
# $ git tag --contains c2a57034eff048cd36c563c8e0051db3a70991b3 | tail -1
|
||||
# llvmorg-18.1.3
|
||||
#
|
||||
# [1]: https://github.com/llvm/llvm-project/commit/c2a57034eff048cd36c563c8e0051db3a70991b3
|
||||
# [2]: https://github.com/ClickHouse/ClickHouse/issues/64086
|
||||
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=18
|
||||
|
||||
RUN apt-get update \
|
||||
|
||||
@ -161,11 +161,11 @@ def process_result(result_path, broken_tests):
|
||||
retries,
|
||||
test_results,
|
||||
) = process_test_log(result_path, broken_tests)
|
||||
is_flacky_check = 1 < int(os.environ.get("NUM_TRIES", 1))
|
||||
logging.info("Is flaky check: %s", is_flacky_check)
|
||||
is_flaky_check = 1 < int(os.environ.get("NUM_TRIES", 1))
|
||||
logging.info("Is flaky check: %s", is_flaky_check)
|
||||
# If no tests were run (success == 0) it indicates an error (e.g. server did not start or crashed immediately)
|
||||
# But it's Ok for "flaky checks" - they can contain just one test for check which is marked as skipped.
|
||||
if failed != 0 or unknown != 0 or (success == 0 and (not is_flacky_check)):
|
||||
if failed != 0 or unknown != 0 or (success == 0 and (not is_flaky_check)):
|
||||
state = "failure"
|
||||
|
||||
if hung:
|
||||
|
||||
@ -331,7 +331,7 @@
|
||||
* Fix several non significant errors in unit tests. [#11262](https://github.com/ClickHouse/ClickHouse/pull/11262) ([alesapin](https://github.com/alesapin)).
|
||||
* Add a test for Join table engine from @donmikel. This closes [#9158](https://github.com/ClickHouse/ClickHouse/issues/9158). [#11265](https://github.com/ClickHouse/ClickHouse/pull/11265) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Repeat test in CI if `curl` invocation was timed out. It is possible due to system hangups for 10+ seconds that are typical in our CI infrastructure. This fixes [#11267](https://github.com/ClickHouse/ClickHouse/issues/11267). [#11268](https://github.com/ClickHouse/ClickHouse/pull/11268) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix potentially flacky test `00731_long_merge_tree_select_opened_files.sh`. It does not fail frequently but we have discovered potential race condition in this test while experimenting with ThreadFuzzer: [#9814](https://github.com/ClickHouse/ClickHouse/issues/9814) See [link](https://clickhouse-test-reports.s3.yandex.net/9814/40e3023e215df22985d275bf85f4d2290897b76b/functional_stateless_tests_(unbundled).html#fail1) for the example. [#11270](https://github.com/ClickHouse/ClickHouse/pull/11270) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix potentially flaky test `00731_long_merge_tree_select_opened_files.sh`. It does not fail frequently but we have discovered potential race condition in this test while experimenting with ThreadFuzzer: [#9814](https://github.com/ClickHouse/ClickHouse/issues/9814) See [link](https://clickhouse-test-reports.s3.yandex.net/9814/40e3023e215df22985d275bf85f4d2290897b76b/functional_stateless_tests_(unbundled).html#fail1) for the example. [#11270](https://github.com/ClickHouse/ClickHouse/pull/11270) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Now clickhouse-test check the server aliveness before tests run. [#11285](https://github.com/ClickHouse/ClickHouse/pull/11285) ([alesapin](https://github.com/alesapin)).
|
||||
* Emit a warning if server was build in debug or with sanitizers. [#11304](https://github.com/ClickHouse/ClickHouse/pull/11304) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Better check for hung queries in clickhouse-test. [#11321](https://github.com/ClickHouse/ClickHouse/pull/11321) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
|
||||
@ -280,7 +280,7 @@ sidebar_label: 2022
|
||||
* Cleanup unbundled image [#29689](https://github.com/ClickHouse/ClickHouse/pull/29689) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix memory tracking for merges and mutations [#29691](https://github.com/ClickHouse/ClickHouse/pull/29691) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix data-race in WriteIndirectBuffer (used in DiskMemory) [#29692](https://github.com/ClickHouse/ClickHouse/pull/29692) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix flacky test [#29706](https://github.com/ClickHouse/ClickHouse/pull/29706) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix flaky test [#29706](https://github.com/ClickHouse/ClickHouse/pull/29706) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* BorrowedObjectPool condition variable notify fix [#29722](https://github.com/ClickHouse/ClickHouse/pull/29722) ([Maksim Kita](https://github.com/kitaisreal)).
|
||||
* Better exception message for local interactive [#29737](https://github.com/ClickHouse/ClickHouse/pull/29737) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix --stage for clickhouse-local [#29745](https://github.com/ClickHouse/ClickHouse/pull/29745) ([Azat Khuzhin](https://github.com/azat)).
|
||||
@ -308,7 +308,7 @@ sidebar_label: 2022
|
||||
* Fix client [#29864](https://github.com/ClickHouse/ClickHouse/pull/29864) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Remove some more streams. [#29898](https://github.com/ClickHouse/ClickHouse/pull/29898) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Add logging in ZooKeeper client [#29901](https://github.com/ClickHouse/ClickHouse/pull/29901) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Fix some flacky tests [#29902](https://github.com/ClickHouse/ClickHouse/pull/29902) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix some flaky tests [#29902](https://github.com/ClickHouse/ClickHouse/pull/29902) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Grep server log even if it contains binary data [#29903](https://github.com/ClickHouse/ClickHouse/pull/29903) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Cosmetic refactoring of server constants. [#29913](https://github.com/ClickHouse/ClickHouse/pull/29913) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Format improvement of AlterQuery [#29916](https://github.com/ClickHouse/ClickHouse/pull/29916) ([flynn](https://github.com/ucasfl)).
|
||||
@ -465,7 +465,7 @@ sidebar_label: 2022
|
||||
* Fix docs release [#30933](https://github.com/ClickHouse/ClickHouse/pull/30933) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix style check [#30937](https://github.com/ClickHouse/ClickHouse/pull/30937) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix file progress for clickhouse-local [#30938](https://github.com/ClickHouse/ClickHouse/pull/30938) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix flacky test [#30940](https://github.com/ClickHouse/ClickHouse/pull/30940) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix flaky test [#30940](https://github.com/ClickHouse/ClickHouse/pull/30940) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix reading from TinyLog [#30941](https://github.com/ClickHouse/ClickHouse/pull/30941) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Add github to known hosts in docs release [#30947](https://github.com/ClickHouse/ClickHouse/pull/30947) ([alesapin](https://github.com/alesapin)).
|
||||
* Parse json from response in ci checks [#30948](https://github.com/ClickHouse/ClickHouse/pull/30948) ([alesapin](https://github.com/alesapin)).
|
||||
|
||||
@ -220,7 +220,7 @@ sidebar_label: 2022
|
||||
* Fix test_backward_compatibility [#30950](https://github.com/ClickHouse/ClickHouse/pull/30950) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
* Add stress test to github actions [#30952](https://github.com/ClickHouse/ClickHouse/pull/30952) ([alesapin](https://github.com/alesapin)).
|
||||
* Try smaller blacklist of non parallel integration tests [#30963](https://github.com/ClickHouse/ClickHouse/pull/30963) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
* Fix flacky test [#30967](https://github.com/ClickHouse/ClickHouse/pull/30967) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix flaky test [#30967](https://github.com/ClickHouse/ClickHouse/pull/30967) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Move access-rights source code [#30973](https://github.com/ClickHouse/ClickHouse/pull/30973) ([Vitaly Baranov](https://github.com/vitlibar)).
|
||||
* Set output_format_avro_rows_in_file default to 1 [#30990](https://github.com/ClickHouse/ClickHouse/pull/30990) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Remove remaining usages of Y_IGNORE [#30993](https://github.com/ClickHouse/ClickHouse/pull/30993) ([Yuriy Chernyshov](https://github.com/georgthegreat)).
|
||||
@ -353,7 +353,7 @@ sidebar_label: 2022
|
||||
* Support toUInt8/toInt8 for if constant condition optimization. [#31866](https://github.com/ClickHouse/ClickHouse/pull/31866) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Added -no-sanitize=unsigned-integer-overflow build flag [#31881](https://github.com/ClickHouse/ClickHouse/pull/31881) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
* Fix typos [#31886](https://github.com/ClickHouse/ClickHouse/pull/31886) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Try to fix flacky test. [#31889](https://github.com/ClickHouse/ClickHouse/pull/31889) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Try to fix flaky test. [#31889](https://github.com/ClickHouse/ClickHouse/pull/31889) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Reduce the files that depend on parser headers [#31896](https://github.com/ClickHouse/ClickHouse/pull/31896) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Fix magic_enum for debug helpers (fixes build w/ USE_DEBUG_HELPERS) [#31922](https://github.com/ClickHouse/ClickHouse/pull/31922) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Remove some trash from build [#31923](https://github.com/ClickHouse/ClickHouse/pull/31923) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
@ -387,7 +387,7 @@ sidebar_label: 2022
|
||||
* make looping in H3 funcs uniform [#32110](https://github.com/ClickHouse/ClickHouse/pull/32110) ([Bharat Nallan](https://github.com/bharatnc)).
|
||||
* Remove PVS check from master [#32114](https://github.com/ClickHouse/ClickHouse/pull/32114) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix flaky keeper whitelist test [#32115](https://github.com/ClickHouse/ClickHouse/pull/32115) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix flacky test test_executable_storage_input [#32118](https://github.com/ClickHouse/ClickHouse/pull/32118) ([Maksim Kita](https://github.com/kitaisreal)).
|
||||
* Fix flaky test test_executable_storage_input [#32118](https://github.com/ClickHouse/ClickHouse/pull/32118) ([Maksim Kita](https://github.com/kitaisreal)).
|
||||
* Fix data race in `removePartAndEnqueueFetch(...)` [#32119](https://github.com/ClickHouse/ClickHouse/pull/32119) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Move fuzzers and unit tests to another group [#32120](https://github.com/ClickHouse/ClickHouse/pull/32120) ([alesapin](https://github.com/alesapin)).
|
||||
* Add a test with 20000 mutations in one query [#32122](https://github.com/ClickHouse/ClickHouse/pull/32122) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
@ -411,11 +411,11 @@ sidebar_label: 2022
|
||||
* Add test for [#32186](https://github.com/ClickHouse/ClickHouse/issues/32186) [#32203](https://github.com/ClickHouse/ClickHouse/pull/32203) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Fix uncaught exception in DatabaseLazy [#32206](https://github.com/ClickHouse/ClickHouse/pull/32206) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Update ASTCreateQuery.cpp [#32208](https://github.com/ClickHouse/ClickHouse/pull/32208) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix flacky fileLog test (probably) [#32209](https://github.com/ClickHouse/ClickHouse/pull/32209) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix flaky fileLog test (probably) [#32209](https://github.com/ClickHouse/ClickHouse/pull/32209) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix jemalloc under osx [#32219](https://github.com/ClickHouse/ClickHouse/pull/32219) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Add missing timezones to some tests [#32222](https://github.com/ClickHouse/ClickHouse/pull/32222) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix versioning of aggregate functions (fixes performance tests) [#32236](https://github.com/ClickHouse/ClickHouse/pull/32236) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Disable window view tests temporarily because still flacky [#32257](https://github.com/ClickHouse/ClickHouse/pull/32257) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Disable window view tests temporarily because still flaky [#32257](https://github.com/ClickHouse/ClickHouse/pull/32257) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix typo in tupleToNameValuePairs doc [#32262](https://github.com/ClickHouse/ClickHouse/pull/32262) ([Vladimir C](https://github.com/vdimir)).
|
||||
* Fix possible Pipeline stuck in case of StrictResize processor. [#32270](https://github.com/ClickHouse/ClickHouse/pull/32270) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix possible crash in DataTypeAggregateFunction [#32287](https://github.com/ClickHouse/ClickHouse/pull/32287) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
|
||||
@ -158,7 +158,7 @@ sidebar_label: 2022
|
||||
* MemoryStorage sync comments and code [#22721](https://github.com/ClickHouse/ClickHouse/pull/22721) ([Maksim Kita](https://github.com/kitaisreal)).
|
||||
* Fix potential segfault on Keeper startup [#22743](https://github.com/ClickHouse/ClickHouse/pull/22743) ([alesapin](https://github.com/alesapin)).
|
||||
* Avoid using harmful function rand() [#22744](https://github.com/ClickHouse/ClickHouse/pull/22744) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Fix flacky hedged tests [#22746](https://github.com/ClickHouse/ClickHouse/pull/22746) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Fix flaky hedged tests [#22746](https://github.com/ClickHouse/ClickHouse/pull/22746) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* add more messages when flushing the logs [#22761](https://github.com/ClickHouse/ClickHouse/pull/22761) ([Alexander Kuzmenkov](https://github.com/akuzm)).
|
||||
* Moved BorrowedObjectPool to common [#22764](https://github.com/ClickHouse/ClickHouse/pull/22764) ([Maksim Kita](https://github.com/kitaisreal)).
|
||||
* Functions ExternalDictionaries standardize exception throw [#22821](https://github.com/ClickHouse/ClickHouse/pull/22821) ([Maksim Kita](https://github.com/kitaisreal)).
|
||||
|
||||
@ -55,7 +55,7 @@ sidebar_label: 2022
|
||||
* Try fix rabbitmq tests [#26826](https://github.com/ClickHouse/ClickHouse/pull/26826) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* One more library bridge fix [#26873](https://github.com/ClickHouse/ClickHouse/pull/26873) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Update PVS checksum [#27317](https://github.com/ClickHouse/ClickHouse/pull/27317) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Fix flacky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix flaky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix throw without exception in MySQL source. [#28027](https://github.com/ClickHouse/ClickHouse/pull/28027) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
|
||||
@ -35,7 +35,7 @@ sidebar_label: 2022
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Fix prometheus metric name [#26140](https://github.com/ClickHouse/ClickHouse/pull/26140) ([Vladimir C](https://github.com/vdimir)).
|
||||
* Fix flacky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix flaky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix throw without exception in MySQL source. [#28027](https://github.com/ClickHouse/ClickHouse/pull/28027) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
|
||||
@ -101,7 +101,7 @@ sidebar_label: 2022
|
||||
* Separate log files for separate runs in stress test [#25741](https://github.com/ClickHouse/ClickHouse/pull/25741) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Fix slow performance test [#25742](https://github.com/ClickHouse/ClickHouse/pull/25742) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* DatabaseAtomic EXCHANGE DICTIONARIES fix test [#25753](https://github.com/ClickHouse/ClickHouse/pull/25753) ([Maksim Kita](https://github.com/kitaisreal)).
|
||||
* Try fix flacky rabbitmq test [#25756](https://github.com/ClickHouse/ClickHouse/pull/25756) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Try fix flaky rabbitmq test [#25756](https://github.com/ClickHouse/ClickHouse/pull/25756) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Add a test for [#13993](https://github.com/ClickHouse/ClickHouse/issues/13993) [#25758](https://github.com/ClickHouse/ClickHouse/pull/25758) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Set follow-fork-mode child for gdb in stress/fasttest/fuzzer [#25769](https://github.com/ClickHouse/ClickHouse/pull/25769) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Ignore TOO_DEEP_RECURSION server exception during fuzzing [#25770](https://github.com/ClickHouse/ClickHouse/pull/25770) ([Azat Khuzhin](https://github.com/azat)).
|
||||
|
||||
@ -40,7 +40,7 @@ sidebar_label: 2022
|
||||
|
||||
* Fix several bugs in ZooKeeper snapshots deserialization [#26127](https://github.com/ClickHouse/ClickHouse/pull/26127) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix prometheus metric name [#26140](https://github.com/ClickHouse/ClickHouse/pull/26140) ([Vladimir C](https://github.com/vdimir)).
|
||||
* Fix flacky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix flaky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix throw without exception in MySQL source. [#28027](https://github.com/ClickHouse/ClickHouse/pull/28027) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
|
||||
@ -346,7 +346,7 @@ sidebar_label: 2022
|
||||
* Update PVS checksum [#27317](https://github.com/ClickHouse/ClickHouse/pull/27317) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Fix 01300_client_save_history_when_terminated_long [#27324](https://github.com/ClickHouse/ClickHouse/pull/27324) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Try update contrib/zlib-ng [#27327](https://github.com/ClickHouse/ClickHouse/pull/27327) ([Ilya Yatsishin](https://github.com/qoega)).
|
||||
* Fix flacky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix flaky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Add and check system.mutations for database filter [#27384](https://github.com/ClickHouse/ClickHouse/pull/27384) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Correct the key data type used in mapContains [#27423](https://github.com/ClickHouse/ClickHouse/pull/27423) ([Fuwang Hu](https://github.com/fuwhu)).
|
||||
* Fix tests for WithMergeableStateAfterAggregationAndLimit [#27424](https://github.com/ClickHouse/ClickHouse/pull/27424) ([Azat Khuzhin](https://github.com/azat)).
|
||||
|
||||
@ -398,7 +398,7 @@ sidebar_label: 2022
|
||||
* test for [#24410](https://github.com/ClickHouse/ClickHouse/issues/24410) [#33265](https://github.com/ClickHouse/ClickHouse/pull/33265) ([Denny Crane](https://github.com/den-crane)).
|
||||
* Wait for RabbitMQ container to actually start when it was restarted in test on purpose [#33266](https://github.com/ClickHouse/ClickHouse/pull/33266) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Mark max_alter_threads as obsolete [#33268](https://github.com/ClickHouse/ClickHouse/pull/33268) ([Denny Crane](https://github.com/den-crane)).
|
||||
* Fix azure tests flackyness because of azure server closing connection [#33269](https://github.com/ClickHouse/ClickHouse/pull/33269) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix azure tests flakyness because of azure server closing connection [#33269](https://github.com/ClickHouse/ClickHouse/pull/33269) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Test for [#26920](https://github.com/ClickHouse/ClickHouse/issues/26920) [#33272](https://github.com/ClickHouse/ClickHouse/pull/33272) ([Denny Crane](https://github.com/den-crane)).
|
||||
* Fix test_storage_kafka failures by adjusting retention.ms [#33278](https://github.com/ClickHouse/ClickHouse/pull/33278) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Disable FunctionConvertFromString::canBeExecutedOnDefaultArguments [#33286](https://github.com/ClickHouse/ClickHouse/pull/33286) ([Vladimir C](https://github.com/vdimir)).
|
||||
@ -447,7 +447,7 @@ sidebar_label: 2022
|
||||
* Update mongodb.md [#33585](https://github.com/ClickHouse/ClickHouse/pull/33585) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Restore existing static builds links [#33597](https://github.com/ClickHouse/ClickHouse/pull/33597) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Fix pylint for run_check.py [#33600](https://github.com/ClickHouse/ClickHouse/pull/33600) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Fix flacky test_dictionaries_postgresql/ [#33601](https://github.com/ClickHouse/ClickHouse/pull/33601) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix flaky test_dictionaries_postgresql/ [#33601](https://github.com/ClickHouse/ClickHouse/pull/33601) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Make ZooKeeper client better interpret keeper server connection reject [#33602](https://github.com/ClickHouse/ClickHouse/pull/33602) ([alesapin](https://github.com/alesapin)).
|
||||
* Fix broken workflow dependencies [#33608](https://github.com/ClickHouse/ClickHouse/pull/33608) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Force rebuild images in CI [#33609](https://github.com/ClickHouse/ClickHouse/pull/33609) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
|
||||
@ -410,7 +410,7 @@ sidebar_label: 2022
|
||||
* Fix mongodb test with new cert [#36161](https://github.com/ClickHouse/ClickHouse/pull/36161) ([alesapin](https://github.com/alesapin)).
|
||||
* Some fixes for ReplicatedMergeTree [#36163](https://github.com/ClickHouse/ClickHouse/pull/36163) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* clickhouse-client: properly cancel query in case of error during formatting data [#36164](https://github.com/ClickHouse/ClickHouse/pull/36164) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Fix flacky test 01161_all_system_tables under s3 storage [#36175](https://github.com/ClickHouse/ClickHouse/pull/36175) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Fix flaky test 01161_all_system_tables under s3 storage [#36175](https://github.com/ClickHouse/ClickHouse/pull/36175) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Revert "Fix possible mutation stuck due to race with DROP_RANGE" [#36190](https://github.com/ClickHouse/ClickHouse/pull/36190) ([Azat Khuzhin](https://github.com/azat)).
|
||||
* Use atomic instead of mutex + condvar in ParallelReadBuffer [#36192](https://github.com/ClickHouse/ClickHouse/pull/36192) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Follow-up to [#36138](https://github.com/ClickHouse/ClickHouse/issues/36138) [#36194](https://github.com/ClickHouse/ClickHouse/pull/36194) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
|
||||
@ -321,7 +321,7 @@ sidebar_label: 2023
|
||||
* Add a test for [#38128](https://github.com/ClickHouse/ClickHouse/issues/38128) [#48817](https://github.com/ClickHouse/ClickHouse/pull/48817) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Remove excessive logging [#48826](https://github.com/ClickHouse/ClickHouse/pull/48826) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* remove duplicate indentwith in clang-format [#48834](https://github.com/ClickHouse/ClickHouse/pull/48834) ([cluster](https://github.com/infdahai)).
|
||||
* Try fix flacky test_concurrent_alter_move_and_drop [#48843](https://github.com/ClickHouse/ClickHouse/pull/48843) ([Sergei Trifonov](https://github.com/serxa)).
|
||||
* Try fix flaky test_concurrent_alter_move_and_drop [#48843](https://github.com/ClickHouse/ClickHouse/pull/48843) ([Sergei Trifonov](https://github.com/serxa)).
|
||||
* fix the race wait loading parts [#48844](https://github.com/ClickHouse/ClickHouse/pull/48844) ([Sema Checherinda](https://github.com/CheSema)).
|
||||
* suppress assert of progress for test_system_replicated_fetches [#48856](https://github.com/ClickHouse/ClickHouse/pull/48856) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* Fix: do not run test_store_cleanup_disk_s3 in parallel [#48863](https://github.com/ClickHouse/ClickHouse/pull/48863) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
@ -372,4 +372,3 @@ sidebar_label: 2023
|
||||
* suppress two timeout tests [#49175](https://github.com/ClickHouse/ClickHouse/pull/49175) ([Han Fei](https://github.com/hanfei1991)).
|
||||
* Document makeDateTime() and its variants [#49183](https://github.com/ClickHouse/ClickHouse/pull/49183) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Fix after [#49110](https://github.com/ClickHouse/ClickHouse/issues/49110) [#49206](https://github.com/ClickHouse/ClickHouse/pull/49206) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
|
||||
|
||||
@ -263,7 +263,7 @@ sidebar_label: 2023
|
||||
* Fix broken labeling for `manual approve` [#51405](https://github.com/ClickHouse/ClickHouse/pull/51405) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Fix parts lifetime in `MergeTreeTransaction` [#51407](https://github.com/ClickHouse/ClickHouse/pull/51407) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Fix flaky test test_skip_empty_files [#51409](https://github.com/ClickHouse/ClickHouse/pull/51409) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* fix flacky test test_profile_events_s3 [#51412](https://github.com/ClickHouse/ClickHouse/pull/51412) ([Sema Checherinda](https://github.com/CheSema)).
|
||||
* fix flaky test test_profile_events_s3 [#51412](https://github.com/ClickHouse/ClickHouse/pull/51412) ([Sema Checherinda](https://github.com/CheSema)).
|
||||
* Update README.md [#51413](https://github.com/ClickHouse/ClickHouse/pull/51413) ([Tyler Hannan](https://github.com/tylerhannan)).
|
||||
* Replace try/catch logic in hasTokenOrNull() by something more lightweight [#51425](https://github.com/ClickHouse/ClickHouse/pull/51425) ([Robert Schulze](https://github.com/rschu1ze)).
|
||||
* Add retries to `tlsv1_3` tests [#51434](https://github.com/ClickHouse/ClickHouse/pull/51434) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
|
||||
55
docs/changelogs/v24.5.5.78-stable.md
Normal file
55
docs/changelogs/v24.5.5.78-stable.md
Normal file
@ -0,0 +1,55 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2024
|
||||
---
|
||||
|
||||
# 2024 Changelog
|
||||
|
||||
### ClickHouse release v24.5.5.78-stable (0138248cb62) FIXME as compared to v24.5.4.49-stable (63b760955a0)
|
||||
|
||||
#### Improvement
|
||||
* Backported in [#66768](https://github.com/ClickHouse/ClickHouse/issues/66768): Make allow_experimental_analyzer be controlled by the initiator for distributed queries. This ensures compatibility and correctness during operations in mixed version clusters. [#65777](https://github.com/ClickHouse/ClickHouse/pull/65777) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
|
||||
|
||||
#### Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
|
||||
* Backported in [#66884](https://github.com/ClickHouse/ClickHouse/issues/66884): Fix unexpeced size of low cardinality column in function calls. [#65298](https://github.com/ClickHouse/ClickHouse/pull/65298) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#66691](https://github.com/ClickHouse/ClickHouse/issues/66691): Fix the VALID UNTIL clause in the user definition resetting after a restart. Closes [#66405](https://github.com/ClickHouse/ClickHouse/issues/66405). [#66409](https://github.com/ClickHouse/ClickHouse/pull/66409) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
* Backported in [#67814](https://github.com/ClickHouse/ClickHouse/issues/67814): Only relevant to the experimental Variant data type. Fix crash with Variant + AggregateFunction type. [#67122](https://github.com/ClickHouse/ClickHouse/pull/67122) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Backported in [#67501](https://github.com/ClickHouse/ClickHouse/issues/67501): Fix crash in DistributedAsyncInsert when connection is empty. [#67219](https://github.com/ClickHouse/ClickHouse/pull/67219) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Backported in [#67850](https://github.com/ClickHouse/ClickHouse/issues/67850): Fixes [#66026](https://github.com/ClickHouse/ClickHouse/issues/66026). Avoid unresolved table function arguments traversal in `ReplaceTableNodeToDummyVisitor`. [#67522](https://github.com/ClickHouse/ClickHouse/pull/67522) ([Dmitry Novik](https://github.com/novikd)).
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in an official stable release)
|
||||
* Backported in [#65350](https://github.com/ClickHouse/ClickHouse/issues/65350): Fix possible abort on uncaught exception in ~WriteBufferFromFileDescriptor in StatusFile. [#64206](https://github.com/ClickHouse/ClickHouse/pull/64206) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Backported in [#65621](https://github.com/ClickHouse/ClickHouse/issues/65621): Fix `Cannot find column` in distributed query with `ARRAY JOIN` by `Nested` column. Fixes [#64755](https://github.com/ClickHouse/ClickHouse/issues/64755). [#64801](https://github.com/ClickHouse/ClickHouse/pull/64801) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#65933](https://github.com/ClickHouse/ClickHouse/issues/65933): For queries that read from `PostgreSQL`, cancel the internal `PostgreSQL` query if the ClickHouse query is finished. Otherwise, `ClickHouse` query cannot be canceled until the internal `PostgreSQL` query is finished. [#65771](https://github.com/ClickHouse/ClickHouse/pull/65771) ([Maksim Kita](https://github.com/kitaisreal)).
|
||||
* Backported in [#66301](https://github.com/ClickHouse/ClickHouse/issues/66301): Better handling of join conditions involving `IS NULL` checks (for example `ON (a = b AND (a IS NOT NULL) AND (b IS NOT NULL) ) OR ( (a IS NULL) AND (b IS NULL) )` is rewritten to `ON a <=> b`), fix incorrect optimization when condition other then `IS NULL` are present. [#65835](https://github.com/ClickHouse/ClickHouse/pull/65835) ([vdimir](https://github.com/vdimir)).
|
||||
* Backported in [#66328](https://github.com/ClickHouse/ClickHouse/issues/66328): Add missing settings `input_format_csv_skip_first_lines/input_format_tsv_skip_first_lines/input_format_csv_try_infer_numbers_from_strings/input_format_csv_try_infer_strings_from_quoted_tuples` in schema inference cache because they can change the resulting schema. It prevents from incorrect result of schema inference with these settings changed. [#65980](https://github.com/ClickHouse/ClickHouse/pull/65980) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
* Backported in [#66155](https://github.com/ClickHouse/ClickHouse/issues/66155): Fixed buffer overflow bug in `unbin`/`unhex` implementation. [#66106](https://github.com/ClickHouse/ClickHouse/pull/66106) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Backported in [#66454](https://github.com/ClickHouse/ClickHouse/issues/66454): Fixed a bug in ZooKeeper client: a session could get stuck in unusable state after receiving a hardware error from ZooKeeper. For example, this might happen due to "soft memory limit" in ClickHouse Keeper. [#66140](https://github.com/ClickHouse/ClickHouse/pull/66140) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Backported in [#66226](https://github.com/ClickHouse/ClickHouse/issues/66226): Fix issue in SumIfToCountIfVisitor and signed integers. [#66146](https://github.com/ClickHouse/ClickHouse/pull/66146) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#66680](https://github.com/ClickHouse/ClickHouse/issues/66680): Fix handling limit for `system.numbers_mt` when no index can be used. [#66231](https://github.com/ClickHouse/ClickHouse/pull/66231) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
|
||||
* Backported in [#66604](https://github.com/ClickHouse/ClickHouse/issues/66604): Fixed how the ClickHouse server detects the maximum number of usable CPU cores as specified by cgroups v2 if the server runs in a container such as Docker. In more detail, containers often run their process in the root cgroup which has an empty name. In that case, ClickHouse ignored the CPU limits set by cgroups v2. [#66237](https://github.com/ClickHouse/ClickHouse/pull/66237) ([filimonov](https://github.com/filimonov)).
|
||||
* Backported in [#66360](https://github.com/ClickHouse/ClickHouse/issues/66360): Fix the `Not-ready set` error when a subquery with `IN` is used in the constraint. [#66261](https://github.com/ClickHouse/ClickHouse/pull/66261) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#66972](https://github.com/ClickHouse/ClickHouse/issues/66972): Fix `Column identifier is already registered` error with `group_by_use_nulls=true` and new analyzer. [#66400](https://github.com/ClickHouse/ClickHouse/pull/66400) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#66969](https://github.com/ClickHouse/ClickHouse/issues/66969): Fix `Cannot find column` error for queries with constant expression in `GROUP BY` key and new analyzer enabled. [#66433](https://github.com/ClickHouse/ClickHouse/pull/66433) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#66720](https://github.com/ClickHouse/ClickHouse/issues/66720): Correctly track memory for `Allocator::realloc`. [#66548](https://github.com/ClickHouse/ClickHouse/pull/66548) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Backported in [#66951](https://github.com/ClickHouse/ClickHouse/issues/66951): Fix an invalid result for queries with `WINDOW`. This could happen when `PARTITION` columns have sparse serialization and window functions are executed in parallel. [#66579](https://github.com/ClickHouse/ClickHouse/pull/66579) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#66757](https://github.com/ClickHouse/ClickHouse/issues/66757): Fix `Unknown identifier` and `Column is not under aggregate function` errors for queries with the expression `(column IS NULL).` The bug was triggered by [#65088](https://github.com/ClickHouse/ClickHouse/issues/65088), with the disabled analyzer only. [#66654](https://github.com/ClickHouse/ClickHouse/pull/66654) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#66948](https://github.com/ClickHouse/ClickHouse/issues/66948): Fix `Method getResultType is not supported for QUERY query node` error when scalar subquery was used as the first argument of IN (with new analyzer). [#66655](https://github.com/ClickHouse/ClickHouse/pull/66655) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#67633](https://github.com/ClickHouse/ClickHouse/issues/67633): Fix for occasional deadlock in Context::getDDLWorker. [#66843](https://github.com/ClickHouse/ClickHouse/pull/66843) ([Alexander Gololobov](https://github.com/davenger)).
|
||||
* Backported in [#67481](https://github.com/ClickHouse/ClickHouse/issues/67481): In rare cases ClickHouse could consider parts as broken because of some unexpected projections on disk. Now it's fixed. [#66898](https://github.com/ClickHouse/ClickHouse/pull/66898) ([alesapin](https://github.com/alesapin)).
|
||||
* Backported in [#67197](https://github.com/ClickHouse/ClickHouse/issues/67197): TRUNCATE DATABASE used to stop replication as if it was a DROP DATABASE query, it's fixed. [#67129](https://github.com/ClickHouse/ClickHouse/pull/67129) ([Alexander Tokmakov](https://github.com/tavplubix)).
|
||||
* Backported in [#67379](https://github.com/ClickHouse/ClickHouse/issues/67379): Fix error `Cannot convert column because it is non constant in source stream but must be constant in result.` for a query that reads from the `Merge` table over the `Distriburted` table with one shard. [#67146](https://github.com/ClickHouse/ClickHouse/pull/67146) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#67576](https://github.com/ClickHouse/ClickHouse/issues/67576): Fix execution of nested short-circuit functions. [#67520](https://github.com/ClickHouse/ClickHouse/pull/67520) ([Kruglov Pavel](https://github.com/Avogar)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Backported in [#66387](https://github.com/ClickHouse/ClickHouse/issues/66387): Disable broken cases from 02911_join_on_nullsafe_optimization. [#66310](https://github.com/ClickHouse/ClickHouse/pull/66310) ([vdimir](https://github.com/vdimir)).
|
||||
* Backported in [#66426](https://github.com/ClickHouse/ClickHouse/issues/66426): Ignore subquery for IN in DDLLoadingDependencyVisitor. [#66395](https://github.com/ClickHouse/ClickHouse/pull/66395) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#66544](https://github.com/ClickHouse/ClickHouse/issues/66544): Add additional log masking in CI. [#66523](https://github.com/ClickHouse/ClickHouse/pull/66523) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#66859](https://github.com/ClickHouse/ClickHouse/issues/66859): Fix data race in S3::ClientCache. [#66644](https://github.com/ClickHouse/ClickHouse/pull/66644) ([Konstantin Morozov](https://github.com/k-morozov)).
|
||||
* Backported in [#66875](https://github.com/ClickHouse/ClickHouse/issues/66875): Support one more case in JOIN ON ... IS NULL. [#66725](https://github.com/ClickHouse/ClickHouse/pull/66725) ([vdimir](https://github.com/vdimir)).
|
||||
* Backported in [#67059](https://github.com/ClickHouse/ClickHouse/issues/67059): Increase asio pool size in case the server is tiny. [#66761](https://github.com/ClickHouse/ClickHouse/pull/66761) ([alesapin](https://github.com/alesapin)).
|
||||
* Backported in [#66945](https://github.com/ClickHouse/ClickHouse/issues/66945): Small fix in realloc memory tracking. [#66820](https://github.com/ClickHouse/ClickHouse/pull/66820) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Backported in [#67252](https://github.com/ClickHouse/ClickHouse/issues/67252): Followup [#66725](https://github.com/ClickHouse/ClickHouse/issues/66725). [#66869](https://github.com/ClickHouse/ClickHouse/pull/66869) ([vdimir](https://github.com/vdimir)).
|
||||
* Backported in [#67412](https://github.com/ClickHouse/ClickHouse/issues/67412): CI: Fix build results for release branches. [#67402](https://github.com/ClickHouse/ClickHouse/pull/67402) ([Max K.](https://github.com/maxknv)).
|
||||
|
||||
@ -85,6 +85,7 @@ The BACKUP and RESTORE statements take a list of DATABASE and TABLE names, a des
|
||||
- `password` for the file on disk
|
||||
- `base_backup`: the destination of the previous backup of this source. For example, `Disk('backups', '1.zip')`
|
||||
- `use_same_s3_credentials_for_base_backup`: whether base backup to S3 should inherit credentials from the query. Only works with `S3`.
|
||||
- `use_same_password_for_base_backup`: whether base backup archive should inherit the password from the query.
|
||||
- `structure_only`: if enabled, allows to only backup or restore the CREATE statements without the data of tables
|
||||
- `storage_policy`: storage policy for the tables being restored. See [Using Multiple Block Devices for Data Storage](../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes). This setting is only applicable to the `RESTORE` command. The specified storage policy applies only to tables with an engine from the `MergeTree` family.
|
||||
- `s3_storage_class`: the storage class used for S3 backup. For example, `STANDARD`
|
||||
|
||||
@ -223,3 +223,28 @@ SELECT translateUTF8('Münchener Straße', 'üß', 'us') AS res;
|
||||
│ Munchener Strase │
|
||||
└──────────────────┘
|
||||
```
|
||||
|
||||
## printf
|
||||
|
||||
The `printf` function formats the given string with the values (strings, integers, floating-points etc.) listed in the arguments, similar to printf function in C++. The format string can contain format specifiers starting with `%` character. Anything not contained in `%` and the following format specifier is considered literal text and copied verbatim into the output. Literal `%` character can be escaped by `%%`.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
printf(format, arg1, arg2, ...)
|
||||
```
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
select printf('%%%s %s %d', 'Hello', 'World', 2024);
|
||||
```
|
||||
|
||||
|
||||
``` response
|
||||
┌─printf('%%%s %s %d', 'Hello', 'World', 2024)─┐
|
||||
│ %Hello World 2024 │
|
||||
└──────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
@ -252,7 +252,7 @@ sidebar_label: "\u53D8\u66F4\u65E5\u5FD7"
|
||||
- 抑制MSan下的一些测试失败。 [#8780](https://github.com/ClickHouse/ClickHouse/pull/8780) ([Alexander Kuzmenkov](https://github.com/akuzm))
|
||||
- 加速 “exception while insert” 测试 此测试通常在具有复盖率的调试版本中超时。 [#8711](https://github.com/ClickHouse/ClickHouse/pull/8711) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov))
|
||||
- 更新 `libcxx` 和 `libcxxabi` 为了主人 在准备 [#9304](https://github.com/ClickHouse/ClickHouse/issues/9304) [#9308](https://github.com/ClickHouse/ClickHouse/pull/9308) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov))
|
||||
- 修复flacky测试 `00910_zookeeper_test_alter_compression_codecs`. [#9525](https://github.com/ClickHouse/ClickHouse/pull/9525) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov))
|
||||
- 修复flaky测试 `00910_zookeeper_test_alter_compression_codecs`. [#9525](https://github.com/ClickHouse/ClickHouse/pull/9525) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov))
|
||||
- 清理重复的链接器标志。 确保链接器不会查找意想不到的符号。 [#9433](https://github.com/ClickHouse/ClickHouse/pull/9433) ([阿莫斯鸟](https://github.com/amosbird))
|
||||
- 添加 `clickhouse-odbc` 驱动程序进入测试图像。 这允许通过自己的ODBC驱动程序测试ClickHouse与ClickHouse的交互。 [#9348](https://github.com/ClickHouse/ClickHouse/pull/9348) ([filimonov](https://github.com/filimonov))
|
||||
- 修复单元测试中的几个错误。 [#9047](https://github.com/ClickHouse/ClickHouse/pull/9047) ([阿利沙平](https://github.com/alesapin))
|
||||
|
||||
@ -66,14 +66,14 @@
|
||||
<server>
|
||||
<!-- Used for secure tcp port -->
|
||||
<!-- openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt -->
|
||||
<certificateFile>/etc/clickhouse-keeper/server.crt</certificateFile>
|
||||
<privateKeyFile>/etc/clickhouse-keeper/server.key</privateKeyFile>
|
||||
<!-- <certificateFile>/etc/clickhouse-keeper/server.crt</certificateFile> -->
|
||||
<!-- <privateKeyFile>/etc/clickhouse-keeper/server.key</privateKeyFile> -->
|
||||
<!-- dhparams are optional. You can delete the <dhParamsFile> element.
|
||||
To generate dhparams, use the following command:
|
||||
openssl dhparam -out /etc/clickhouse-keeper/dhparam.pem 4096
|
||||
Only file format with BEGIN DH PARAMETERS is supported.
|
||||
-->
|
||||
<dhParamsFile>/etc/clickhouse-keeper/dhparam.pem</dhParamsFile>
|
||||
<!-- <dhParamsFile>/etc/clickhouse-keeper/dhparam.pem</dhParamsFile> -->
|
||||
<verificationMode>none</verificationMode>
|
||||
<loadDefaultCAFile>true</loadDefaultCAFile>
|
||||
<cacheSessions>true</cacheSessions>
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#include "LocalServer.h"
|
||||
|
||||
#include <sys/resource.h>
|
||||
#include <Common/Config/getLocalConfigPath.h>
|
||||
#include <Common/logger_useful.h>
|
||||
#include <Common/formatReadable.h>
|
||||
#include <Core/UUID.h>
|
||||
@ -127,10 +128,21 @@ void LocalServer::initialize(Poco::Util::Application & self)
|
||||
{
|
||||
Poco::Util::Application::initialize(self);
|
||||
|
||||
const char * home_path_cstr = getenv("HOME"); // NOLINT(concurrency-mt-unsafe)
|
||||
if (home_path_cstr)
|
||||
home_path = home_path_cstr;
|
||||
|
||||
/// Load config files if exists
|
||||
if (getClientConfiguration().has("config-file") || fs::exists("config.xml"))
|
||||
std::string config_path;
|
||||
if (getClientConfiguration().has("config-file"))
|
||||
config_path = getClientConfiguration().getString("config-file");
|
||||
else if (config_path.empty() && fs::exists("config.xml"))
|
||||
config_path = "config.xml";
|
||||
else if (config_path.empty())
|
||||
config_path = getLocalConfigPath(home_path).value_or("");
|
||||
|
||||
if (fs::exists(config_path))
|
||||
{
|
||||
const auto config_path = getClientConfiguration().getString("config-file", "config.xml");
|
||||
ConfigProcessor config_processor(config_path, false, true);
|
||||
ConfigProcessor::setConfigPath(fs::path(config_path).parent_path());
|
||||
auto loaded_config = config_processor.loadConfig();
|
||||
|
||||
@ -11,6 +11,7 @@
|
||||
#include <Analyzer/InDepthQueryTreeVisitor.h>
|
||||
#include <Analyzer/ConstantNode.h>
|
||||
#include <Analyzer/FunctionNode.h>
|
||||
#include <Analyzer/JoinNode.h>
|
||||
#include <Analyzer/Utils.h>
|
||||
|
||||
namespace DB
|
||||
@ -25,8 +26,15 @@ public:
|
||||
using Base = InDepthQueryTreeVisitorWithContext<ComparisonTupleEliminationPassVisitor>;
|
||||
using Base::Base;
|
||||
|
||||
static bool needChildVisit(QueryTreeNodePtr &, QueryTreeNodePtr & child)
|
||||
static bool needChildVisit(QueryTreeNodePtr & parent, QueryTreeNodePtr & child)
|
||||
{
|
||||
if (parent->getNodeType() == QueryTreeNodeType::JOIN)
|
||||
{
|
||||
/// In JOIN ON section comparison of tuples works a bit differently.
|
||||
/// For example we can join on tuple(NULL) = tuple(NULL), join algorithms consider only NULLs on the top level.
|
||||
if (parent->as<const JoinNode &>().getJoinExpression().get() == child.get())
|
||||
return false;
|
||||
}
|
||||
return child->getNodeType() != QueryTreeNodeType::TABLE_FUNCTION;
|
||||
}
|
||||
|
||||
|
||||
@ -4541,7 +4541,15 @@ void QueryAnalyzer::resolveTableFunction(QueryTreeNodePtr & table_function_node,
|
||||
resolveExpressionNode(nodes[1], scope, /* allow_lambda_expression */false, /* allow_table_function */false);
|
||||
if (auto * constant = nodes[1]->as<ConstantNode>())
|
||||
{
|
||||
view_params[identifier_node->getIdentifier().getFullName()] = convertFieldToString(constant->getValue());
|
||||
/// Serialize the constant value using datatype specific
|
||||
/// interfaces to match the deserialization in ReplaceQueryParametersVistor.
|
||||
WriteBufferFromOwnString buf;
|
||||
const auto & value = constant->getValue();
|
||||
auto real_type = constant->getResultType();
|
||||
auto temporary_column = real_type->createColumn();
|
||||
temporary_column->insert(value);
|
||||
real_type->getDefaultSerialization()->serializeTextEscaped(*temporary_column, 0, buf, {});
|
||||
view_params[identifier_node->getIdentifier().getFullName()] = buf.str();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -41,6 +41,7 @@ public:
|
||||
bool allow_s3_native_copy = true;
|
||||
bool allow_azure_native_copy = true;
|
||||
bool use_same_s3_credentials_for_base_backup = false;
|
||||
bool use_same_password_for_base_backup = false;
|
||||
bool azure_attempt_to_create_container = true;
|
||||
ReadSettings read_settings;
|
||||
WriteSettings write_settings;
|
||||
|
||||
@ -92,7 +92,8 @@ BackupImpl::BackupImpl(
|
||||
std::shared_ptr<IBackupReader> reader_,
|
||||
const ContextPtr & context_,
|
||||
bool is_internal_backup_,
|
||||
bool use_same_s3_credentials_for_base_backup_)
|
||||
bool use_same_s3_credentials_for_base_backup_,
|
||||
bool use_same_password_for_base_backup_)
|
||||
: backup_info(backup_info_)
|
||||
, backup_name_for_logging(backup_info.toStringForLogging())
|
||||
, use_archive(!archive_params_.archive_name.empty())
|
||||
@ -104,6 +105,7 @@ BackupImpl::BackupImpl(
|
||||
, version(INITIAL_BACKUP_VERSION)
|
||||
, base_backup_info(base_backup_info_)
|
||||
, use_same_s3_credentials_for_base_backup(use_same_s3_credentials_for_base_backup_)
|
||||
, use_same_password_for_base_backup(use_same_password_for_base_backup_)
|
||||
, log(getLogger("BackupImpl"))
|
||||
{
|
||||
open();
|
||||
@ -120,7 +122,8 @@ BackupImpl::BackupImpl(
|
||||
const std::shared_ptr<IBackupCoordination> & coordination_,
|
||||
const std::optional<UUID> & backup_uuid_,
|
||||
bool deduplicate_files_,
|
||||
bool use_same_s3_credentials_for_base_backup_)
|
||||
bool use_same_s3_credentials_for_base_backup_,
|
||||
bool use_same_password_for_base_backup_)
|
||||
: backup_info(backup_info_)
|
||||
, backup_name_for_logging(backup_info.toStringForLogging())
|
||||
, use_archive(!archive_params_.archive_name.empty())
|
||||
@ -135,6 +138,7 @@ BackupImpl::BackupImpl(
|
||||
, base_backup_info(base_backup_info_)
|
||||
, deduplicate_files(deduplicate_files_)
|
||||
, use_same_s3_credentials_for_base_backup(use_same_s3_credentials_for_base_backup_)
|
||||
, use_same_password_for_base_backup(use_same_password_for_base_backup_)
|
||||
, log(getLogger("BackupImpl"))
|
||||
{
|
||||
open();
|
||||
@ -258,6 +262,11 @@ std::shared_ptr<const IBackup> BackupImpl::getBaseBackupUnlocked() const
|
||||
params.is_internal_backup = is_internal_backup;
|
||||
/// use_same_s3_credentials_for_base_backup should be inherited for base backups
|
||||
params.use_same_s3_credentials_for_base_backup = use_same_s3_credentials_for_base_backup;
|
||||
/// use_same_password_for_base_backup should be inherited for base backups
|
||||
params.use_same_password_for_base_backup = use_same_password_for_base_backup;
|
||||
|
||||
if (params.use_same_password_for_base_backup)
|
||||
params.password = archive_params.password;
|
||||
|
||||
base_backup = BackupFactory::instance().createBackup(params);
|
||||
|
||||
|
||||
@ -41,7 +41,8 @@ public:
|
||||
std::shared_ptr<IBackupReader> reader_,
|
||||
const ContextPtr & context_,
|
||||
bool is_internal_backup_,
|
||||
bool use_same_s3_credentials_for_base_backup_);
|
||||
bool use_same_s3_credentials_for_base_backup_,
|
||||
bool use_same_password_for_base_backup_);
|
||||
|
||||
BackupImpl(
|
||||
const BackupInfo & backup_info_,
|
||||
@ -53,7 +54,8 @@ public:
|
||||
const std::shared_ptr<IBackupCoordination> & coordination_,
|
||||
const std::optional<UUID> & backup_uuid_,
|
||||
bool deduplicate_files_,
|
||||
bool use_same_s3_credentials_for_base_backup_);
|
||||
bool use_same_s3_credentials_for_base_backup_,
|
||||
bool use_same_password_for_base_backup_);
|
||||
|
||||
~BackupImpl() override;
|
||||
|
||||
@ -153,6 +155,7 @@ private:
|
||||
bool writing_finalized = false;
|
||||
bool deduplicate_files = true;
|
||||
bool use_same_s3_credentials_for_base_backup = false;
|
||||
bool use_same_password_for_base_backup = false;
|
||||
const LoggerPtr log;
|
||||
};
|
||||
|
||||
|
||||
@ -29,6 +29,7 @@ namespace ErrorCodes
|
||||
M(Bool, allow_s3_native_copy) \
|
||||
M(Bool, allow_azure_native_copy) \
|
||||
M(Bool, use_same_s3_credentials_for_base_backup) \
|
||||
M(Bool, use_same_password_for_base_backup) \
|
||||
M(Bool, azure_attempt_to_create_container) \
|
||||
M(Bool, read_from_filesystem_cache) \
|
||||
M(UInt64, shard_num) \
|
||||
|
||||
@ -50,6 +50,9 @@ struct BackupSettings
|
||||
/// Whether base backup to S3 should inherit credentials from the BACKUP query.
|
||||
bool use_same_s3_credentials_for_base_backup = false;
|
||||
|
||||
/// Whether base backup archive should be unlocked using the same password as the incremental archive
|
||||
bool use_same_password_for_base_backup = false;
|
||||
|
||||
/// Whether a new Azure container should be created if it does not exist (requires permissions at storage account level)
|
||||
bool azure_attempt_to_create_container = true;
|
||||
|
||||
|
||||
@ -602,6 +602,7 @@ void BackupsWorker::doBackup(
|
||||
backup_create_params.allow_s3_native_copy = backup_settings.allow_s3_native_copy;
|
||||
backup_create_params.allow_azure_native_copy = backup_settings.allow_azure_native_copy;
|
||||
backup_create_params.use_same_s3_credentials_for_base_backup = backup_settings.use_same_s3_credentials_for_base_backup;
|
||||
backup_create_params.use_same_password_for_base_backup = backup_settings.use_same_password_for_base_backup;
|
||||
backup_create_params.azure_attempt_to_create_container = backup_settings.azure_attempt_to_create_container;
|
||||
backup_create_params.read_settings = getReadSettingsForBackup(context, backup_settings);
|
||||
backup_create_params.write_settings = getWriteSettingsForBackup(context);
|
||||
@ -924,6 +925,7 @@ void BackupsWorker::doRestore(
|
||||
backup_open_params.password = restore_settings.password;
|
||||
backup_open_params.allow_s3_native_copy = restore_settings.allow_s3_native_copy;
|
||||
backup_open_params.use_same_s3_credentials_for_base_backup = restore_settings.use_same_s3_credentials_for_base_backup;
|
||||
backup_open_params.use_same_password_for_base_backup = restore_settings.use_same_password_for_base_backup;
|
||||
backup_open_params.read_settings = getReadSettingsForRestore(context);
|
||||
backup_open_params.write_settings = getWriteSettingsForRestore(context);
|
||||
backup_open_params.is_internal_backup = restore_settings.internal;
|
||||
|
||||
@ -164,6 +164,7 @@ namespace
|
||||
M(RestoreUDFCreationMode, create_function) \
|
||||
M(Bool, allow_s3_native_copy) \
|
||||
M(Bool, use_same_s3_credentials_for_base_backup) \
|
||||
M(Bool, use_same_password_for_base_backup) \
|
||||
M(Bool, restore_broken_parts_as_detached) \
|
||||
M(Bool, internal) \
|
||||
M(String, host_id) \
|
||||
|
||||
@ -113,6 +113,9 @@ struct RestoreSettings
|
||||
/// Whether base backup from S3 should inherit credentials from the RESTORE query.
|
||||
bool use_same_s3_credentials_for_base_backup = false;
|
||||
|
||||
/// Whether base backup archive should be unlocked using the same password as the incremental archive
|
||||
bool use_same_password_for_base_backup = false;
|
||||
|
||||
/// If it's true RESTORE won't stop on broken parts while restoring, instead they will be restored as detached parts
|
||||
/// to the `detached` folder with names starting with `broken-from-backup'.
|
||||
bool restore_broken_parts_as_detached = false;
|
||||
|
||||
@ -141,7 +141,8 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory)
|
||||
reader,
|
||||
params.context,
|
||||
params.is_internal_backup,
|
||||
/* use_same_s3_credentials_for_base_backup*/ false);
|
||||
/* use_same_s3_credentials_for_base_backup*/ false,
|
||||
params.use_same_password_for_base_backup);
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -164,7 +165,8 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory)
|
||||
params.backup_coordination,
|
||||
params.backup_uuid,
|
||||
params.deduplicate_files,
|
||||
/* use_same_s3_credentials_for_base_backup */ false);
|
||||
/* use_same_s3_credentials_for_base_backup */ false,
|
||||
params.use_same_password_for_base_backup);
|
||||
}
|
||||
#else
|
||||
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "AzureBlobStorage support is disabled");
|
||||
|
||||
@ -120,7 +120,8 @@ void registerBackupEngineS3(BackupFactory & factory)
|
||||
reader,
|
||||
params.context,
|
||||
params.is_internal_backup,
|
||||
params.use_same_s3_credentials_for_base_backup);
|
||||
params.use_same_s3_credentials_for_base_backup,
|
||||
params.use_same_password_for_base_backup);
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -144,7 +145,8 @@ void registerBackupEngineS3(BackupFactory & factory)
|
||||
params.backup_coordination,
|
||||
params.backup_uuid,
|
||||
params.deduplicate_files,
|
||||
params.use_same_s3_credentials_for_base_backup);
|
||||
params.use_same_s3_credentials_for_base_backup,
|
||||
params.use_same_password_for_base_backup);
|
||||
}
|
||||
#else
|
||||
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "S3 support is disabled");
|
||||
|
||||
@ -178,7 +178,8 @@ void registerBackupEnginesFileAndDisk(BackupFactory & factory)
|
||||
reader,
|
||||
params.context,
|
||||
params.is_internal_backup,
|
||||
params.use_same_s3_credentials_for_base_backup);
|
||||
params.use_same_s3_credentials_for_base_backup,
|
||||
params.use_same_password_for_base_backup);
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -197,7 +198,8 @@ void registerBackupEnginesFileAndDisk(BackupFactory & factory)
|
||||
params.backup_coordination,
|
||||
params.backup_uuid,
|
||||
params.deduplicate_files,
|
||||
params.use_same_s3_credentials_for_base_backup);
|
||||
params.use_same_s3_credentials_for_base_backup,
|
||||
params.use_same_password_for_base_backup);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -330,7 +330,38 @@ ColumnPtr ColumnAggregateFunction::filter(const Filter & filter, ssize_t result_
|
||||
|
||||
void ColumnAggregateFunction::expand(const Filter & mask, bool inverted)
|
||||
{
|
||||
expandDataByMask<char *>(data, mask, inverted);
|
||||
ensureOwnership();
|
||||
Arena & arena = createOrGetArena();
|
||||
|
||||
if (mask.size() < data.size())
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Mask size should be no less than data size.");
|
||||
|
||||
ssize_t from = data.size() - 1;
|
||||
ssize_t index = mask.size() - 1;
|
||||
data.resize(mask.size());
|
||||
while (index >= 0)
|
||||
{
|
||||
if (!!mask[index] ^ inverted)
|
||||
{
|
||||
if (from < 0)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Too many bytes in mask");
|
||||
|
||||
/// Copy only if it makes sense.
|
||||
if (index != from)
|
||||
data[index] = data[from];
|
||||
--from;
|
||||
}
|
||||
else
|
||||
{
|
||||
data[index] = arena.alignedAlloc(func->sizeOfData(), func->alignOfData());
|
||||
func->create(data[index]);
|
||||
}
|
||||
|
||||
--index;
|
||||
}
|
||||
|
||||
if (from != -1)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Not enough bytes in mask");
|
||||
}
|
||||
|
||||
ColumnPtr ColumnAggregateFunction::permute(const Permutation & perm, size_t limit) const
|
||||
|
||||
@ -2,6 +2,7 @@ set (SRCS
|
||||
AbstractConfigurationComparison.cpp
|
||||
ConfigProcessor.cpp
|
||||
getClientConfigPath.cpp
|
||||
getLocalConfigPath.cpp
|
||||
ConfigReloader.cpp
|
||||
YAMLParser.cpp
|
||||
ConfigHelper.cpp
|
||||
|
||||
@ -138,9 +138,14 @@ static Node * getRootNode(Document * document)
|
||||
return XMLUtils::getRootNode(document);
|
||||
}
|
||||
|
||||
static size_t firstNonWhitespacePos(const std::string & s)
|
||||
{
|
||||
return s.find_first_not_of(" \t\n\r");
|
||||
}
|
||||
|
||||
static bool allWhitespace(const std::string & s)
|
||||
{
|
||||
return s.find_first_not_of(" \t\n\r") == std::string::npos;
|
||||
return firstNonWhitespacePos(s) == std::string::npos;
|
||||
}
|
||||
|
||||
static void deleteAttributesRecursive(Node * root)
|
||||
@ -622,6 +627,49 @@ ConfigProcessor::Files ConfigProcessor::getConfigMergeFiles(const std::string &
|
||||
return files;
|
||||
}
|
||||
|
||||
XMLDocumentPtr ConfigProcessor::parseConfig(const std::string & config_path)
|
||||
{
|
||||
fs::path p(config_path);
|
||||
std::string extension = p.extension();
|
||||
boost::algorithm::to_lower(extension);
|
||||
|
||||
if (extension == ".xml")
|
||||
return dom_parser.parse(config_path);
|
||||
else if (extension == ".yaml" || extension == ".yml")
|
||||
return YAMLParser::parse(config_path);
|
||||
else
|
||||
{
|
||||
/// Suppose non regular file parsed as XML, such as pipe: /dev/fd/X (regardless it has .xml extension or not)
|
||||
if (!fs::is_regular_file(config_path))
|
||||
return dom_parser.parse(config_path);
|
||||
|
||||
/// If the regular file begins with < it might be XML, otherwise it might be YAML.
|
||||
bool maybe_xml = false;
|
||||
{
|
||||
std::ifstream file(config_path);
|
||||
if (!file.is_open())
|
||||
throw Exception(ErrorCodes::CANNOT_LOAD_CONFIG, "Unknown format of '{}' config", config_path);
|
||||
|
||||
std::string line;
|
||||
while (std::getline(file, line))
|
||||
{
|
||||
const size_t pos = firstNonWhitespacePos(line);
|
||||
|
||||
if (pos < line.size() && '<' == line[pos])
|
||||
{
|
||||
maybe_xml = true;
|
||||
break;
|
||||
}
|
||||
else if (pos != std::string::npos)
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (maybe_xml)
|
||||
return dom_parser.parse(config_path);
|
||||
return YAMLParser::parse(config_path);
|
||||
}
|
||||
}
|
||||
|
||||
XMLDocumentPtr ConfigProcessor::processConfig(
|
||||
bool * has_zk_includes,
|
||||
zkutil::ZooKeeperNodeCache * zk_node_cache,
|
||||
@ -633,23 +681,7 @@ XMLDocumentPtr ConfigProcessor::processConfig(
|
||||
|
||||
if (fs::exists(path))
|
||||
{
|
||||
fs::path p(path);
|
||||
|
||||
std::string extension = p.extension();
|
||||
boost::algorithm::to_lower(extension);
|
||||
|
||||
if (extension == ".yaml" || extension == ".yml")
|
||||
{
|
||||
config = YAMLParser::parse(path);
|
||||
}
|
||||
else if (extension == ".xml" || extension == ".conf" || extension.empty())
|
||||
{
|
||||
config = dom_parser.parse(path);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw Exception(ErrorCodes::CANNOT_LOAD_CONFIG, "Unknown format of '{}' config", path);
|
||||
}
|
||||
config = parseConfig(path);
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -673,20 +705,7 @@ XMLDocumentPtr ConfigProcessor::processConfig(
|
||||
LOG_DEBUG(log, "Merging configuration file '{}'.", merge_file);
|
||||
|
||||
XMLDocumentPtr with;
|
||||
|
||||
fs::path p(merge_file);
|
||||
std::string extension = p.extension();
|
||||
boost::algorithm::to_lower(extension);
|
||||
|
||||
if (extension == ".yaml" || extension == ".yml")
|
||||
{
|
||||
with = YAMLParser::parse(merge_file);
|
||||
}
|
||||
else
|
||||
{
|
||||
with = dom_parser.parse(merge_file);
|
||||
}
|
||||
|
||||
with = parseConfig(merge_file);
|
||||
if (!merge(config, with))
|
||||
{
|
||||
LOG_DEBUG(log, "Merging bypassed - configuration file '{}' doesn't belong to configuration '{}' - merging root node name '{}' doesn't match '{}'",
|
||||
@ -730,19 +749,7 @@ XMLDocumentPtr ConfigProcessor::processConfig(
|
||||
{
|
||||
LOG_DEBUG(log, "Including configuration file '{}'.", include_from_path);
|
||||
|
||||
fs::path p(include_from_path);
|
||||
std::string extension = p.extension();
|
||||
boost::algorithm::to_lower(extension);
|
||||
|
||||
if (extension == ".yaml" || extension == ".yml")
|
||||
{
|
||||
include_from = YAMLParser::parse(include_from_path);
|
||||
}
|
||||
else
|
||||
{
|
||||
include_from = dom_parser.parse(include_from_path);
|
||||
}
|
||||
|
||||
include_from = parseConfig(include_from_path);
|
||||
contributing_files.push_back(include_from_path);
|
||||
}
|
||||
|
||||
|
||||
@ -65,6 +65,8 @@ public:
|
||||
zkutil::ZooKeeperNodeCache * zk_node_cache = nullptr,
|
||||
const zkutil::EventPtr & zk_changed_event = nullptr);
|
||||
|
||||
XMLDocumentPtr parseConfig(const std::string & config_path);
|
||||
|
||||
/// These configurations will be used if there is no configuration file.
|
||||
static void registerEmbeddedConfig(std::string name, std::string_view content);
|
||||
|
||||
|
||||
@ -12,7 +12,6 @@ namespace DB
|
||||
std::optional<std::string> getClientConfigPath(const std::string & home_path)
|
||||
{
|
||||
std::string config_path;
|
||||
bool found = false;
|
||||
|
||||
std::vector<std::string> names;
|
||||
names.emplace_back("./clickhouse-client");
|
||||
@ -28,18 +27,10 @@ std::optional<std::string> getClientConfigPath(const std::string & home_path)
|
||||
|
||||
std::error_code ec;
|
||||
if (fs::exists(config_path, ec))
|
||||
{
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
return config_path;
|
||||
}
|
||||
if (found)
|
||||
break;
|
||||
}
|
||||
|
||||
if (found)
|
||||
return config_path;
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
|
||||
37
src/Common/Config/getLocalConfigPath.cpp
Normal file
37
src/Common/Config/getLocalConfigPath.cpp
Normal file
@ -0,0 +1,37 @@
|
||||
#include <Common/Config/getLocalConfigPath.h>
|
||||
|
||||
#include <filesystem>
|
||||
#include <vector>
|
||||
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
std::optional<std::string> getLocalConfigPath(const std::string & home_path)
|
||||
{
|
||||
std::string config_path;
|
||||
|
||||
std::vector<std::string> names;
|
||||
names.emplace_back("./clickhouse-local");
|
||||
if (!home_path.empty())
|
||||
names.emplace_back(home_path + "/.clickhouse-local/config");
|
||||
names.emplace_back("/etc/clickhouse-local/config");
|
||||
|
||||
for (const auto & name : names)
|
||||
{
|
||||
for (const auto & extension : {".xml", ".yaml", ".yml"})
|
||||
{
|
||||
config_path = name + extension;
|
||||
|
||||
std::error_code ec;
|
||||
if (fs::exists(config_path, ec))
|
||||
return config_path;
|
||||
}
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
}
|
||||
12
src/Common/Config/getLocalConfigPath.h
Normal file
12
src/Common/Config/getLocalConfigPath.h
Normal file
@ -0,0 +1,12 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <optional>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// Return path to existing configuration file.
|
||||
std::optional<std::string> getLocalConfigPath(const std::string & home_path);
|
||||
|
||||
}
|
||||
@ -3,10 +3,14 @@
|
||||
#include "config.h"
|
||||
|
||||
#if USE_RAPIDJSON
|
||||
# include <base/types.h>
|
||||
# include <base/defines.h>
|
||||
# include <rapidjson/document.h>
|
||||
# include "ElementTypes.h"
|
||||
|
||||
/// Prevent stack overflow:
|
||||
#define RAPIDJSON_PARSE_DEFAULT_FLAGS (kParseIterativeFlag)
|
||||
|
||||
#include <base/types.h>
|
||||
#include <base/defines.h>
|
||||
#include <rapidjson/document.h>
|
||||
#include "ElementTypes.h"
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
@ -193,8 +193,10 @@
|
||||
M(ReplicaPartialShutdown, "How many times Replicated table has to deinitialize its state due to session expiration in ZooKeeper. The state is reinitialized every time when ZooKeeper is available again.") \
|
||||
\
|
||||
M(SelectedParts, "Number of data parts selected to read from a MergeTree table.") \
|
||||
M(SelectedPartsTotal, "Number of total data parts before selecting which ones to read from a MergeTree table.") \
|
||||
M(SelectedRanges, "Number of (non-adjacent) ranges in all data parts selected to read from a MergeTree table.") \
|
||||
M(SelectedMarks, "Number of marks (index granules) selected to read from a MergeTree table.") \
|
||||
M(SelectedMarksTotal, "Number of total marks (index granules) before selecting which ones to read from a MergeTree table.") \
|
||||
M(SelectedRows, "Number of rows SELECTed from all tables.") \
|
||||
M(SelectedBytes, "Number of bytes (uncompressed; for columns as they stored in memory) SELECTed from all tables.") \
|
||||
M(RowsReadByMainReader, "Number of rows read from MergeTree tables by the main reader (after PREWHERE step).") \
|
||||
|
||||
@ -65,7 +65,7 @@ void SystemLogQueue<LogElement>::push(LogElement&& element)
|
||||
/// Memory can be allocated while resizing on queue.push_back.
|
||||
/// The size of allocation can be in order of a few megabytes.
|
||||
/// But this should not be accounted for query memory usage.
|
||||
/// Otherwise the tests like 01017_uniqCombined_memory_usage.sql will be flacky.
|
||||
/// Otherwise the tests like 01017_uniqCombined_memory_usage.sql will be flaky.
|
||||
MemoryTrackerBlockerInThread temporarily_disable_memory_tracker;
|
||||
|
||||
/// Should not log messages under mutex.
|
||||
|
||||
@ -123,6 +123,9 @@ void TimerDescriptor::drain() const
|
||||
throw ErrnoException(ErrorCodes::CANNOT_READ_FROM_SOCKET, "Cannot readlink for a timer_fd {}", timer_fd);
|
||||
|
||||
LOG_TRACE(log, "Received EINTR while trying to drain a TimerDescriptor, fd {}: {}", timer_fd, std::string_view(link_path, link_path_length));
|
||||
|
||||
/// Check that it's actually a timerfd.
|
||||
chassert(std::string_view(link_path, link_path_length).contains("timerfd"));
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
@ -51,7 +51,7 @@ DatabaseHDFS::DatabaseHDFS(const String & name_, const String & source_url, Cont
|
||||
if (!source.empty())
|
||||
{
|
||||
if (!re2::RE2::FullMatch(source, std::string(HDFS_HOST_REGEXP)))
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad hdfs host: {}. "
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad HDFS host: {}. "
|
||||
"It should have structure 'hdfs://<host_name>:<port>'", source);
|
||||
|
||||
context_->getGlobalContext()->getRemoteHostFilter().checkURL(Poco::URI(source));
|
||||
@ -75,8 +75,8 @@ std::string DatabaseHDFS::getTablePath(const std::string & table_name) const
|
||||
return table_name;
|
||||
|
||||
if (source.empty())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad hdfs url: {}. "
|
||||
"It should have structure 'hdfs://<host_name>:<port>/path'", table_name);
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad HDFS URL: {}. "
|
||||
"It should have the following structure 'hdfs://<host_name>:<port>/path'", table_name);
|
||||
|
||||
return fs::path(source) / table_name;
|
||||
}
|
||||
|
||||
@ -37,10 +37,11 @@ struct CountSubstringsImpl
|
||||
const std::string & needle,
|
||||
const ColumnPtr & start_pos,
|
||||
PaddedPODArray<UInt64> & res,
|
||||
[[maybe_unused]] ColumnUInt8 * res_null)
|
||||
[[maybe_unused]] ColumnUInt8 * res_null,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
/// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
|
||||
assert(!res_null);
|
||||
chassert(!res_null);
|
||||
|
||||
const UInt8 * const begin = haystack_data.data();
|
||||
const UInt8 * const end = haystack_data.data() + haystack_data.size();
|
||||
@ -80,6 +81,8 @@ struct CountSubstringsImpl
|
||||
}
|
||||
pos = begin + haystack_offsets[i];
|
||||
++i;
|
||||
|
||||
chassert(i < input_rows_count);
|
||||
}
|
||||
}
|
||||
|
||||
@ -115,7 +118,7 @@ struct CountSubstringsImpl
|
||||
[[maybe_unused]] ColumnUInt8 * res_null)
|
||||
{
|
||||
/// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
|
||||
assert(!res_null);
|
||||
chassert(!res_null);
|
||||
|
||||
Impl::toLowerIfNeed(haystack);
|
||||
Impl::toLowerIfNeed(needle);
|
||||
@ -150,17 +153,18 @@ struct CountSubstringsImpl
|
||||
const ColumnString::Offsets & needle_offsets,
|
||||
const ColumnPtr & start_pos,
|
||||
PaddedPODArray<UInt64> & res,
|
||||
[[maybe_unused]] ColumnUInt8 * res_null)
|
||||
[[maybe_unused]] ColumnUInt8 * res_null,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
chassert(input_rows_count == haystack_offsets.size());
|
||||
|
||||
/// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
|
||||
assert(!res_null);
|
||||
chassert(!res_null);
|
||||
|
||||
ColumnString::Offset prev_haystack_offset = 0;
|
||||
ColumnString::Offset prev_needle_offset = 0;
|
||||
|
||||
size_t size = haystack_offsets.size();
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
size_t needle_size = needle_offsets[i] - prev_needle_offset - 1;
|
||||
size_t haystack_size = haystack_offsets[i] - prev_haystack_offset - 1;
|
||||
@ -207,17 +211,18 @@ struct CountSubstringsImpl
|
||||
const ColumnString::Offsets & needle_offsets,
|
||||
const ColumnPtr & start_pos,
|
||||
PaddedPODArray<UInt64> & res,
|
||||
[[maybe_unused]] ColumnUInt8 * res_null)
|
||||
[[maybe_unused]] ColumnUInt8 * res_null,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
chassert(input_rows_count == needle_offsets.size());
|
||||
|
||||
/// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
|
||||
assert(!res_null);
|
||||
chassert(!res_null);
|
||||
|
||||
/// NOTE You could use haystack indexing. But this is a rare case.
|
||||
ColumnString::Offset prev_needle_offset = 0;
|
||||
|
||||
size_t size = needle_offsets.size();
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
res[i] = 0;
|
||||
auto start = start_pos != nullptr ? std::max(start_pos->getUInt(i), UInt64(1)) : UInt64(1);
|
||||
|
||||
@ -71,7 +71,7 @@ public:
|
||||
return Impl::getReturnType();
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
const ColumnPtr & haystack_ptr = arguments[0].column;
|
||||
const ColumnPtr & edit_distance_ptr = arguments[1].column;
|
||||
@ -114,14 +114,16 @@ public:
|
||||
col_needles_const->getValue<Array>(),
|
||||
vec_res, offsets_res,
|
||||
edit_distance,
|
||||
allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
|
||||
allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps,
|
||||
input_rows_count);
|
||||
else
|
||||
Impl::vectorVector(
|
||||
col_haystack_vector->getChars(), col_haystack_vector->getOffsets(),
|
||||
col_needles_vector->getData(), col_needles_vector->getOffsets(),
|
||||
vec_res, offsets_res,
|
||||
edit_distance,
|
||||
allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
|
||||
allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps,
|
||||
input_rows_count);
|
||||
|
||||
// the combination of const haystack + const needle is not implemented because
|
||||
// useDefaultImplementationForConstants() == true makes upper layers convert both to
|
||||
|
||||
@ -81,7 +81,7 @@ public:
|
||||
return Impl::getReturnType();
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
const ColumnPtr & haystack_ptr = arguments[0].column;
|
||||
const ColumnPtr & needles_ptr = arguments[1].column;
|
||||
@ -110,13 +110,15 @@ public:
|
||||
col_haystack_vector->getChars(), col_haystack_vector->getOffsets(),
|
||||
col_needles_const->getValue<Array>(),
|
||||
vec_res, offsets_res,
|
||||
allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
|
||||
allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps,
|
||||
input_rows_count);
|
||||
else
|
||||
Impl::vectorVector(
|
||||
col_haystack_vector->getChars(), col_haystack_vector->getOffsets(),
|
||||
col_needles_vector->getData(), col_needles_vector->getOffsets(),
|
||||
vec_res, offsets_res,
|
||||
allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
|
||||
allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps,
|
||||
input_rows_count);
|
||||
|
||||
// the combination of const haystack + const needle is not implemented because
|
||||
// useDefaultImplementationForConstants() == true makes upper layers convert both to
|
||||
|
||||
@ -163,7 +163,7 @@ public:
|
||||
return return_type;
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
|
||||
{
|
||||
const ColumnPtr & column_haystack = (argument_order == ArgumentOrder::HaystackNeedle) ? arguments[0].column : arguments[1].column;
|
||||
const ColumnPtr & column_needle = (argument_order == ArgumentOrder::HaystackNeedle) ? arguments[1].column : arguments[0].column;
|
||||
@ -236,7 +236,8 @@ public:
|
||||
col_needle_vector->getOffsets(),
|
||||
column_start_pos,
|
||||
vec_res,
|
||||
null_map.get());
|
||||
null_map.get(),
|
||||
input_rows_count);
|
||||
else if (col_haystack_vector && col_needle_const)
|
||||
Impl::vectorConstant(
|
||||
col_haystack_vector->getChars(),
|
||||
@ -244,7 +245,8 @@ public:
|
||||
col_needle_const->getValue<String>(),
|
||||
column_start_pos,
|
||||
vec_res,
|
||||
null_map.get());
|
||||
null_map.get(),
|
||||
input_rows_count);
|
||||
else if (col_haystack_vector_fixed && col_needle_vector)
|
||||
Impl::vectorFixedVector(
|
||||
col_haystack_vector_fixed->getChars(),
|
||||
@ -253,14 +255,16 @@ public:
|
||||
col_needle_vector->getOffsets(),
|
||||
column_start_pos,
|
||||
vec_res,
|
||||
null_map.get());
|
||||
null_map.get(),
|
||||
input_rows_count);
|
||||
else if (col_haystack_vector_fixed && col_needle_const)
|
||||
Impl::vectorFixedConstant(
|
||||
col_haystack_vector_fixed->getChars(),
|
||||
col_haystack_vector_fixed->getN(),
|
||||
col_needle_const->getValue<String>(),
|
||||
vec_res,
|
||||
null_map.get());
|
||||
null_map.get(),
|
||||
input_rows_count);
|
||||
else if (col_haystack_const && col_needle_vector)
|
||||
Impl::constantVector(
|
||||
col_haystack_const->getValue<String>(),
|
||||
@ -268,7 +272,8 @@ public:
|
||||
col_needle_vector->getOffsets(),
|
||||
column_start_pos,
|
||||
vec_res,
|
||||
null_map.get());
|
||||
null_map.get(),
|
||||
input_rows_count);
|
||||
else
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_COLUMN,
|
||||
|
||||
@ -60,7 +60,7 @@ public:
|
||||
return std::make_shared<DataTypeString>();
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
const ColumnPtr column = arguments[0].column;
|
||||
const ColumnPtr column_needle = arguments[1].column;
|
||||
@ -75,7 +75,7 @@ public:
|
||||
|
||||
ColumnString::Chars & vec_res = col_res->getChars();
|
||||
ColumnString::Offsets & offsets_res = col_res->getOffsets();
|
||||
Impl::vector(col->getChars(), col->getOffsets(), col_needle->getValue<String>(), vec_res, offsets_res);
|
||||
Impl::vector(col->getChars(), col->getOffsets(), col_needle->getValue<String>(), vec_res, offsets_res, input_rows_count);
|
||||
|
||||
return col_res;
|
||||
}
|
||||
|
||||
@ -93,7 +93,8 @@ struct ExtractParamImpl
|
||||
std::string needle,
|
||||
const ColumnPtr & start_pos,
|
||||
PaddedPODArray<ResultType> & res,
|
||||
[[maybe_unused]] ColumnUInt8 * res_null)
|
||||
[[maybe_unused]] ColumnUInt8 * res_null,
|
||||
size_t /*input_rows_count*/)
|
||||
{
|
||||
/// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
|
||||
assert(!res_null);
|
||||
@ -168,11 +169,12 @@ struct ExtractParamToStringImpl
|
||||
{
|
||||
static void vector(const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets,
|
||||
std::string needle,
|
||||
ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets)
|
||||
ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
/// Constant 5 is taken from a function that performs a similar task FunctionsStringSearch.h::ExtractImpl
|
||||
res_data.reserve(haystack_data.size() / 5);
|
||||
res_offsets.resize(haystack_offsets.size());
|
||||
res_offsets.resize(input_rows_count);
|
||||
|
||||
/// We are looking for a parameter simply as a substring of the form "name"
|
||||
needle = "\"" + needle + "\":";
|
||||
|
||||
@ -35,12 +35,13 @@ struct HasTokenImpl
|
||||
const std::string & pattern,
|
||||
const ColumnPtr & start_pos,
|
||||
PaddedPODArray<UInt8> & res,
|
||||
ColumnUInt8 * res_null)
|
||||
ColumnUInt8 * res_null,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
if (start_pos != nullptr)
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function '{}' does not support start_pos argument", name);
|
||||
|
||||
if (haystack_offsets.empty())
|
||||
if (input_rows_count == 0)
|
||||
return;
|
||||
|
||||
const UInt8 * const begin = haystack_data.data();
|
||||
|
||||
@ -127,17 +127,17 @@ struct MatchImpl
|
||||
const String & needle,
|
||||
[[maybe_unused]] const ColumnPtr & start_pos_,
|
||||
PaddedPODArray<UInt8> & res,
|
||||
[[maybe_unused]] ColumnUInt8 * res_null)
|
||||
[[maybe_unused]] ColumnUInt8 * res_null,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
/// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
|
||||
assert(!res_null);
|
||||
chassert(!res_null);
|
||||
|
||||
const size_t haystack_size = haystack_offsets.size();
|
||||
chassert(res.size() == haystack_offsets.size());
|
||||
chassert(res.size() == input_rows_count);
|
||||
chassert(start_pos_ == nullptr);
|
||||
|
||||
assert(haystack_size == res.size());
|
||||
assert(start_pos_ == nullptr);
|
||||
|
||||
if (haystack_offsets.empty())
|
||||
if (input_rows_count == 0)
|
||||
return;
|
||||
|
||||
/// Shortcut for the silly but practical case that the pattern matches everything/nothing independently of the haystack:
|
||||
@ -202,11 +202,11 @@ struct MatchImpl
|
||||
if (required_substring.empty())
|
||||
{
|
||||
if (!regexp.getRE2()) /// An empty regexp. Always matches.
|
||||
memset(res.data(), !negate, haystack_size * sizeof(res[0]));
|
||||
memset(res.data(), !negate, input_rows_count * sizeof(res[0]));
|
||||
else
|
||||
{
|
||||
size_t prev_offset = 0;
|
||||
for (size_t i = 0; i < haystack_size; ++i)
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
const bool match = regexp.getRE2()->Match(
|
||||
{reinterpret_cast<const char *>(&haystack_data[prev_offset]), haystack_offsets[i] - prev_offset - 1},
|
||||
@ -291,16 +291,16 @@ struct MatchImpl
|
||||
size_t N,
|
||||
const String & needle,
|
||||
PaddedPODArray<UInt8> & res,
|
||||
[[maybe_unused]] ColumnUInt8 * res_null)
|
||||
[[maybe_unused]] ColumnUInt8 * res_null,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
/// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
|
||||
assert(!res_null);
|
||||
chassert(!res_null);
|
||||
|
||||
const size_t haystack_size = haystack.size() / N;
|
||||
chassert(res.size() == haystack.size() / N);
|
||||
chassert(res.size() == input_rows_count);
|
||||
|
||||
assert(haystack_size == res.size());
|
||||
|
||||
if (haystack.empty())
|
||||
if (input_rows_count == 0)
|
||||
return;
|
||||
|
||||
/// Shortcut for the silly but practical case that the pattern matches everything/nothing independently of the haystack:
|
||||
@ -370,11 +370,11 @@ struct MatchImpl
|
||||
if (required_substring.empty())
|
||||
{
|
||||
if (!regexp.getRE2()) /// An empty regexp. Always matches.
|
||||
memset(res.data(), !negate, haystack_size * sizeof(res[0]));
|
||||
memset(res.data(), !negate, input_rows_count * sizeof(res[0]));
|
||||
else
|
||||
{
|
||||
size_t offset = 0;
|
||||
for (size_t i = 0; i < haystack_size; ++i)
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
const bool match = regexp.getRE2()->Match(
|
||||
{reinterpret_cast<const char *>(&haystack[offset]), N},
|
||||
@ -464,18 +464,18 @@ struct MatchImpl
|
||||
const ColumnString::Offsets & needle_offset,
|
||||
[[maybe_unused]] const ColumnPtr & start_pos_,
|
||||
PaddedPODArray<UInt8> & res,
|
||||
[[maybe_unused]] ColumnUInt8 * res_null)
|
||||
[[maybe_unused]] ColumnUInt8 * res_null,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
/// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
|
||||
assert(!res_null);
|
||||
chassert(!res_null);
|
||||
|
||||
const size_t haystack_size = haystack_offsets.size();
|
||||
chassert(haystack_offsets.size() == needle_offset.size());
|
||||
chassert(res.size() == haystack_offsets.size());
|
||||
chassert(res.size() == input_rows_count);
|
||||
chassert(start_pos_ == nullptr);
|
||||
|
||||
assert(haystack_size == needle_offset.size());
|
||||
assert(haystack_size == res.size());
|
||||
assert(start_pos_ == nullptr);
|
||||
|
||||
if (haystack_offsets.empty())
|
||||
if (input_rows_count == 0)
|
||||
return;
|
||||
|
||||
String required_substr;
|
||||
@ -488,7 +488,7 @@ struct MatchImpl
|
||||
Regexps::LocalCacheTable cache;
|
||||
Regexps::RegexpPtr regexp;
|
||||
|
||||
for (size_t i = 0; i < haystack_size; ++i)
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
const auto * const cur_haystack_data = &haystack_data[prev_haystack_offset];
|
||||
const size_t cur_haystack_length = haystack_offsets[i] - prev_haystack_offset - 1;
|
||||
@ -573,16 +573,16 @@ struct MatchImpl
|
||||
const ColumnString::Offsets & needle_offset,
|
||||
[[maybe_unused]] const ColumnPtr & start_pos_,
|
||||
PaddedPODArray<UInt8> & res,
|
||||
[[maybe_unused]] ColumnUInt8 * res_null)
|
||||
[[maybe_unused]] ColumnUInt8 * res_null,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
/// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
|
||||
assert(!res_null);
|
||||
chassert(!res_null);
|
||||
|
||||
const size_t haystack_size = haystack.size()/N;
|
||||
|
||||
assert(haystack_size == needle_offset.size());
|
||||
assert(haystack_size == res.size());
|
||||
assert(start_pos_ == nullptr);
|
||||
chassert(res.size() == input_rows_count);
|
||||
chassert(res.size() == haystack.size() / N);
|
||||
chassert(res.size() == needle_offset.size());
|
||||
chassert(start_pos_ == nullptr);
|
||||
|
||||
if (haystack.empty())
|
||||
return;
|
||||
@ -597,7 +597,7 @@ struct MatchImpl
|
||||
Regexps::LocalCacheTable cache;
|
||||
Regexps::RegexpPtr regexp;
|
||||
|
||||
for (size_t i = 0; i < haystack_size; ++i)
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
const auto * const cur_haystack_data = &haystack[prev_haystack_offset];
|
||||
const size_t cur_haystack_length = N;
|
||||
|
||||
@ -52,9 +52,10 @@ struct MultiMatchAllIndicesImpl
|
||||
bool allow_hyperscan,
|
||||
size_t max_hyperscan_regexp_length,
|
||||
size_t max_hyperscan_regexp_total_length,
|
||||
bool reject_expensive_hyperscan_regexps)
|
||||
bool reject_expensive_hyperscan_regexps,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
vectorConstant(haystack_data, haystack_offsets, needles_arr, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
|
||||
vectorConstant(haystack_data, haystack_offsets, needles_arr, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps, input_rows_count);
|
||||
}
|
||||
|
||||
static void vectorConstant(
|
||||
@ -67,7 +68,8 @@ struct MultiMatchAllIndicesImpl
|
||||
bool allow_hyperscan,
|
||||
size_t max_hyperscan_regexp_length,
|
||||
size_t max_hyperscan_regexp_total_length,
|
||||
bool reject_expensive_hyperscan_regexps)
|
||||
bool reject_expensive_hyperscan_regexps,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
if (!allow_hyperscan)
|
||||
throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0");
|
||||
@ -87,7 +89,7 @@ struct MultiMatchAllIndicesImpl
|
||||
throw Exception(ErrorCodes::HYPERSCAN_CANNOT_SCAN_TEXT, "Regular expression evaluation in vectorscan will be too slow. To ignore this error, disable setting 'reject_expensive_hyperscan_regexps'.");
|
||||
}
|
||||
|
||||
offsets.resize(haystack_offsets.size());
|
||||
offsets.resize(input_rows_count);
|
||||
|
||||
if (needles_arr.empty())
|
||||
{
|
||||
@ -114,9 +116,8 @@ struct MultiMatchAllIndicesImpl
|
||||
static_cast<PaddedPODArray<ResultType>*>(context)->push_back(id);
|
||||
return 0;
|
||||
};
|
||||
const size_t haystack_offsets_size = haystack_offsets.size();
|
||||
UInt64 offset = 0;
|
||||
for (size_t i = 0; i < haystack_offsets_size; ++i)
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
UInt64 length = haystack_offsets[i] - offset - 1;
|
||||
/// vectorscan restriction.
|
||||
@ -146,6 +147,7 @@ struct MultiMatchAllIndicesImpl
|
||||
(void)max_hyperscan_regexp_length;
|
||||
(void)max_hyperscan_regexp_total_length;
|
||||
(void)reject_expensive_hyperscan_regexps;
|
||||
(void)input_rows_count;
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "multi-search all indices is not implemented when vectorscan is off");
|
||||
#endif // USE_VECTORSCAN
|
||||
}
|
||||
@ -160,9 +162,10 @@ struct MultiMatchAllIndicesImpl
|
||||
bool allow_hyperscan,
|
||||
size_t max_hyperscan_regexp_length,
|
||||
size_t max_hyperscan_regexp_total_length,
|
||||
bool reject_expensive_hyperscan_regexps)
|
||||
bool reject_expensive_hyperscan_regexps,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
vectorVector(haystack_data, haystack_offsets, needles_data, needles_offsets, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
|
||||
vectorVector(haystack_data, haystack_offsets, needles_data, needles_offsets, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps, input_rows_count);
|
||||
}
|
||||
|
||||
static void vectorVector(
|
||||
@ -176,12 +179,13 @@ struct MultiMatchAllIndicesImpl
|
||||
bool allow_hyperscan,
|
||||
size_t max_hyperscan_regexp_length,
|
||||
size_t max_hyperscan_regexp_total_length,
|
||||
bool reject_expensive_hyperscan_regexps)
|
||||
bool reject_expensive_hyperscan_regexps,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
if (!allow_hyperscan)
|
||||
throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0");
|
||||
#if USE_VECTORSCAN
|
||||
offsets.resize(haystack_offsets.size());
|
||||
offsets.resize(input_rows_count);
|
||||
size_t prev_haystack_offset = 0;
|
||||
size_t prev_needles_offset = 0;
|
||||
|
||||
@ -189,7 +193,7 @@ struct MultiMatchAllIndicesImpl
|
||||
|
||||
std::vector<std::string_view> needles;
|
||||
|
||||
for (size_t i = 0; i < haystack_offsets.size(); ++i)
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
needles.reserve(needles_offsets[i] - prev_needles_offset);
|
||||
|
||||
@ -271,6 +275,7 @@ struct MultiMatchAllIndicesImpl
|
||||
(void)max_hyperscan_regexp_length;
|
||||
(void)max_hyperscan_regexp_total_length;
|
||||
(void)reject_expensive_hyperscan_regexps;
|
||||
(void)input_rows_count;
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "multi-search all indices is not implemented when vectorscan is off");
|
||||
#endif // USE_VECTORSCAN
|
||||
}
|
||||
|
||||
@ -66,9 +66,10 @@ struct MultiMatchAnyImpl
|
||||
bool allow_hyperscan,
|
||||
size_t max_hyperscan_regexp_length,
|
||||
size_t max_hyperscan_regexp_total_length,
|
||||
bool reject_expensive_hyperscan_regexps)
|
||||
bool reject_expensive_hyperscan_regexps,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
vectorConstant(haystack_data, haystack_offsets, needles_arr, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
|
||||
vectorConstant(haystack_data, haystack_offsets, needles_arr, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps, input_rows_count);
|
||||
}
|
||||
|
||||
static void vectorConstant(
|
||||
@ -81,7 +82,8 @@ struct MultiMatchAnyImpl
|
||||
bool allow_hyperscan,
|
||||
size_t max_hyperscan_regexp_length,
|
||||
size_t max_hyperscan_regexp_total_length,
|
||||
bool reject_expensive_hyperscan_regexps)
|
||||
bool reject_expensive_hyperscan_regexps,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
if (!allow_hyperscan)
|
||||
throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0");
|
||||
@ -101,7 +103,7 @@ struct MultiMatchAnyImpl
|
||||
throw Exception(ErrorCodes::HYPERSCAN_CANNOT_SCAN_TEXT, "Regular expression evaluation in vectorscan will be too slow. To ignore this error, disable setting 'reject_expensive_hyperscan_regexps'.");
|
||||
}
|
||||
|
||||
res.resize(haystack_offsets.size());
|
||||
res.resize(input_rows_count);
|
||||
|
||||
if (needles_arr.empty())
|
||||
{
|
||||
@ -133,9 +135,8 @@ struct MultiMatchAnyImpl
|
||||
/// Once we hit the callback, there is no need to search for others.
|
||||
return 1;
|
||||
};
|
||||
const size_t haystack_offsets_size = haystack_offsets.size();
|
||||
UInt64 offset = 0;
|
||||
for (size_t i = 0; i < haystack_offsets_size; ++i)
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
UInt64 length = haystack_offsets[i] - offset - 1;
|
||||
/// vectorscan restriction.
|
||||
@ -164,7 +165,7 @@ struct MultiMatchAnyImpl
|
||||
memset(accum.data(), 0, accum.size());
|
||||
for (size_t j = 0; j < needles.size(); ++j)
|
||||
{
|
||||
MatchImpl<Name, MatchTraits::Syntax::Re2, MatchTraits::Case::Sensitive, MatchTraits::Result::DontNegate>::vectorConstant(haystack_data, haystack_offsets, String(needles[j].data(), needles[j].size()), nullptr, accum, nullptr);
|
||||
MatchImpl<Name, MatchTraits::Syntax::Re2, MatchTraits::Case::Sensitive, MatchTraits::Result::DontNegate>::vectorConstant(haystack_data, haystack_offsets, String(needles[j].data(), needles[j].size()), nullptr, accum, nullptr, input_rows_count);
|
||||
for (size_t i = 0; i < res.size(); ++i)
|
||||
{
|
||||
if constexpr (FindAny)
|
||||
@ -186,9 +187,10 @@ struct MultiMatchAnyImpl
|
||||
bool allow_hyperscan,
|
||||
size_t max_hyperscan_regexp_length,
|
||||
size_t max_hyperscan_regexp_total_length,
|
||||
bool reject_expensive_hyperscan_regexps)
|
||||
bool reject_expensive_hyperscan_regexps,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
vectorVector(haystack_data, haystack_offsets, needles_data, needles_offsets, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
|
||||
vectorVector(haystack_data, haystack_offsets, needles_data, needles_offsets, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps, input_rows_count);
|
||||
}
|
||||
|
||||
static void vectorVector(
|
||||
@ -202,12 +204,13 @@ struct MultiMatchAnyImpl
|
||||
bool allow_hyperscan,
|
||||
size_t max_hyperscan_regexp_length,
|
||||
size_t max_hyperscan_regexp_total_length,
|
||||
bool reject_expensive_hyperscan_regexps)
|
||||
bool reject_expensive_hyperscan_regexps,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
if (!allow_hyperscan)
|
||||
throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0");
|
||||
|
||||
res.resize(haystack_offsets.size());
|
||||
res.resize(input_rows_count);
|
||||
#if USE_VECTORSCAN
|
||||
size_t prev_haystack_offset = 0;
|
||||
size_t prev_needles_offset = 0;
|
||||
@ -216,7 +219,7 @@ struct MultiMatchAnyImpl
|
||||
|
||||
std::vector<std::string_view> needles;
|
||||
|
||||
for (size_t i = 0; i < haystack_offsets.size(); ++i)
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
needles.reserve(needles_offsets[i] - prev_needles_offset);
|
||||
|
||||
@ -306,7 +309,7 @@ struct MultiMatchAnyImpl
|
||||
|
||||
std::vector<std::string_view> needles;
|
||||
|
||||
for (size_t i = 0; i < haystack_offsets.size(); ++i)
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
const auto * const cur_haystack_data = &haystack_data[prev_haystack_offset];
|
||||
const size_t cur_haystack_length = haystack_offsets[i] - prev_haystack_offset - 1;
|
||||
|
||||
@ -33,7 +33,8 @@ struct MultiSearchFirstIndexImpl
|
||||
bool /*allow_hyperscan*/,
|
||||
size_t /*max_hyperscan_regexp_length*/,
|
||||
size_t /*max_hyperscan_regexp_total_length*/,
|
||||
bool /*reject_expensive_hyperscan_regexps*/)
|
||||
bool /*reject_expensive_hyperscan_regexps*/,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
// For performance of Volnitsky search, it is crucial to save only one byte for pattern number.
|
||||
if (needles_arr.size() > std::numeric_limits<UInt8>::max())
|
||||
@ -48,14 +49,13 @@ struct MultiSearchFirstIndexImpl
|
||||
|
||||
auto searcher = Impl::createMultiSearcherInBigHaystack(needles);
|
||||
|
||||
const size_t haystack_size = haystack_offsets.size();
|
||||
res.resize(haystack_size);
|
||||
res.resize(input_rows_count);
|
||||
|
||||
size_t iteration = 0;
|
||||
while (searcher.hasMoreToSearch())
|
||||
{
|
||||
size_t prev_haystack_offset = 0;
|
||||
for (size_t j = 0; j < haystack_size; ++j)
|
||||
for (size_t j = 0; j < input_rows_count; ++j)
|
||||
{
|
||||
const auto * haystack = &haystack_data[prev_haystack_offset];
|
||||
const auto * haystack_end = haystack + haystack_offsets[j] - prev_haystack_offset - 1;
|
||||
@ -80,10 +80,10 @@ struct MultiSearchFirstIndexImpl
|
||||
bool /*allow_hyperscan*/,
|
||||
size_t /*max_hyperscan_regexp_length*/,
|
||||
size_t /*max_hyperscan_regexp_total_length*/,
|
||||
bool /*reject_expensive_hyperscan_regexps*/)
|
||||
bool /*reject_expensive_hyperscan_regexps*/,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
const size_t haystack_size = haystack_offsets.size();
|
||||
res.resize(haystack_size);
|
||||
res.resize(input_rows_count);
|
||||
|
||||
size_t prev_haystack_offset = 0;
|
||||
size_t prev_needles_offset = 0;
|
||||
@ -92,7 +92,7 @@ struct MultiSearchFirstIndexImpl
|
||||
|
||||
std::vector<std::string_view> needles;
|
||||
|
||||
for (size_t i = 0; i < haystack_size; ++i)
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
needles.reserve(needles_offsets[i] - prev_needles_offset);
|
||||
|
||||
|
||||
@ -33,7 +33,8 @@ struct MultiSearchFirstPositionImpl
|
||||
bool /*allow_hyperscan*/,
|
||||
size_t /*max_hyperscan_regexp_length*/,
|
||||
size_t /*max_hyperscan_regexp_total_length*/,
|
||||
bool /*reject_expensive_hyperscan_regexps*/)
|
||||
bool /*reject_expensive_hyperscan_regexps*/,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
// For performance of Volnitsky search, it is crucial to save only one byte for pattern number.
|
||||
if (needles_arr.size() > std::numeric_limits<UInt8>::max())
|
||||
@ -52,14 +53,13 @@ struct MultiSearchFirstPositionImpl
|
||||
};
|
||||
auto searcher = Impl::createMultiSearcherInBigHaystack(needles);
|
||||
|
||||
const size_t haystack_size = haystack_offsets.size();
|
||||
res.resize(haystack_size);
|
||||
res.resize(input_rows_count);
|
||||
|
||||
size_t iteration = 0;
|
||||
while (searcher.hasMoreToSearch())
|
||||
{
|
||||
size_t prev_haystack_offset = 0;
|
||||
for (size_t j = 0; j < haystack_size; ++j)
|
||||
for (size_t j = 0; j < input_rows_count; ++j)
|
||||
{
|
||||
const auto * haystack = &haystack_data[prev_haystack_offset];
|
||||
const auto * haystack_end = haystack + haystack_offsets[j] - prev_haystack_offset - 1;
|
||||
@ -89,10 +89,10 @@ struct MultiSearchFirstPositionImpl
|
||||
bool /*allow_hyperscan*/,
|
||||
size_t /*max_hyperscan_regexp_length*/,
|
||||
size_t /*max_hyperscan_regexp_total_length*/,
|
||||
bool /*reject_expensive_hyperscan_regexps*/)
|
||||
bool /*reject_expensive_hyperscan_regexps*/,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
const size_t haystack_size = haystack_offsets.size();
|
||||
res.resize(haystack_size);
|
||||
res.resize(input_rows_count);
|
||||
|
||||
size_t prev_haystack_offset = 0;
|
||||
size_t prev_needles_offset = 0;
|
||||
@ -106,14 +106,12 @@ struct MultiSearchFirstPositionImpl
|
||||
return 1 + Impl::countChars(reinterpret_cast<const char *>(start), reinterpret_cast<const char *>(end));
|
||||
};
|
||||
|
||||
for (size_t i = 0; i < haystack_size; ++i)
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
needles.reserve(needles_offsets[i] - prev_needles_offset);
|
||||
|
||||
for (size_t j = prev_needles_offset; j < needles_offsets[i]; ++j)
|
||||
{
|
||||
needles.emplace_back(needles_data_string.getDataAt(j).toView());
|
||||
}
|
||||
|
||||
auto searcher = Impl::createMultiSearcherInBigHaystack(needles); // sub-optimal
|
||||
|
||||
|
||||
@ -33,7 +33,8 @@ struct MultiSearchImpl
|
||||
bool /*allow_hyperscan*/,
|
||||
size_t /*max_hyperscan_regexp_length*/,
|
||||
size_t /*max_hyperscan_regexp_total_length*/,
|
||||
bool /*reject_expensive_hyperscan_regexps*/)
|
||||
bool /*reject_expensive_hyperscan_regexps*/,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
// For performance of Volnitsky search, it is crucial to save only one byte for pattern number.
|
||||
if (needles_arr.size() > std::numeric_limits<UInt8>::max())
|
||||
@ -48,14 +49,13 @@ struct MultiSearchImpl
|
||||
|
||||
auto searcher = Impl::createMultiSearcherInBigHaystack(needles);
|
||||
|
||||
const size_t haystack_size = haystack_offsets.size();
|
||||
res.resize(haystack_size);
|
||||
res.resize(input_rows_count);
|
||||
|
||||
size_t iteration = 0;
|
||||
while (searcher.hasMoreToSearch())
|
||||
{
|
||||
size_t prev_haystack_offset = 0;
|
||||
for (size_t j = 0; j < haystack_size; ++j)
|
||||
for (size_t j = 0; j < input_rows_count; ++j)
|
||||
{
|
||||
const auto * haystack = &haystack_data[prev_haystack_offset];
|
||||
const auto * haystack_end = haystack + haystack_offsets[j] - prev_haystack_offset - 1;
|
||||
@ -79,10 +79,10 @@ struct MultiSearchImpl
|
||||
bool /*allow_hyperscan*/,
|
||||
size_t /*max_hyperscan_regexp_length*/,
|
||||
size_t /*max_hyperscan_regexp_total_length*/,
|
||||
bool /*reject_expensive_hyperscan_regexps*/)
|
||||
bool /*reject_expensive_hyperscan_regexps*/,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
const size_t haystack_size = haystack_offsets.size();
|
||||
res.resize(haystack_size);
|
||||
res.resize(input_rows_count);
|
||||
|
||||
size_t prev_haystack_offset = 0;
|
||||
size_t prev_needles_offset = 0;
|
||||
@ -91,14 +91,12 @@ struct MultiSearchImpl
|
||||
|
||||
std::vector<std::string_view> needles;
|
||||
|
||||
for (size_t i = 0; i < haystack_size; ++i)
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
needles.reserve(needles_offsets[i] - prev_needles_offset);
|
||||
|
||||
for (size_t j = prev_needles_offset; j < needles_offsets[i]; ++j)
|
||||
{
|
||||
needles.emplace_back(needles_data_string.getDataAt(j).toView());
|
||||
}
|
||||
|
||||
const auto * const haystack = &haystack_data[prev_haystack_offset];
|
||||
const size_t haystack_length = haystack_offsets[i] - prev_haystack_offset - 1;
|
||||
|
||||
@ -193,7 +193,8 @@ struct PositionImpl
|
||||
const std::string & needle,
|
||||
const ColumnPtr & start_pos,
|
||||
PaddedPODArray<UInt64> & res,
|
||||
[[maybe_unused]] ColumnUInt8 * res_null)
|
||||
[[maybe_unused]] ColumnUInt8 * res_null,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
/// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
|
||||
assert(!res_null);
|
||||
@ -214,13 +215,12 @@ struct PositionImpl
|
||||
}
|
||||
|
||||
ColumnString::Offset prev_offset = 0;
|
||||
size_t rows = haystack_offsets.size();
|
||||
|
||||
if (const ColumnConst * start_pos_const = typeid_cast<const ColumnConst *>(&*start_pos))
|
||||
{
|
||||
/// Needle is empty and start_pos is constant
|
||||
UInt64 start = std::max(start_pos_const->getUInt(0), static_cast<UInt64>(1));
|
||||
for (size_t i = 0; i < rows; ++i)
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
size_t haystack_size = Impl::countChars(
|
||||
reinterpret_cast<const char *>(pos), reinterpret_cast<const char *>(pos + haystack_offsets[i] - prev_offset - 1));
|
||||
@ -234,7 +234,7 @@ struct PositionImpl
|
||||
else
|
||||
{
|
||||
/// Needle is empty and start_pos is not constant
|
||||
for (size_t i = 0; i < rows; ++i)
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
size_t haystack_size = Impl::countChars(
|
||||
reinterpret_cast<const char *>(pos), reinterpret_cast<const char *>(pos + haystack_offsets[i] - prev_offset - 1));
|
||||
@ -359,7 +359,8 @@ struct PositionImpl
|
||||
const ColumnString::Offsets & needle_offsets,
|
||||
const ColumnPtr & start_pos,
|
||||
PaddedPODArray<UInt64> & res,
|
||||
[[maybe_unused]] ColumnUInt8 * res_null)
|
||||
[[maybe_unused]] ColumnUInt8 * res_null,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
/// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
|
||||
assert(!res_null);
|
||||
@ -367,9 +368,7 @@ struct PositionImpl
|
||||
ColumnString::Offset prev_haystack_offset = 0;
|
||||
ColumnString::Offset prev_needle_offset = 0;
|
||||
|
||||
size_t size = haystack_offsets.size();
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
size_t needle_size = needle_offsets[i] - prev_needle_offset - 1;
|
||||
size_t haystack_size = haystack_offsets[i] - prev_haystack_offset - 1;
|
||||
@ -423,7 +422,8 @@ struct PositionImpl
|
||||
const ColumnString::Offsets & needle_offsets,
|
||||
const ColumnPtr & start_pos,
|
||||
PaddedPODArray<UInt64> & res,
|
||||
[[maybe_unused]] ColumnUInt8 * res_null)
|
||||
[[maybe_unused]] ColumnUInt8 * res_null,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
/// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
|
||||
assert(!res_null);
|
||||
@ -431,9 +431,7 @@ struct PositionImpl
|
||||
/// NOTE You could use haystack indexing. But this is a rare case.
|
||||
ColumnString::Offset prev_needle_offset = 0;
|
||||
|
||||
size_t size = needle_offsets.size();
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
size_t needle_size = needle_offsets[i] - prev_needle_offset - 1;
|
||||
|
||||
|
||||
@ -10,10 +10,11 @@ struct ExtractURLParameterImpl
|
||||
static void vector(const ColumnString::Chars & data,
|
||||
const ColumnString::Offsets & offsets,
|
||||
std::string pattern,
|
||||
ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets)
|
||||
ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
res_data.reserve(data.size() / 5);
|
||||
res_offsets.resize(offsets.size());
|
||||
res_offsets.resize(input_rows_count);
|
||||
|
||||
pattern += '=';
|
||||
const char * param_str = pattern.c_str();
|
||||
@ -22,7 +23,7 @@ struct ExtractURLParameterImpl
|
||||
ColumnString::Offset prev_offset = 0;
|
||||
ColumnString::Offset res_offset = 0;
|
||||
|
||||
for (size_t i = 0; i < offsets.size(); ++i)
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
ColumnString::Offset cur_offset = offsets[i];
|
||||
|
||||
|
||||
@ -18,9 +18,7 @@ using namespace GatherUtils;
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_COLUMN;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int ZERO_ARRAY_OR_TUPLE_INDEX;
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
}
|
||||
|
||||
class FunctionBitSlice : public IFunction
|
||||
@ -40,28 +38,22 @@ public:
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
|
||||
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
|
||||
{
|
||||
const size_t number_of_arguments = arguments.size();
|
||||
FunctionArgumentDescriptors mandatory_args{
|
||||
{"s", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "String"},
|
||||
{"offset", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeNumber), nullptr, "(U)Int8, (U)Int16, (U)Int32, (U)Int64 or Float"},
|
||||
};
|
||||
|
||||
if (number_of_arguments < 2 || number_of_arguments > 3)
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
|
||||
"Number of arguments for function {} doesn't match: passed {}, should be 2 or 3",
|
||||
getName(), number_of_arguments);
|
||||
FunctionArgumentDescriptors optional_args{
|
||||
{"length", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeNumber), nullptr, "(U)Int8, (U)Int16, (U)Int32, (U)Int64 or Float"},
|
||||
};
|
||||
|
||||
if (!isString(arguments[0]) && !isStringOrFixedString(arguments[0]))
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}",
|
||||
arguments[0]->getName(), getName());
|
||||
if (arguments[0]->onlyNull())
|
||||
return arguments[0];
|
||||
validateFunctionArguments(*this, arguments, mandatory_args, optional_args);
|
||||
|
||||
if (!isNativeNumber(arguments[1]))
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of second argument of function {}",
|
||||
arguments[1]->getName(), getName());
|
||||
|
||||
if (number_of_arguments == 3 && !isNativeNumber(arguments[2]))
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of second argument of function {}",
|
||||
arguments[2]->getName(), getName());
|
||||
const auto & type = arguments[0].type;
|
||||
if (type->onlyNull())
|
||||
return type;
|
||||
|
||||
return std::make_shared<DataTypeString>();
|
||||
}
|
||||
|
||||
@ -16,10 +16,11 @@ struct ExtractImpl
|
||||
const ColumnString::Offsets & offsets,
|
||||
const std::string & pattern,
|
||||
ColumnString::Chars & res_data,
|
||||
ColumnString::Offsets & res_offsets)
|
||||
ColumnString::Offsets & res_offsets,
|
||||
size_t input_rows_count)
|
||||
{
|
||||
res_data.reserve(data.size() / 5);
|
||||
res_offsets.resize(offsets.size());
|
||||
res_offsets.resize(input_rows_count);
|
||||
|
||||
const OptimizedRegularExpression regexp = Regexps::createRegexp<false, false, false>(pattern);
|
||||
|
||||
@ -29,7 +30,7 @@ struct ExtractImpl
|
||||
size_t prev_offset = 0;
|
||||
size_t res_offset = 0;
|
||||
|
||||
for (size_t i = 0; i < offsets.size(); ++i)
|
||||
for (size_t i = 0; i < input_rows_count; ++i)
|
||||
{
|
||||
size_t cur_offset = offsets[i];
|
||||
|
||||
|
||||
@ -10,12 +10,14 @@
|
||||
|
||||
#if USE_RAPIDJSON
|
||||
|
||||
#include "rapidjson/document.h"
|
||||
#include "rapidjson/writer.h"
|
||||
#include "rapidjson/stringbuffer.h"
|
||||
#include "rapidjson/filewritestream.h"
|
||||
#include "rapidjson/prettywriter.h"
|
||||
#include "rapidjson/filereadstream.h"
|
||||
/// Prevent stack overflow:
|
||||
#define RAPIDJSON_PARSE_DEFAULT_FLAGS (kParseIterativeFlag)
|
||||
|
||||
#include <rapidjson/document.h>
|
||||
#include <rapidjson/writer.h>
|
||||
#include <rapidjson/stringbuffer.h>
|
||||
#include <rapidjson/filereadstream.h>
|
||||
#include <rapidjson/error/en.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
@ -31,17 +33,17 @@ namespace ErrorCodes
|
||||
|
||||
namespace
|
||||
{
|
||||
// select jsonMergePatch('{"a":1}','{"name": "joey"}','{"name": "tom"}','{"name": "zoey"}');
|
||||
// select JSONMergePatch('{"a":1}','{"name": "joey"}','{"name": "tom"}','{"name": "zoey"}');
|
||||
// ||
|
||||
// \/
|
||||
// ┌───────────────────────┐
|
||||
// │ {"a":1,"name":"zoey"} │
|
||||
// └───────────────────────┘
|
||||
class FunctionjsonMergePatch : public IFunction
|
||||
class FunctionJSONMergePatch : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "jsonMergePatch";
|
||||
static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionjsonMergePatch>(); }
|
||||
static constexpr auto name = "JSONMergePatch";
|
||||
static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionJSONMergePatch>(); }
|
||||
|
||||
String getName() const override { return name; }
|
||||
bool isVariadic() const override { return true; }
|
||||
@ -98,7 +100,11 @@ namespace
|
||||
const char * json = str_ref.data;
|
||||
|
||||
document.Parse(json);
|
||||
if (document.HasParseError() || !document.IsObject())
|
||||
|
||||
if (document.HasParseError())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Wrong JSON string to merge: {}", rapidjson::GetParseError_En(document.GetParseError()));
|
||||
|
||||
if (!document.IsObject())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Wrong JSON string to merge. Expected JSON object");
|
||||
};
|
||||
|
||||
@ -162,10 +168,12 @@ namespace
|
||||
|
||||
}
|
||||
|
||||
REGISTER_FUNCTION(jsonMergePatch)
|
||||
REGISTER_FUNCTION(JSONMergePatch)
|
||||
{
|
||||
factory.registerFunction<FunctionjsonMergePatch>(FunctionDocumentation{
|
||||
factory.registerFunction<FunctionJSONMergePatch>(FunctionDocumentation{
|
||||
.description="Returns the merged JSON object string, which is formed by merging multiple JSON objects."});
|
||||
|
||||
factory.registerAlias("jsonMergePatch", "JSONMergePatch");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
364
src/Functions/printf.cpp
Normal file
364
src/Functions/printf.cpp
Normal file
@ -0,0 +1,364 @@
|
||||
#include <Columns/ColumnFixedString.h>
|
||||
#include <Columns/ColumnString.h>
|
||||
#include <Columns/ColumnStringHelpers.h>
|
||||
#include <DataTypes/DataTypeString.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/FunctionHelpers.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Functions/formatString.h>
|
||||
#include <IO/Operators.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include <fmt/format.h>
|
||||
#include <fmt/printf.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_COLUMN;
|
||||
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
class FunctionPrintf : public IFunction
|
||||
{
|
||||
private:
|
||||
ContextPtr context;
|
||||
FunctionOverloadResolverPtr function_concat;
|
||||
|
||||
struct Instruction
|
||||
{
|
||||
std::string_view format;
|
||||
size_t rows;
|
||||
bool is_literal; /// format is literal string without any argument
|
||||
ColumnWithTypeAndName input; /// Only used when is_literal is false
|
||||
|
||||
ColumnWithTypeAndName execute() const
|
||||
{
|
||||
if (is_literal)
|
||||
return executeLiteral(format);
|
||||
else if (isColumnConst(*input.column))
|
||||
return executeConstant(input);
|
||||
else
|
||||
return executeNonconstant(input);
|
||||
}
|
||||
|
||||
[[maybe_unused]] String toString() const
|
||||
{
|
||||
WriteBufferFromOwnString buf;
|
||||
buf << "format:" << format << ", rows:" << rows << ", is_literal:" << is_literal << ", input:" << input.dumpStructure() << "\n";
|
||||
return buf.str();
|
||||
}
|
||||
|
||||
private:
|
||||
ColumnWithTypeAndName executeLiteral(std::string_view literal) const
|
||||
{
|
||||
ColumnWithTypeAndName res;
|
||||
auto str_col = ColumnString::create();
|
||||
str_col->insert(fmt::sprintf(literal));
|
||||
res.column = ColumnConst::create(std::move(str_col), rows);
|
||||
res.type = std::make_shared<DataTypeString>();
|
||||
return res;
|
||||
}
|
||||
|
||||
ColumnWithTypeAndName executeConstant(const ColumnWithTypeAndName & arg) const
|
||||
{
|
||||
ColumnWithTypeAndName tmp_arg = arg;
|
||||
const auto & const_col = static_cast<const ColumnConst &>(*arg.column);
|
||||
tmp_arg.column = const_col.getDataColumnPtr();
|
||||
|
||||
ColumnWithTypeAndName tmp_res = executeNonconstant(tmp_arg);
|
||||
return ColumnWithTypeAndName{ColumnConst::create(tmp_res.column, arg.column->size()), tmp_res.type, tmp_res.name};
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
bool executeNumber(const IColumn & column, ColumnString::Chars & res_chars, ColumnString::Offsets & res_offsets) const
|
||||
{
|
||||
const ColumnVector<T> * concrete_column = checkAndGetColumn<ColumnVector<T>>(&column);
|
||||
if (!concrete_column)
|
||||
return false;
|
||||
|
||||
String s;
|
||||
size_t curr_offset = 0;
|
||||
const auto & data = concrete_column->getData();
|
||||
for (size_t i = 0; i < data.size(); ++i)
|
||||
{
|
||||
T a = data[i];
|
||||
s = fmt::sprintf(format, static_cast<NearestFieldType<T>>(a));
|
||||
|
||||
res_chars.resize(curr_offset + s.size() + 1);
|
||||
memcpy(&res_chars[curr_offset], s.data(), s.size());
|
||||
res_chars[curr_offset + s.size()] = 0;
|
||||
|
||||
curr_offset += s.size() + 1;
|
||||
res_offsets[i] = curr_offset;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
template <typename COLUMN>
|
||||
bool executeString(const IColumn & column, ColumnString::Chars & res_chars, ColumnString::Offsets & res_offsets) const
|
||||
{
|
||||
const COLUMN * concrete_column = checkAndGetColumn<COLUMN>(&column);
|
||||
if (!concrete_column)
|
||||
return false;
|
||||
|
||||
String s;
|
||||
size_t curr_offset = 0;
|
||||
for (size_t i = 0; i < concrete_column->size(); ++i)
|
||||
{
|
||||
auto a = concrete_column->getDataAt(i).toView();
|
||||
s = fmt::sprintf(format, a);
|
||||
|
||||
res_chars.resize(curr_offset + s.size() + 1);
|
||||
memcpy(&res_chars[curr_offset], s.data(), s.size());
|
||||
res_chars[curr_offset + s.size()] = 0;
|
||||
|
||||
curr_offset += s.size() + 1;
|
||||
res_offsets[i] = curr_offset;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
ColumnWithTypeAndName executeNonconstant(const ColumnWithTypeAndName & arg) const
|
||||
{
|
||||
size_t size = arg.column->size();
|
||||
auto res_col = ColumnString::create();
|
||||
auto & res_str = static_cast<ColumnString &>(*res_col);
|
||||
auto & res_offsets = res_str.getOffsets();
|
||||
auto & res_chars = res_str.getChars();
|
||||
res_offsets.resize_exact(size);
|
||||
res_chars.reserve(format.size() * size);
|
||||
|
||||
WhichDataType which(arg.type);
|
||||
if (which.isNativeNumber()
|
||||
&& (executeNumber<UInt8>(*arg.column, res_chars, res_offsets) || executeNumber<UInt16>(*arg.column, res_chars, res_offsets)
|
||||
|| executeNumber<UInt32>(*arg.column, res_chars, res_offsets)
|
||||
|| executeNumber<UInt64>(*arg.column, res_chars, res_offsets)
|
||||
|| executeNumber<Int8>(*arg.column, res_chars, res_offsets) || executeNumber<Int16>(*arg.column, res_chars, res_offsets)
|
||||
|| executeNumber<Int32>(*arg.column, res_chars, res_offsets)
|
||||
|| executeNumber<Int64>(*arg.column, res_chars, res_offsets)
|
||||
|| executeNumber<Float32>(*arg.column, res_chars, res_offsets)
|
||||
|| executeNumber<Float64>(*arg.column, res_chars, res_offsets)))
|
||||
{
|
||||
return {std::move(res_col), std::make_shared<DataTypeString>(), arg.name};
|
||||
}
|
||||
else if (
|
||||
which.isStringOrFixedString()
|
||||
&& (executeString<ColumnString>(*arg.column, res_chars, res_offsets)
|
||||
|| executeString<ColumnFixedString>(*arg.column, res_chars, res_offsets)))
|
||||
{
|
||||
return {std::move(res_col), std::make_shared<DataTypeString>(), arg.name};
|
||||
}
|
||||
else
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"The argument type of function {} is {}, but native numeric or string type is expected",
|
||||
FunctionPrintf::name,
|
||||
arg.type->getName());
|
||||
}
|
||||
};
|
||||
|
||||
public:
|
||||
static constexpr auto name = "printf";
|
||||
|
||||
static FunctionPtr create(ContextPtr context) { return std::make_shared<FunctionPrintf>(context); }
|
||||
|
||||
explicit FunctionPrintf(ContextPtr context_)
|
||||
: context(context_), function_concat(FunctionFactory::instance().get("concat", context)) { }
|
||||
|
||||
String getName() const override { return name; }
|
||||
|
||||
bool isVariadic() const override { return true; }
|
||||
|
||||
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
|
||||
|
||||
size_t getNumberOfArguments() const override { return 0; }
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return false; }
|
||||
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
|
||||
{
|
||||
if (arguments.empty())
|
||||
throw Exception(
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
|
||||
"Number of arguments for function {} doesn't match: passed {}, should be at least 1",
|
||||
getName(),
|
||||
arguments.size());
|
||||
|
||||
/// First pattern argument must have string type
|
||||
if (!isString(arguments[0]))
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"The first argument type of function {} is {}, but String type is expected",
|
||||
getName(),
|
||||
arguments[0]->getName());
|
||||
|
||||
for (size_t i = 1; i < arguments.size(); ++i)
|
||||
{
|
||||
if (!isNativeNumber(arguments[i]) && !isStringOrFixedString(arguments[i]))
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"The {}-th argument type of function {} is {}, but native numeric or string type is expected",
|
||||
i + 1,
|
||||
getName(),
|
||||
arguments[i]->getName());
|
||||
}
|
||||
return std::make_shared<DataTypeString>();
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
const ColumnPtr & c0 = arguments[0].column;
|
||||
const ColumnConst * c0_const_string = typeid_cast<const ColumnConst *>(&*c0);
|
||||
if (!c0_const_string)
|
||||
throw Exception(ErrorCodes::ILLEGAL_COLUMN, "First argument of function {} must be constant string", getName());
|
||||
|
||||
String format = c0_const_string->getValue<String>();
|
||||
auto instructions = buildInstructions(format, arguments, input_rows_count);
|
||||
|
||||
ColumnsWithTypeAndName concat_args(instructions.size());
|
||||
for (size_t i = 0; i < instructions.size(); ++i)
|
||||
{
|
||||
const auto & instruction = instructions[i];
|
||||
try
|
||||
{
|
||||
// std::cout << "instruction[" << i << "]:" << instructions[i].toString() << std::endl;
|
||||
concat_args[i] = instruction.execute();
|
||||
// std::cout << "concat_args[" << i << "]:" << concat_args[i].dumpStructure() << std::endl;
|
||||
}
|
||||
catch (const fmt::v9::format_error & e)
|
||||
{
|
||||
if (instruction.is_literal)
|
||||
throw Exception(
|
||||
ErrorCodes::BAD_ARGUMENTS,
|
||||
"Bad format {} in function {} without input argument, reason: {}",
|
||||
instruction.format,
|
||||
getName(),
|
||||
e.what());
|
||||
else
|
||||
throw Exception(
|
||||
ErrorCodes::BAD_ARGUMENTS,
|
||||
"Bad format {} in function {} with {} as input argument, reason: {}",
|
||||
instructions[i].format,
|
||||
getName(),
|
||||
instruction.input.dumpStructure(),
|
||||
e.what());
|
||||
}
|
||||
}
|
||||
|
||||
auto res = function_concat->build(concat_args)->execute(concat_args, std::make_shared<DataTypeString>(), input_rows_count);
|
||||
return res;
|
||||
}
|
||||
|
||||
private:
|
||||
std::vector<Instruction>
|
||||
buildInstructions(const String & format, const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const
|
||||
{
|
||||
std::vector<Instruction> instructions;
|
||||
instructions.reserve(arguments.size());
|
||||
|
||||
auto append_instruction = [&](const char * begin, const char * end, const ColumnWithTypeAndName & arg)
|
||||
{
|
||||
Instruction instr;
|
||||
instr.rows = input_rows_count;
|
||||
instr.format = std::string_view(begin, end - begin);
|
||||
|
||||
size_t size = end - begin;
|
||||
if (size > 1 && begin[0] == '%' and begin[1] != '%')
|
||||
{
|
||||
instr.is_literal = false;
|
||||
instr.input = arg;
|
||||
}
|
||||
else
|
||||
{
|
||||
instr.is_literal = true;
|
||||
}
|
||||
instructions.emplace_back(std::move(instr));
|
||||
};
|
||||
|
||||
auto check_index_range = [&](size_t idx)
|
||||
{
|
||||
if (idx >= arguments.size())
|
||||
throw Exception(
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
|
||||
"Number of arguments for function {} doesn't match: passed {}, but format is {}",
|
||||
getName(),
|
||||
arguments.size(),
|
||||
format);
|
||||
};
|
||||
|
||||
const char * begin = format.data();
|
||||
const char * end = format.data() + format.size();
|
||||
const char * curr = begin;
|
||||
size_t idx = 0;
|
||||
while (curr < end)
|
||||
{
|
||||
const char * tmp = curr;
|
||||
bool is_first = curr == begin; /// If current instruction is the first one
|
||||
bool is_literal = false; /// If current instruction is literal string without any argument
|
||||
if (is_first)
|
||||
{
|
||||
if (*curr != '%')
|
||||
is_literal = true;
|
||||
else if (curr + 1 < end && *(curr + 1) == '%')
|
||||
is_literal = true;
|
||||
else
|
||||
++idx; /// Skip first argument if first instruction is not literal
|
||||
}
|
||||
|
||||
if (!is_literal)
|
||||
++curr;
|
||||
|
||||
while (curr < end)
|
||||
{
|
||||
if (*curr != '%')
|
||||
++curr;
|
||||
else if (curr + 1 < end && *(curr + 1) == '%')
|
||||
curr += 2;
|
||||
else
|
||||
{
|
||||
check_index_range(idx);
|
||||
append_instruction(tmp, curr, arguments[idx]);
|
||||
++idx;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (curr == end)
|
||||
{
|
||||
check_index_range(idx);
|
||||
append_instruction(tmp, curr, arguments[idx]);
|
||||
++idx;
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if all arguments are used
|
||||
if (idx != arguments.size())
|
||||
throw Exception(
|
||||
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
|
||||
"Number of arguments for function {} doesn't match: passed {}, but format is {}",
|
||||
getName(),
|
||||
arguments.size(),
|
||||
format);
|
||||
|
||||
return instructions;
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
REGISTER_FUNCTION(Printf)
|
||||
{
|
||||
factory.registerFunction<FunctionPrintf>();
|
||||
}
|
||||
|
||||
}
|
||||
@ -326,6 +326,8 @@ std::vector<FileSegment::Range> FileCache::splitRange(size_t offset, size_t size
|
||||
/// ^ ^
|
||||
/// right offset aligned_right_offset
|
||||
/// [_________] <-- last cached file segment, e.g. we have uncovered suffix of the requested range
|
||||
/// ^
|
||||
/// last_file_segment_right_offset
|
||||
/// [________________]
|
||||
/// size
|
||||
/// [____________________________________]
|
||||
@ -335,9 +337,10 @@ std::vector<FileSegment::Range> FileCache::splitRange(size_t offset, size_t size
|
||||
/// and get something like this:
|
||||
///
|
||||
/// [________________________]
|
||||
/// ^ ^
|
||||
/// right_offset right_offset + max_file_segment_size
|
||||
/// e.g. there is no need to create sub-segment for range (right_offset + max_file_segment_size, aligned_right_offset].
|
||||
/// ^ ^
|
||||
/// | last_file_segment_right_offset + max_file_segment_size
|
||||
/// last_file_segment_right_offset
|
||||
/// e.g. there is no need to create sub-segment for range (last_file_segment_right_offset + max_file_segment_size, aligned_right_offset].
|
||||
/// Because its left offset would be bigger than right_offset.
|
||||
/// Therefore, we set end_pos_non_included as offset+size, but remaining_size as aligned_size.
|
||||
|
||||
@ -557,7 +560,7 @@ FileCache::getOrSet(
|
||||
|
||||
FileSegment::Range initial_range(offset, offset + size - 1);
|
||||
/// result_range is initial range, which will be adjusted according to
|
||||
/// 1. aligned offset, alighed_end_offset
|
||||
/// 1. aligned_offset, aligned_end_offset
|
||||
/// 2. max_file_segments_limit
|
||||
FileSegment::Range result_range = initial_range;
|
||||
|
||||
@ -997,18 +1000,19 @@ void FileCache::freeSpaceRatioKeepingThreadFunc()
|
||||
FileCacheReserveStat stat;
|
||||
EvictionCandidates eviction_candidates;
|
||||
|
||||
bool limits_satisfied = true;
|
||||
IFileCachePriority::CollectStatus desired_size_status;
|
||||
try
|
||||
{
|
||||
/// Collect at most `keep_up_free_space_remove_batch` elements to evict,
|
||||
/// (we use batches to make sure we do not block cache for too long,
|
||||
/// by default the batch size is quite small).
|
||||
limits_satisfied = main_priority->collectCandidatesForEviction(
|
||||
desired_size_status = main_priority->collectCandidatesForEviction(
|
||||
desired_size, desired_elements_num, keep_up_free_space_remove_batch, stat, eviction_candidates, lock);
|
||||
|
||||
#ifdef DEBUG_OR_SANITIZER_BUILD
|
||||
/// Let's make sure that we correctly processed the limits.
|
||||
if (limits_satisfied && eviction_candidates.size() < keep_up_free_space_remove_batch)
|
||||
if (desired_size_status == IFileCachePriority::CollectStatus::SUCCESS
|
||||
&& eviction_candidates.size() < keep_up_free_space_remove_batch)
|
||||
{
|
||||
const auto current_size = main_priority->getSize(lock);
|
||||
chassert(current_size >= stat.total_stat.releasable_size);
|
||||
@ -1062,13 +1066,24 @@ void FileCache::freeSpaceRatioKeepingThreadFunc()
|
||||
watch.stop();
|
||||
ProfileEvents::increment(ProfileEvents::FilesystemCacheFreeSpaceKeepingThreadWorkMilliseconds, watch.elapsedMilliseconds());
|
||||
|
||||
LOG_TRACE(log, "Free space ratio keeping thread finished in {} ms", watch.elapsedMilliseconds());
|
||||
LOG_TRACE(log, "Free space ratio keeping thread finished in {} ms (status: {})",
|
||||
watch.elapsedMilliseconds(), desired_size_status);
|
||||
|
||||
[[maybe_unused]] bool scheduled = false;
|
||||
if (limits_satisfied)
|
||||
scheduled = keep_up_free_space_ratio_task->scheduleAfter(general_reschedule_ms);
|
||||
else
|
||||
scheduled = keep_up_free_space_ratio_task->schedule();
|
||||
switch (desired_size_status)
|
||||
{
|
||||
case IFileCachePriority::CollectStatus::SUCCESS: [[fallthrough]];
|
||||
case IFileCachePriority::CollectStatus::CANNOT_EVICT:
|
||||
{
|
||||
scheduled = keep_up_free_space_ratio_task->scheduleAfter(general_reschedule_ms);
|
||||
break;
|
||||
}
|
||||
case IFileCachePriority::CollectStatus::REACHED_MAX_CANDIDATES_LIMIT:
|
||||
{
|
||||
scheduled = keep_up_free_space_ratio_task->schedule();
|
||||
break;
|
||||
}
|
||||
}
|
||||
chassert(scheduled);
|
||||
}
|
||||
|
||||
@ -1545,7 +1560,7 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings,
|
||||
FileCacheReserveStat stat;
|
||||
if (main_priority->collectCandidatesForEviction(
|
||||
new_settings.max_size, new_settings.max_elements, 0/* max_candidates_to_evict */,
|
||||
stat, eviction_candidates, cache_lock))
|
||||
stat, eviction_candidates, cache_lock) == IFileCachePriority::CollectStatus::SUCCESS)
|
||||
{
|
||||
if (eviction_candidates.size() == 0)
|
||||
{
|
||||
|
||||
@ -150,8 +150,14 @@ public:
|
||||
/// Collect eviction candidates sufficient to have `desired_size`
|
||||
/// and `desired_elements_num` as current cache state.
|
||||
/// Collect no more than `max_candidates_to_evict` elements.
|
||||
/// Return `true` if the first condition is satisfied.
|
||||
virtual bool collectCandidatesForEviction(
|
||||
/// Return SUCCESS status if the first condition is satisfied.
|
||||
enum class CollectStatus
|
||||
{
|
||||
SUCCESS,
|
||||
CANNOT_EVICT,
|
||||
REACHED_MAX_CANDIDATES_LIMIT,
|
||||
};
|
||||
virtual CollectStatus collectCandidatesForEviction(
|
||||
size_t desired_size,
|
||||
size_t desired_elements_count,
|
||||
size_t max_candidates_to_evict,
|
||||
|
||||
@ -323,7 +323,7 @@ bool LRUFileCachePriority::collectCandidatesForEviction(
|
||||
}
|
||||
}
|
||||
|
||||
bool LRUFileCachePriority::collectCandidatesForEviction(
|
||||
IFileCachePriority::CollectStatus LRUFileCachePriority::collectCandidatesForEviction(
|
||||
size_t desired_size,
|
||||
size_t desired_elements_count,
|
||||
size_t max_candidates_to_evict,
|
||||
@ -336,12 +336,24 @@ bool LRUFileCachePriority::collectCandidatesForEviction(
|
||||
return canFit(0, 0, stat.total_stat.releasable_size, stat.total_stat.releasable_count,
|
||||
lock, &desired_size, &desired_elements_count);
|
||||
};
|
||||
auto status = CollectStatus::CANNOT_EVICT;
|
||||
auto stop_condition = [&]()
|
||||
{
|
||||
return desired_limits_satisfied() || (max_candidates_to_evict && res.size() >= max_candidates_to_evict);
|
||||
if (desired_limits_satisfied())
|
||||
{
|
||||
status = CollectStatus::SUCCESS;
|
||||
return true;
|
||||
}
|
||||
if (max_candidates_to_evict && res.size() >= max_candidates_to_evict)
|
||||
{
|
||||
status = CollectStatus::REACHED_MAX_CANDIDATES_LIMIT;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
iterateForEviction(res, stat, stop_condition, lock);
|
||||
return desired_limits_satisfied();
|
||||
chassert(status != CollectStatus::SUCCESS || stop_condition());
|
||||
return status;
|
||||
}
|
||||
|
||||
void LRUFileCachePriority::iterateForEviction(
|
||||
@ -350,6 +362,9 @@ void LRUFileCachePriority::iterateForEviction(
|
||||
StopConditionFunc stop_condition,
|
||||
const CachePriorityGuard::Lock & lock)
|
||||
{
|
||||
if (stop_condition())
|
||||
return;
|
||||
|
||||
ProfileEvents::increment(ProfileEvents::FilesystemCacheEvictionTries);
|
||||
|
||||
IterateFunc iterate_func = [&](LockedKey & locked_key, const FileSegmentMetadataPtr & segment_metadata)
|
||||
|
||||
@ -63,7 +63,7 @@ public:
|
||||
const UserID & user_id,
|
||||
const CachePriorityGuard::Lock &) override;
|
||||
|
||||
bool collectCandidatesForEviction(
|
||||
CollectStatus collectCandidatesForEviction(
|
||||
size_t desired_size,
|
||||
size_t desired_elements_count,
|
||||
size_t max_candidates_to_evict,
|
||||
|
||||
@ -256,7 +256,7 @@ bool SLRUFileCachePriority::collectCandidatesForEvictionInProtected(
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SLRUFileCachePriority::collectCandidatesForEviction(
|
||||
IFileCachePriority::CollectStatus SLRUFileCachePriority::collectCandidatesForEviction(
|
||||
size_t desired_size,
|
||||
size_t desired_elements_count,
|
||||
size_t max_candidates_to_evict,
|
||||
@ -268,7 +268,7 @@ bool SLRUFileCachePriority::collectCandidatesForEviction(
|
||||
const auto desired_probationary_elements_num = getRatio(desired_elements_count, 1 - size_ratio);
|
||||
|
||||
FileCacheReserveStat probationary_stat;
|
||||
const bool probationary_limit_satisfied = probationary_queue.collectCandidatesForEviction(
|
||||
const auto probationary_desired_size_status = probationary_queue.collectCandidatesForEviction(
|
||||
desired_probationary_size, desired_probationary_elements_num,
|
||||
max_candidates_to_evict, probationary_stat, res, lock);
|
||||
|
||||
@ -285,14 +285,14 @@ bool SLRUFileCachePriority::collectCandidatesForEviction(
|
||||
chassert(!max_candidates_to_evict || res.size() <= max_candidates_to_evict);
|
||||
chassert(res.size() == stat.total_stat.releasable_count);
|
||||
|
||||
if (max_candidates_to_evict && res.size() >= max_candidates_to_evict)
|
||||
return probationary_limit_satisfied;
|
||||
if (probationary_desired_size_status == CollectStatus::REACHED_MAX_CANDIDATES_LIMIT)
|
||||
return probationary_desired_size_status;
|
||||
|
||||
const auto desired_protected_size = getRatio(desired_size, size_ratio);
|
||||
const auto desired_protected_elements_num = getRatio(desired_elements_count, size_ratio);
|
||||
|
||||
FileCacheReserveStat protected_stat;
|
||||
const bool protected_limit_satisfied = protected_queue.collectCandidatesForEviction(
|
||||
const auto protected_desired_size_status = protected_queue.collectCandidatesForEviction(
|
||||
desired_protected_size, desired_protected_elements_num,
|
||||
max_candidates_to_evict - res.size(), protected_stat, res, lock);
|
||||
|
||||
@ -306,7 +306,10 @@ bool SLRUFileCachePriority::collectCandidatesForEviction(
|
||||
desired_protected_size, desired_protected_elements_num,
|
||||
protected_queue.getStateInfoForLog(lock));
|
||||
|
||||
return probationary_limit_satisfied && protected_limit_satisfied;
|
||||
if (probationary_desired_size_status == CollectStatus::SUCCESS)
|
||||
return protected_desired_size_status;
|
||||
else
|
||||
return probationary_desired_size_status;
|
||||
}
|
||||
|
||||
void SLRUFileCachePriority::downgrade(IteratorPtr iterator, const CachePriorityGuard::Lock & lock)
|
||||
|
||||
@ -58,7 +58,7 @@ public:
|
||||
const UserID & user_id,
|
||||
const CachePriorityGuard::Lock &) override;
|
||||
|
||||
bool collectCandidatesForEviction(
|
||||
CollectStatus collectCandidatesForEviction(
|
||||
size_t desired_size,
|
||||
size_t desired_elements_count,
|
||||
size_t max_candidates_to_evict,
|
||||
|
||||
@ -796,10 +796,9 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
|
||||
catch (const Exception & e)
|
||||
{
|
||||
if (e.code() == ErrorCodes::SYNTAX_ERROR)
|
||||
/// Don't print the original query text because it may contain sensitive data.
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR,
|
||||
"Inconsistent AST formatting: the query:\n{}\ncannot parse.",
|
||||
formatted1);
|
||||
"Inconsistent AST formatting: the query:\n{}\ncannot parse query back from {}",
|
||||
formatted1, std::string_view(begin, end-begin));
|
||||
else
|
||||
throw;
|
||||
}
|
||||
|
||||
@ -323,9 +323,7 @@ void ASTColumnsReplaceTransformer::formatImpl(const FormatSettings & settings, F
|
||||
{
|
||||
settings.ostr << (settings.hilite ? hilite_keyword : "") << "REPLACE" << (is_strict ? " STRICT " : " ") << (settings.hilite ? hilite_none : "");
|
||||
|
||||
if (children.size() > 1)
|
||||
settings.ostr << "(";
|
||||
|
||||
settings.ostr << "(";
|
||||
for (ASTs::const_iterator it = children.begin(); it != children.end(); ++it)
|
||||
{
|
||||
if (it != children.begin())
|
||||
@ -333,9 +331,7 @@ void ASTColumnsReplaceTransformer::formatImpl(const FormatSettings & settings, F
|
||||
|
||||
(*it)->formatImpl(settings, state, frame);
|
||||
}
|
||||
|
||||
if (children.size() > 1)
|
||||
settings.ostr << ")";
|
||||
settings.ostr << ")";
|
||||
}
|
||||
|
||||
void ASTColumnsReplaceTransformer::appendColumnName(WriteBuffer & ostr) const
|
||||
|
||||
@ -22,7 +22,7 @@ public:
|
||||
};
|
||||
|
||||
|
||||
template<class Derived>
|
||||
template <typename Derived>
|
||||
class ChunkInfoCloneable : public ChunkInfo
|
||||
{
|
||||
public:
|
||||
|
||||
@ -119,8 +119,10 @@ bool restorePrewhereInputs(PrewhereInfo & info, const NameSet & inputs)
|
||||
namespace ProfileEvents
|
||||
{
|
||||
extern const Event SelectedParts;
|
||||
extern const Event SelectedPartsTotal;
|
||||
extern const Event SelectedRanges;
|
||||
extern const Event SelectedMarks;
|
||||
extern const Event SelectedMarksTotal;
|
||||
extern const Event SelectQueriesWithPrimaryKeyUsage;
|
||||
}
|
||||
|
||||
@ -1970,8 +1972,10 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons
|
||||
}
|
||||
|
||||
ProfileEvents::increment(ProfileEvents::SelectedParts, result.selected_parts);
|
||||
ProfileEvents::increment(ProfileEvents::SelectedPartsTotal, result.total_parts);
|
||||
ProfileEvents::increment(ProfileEvents::SelectedRanges, result.selected_ranges);
|
||||
ProfileEvents::increment(ProfileEvents::SelectedMarks, result.selected_marks);
|
||||
ProfileEvents::increment(ProfileEvents::SelectedMarksTotal, result.total_marks_pk);
|
||||
|
||||
auto query_id_holder = MergeTreeDataSelectExecutor::checkLimits(data, result, context);
|
||||
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
#include <vector>
|
||||
|
||||
#include <Poco/MongoDB/Array.h>
|
||||
#include <Poco/MongoDB/Binary.h>
|
||||
#include <Poco/MongoDB/Database.h>
|
||||
#include <Poco/MongoDB/Connection.h>
|
||||
#include <Poco/MongoDB/Cursor.h>
|
||||
@ -17,6 +18,7 @@
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <Common/assert_cast.h>
|
||||
#include <Common/quoteString.h>
|
||||
#include "base/types.h"
|
||||
#include <base/range.h>
|
||||
#include <Poco/URI.h>
|
||||
|
||||
@ -45,8 +47,28 @@ namespace
|
||||
using ValueType = ExternalResultDescription::ValueType;
|
||||
using ObjectId = Poco::MongoDB::ObjectId;
|
||||
using MongoArray = Poco::MongoDB::Array;
|
||||
using MongoUUID = Poco::MongoDB::Binary::Ptr;
|
||||
|
||||
|
||||
UUID parsePocoUUID(const Poco::UUID & src)
|
||||
{
|
||||
UUID uuid;
|
||||
|
||||
std::array<Poco::UInt8, 6> src_node = src.getNode();
|
||||
UInt64 node = 0;
|
||||
node |= UInt64(src_node[0]) << 40;
|
||||
node |= UInt64(src_node[1]) << 32;
|
||||
node |= UInt64(src_node[2]) << 24;
|
||||
node |= UInt64(src_node[3]) << 16;
|
||||
node |= UInt64(src_node[4]) << 8;
|
||||
node |= src_node[5];
|
||||
|
||||
UUIDHelpers::getHighBytes(uuid) = UInt64(src.getTimeLow()) << 32 | UInt32(src.getTimeMid() << 16 | src.getTimeHiAndVersion());
|
||||
UUIDHelpers::getLowBytes(uuid) = UInt64(src.getClockSeq()) << 48 | node;
|
||||
|
||||
return uuid;
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
Field getNumber(const Poco::MongoDB::Element & value, const std::string & name)
|
||||
{
|
||||
@ -149,12 +171,20 @@ namespace
|
||||
else if (which.isUUID())
|
||||
parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field
|
||||
{
|
||||
if (value.type() != Poco::MongoDB::ElementTraits<String>::TypeId)
|
||||
throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected String (UUID), got type id = {} for column {}",
|
||||
if (value.type() == Poco::MongoDB::ElementTraits<String>::TypeId)
|
||||
{
|
||||
String string = static_cast<const Poco::MongoDB::ConcreteElement<String> &>(value).value();
|
||||
return parse<UUID>(string);
|
||||
}
|
||||
else if (value.type() == Poco::MongoDB::ElementTraits<MongoUUID>::TypeId)
|
||||
{
|
||||
const Poco::UUID & poco_uuid = static_cast<const Poco::MongoDB::ConcreteElement<MongoUUID> &>(value).value()->uuid();
|
||||
return parsePocoUUID(poco_uuid);
|
||||
}
|
||||
else
|
||||
throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected String/UUID, got type id = {} for column {}",
|
||||
toString(value.type()), name);
|
||||
|
||||
String string = static_cast<const Poco::MongoDB::ConcreteElement<String> &>(value).value();
|
||||
return parse<UUID>(string);
|
||||
};
|
||||
else
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Type conversion to {} is not supported", nested->getName());
|
||||
@ -286,8 +316,14 @@ namespace
|
||||
String string = static_cast<const Poco::MongoDB::ConcreteElement<String> &>(value).value();
|
||||
assert_cast<ColumnUUID &>(column).getData().push_back(parse<UUID>(string));
|
||||
}
|
||||
else if (value.type() == Poco::MongoDB::ElementTraits<MongoUUID>::TypeId)
|
||||
{
|
||||
const Poco::UUID & poco_uuid = static_cast<const Poco::MongoDB::ConcreteElement<MongoUUID> &>(value).value()->uuid();
|
||||
UUID uuid = parsePocoUUID(poco_uuid);
|
||||
assert_cast<ColumnUUID &>(column).getData().push_back(uuid);
|
||||
}
|
||||
else
|
||||
throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected String (UUID), got type id = {} for column {}",
|
||||
throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected String/UUID, got type id = {} for column {}",
|
||||
toString(value.type()), name);
|
||||
break;
|
||||
}
|
||||
|
||||
@ -142,11 +142,11 @@ void StorageHDFSConfiguration::setURL(const std::string & url_)
|
||||
{
|
||||
auto pos = url_.find("//");
|
||||
if (pos == std::string::npos)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad hdfs url: {}", url_);
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad HDFS URL: {}. It should have the following structure 'hdfs://<host_name>:<port>/path'", url_);
|
||||
|
||||
pos = url_.find('/', pos + 2);
|
||||
if (pos == std::string::npos)
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad hdfs url: {}", url_);
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad HDFS URL: {}. It should have the following structure 'hdfs://<host_name>:<port>/path'", url_);
|
||||
|
||||
path = url_.substr(pos + 1);
|
||||
if (!path.starts_with('/'))
|
||||
@ -155,7 +155,7 @@ void StorageHDFSConfiguration::setURL(const std::string & url_)
|
||||
url = url_.substr(0, pos);
|
||||
paths = {path};
|
||||
|
||||
LOG_TRACE(getLogger("StorageHDFSConfiguration"), "Using url: {}, path: {}", url, path);
|
||||
LOG_TRACE(getLogger("StorageHDFSConfiguration"), "Using URL: {}, path: {}", url, path);
|
||||
}
|
||||
|
||||
void StorageHDFSConfiguration::addStructureAndFormatToArgs(
|
||||
|
||||
@ -192,7 +192,7 @@ String getNameNodeCluster(const String &hdfs_url)
|
||||
void checkHDFSURL(const String & url)
|
||||
{
|
||||
if (!re2::RE2::FullMatch(url, std::string(HDFS_URL_REGEXP)))
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad hdfs url: {}. It should have structure 'hdfs://<host_name>:<port>/<path>'", url);
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad HDFS URL: {}. It should have structure 'hdfs://<host_name>:<port>/<path>'", url);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -166,6 +166,7 @@ def _get_statless_tests_to_run(pr_info: PRInfo) -> List[str]:
|
||||
|
||||
|
||||
def process_results(
|
||||
ret_code: int,
|
||||
result_directory: Path,
|
||||
server_log_path: Path,
|
||||
) -> Tuple[StatusType, str, TestResults, List[Path]]:
|
||||
@ -192,6 +193,9 @@ def process_results(
|
||||
logging.info("Files in result folder %s", os.listdir(result_directory))
|
||||
return ERROR, "Invalid check_status.tsv", test_results, additional_files
|
||||
state, description = status[0][0], status[0][1]
|
||||
if ret_code != 0:
|
||||
state = ERROR
|
||||
description += " (but script exited with an error)"
|
||||
|
||||
try:
|
||||
results_path = result_directory / "test_results.tsv"
|
||||
@ -339,7 +343,7 @@ def main():
|
||||
ci_logs_credentials.clean_ci_logs_from_credentials(run_log_path)
|
||||
|
||||
state, description, test_results, additional_logs = process_results(
|
||||
result_path, server_log_path
|
||||
retcode, result_path, server_log_path
|
||||
)
|
||||
else:
|
||||
print(
|
||||
|
||||
@ -1232,6 +1232,11 @@ class TestCase:
|
||||
):
|
||||
return FailureReason.SKIP
|
||||
|
||||
elif "no-flaky-check" in tags and (
|
||||
1 == int(os.environ.get("IS_FLAKY_CHECK", 0))
|
||||
):
|
||||
return FailureReason.SKIP
|
||||
|
||||
elif tags:
|
||||
for build_flag in args.build_flags:
|
||||
if "no-" + build_flag in tags:
|
||||
|
||||
@ -19,6 +19,7 @@ services:
|
||||
ldapsearch -x -H ldap://localhost:$$LDAP_PORT_NUMBER -D $$LDAP_ADMIN_DN -w $$LDAP_ADMIN_PASSWORD -b $$LDAP_ROOT
|
||||
| grep -c -E "member: cn=j(ohn|ane)doe"
|
||||
| grep 2 >> /dev/null
|
||||
&& cat /run/slapd/slapd.pid
|
||||
interval: 10s
|
||||
retries: 10
|
||||
timeout: 2s
|
||||
|
||||
@ -157,7 +157,7 @@ def test_bad_hdfs_uri(started_cluster):
|
||||
)
|
||||
except Exception as ex:
|
||||
print(ex)
|
||||
assert "Bad hdfs url" in str(ex)
|
||||
assert "Bad HDFS URL" in str(ex)
|
||||
try:
|
||||
node1.query(
|
||||
"create table BadStorage2 (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs100500:9000/other_storage', 'TSV')"
|
||||
|
||||
@ -4771,7 +4771,7 @@ def test_system_kafka_consumers_rebalance(kafka_cluster, max_retries=15):
|
||||
assignments.current_offset,
|
||||
if(length(exceptions.time)>0, exceptions.time[1]::String, 'never') as last_exception_time_,
|
||||
if(length(exceptions.text)>0, exceptions.text[1], 'no exception') as last_exception_,
|
||||
stable_timestamp(last_poll_time) as last_poll_time_, num_messages_read, stable_timestamp(last_commit_time) as last_commit_time_,
|
||||
stable_timestamp(last_poll_time) as last_poll_time_, stable_timestamp(last_commit_time) as last_commit_time_,
|
||||
num_commits, stable_timestamp(last_rebalance_time) as last_rebalance_time_,
|
||||
num_rebalance_revocations, num_rebalance_assignments, is_currently_used
|
||||
FROM system.kafka_consumers WHERE database='test' and table IN ('kafka', 'kafka2') format Vertical;
|
||||
@ -4791,7 +4791,6 @@ assignments.current_offset: [2]
|
||||
last_exception_time_: never
|
||||
last_exception_: no exception
|
||||
last_poll_time_: now
|
||||
num_messages_read: 4
|
||||
last_commit_time_: now
|
||||
num_commits: 2
|
||||
last_rebalance_time_: now
|
||||
@ -4810,7 +4809,6 @@ assignments.current_offset: [2]
|
||||
last_exception_time_: never
|
||||
last_exception_: no exception
|
||||
last_poll_time_: now
|
||||
num_messages_read: 1
|
||||
last_commit_time_: now
|
||||
num_commits: 1
|
||||
last_rebalance_time_: never
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import pymongo
|
||||
from uuid import UUID
|
||||
|
||||
import pytest
|
||||
from helpers.client import QueryRuntimeException
|
||||
@ -72,6 +73,28 @@ def test_simple_select(started_cluster):
|
||||
simple_mongo_table.drop()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"])
|
||||
def test_uuid(started_cluster):
|
||||
mongo_connection = get_mongo_connection(started_cluster)
|
||||
db = mongo_connection["test"]
|
||||
db.add_user("root", "clickhouse")
|
||||
mongo_table = db["uuid_table"]
|
||||
mongo_table.insert({"key": 0, "data": UUID("f0e77736-91d1-48ce-8f01-15123ca1c7ed")})
|
||||
|
||||
node = started_cluster.instances["node"]
|
||||
node.query(
|
||||
"CREATE TABLE uuid_mongo_table(key UInt64, data UUID) ENGINE = MongoDB('mongo1:27017', 'test', 'uuid_table', 'root', 'clickhouse')"
|
||||
)
|
||||
|
||||
assert node.query("SELECT COUNT() FROM uuid_mongo_table") == "1\n"
|
||||
assert (
|
||||
node.query("SELECT data from uuid_mongo_table where key = 0")
|
||||
== "f0e77736-91d1-48ce-8f01-15123ca1c7ed\n"
|
||||
)
|
||||
node.query("DROP TABLE uuid_mongo_table")
|
||||
mongo_table.drop()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"])
|
||||
def test_simple_select_from_view(started_cluster):
|
||||
mongo_connection = get_mongo_connection(started_cluster)
|
||||
@ -140,6 +163,10 @@ def test_arrays(started_cluster):
|
||||
"f0e77736-91d1-48ce-8f01-15123ca1c7ed",
|
||||
"93376a07-c044-4281-a76e-ad27cf6973c5",
|
||||
],
|
||||
"arr_mongo_uuid": [
|
||||
UUID("f0e77736-91d1-48ce-8f01-15123ca1c7ed"),
|
||||
UUID("93376a07-c044-4281-a76e-ad27cf6973c5"),
|
||||
],
|
||||
"arr_arr_bool": [
|
||||
[True, False, True],
|
||||
[True],
|
||||
@ -174,6 +201,7 @@ def test_arrays(started_cluster):
|
||||
"arr_datetime Array(DateTime),"
|
||||
"arr_string Array(String),"
|
||||
"arr_uuid Array(UUID),"
|
||||
"arr_mongo_uuid Array(UUID),"
|
||||
"arr_arr_bool Array(Array(Bool)),"
|
||||
"arr_empty Array(UInt64),"
|
||||
"arr_null Array(UInt64),"
|
||||
@ -222,6 +250,11 @@ def test_arrays(started_cluster):
|
||||
== "['f0e77736-91d1-48ce-8f01-15123ca1c7ed','93376a07-c044-4281-a76e-ad27cf6973c5']\n"
|
||||
)
|
||||
|
||||
assert (
|
||||
node.query(f"SELECT arr_mongo_uuid FROM arrays_mongo_table WHERE key = 42")
|
||||
== "['f0e77736-91d1-48ce-8f01-15123ca1c7ed','93376a07-c044-4281-a76e-ad27cf6973c5']\n"
|
||||
)
|
||||
|
||||
assert (
|
||||
node.query(f"SELECT arr_arr_bool FROM arrays_mongo_table WHERE key = 42")
|
||||
== "[[true,false,true],[true],[],[],[false],[false]]\n"
|
||||
@ -377,6 +410,7 @@ def test_no_credentials(started_cluster):
|
||||
simple_mongo_table.insert_many(data)
|
||||
|
||||
node = started_cluster.instances["node"]
|
||||
node.query("drop table if exists simple_mongo_table_2")
|
||||
node.query(
|
||||
"create table simple_mongo_table_2(key UInt64, data String) engine = MongoDB('mongo2:27017', 'test', 'simple_table', '', '')"
|
||||
)
|
||||
@ -406,10 +440,13 @@ def test_auth_source(started_cluster):
|
||||
simple_mongo_table.insert_many(data)
|
||||
|
||||
node = started_cluster.instances["node"]
|
||||
node.query("drop table if exists simple_mongo_table_fail")
|
||||
node.query(
|
||||
"create table simple_mongo_table_fail(key UInt64, data String) engine = MongoDB('mongo2:27017', 'test', 'simple_table', 'root', 'clickhouse')"
|
||||
)
|
||||
node.query_and_get_error("SELECT count() FROM simple_mongo_table_fail")
|
||||
|
||||
node.query("drop table if exists simple_mongo_table_ok")
|
||||
node.query(
|
||||
"create table simple_mongo_table_ok(key UInt64, data String) engine = MongoDB('mongo2:27017', 'test', 'simple_table', 'root', 'clickhouse', 'authSource=admin')"
|
||||
)
|
||||
|
||||
@ -1333,16 +1333,7 @@ def test_shards_distributed(started_cluster, mode, processing_threads):
|
||||
def get_count(node, table_name):
|
||||
return int(run_query(node, f"SELECT count() FROM {table_name}"))
|
||||
|
||||
for _ in range(30):
|
||||
if (
|
||||
get_count(node, dst_table_name) + get_count(node_2, dst_table_name)
|
||||
) == total_rows:
|
||||
break
|
||||
time.sleep(1)
|
||||
|
||||
if (
|
||||
get_count(node, dst_table_name) + get_count(node_2, dst_table_name)
|
||||
) != total_rows:
|
||||
def print_debug_info():
|
||||
processed_files = (
|
||||
node.query(
|
||||
f"""
|
||||
@ -1369,7 +1360,7 @@ select splitByChar('/', file_name)[-1] as file from system.s3queue where zookeep
|
||||
)
|
||||
|
||||
count = get_count(node, dst_table_name) + get_count(node_2, dst_table_name)
|
||||
logging.debug(f"Processed rows: {count}/{files_to_generate}")
|
||||
logging.debug(f"Processed rows: {count}/{total_rows}")
|
||||
|
||||
info = node.query(
|
||||
f"""
|
||||
@ -1406,6 +1397,18 @@ select splitByChar('/', file_name)[-1] as file from system.s3queue where zookeep
|
||||
|
||||
logging.debug(f"Intersecting files: {intersection(files1, files2)}")
|
||||
|
||||
for _ in range(30):
|
||||
if (
|
||||
get_count(node, dst_table_name) + get_count(node_2, dst_table_name)
|
||||
) == total_rows:
|
||||
break
|
||||
time.sleep(1)
|
||||
|
||||
if (
|
||||
get_count(node, dst_table_name) + get_count(node_2, dst_table_name)
|
||||
) != total_rows:
|
||||
print_debug_info()
|
||||
|
||||
assert False
|
||||
|
||||
get_query = f"SELECT column1, column2, column3 FROM {dst_table_name}"
|
||||
@ -1414,6 +1417,12 @@ select splitByChar('/', file_name)[-1] as file from system.s3queue where zookeep
|
||||
list(map(int, l.split())) for l in run_query(node_2, get_query).splitlines()
|
||||
]
|
||||
|
||||
if len(res1) + len(res2) != total_rows or len(res1) <= 0 or len(res2) <= 0 or True:
|
||||
logging.debug(
|
||||
f"res1 size: {len(res1)}, res2 size: {len(res2)}, total_rows: {total_rows}"
|
||||
)
|
||||
print_debug_info()
|
||||
|
||||
assert len(res1) + len(res2) == total_rows
|
||||
|
||||
# Checking that all engines have made progress
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
-- Tags: no-parallel, no-fasttest, no-ubsan, no-batch
|
||||
-- Tags: no-parallel, no-fasttest, no-ubsan, no-batch, no-flaky-check
|
||||
-- no-parallel because we want to run this test when most of the other tests already passed
|
||||
|
||||
-- If this test fails, see the "Top patterns of log messages" diagnostics in the end of run.log
|
||||
@ -165,7 +165,10 @@ create temporary table known_short_messages (s String) as select * from (select
|
||||
'{} -> {}',
|
||||
'{} {}',
|
||||
'{}%',
|
||||
'{}: {}'
|
||||
'{}: {}',
|
||||
'Unknown data type family: {}',
|
||||
'Cannot load time zone {}',
|
||||
'Unknown table engine {}'
|
||||
] as arr) array join arr;
|
||||
|
||||
-- Check that we don't have too many short meaningless message patterns.
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
-- Tags: no-debug
|
||||
-- Tags: no-fasttest
|
||||
|
||||
SET allow_hyperscan = 1;
|
||||
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: no-parallel
|
||||
|
||||
# Creation of a database with Ordinary engine emits a warning.
|
||||
CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=fatal
|
||||
@ -12,38 +11,40 @@ opts=(
|
||||
"--allow_experimental_analyzer=0"
|
||||
)
|
||||
|
||||
$CLICKHOUSE_CLIENT "${opts[@]}" --allow_deprecated_database_ordinary=1 <<EOF
|
||||
SET allow_experimental_window_view = 1;
|
||||
SET window_view_clean_interval = 1;
|
||||
DATABASE_ORDINARY="${CLICKHOUSE_DATABASE}_ordinary"
|
||||
|
||||
DROP DATABASE IF EXISTS test_01086;
|
||||
CREATE DATABASE test_01086 ENGINE=Ordinary;
|
||||
$CLICKHOUSE_CLIENT "${opts[@]}" --allow_deprecated_database_ordinary=1 --multiquery "
|
||||
SET allow_experimental_window_view = 1;
|
||||
SET window_view_clean_interval = 1;
|
||||
|
||||
CREATE TABLE test_01086.dst(count UInt64, market Int32, w_end DateTime) Engine=MergeTree ORDER BY tuple();
|
||||
CREATE TABLE test_01086.mt(a Int32, market Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple();
|
||||
CREATE WINDOW VIEW test_01086.wv TO test_01086.dst WATERMARK=ASCENDING AS SELECT count(a) AS count, market, tumbleEnd(wid) AS w_end FROM test_01086.mt GROUP BY tumble(timestamp, INTERVAL '5' SECOND, 'US/Samoa') AS wid, market;
|
||||
DROP DATABASE IF EXISTS ${DATABASE_ORDINARY};
|
||||
CREATE DATABASE ${DATABASE_ORDINARY} ENGINE=Ordinary;
|
||||
|
||||
INSERT INTO test_01086.mt VALUES (1, 1, toDateTime('1990/01/01 12:00:00', 'US/Samoa'));
|
||||
INSERT INTO test_01086.mt VALUES (1, 2, toDateTime('1990/01/01 12:00:01', 'US/Samoa'));
|
||||
INSERT INTO test_01086.mt VALUES (1, 3, toDateTime('1990/01/01 12:00:02', 'US/Samoa'));
|
||||
INSERT INTO test_01086.mt VALUES (1, 4, toDateTime('1990/01/01 12:00:05', 'US/Samoa'));
|
||||
INSERT INTO test_01086.mt VALUES (1, 5, toDateTime('1990/01/01 12:00:06', 'US/Samoa'));
|
||||
EOF
|
||||
CREATE TABLE ${DATABASE_ORDINARY}.dst(count UInt64, market Int32, w_end DateTime) Engine=MergeTree ORDER BY tuple();
|
||||
CREATE TABLE ${DATABASE_ORDINARY}.mt(a Int32, market Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple();
|
||||
CREATE WINDOW VIEW ${DATABASE_ORDINARY}.wv TO ${DATABASE_ORDINARY}.dst WATERMARK=ASCENDING AS SELECT count(a) AS count, market, tumbleEnd(wid) AS w_end FROM ${DATABASE_ORDINARY}.mt GROUP BY tumble(timestamp, INTERVAL '5' SECOND, 'US/Samoa') AS wid, market;
|
||||
|
||||
INSERT INTO ${DATABASE_ORDINARY}.mt VALUES (1, 1, toDateTime('1990/01/01 12:00:00', 'US/Samoa'));
|
||||
INSERT INTO ${DATABASE_ORDINARY}.mt VALUES (1, 2, toDateTime('1990/01/01 12:00:01', 'US/Samoa'));
|
||||
INSERT INTO ${DATABASE_ORDINARY}.mt VALUES (1, 3, toDateTime('1990/01/01 12:00:02', 'US/Samoa'));
|
||||
INSERT INTO ${DATABASE_ORDINARY}.mt VALUES (1, 4, toDateTime('1990/01/01 12:00:05', 'US/Samoa'));
|
||||
INSERT INTO ${DATABASE_ORDINARY}.mt VALUES (1, 5, toDateTime('1990/01/01 12:00:06', 'US/Samoa'));
|
||||
"
|
||||
|
||||
while true; do
|
||||
$CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT count(*) FROM test_01086.\`.inner.wv\`" | grep -q "5" && break || sleep .5 ||:
|
||||
$CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT count(*) FROM ${DATABASE_ORDINARY}.\`.inner.wv\`" | grep -q "5" && break || sleep .5 ||:
|
||||
done
|
||||
|
||||
$CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT sleep(2);"
|
||||
|
||||
$CLICKHOUSE_CLIENT "${opts[@]}" --query="INSERT INTO test_01086.mt VALUES (1, 6, toDateTime('1990/01/01 12:00:11', 'US/Samoa'));"
|
||||
$CLICKHOUSE_CLIENT "${opts[@]}" --query="INSERT INTO ${DATABASE_ORDINARY}.mt VALUES (1, 6, toDateTime('1990/01/01 12:00:11', 'US/Samoa'));"
|
||||
|
||||
while true; do
|
||||
$CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT count(*) FROM test_01086.\`.inner.wv\`" | grep -q "3" && break || sleep .5 ||:
|
||||
$CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT count(*) FROM ${DATABASE_ORDINARY}.\`.inner.wv\`" | grep -q "3" && break || sleep .5 ||:
|
||||
done
|
||||
|
||||
$CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT market, wid FROM test_01086.\`.inner.wv\` ORDER BY market, \`windowID(timestamp, toIntervalSecond('5'), 'US/Samoa')\` as wid";
|
||||
$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP TABLE test_01086.wv SYNC;"
|
||||
$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP TABLE test_01086.mt SYNC;"
|
||||
$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP TABLE test_01086.dst SYNC;"
|
||||
$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP DATABASE test_01086 SYNC;"
|
||||
$CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT market, wid FROM ${DATABASE_ORDINARY}.\`.inner.wv\` ORDER BY market, \`windowID(timestamp, toIntervalSecond('5'), 'US/Samoa')\` as wid";
|
||||
$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP TABLE ${DATABASE_ORDINARY}.wv SYNC;"
|
||||
$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP TABLE ${DATABASE_ORDINARY}.mt SYNC;"
|
||||
$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP TABLE ${DATABASE_ORDINARY}.dst SYNC;"
|
||||
$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP DATABASE ${DATABASE_ORDINARY} SYNC;"
|
||||
|
||||
@ -12,5 +12,13 @@ yml
|
||||
2
|
||||
yaml
|
||||
2
|
||||
ini
|
||||
Code: 347. Unknown format of '/config_default.ini' config. (CANNOT_LOAD_CONFIG)
|
||||
autodetect xml (with leading whitespaces)
|
||||
2
|
||||
autodetect xml (non leading whitespaces)
|
||||
2
|
||||
autodetect yaml
|
||||
2
|
||||
autodetect invalid xml
|
||||
Correct: invalid xml parsed with exception
|
||||
autodetect invalid yaml
|
||||
Code: 585. Unable to parse YAML configuration file /config_default.badyaml, yaml-cpp: error at line 2, column 12: illegal map value. (CANNOT_PARSE_YAML)
|
||||
|
||||
@ -12,7 +12,11 @@ XML_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.XML
|
||||
conf_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.conf
|
||||
yml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.yml
|
||||
yaml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.yaml
|
||||
ini_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.ini
|
||||
autodetect_xml_with_leading_whitespace_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.config
|
||||
autodetect_xml_non_leading_whitespace_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.cfg
|
||||
autodetect_yaml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.properties
|
||||
autodetect_invalid_xml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.badxml
|
||||
autodetect_invalid_yaml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.badyaml
|
||||
|
||||
function cleanup()
|
||||
{
|
||||
@ -22,7 +26,11 @@ function cleanup()
|
||||
rm "${conf_config:?}"
|
||||
rm "${yml_config:?}"
|
||||
rm "${yaml_config:?}"
|
||||
rm "${ini_config:?}"
|
||||
rm "${autodetect_xml_with_leading_whitespace_config:?}"
|
||||
rm "${autodetect_xml_non_leading_whitespace_config:?}"
|
||||
rm "${autodetect_yaml_config:?}"
|
||||
rm "${autodetect_invalid_xml_config:?}"
|
||||
rm "${autodetect_invalid_yaml_config:?}"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
@ -52,10 +60,29 @@ EOL
|
||||
cat > "$yaml_config" <<EOL
|
||||
max_threads: 2
|
||||
EOL
|
||||
cat > "$ini_config" <<EOL
|
||||
[config]
|
||||
max_threads=2
|
||||
cat > "$autodetect_xml_with_leading_whitespace_config" <<EOL
|
||||
|
||||
<config>
|
||||
<max_threads>2</max_threads>
|
||||
</config>
|
||||
EOL
|
||||
cat > "$autodetect_xml_non_leading_whitespace_config" <<EOL
|
||||
<config>
|
||||
<max_threads>2</max_threads>
|
||||
</config>
|
||||
EOL
|
||||
cat > "$autodetect_yaml_config" <<EOL
|
||||
max_threads: 2
|
||||
EOL
|
||||
cat > "$autodetect_invalid_xml_config" <<EOL
|
||||
<!-- This is a XML file comment -->
|
||||
<invalid tag><invalid tag>
|
||||
EOL
|
||||
cat > "$autodetect_invalid_yaml_config" <<EOL
|
||||
; This is a INI file comment
|
||||
max_threads: 2
|
||||
EOL
|
||||
|
||||
|
||||
echo 'default'
|
||||
$CLICKHOUSE_CLIENT --config "$config" -q "select getSetting('max_threads')"
|
||||
@ -74,5 +101,16 @@ echo 'yml'
|
||||
$CLICKHOUSE_CLIENT --config "$yml_config" -q "select getSetting('max_threads')"
|
||||
echo 'yaml'
|
||||
$CLICKHOUSE_CLIENT --config "$yaml_config" -q "select getSetting('max_threads')"
|
||||
echo 'ini'
|
||||
$CLICKHOUSE_CLIENT --config "$ini_config" -q "select getSetting('max_threads')" 2>&1 |& sed -e "s#$CLICKHOUSE_TMP##" -e "s#DB::Exception: ##"
|
||||
|
||||
echo 'autodetect xml (with leading whitespaces)'
|
||||
$CLICKHOUSE_CLIENT --config "$autodetect_xml_with_leading_whitespace_config" -q "select getSetting('max_threads')"
|
||||
echo 'autodetect xml (non leading whitespaces)'
|
||||
$CLICKHOUSE_CLIENT --config "$autodetect_xml_non_leading_whitespace_config" -q "select getSetting('max_threads')"
|
||||
echo 'autodetect yaml'
|
||||
$CLICKHOUSE_CLIENT --config "$autodetect_yaml_config" -q "select getSetting('max_threads')"
|
||||
|
||||
# Error code is 1000 (Poco::Exception). It is not ignored.
|
||||
echo 'autodetect invalid xml'
|
||||
$CLICKHOUSE_CLIENT --config "$autodetect_invalid_xml_config" -q "select getSetting('max_threads')" 2>&1 |& grep -q "Code: 1000" && echo "Correct: invalid xml parsed with exception" || echo 'Fail: expected error code 1000 but got other'
|
||||
echo 'autodetect invalid yaml'
|
||||
$CLICKHOUSE_CLIENT --config "$autodetect_invalid_yaml_config" -q "select getSetting('max_threads')" 2>&1 |& sed -e "s#$CLICKHOUSE_TMP##" -e "s#DB::Exception: ##"
|
||||
@ -1 +1 @@
|
||||
CREATE VIEW default.my_view\n(\n `Id` UInt32,\n `Object.Key` Array(UInt16),\n `Object.Value` Array(String)\n)\nAS SELECT * REPLACE arrayMap(x -> (x + 1), `Object.Key`) AS `Object.Key`\nFROM default.my_table
|
||||
CREATE VIEW default.my_view\n(\n `Id` UInt32,\n `Object.Key` Array(UInt16),\n `Object.Value` Array(String)\n)\nAS SELECT * REPLACE (arrayMap(x -> (x + 1), `Object.Key`) AS `Object.Key`)\nFROM default.my_table
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: long
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
@ -14,7 +15,7 @@ do
|
||||
echo $format
|
||||
$CLICKHOUSE_CLIENT -q "INSERT INTO test_02099 SELECT number, toString(number), toDate(number) FROM numbers(3)"
|
||||
$CLICKHOUSE_CLIENT -q "SELECT * FROM test_02099 FORMAT $format"
|
||||
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "SELECT * FROM test_02099 FORMAT $format" | $CLICKHOUSE_CLIENT -q "INSERT INTO test_02099 FORMAT $format"
|
||||
$CLICKHOUSE_CLIENT -q "SELECT * FROM test_02099"
|
||||
|
||||
@ -49,13 +50,12 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE test_nullable_string_02099"
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_parallel_parsing_02099"
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE test_parallel_parsing_02099 (x UInt64, a Array(UInt64), s String) ENGINE=Memory()";
|
||||
$CLICKHOUSE_CLIENT -q "SELECT number AS x, range(number % 50) AS a, toString(a) AS s FROM numbers(1000000) FORMAT TSVRaw" | $CLICKHOUSE_CLIENT --input_format_parallel_parsing=0 -q "INSERT INTO test_parallel_parsing_02099 FORMAT TSVRaw"
|
||||
$CLICKHOUSE_CLIENT -q "SELECT number AS x, range(number % 50) AS a, toString(a) AS s FROM numbers(1000000) FORMAT TSVRaw" | $CLICKHOUSE_CLIENT --input_format_parallel_parsing=0 -q "INSERT INTO test_parallel_parsing_02099 FORMAT TSVRaw"
|
||||
$CLICKHOUSE_CLIENT -q "SELECT * FROM test_parallel_parsing_02099 ORDER BY x" | md5sum
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE test_parallel_parsing_02099"
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "SELECT number AS x, range(number % 50) AS a, toString(a) AS s FROM numbers(1000000) FORMAT TSVRaw" | $CLICKHOUSE_CLIENT --input_format_parallel_parsing=1 -q "INSERT INTO test_parallel_parsing_02099 FORMAT TSVRaw"
|
||||
$CLICKHOUSE_CLIENT -q "SELECT number AS x, range(number % 50) AS a, toString(a) AS s FROM numbers(1000000) FORMAT TSVRaw" | $CLICKHOUSE_CLIENT --input_format_parallel_parsing=1 -q "INSERT INTO test_parallel_parsing_02099 FORMAT TSVRaw"
|
||||
$CLICKHOUSE_CLIENT -q "SELECT * FROM test_parallel_parsing_02099 ORDER BY x" | md5sum
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "DROP TABLE test_parallel_parsing_02099"
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
Bloom filter on sort key
|
||||
10000
|
||||
1000
|
||||
0
|
||||
Bloom filter on non-sort key
|
||||
10000
|
||||
1000
|
||||
0
|
||||
|
||||
@ -12,7 +12,7 @@ INSERT INTO bloom_filter_sizing_pk
|
||||
SELECT
|
||||
number % 100 as key, -- 100 unique keys
|
||||
number as value -- whatever
|
||||
FROM numbers(1000 * 1000);
|
||||
FROM numbers(100_000);
|
||||
|
||||
--
|
||||
-- Merge everything into a single part
|
||||
@ -40,7 +40,7 @@ SELECT
|
||||
number % 100 as key1, -- 100 unique keys
|
||||
rand() % 100 as key2, -- 100 unique keys
|
||||
number as value -- whatever
|
||||
FROM numbers(1000 * 1000);
|
||||
FROM numbers(100_000);
|
||||
|
||||
--
|
||||
-- Merge everything into a single part
|
||||
|
||||
@ -562,6 +562,7 @@ positionCaseInsensitive
|
||||
positionCaseInsensitiveUTF8
|
||||
positionUTF8
|
||||
pow
|
||||
printf
|
||||
proportionsZTest
|
||||
protocol
|
||||
queryID
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user