From 6b541aa98f141a92eeeb586f94a5dc79281b575f Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 21 Jul 2022 12:18:37 +0000 Subject: [PATCH 01/14] Fix WriteBuffer finalize when cancel insert into function --- src/Processors/Formats/IOutputFormat.cpp | 4 ++- src/Storages/HDFS/StorageHDFS.cpp | 28 +++++++++++++++--- src/Storages/StorageFile.cpp | 29 ++++++++++++++++--- src/Storages/StorageS3.cpp | 28 +++++++++++++++--- src/Storages/StorageURL.cpp | 24 +++++++++++++-- src/Storages/StorageURL.h | 4 +++ .../02366_cancel_write_into_file.reference | 0 .../02366_cancel_write_into_file.sh | 24 +++++++++++++++ .../02367_cancel_write_into_s3.reference | 0 .../0_stateless/02367_cancel_write_into_s3.sh | 24 +++++++++++++++ .../02368_cancel_write_into_hdfs.reference | 0 .../02368_cancel_write_into_hdfs.sh | 24 +++++++++++++++ .../0_stateless/data_minio/02366_data.jsonl | 0 13 files changed, 173 insertions(+), 16 deletions(-) create mode 100644 tests/queries/0_stateless/02366_cancel_write_into_file.reference create mode 100755 tests/queries/0_stateless/02366_cancel_write_into_file.sh create mode 100644 tests/queries/0_stateless/02367_cancel_write_into_s3.reference create mode 100755 tests/queries/0_stateless/02367_cancel_write_into_s3.sh create mode 100644 tests/queries/0_stateless/02368_cancel_write_into_hdfs.reference create mode 100755 tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh create mode 100644 tests/queries/0_stateless/data_minio/02366_data.jsonl diff --git a/src/Processors/Formats/IOutputFormat.cpp b/src/Processors/Formats/IOutputFormat.cpp index 3c4e6861151..47ebaa9c5f5 100644 --- a/src/Processors/Formats/IOutputFormat.cpp +++ b/src/Processors/Formats/IOutputFormat.cpp @@ -73,7 +73,6 @@ void IOutputFormat::work() setRowsBeforeLimit(rows_before_limit_counter->get()); finalize(); - finalized = true; return; } @@ -120,9 +119,12 @@ void IOutputFormat::write(const Block & block) void IOutputFormat::finalize() { + if (finalized) + return; writePrefixIfNot(); writeSuffixIfNot(); finalizeImpl(); + finalized = true; } } diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index 57e893e9683..04bd58c3a81 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -427,18 +427,37 @@ public: void consume(Chunk chunk) override { + std::lock_guard lock(cancel_mutex); + if (cancelled) + return; writer->write(getHeader().cloneWithColumns(chunk.detachColumns())); } + void onCancel() override + { + std::lock_guard lock(cancel_mutex); + finalize(); + cancelled = false; + } + void onException() override { - if (!writer) - return; - onFinish(); + std::lock_guard lock(cancel_mutex); + finalize(); } void onFinish() override { + std::lock_guard lock(cancel_mutex); + finalize(); + } + +private: + void finalize() + { + if (!writer) + return; + try { writer->finalize(); @@ -454,9 +473,10 @@ public: } } -private: std::unique_ptr write_buf; OutputFormatPtr writer; + std::mutex cancel_mutex; + bool cancelled = false; }; class PartitionedHDFSSink : public PartitionedSink diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index d138104018a..672727b1478 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -810,18 +810,37 @@ public: void consume(Chunk chunk) override { + std::lock_guard cancel_lock(cancel_mutex); + if (cancelled) + return; writer->write(getHeader().cloneWithColumns(chunk.detachColumns())); } + void onCancel() override + { + std::lock_guard cancel_lock(cancel_mutex); + finalize(); + cancelled = true; + } + void onException() override { - if (!writer) - return; - onFinish(); + std::lock_guard cancel_lock(cancel_mutex); + finalize(); } void onFinish() override { + std::lock_guard cancel_lock(cancel_mutex); + finalize(); + } + +private: + void finalize() + { + if (!writer) + return; + try { writer->finalize(); @@ -836,7 +855,6 @@ public: } } -private: StorageMetadataPtr metadata_snapshot; String table_name_for_log; @@ -854,6 +872,9 @@ private: ContextPtr context; int flags; std::unique_lock lock; + + std::mutex cancel_mutex; + bool cancelled = false; }; class PartitionedStorageFileSink : public PartitionedSink diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 130bc75a65c..aed3f541d47 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -599,18 +599,37 @@ public: void consume(Chunk chunk) override { + std::lock_guard lock(cancel_mutex); + if (cancelled) + return; writer->write(getHeader().cloneWithColumns(chunk.detachColumns())); } + void onCancel() override + { + std::lock_guard lock(cancel_mutex); + finalize(); + cancelled = true; + } + void onException() override { - if (!writer) - return; - onFinish(); + std::lock_guard lock(cancel_mutex); + finalize(); } void onFinish() override { + std::lock_guard lock(cancel_mutex); + finalize(); + } + +private: + void finalize() + { + if (!writer) + return; + try { writer->finalize(); @@ -625,11 +644,12 @@ public: } } -private: Block sample_block; std::optional format_settings; std::unique_ptr write_buf; OutputFormatPtr writer; + bool cancelled = false; + std::mutex cancel_mutex; }; diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 15ae23305f3..acf7444dca4 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -447,18 +447,36 @@ StorageURLSink::StorageURLSink( void StorageURLSink::consume(Chunk chunk) { + std::lock_guard lock(cancel_mutex); + if (cancelled) + return; writer->write(getHeader().cloneWithColumns(chunk.detachColumns())); } +void StorageURLSink::onCancel() +{ + std::lock_guard lock(cancel_mutex); + finalize(); + cancelled = true; +} + void StorageURLSink::onException() { - if (!writer) - return; - onFinish(); + std::lock_guard lock(cancel_mutex); + finalize(); } void StorageURLSink::onFinish() { + std::lock_guard lock(cancel_mutex); + finalize(); +} + +void StorageURLSink::finalize() +{ + if (!writer) + return; + try { writer->finalize(); diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index 79371242bb1..0198eda9e67 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -114,12 +114,16 @@ public: std::string getName() const override { return "StorageURLSink"; } void consume(Chunk chunk) override; + void onCancel() override; void onException() override; void onFinish() override; private: + void finalize(); std::unique_ptr write_buf; OutputFormatPtr writer; + std::mutex cancel_mutex; + bool cancelled = false; }; class StorageURL : public IStorageURLBase diff --git a/tests/queries/0_stateless/02366_cancel_write_into_file.reference b/tests/queries/0_stateless/02366_cancel_write_into_file.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02366_cancel_write_into_file.sh b/tests/queries/0_stateless/02366_cancel_write_into_file.sh new file mode 100755 index 00000000000..d9abc69a414 --- /dev/null +++ b/tests/queries/0_stateless/02366_cancel_write_into_file.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +for i in $(seq 1 10); +do + $CLICKHOUSE_CLIENT --query_id="02366_$i" -q "insert into function file('02366_data_$i.jsonl') select range(number % 1000) from numbers(100000) settings output_format_parallel_formatting=1" 2> /dev/null & +done + +sleep 2 + +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" > /dev/null + +for i in $(seq 1 10); +do + $CLICKHOUSE_CLIENT --query_id="02366_$i" -q "insert into function file('02366_data_$i.jsonl') select range(number % 1000) from numbers(100000) settings output_format_parallel_formatting=0" 2> /dev/null & +done + +sleep 2 + +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" > /dev/null diff --git a/tests/queries/0_stateless/02367_cancel_write_into_s3.reference b/tests/queries/0_stateless/02367_cancel_write_into_s3.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02367_cancel_write_into_s3.sh b/tests/queries/0_stateless/02367_cancel_write_into_s3.sh new file mode 100755 index 00000000000..39316bece5f --- /dev/null +++ b/tests/queries/0_stateless/02367_cancel_write_into_s3.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +for i in $(seq 1 10); +do + $CLICKHOUSE_CLIENT --query_id="02366_$i" -q "insert into function s3('http://localhost:11111/test/02367_data_$i.jsonl') select range(number % 1000) from numbers(100000) settings s3_truncate_on_insert=1, output_format_parallel_formatting=1" 2> /dev/null & +done + +sleep 2 + +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" > /dev/null + +for i in $(seq 1 10); +do + $CLICKHOUSE_CLIENT --query_id="02367_$i" -q "insert into function s3('http://localhost:11111/test/02366_data_$i.jsonl') select range(number % 1000) from numbers(100000) settings s3_truncate_on_inser=1, output_format_parallel_formatting=0" 2> /dev/null & +done + +sleep 2 + +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" > /dev/null diff --git a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.reference b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh new file mode 100755 index 00000000000..70fe1a3e150 --- /dev/null +++ b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +for i in $(seq 1 10); +do + $CLICKHOUSE_CLIENT --query_id="02368_$i" -q "insert into function hdfs('hdfs://localhost:12222/02368_data_$i.jsonl') select range(number % 1000) from numbers(100000) settings hdfs_truncate_on_inser=1, toutput_format_parallel_formatting=1" 2> /dev/null & +done + +sleep 2 + +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" > /dev/null + +for i in $(seq 1 10); +do + $CLICKHOUSE_CLIENT --query_id="02368_$i" -q "insert into function hdfs('hdfs://localhost:12222/02368_data_$i.jsonl') select range(number % 1000) from numbers(100000) settings hdfs_truncate_on_insert=1, output_format_parallel_formatting=0" 2> /dev/null & +done + +sleep 2 + +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" > /dev/null diff --git a/tests/queries/0_stateless/data_minio/02366_data.jsonl b/tests/queries/0_stateless/data_minio/02366_data.jsonl new file mode 100644 index 00000000000..e69de29bb2d From 52b3a87ed1e8b5f6fefb4ce69a7a77ef9a85f5cf Mon Sep 17 00:00:00 2001 From: avogar Date: Thu, 21 Jul 2022 12:38:18 +0000 Subject: [PATCH 02/14] Fix typo --- src/Storages/HDFS/StorageHDFS.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index 04bd58c3a81..5098dfd3ef1 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -437,7 +437,7 @@ public: { std::lock_guard lock(cancel_mutex); finalize(); - cancelled = false; + cancelled = true; } void onException() override From 4e995bce8f99e568b2c31a75a9cd9115ddc22390 Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 22 Jul 2022 11:23:30 +0000 Subject: [PATCH 03/14] Update tests --- .../0_stateless/02366_cancel_write_into_file.reference | 2 ++ tests/queries/0_stateless/02366_cancel_write_into_file.sh | 4 ++-- .../0_stateless/02367_cancel_write_into_s3.reference | 2 ++ tests/queries/0_stateless/02367_cancel_write_into_s3.sh | 8 ++++---- .../0_stateless/02368_cancel_write_into_hdfs.reference | 2 ++ tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh | 6 +++--- 6 files changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/queries/0_stateless/02366_cancel_write_into_file.reference b/tests/queries/0_stateless/02366_cancel_write_into_file.reference index e69de29bb2d..b2f7f08c170 100644 --- a/tests/queries/0_stateless/02366_cancel_write_into_file.reference +++ b/tests/queries/0_stateless/02366_cancel_write_into_file.reference @@ -0,0 +1,2 @@ +10 +10 diff --git a/tests/queries/0_stateless/02366_cancel_write_into_file.sh b/tests/queries/0_stateless/02366_cancel_write_into_file.sh index d9abc69a414..4a6844e43ff 100755 --- a/tests/queries/0_stateless/02366_cancel_write_into_file.sh +++ b/tests/queries/0_stateless/02366_cancel_write_into_file.sh @@ -12,7 +12,7 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" > /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" | grep "finished" -c for i in $(seq 1 10); do @@ -21,4 +21,4 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" > /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" | grep "finished" -c diff --git a/tests/queries/0_stateless/02367_cancel_write_into_s3.reference b/tests/queries/0_stateless/02367_cancel_write_into_s3.reference index e69de29bb2d..b2f7f08c170 100644 --- a/tests/queries/0_stateless/02367_cancel_write_into_s3.reference +++ b/tests/queries/0_stateless/02367_cancel_write_into_s3.reference @@ -0,0 +1,2 @@ +10 +10 diff --git a/tests/queries/0_stateless/02367_cancel_write_into_s3.sh b/tests/queries/0_stateless/02367_cancel_write_into_s3.sh index 39316bece5f..4ddaae203a2 100755 --- a/tests/queries/0_stateless/02367_cancel_write_into_s3.sh +++ b/tests/queries/0_stateless/02367_cancel_write_into_s3.sh @@ -7,18 +7,18 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) for i in $(seq 1 10); do - $CLICKHOUSE_CLIENT --query_id="02366_$i" -q "insert into function s3('http://localhost:11111/test/02367_data_$i.jsonl') select range(number % 1000) from numbers(100000) settings s3_truncate_on_insert=1, output_format_parallel_formatting=1" 2> /dev/null & + $CLICKHOUSE_CLIENT --query_id="02367_$i" -q "insert into function s3('http://localhost:11111/test/02367_data_$i.jsonl') select range(number % 1000) from numbers(100000) settings s3_truncate_on_insert=1, output_format_parallel_formatting=1" 2> /dev/null & done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" > /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" | grep "finished" -c for i in $(seq 1 10); do - $CLICKHOUSE_CLIENT --query_id="02367_$i" -q "insert into function s3('http://localhost:11111/test/02366_data_$i.jsonl') select range(number % 1000) from numbers(100000) settings s3_truncate_on_inser=1, output_format_parallel_formatting=0" 2> /dev/null & + $CLICKHOUSE_CLIENT --query_id="02367_$i" -q "insert into function s3('http://localhost:11111/test/02367_data_$i.jsonl') select range(number % 1000) from numbers(100000) settings s3_truncate_on_insert=1, output_format_parallel_formatting=0" 2> /dev/null & done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" > /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" | grep "finished" -c diff --git a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.reference b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.reference index e69de29bb2d..b2f7f08c170 100644 --- a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.reference +++ b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.reference @@ -0,0 +1,2 @@ +10 +10 diff --git a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh index 70fe1a3e150..70815d339d2 100755 --- a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh +++ b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh @@ -7,12 +7,12 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) for i in $(seq 1 10); do - $CLICKHOUSE_CLIENT --query_id="02368_$i" -q "insert into function hdfs('hdfs://localhost:12222/02368_data_$i.jsonl') select range(number % 1000) from numbers(100000) settings hdfs_truncate_on_inser=1, toutput_format_parallel_formatting=1" 2> /dev/null & + $CLICKHOUSE_CLIENT --query_id="02368_$i" -q "insert into function hdfs('hdfs://localhost:12222/02368_data_$i.jsonl') select range(number % 1000) from numbers(100000) settings hdfs_truncate_on_insert=1, output_format_parallel_formatting=1" 2> /dev/null & done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" > /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" | grep "finished" -c for i in $(seq 1 10); do @@ -21,4 +21,4 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" > /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" | grep "finished" -c From 1e5062c769cdefe3803b2b6b18038ea64439f0f2 Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 22 Jul 2022 12:14:54 +0000 Subject: [PATCH 04/14] Fix fasttest --- tests/queries/0_stateless/02366_cancel_write_into_file.sh | 1 + tests/queries/0_stateless/02367_cancel_write_into_s3.sh | 1 + tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh | 1 + 3 files changed, 3 insertions(+) diff --git a/tests/queries/0_stateless/02366_cancel_write_into_file.sh b/tests/queries/0_stateless/02366_cancel_write_into_file.sh index 4a6844e43ff..47fa3094948 100755 --- a/tests/queries/0_stateless/02366_cancel_write_into_file.sh +++ b/tests/queries/0_stateless/02366_cancel_write_into_file.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/02367_cancel_write_into_s3.sh b/tests/queries/0_stateless/02367_cancel_write_into_s3.sh index 4ddaae203a2..d24462bf584 100755 --- a/tests/queries/0_stateless/02367_cancel_write_into_s3.sh +++ b/tests/queries/0_stateless/02367_cancel_write_into_s3.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh index 70815d339d2..6fda8e9dcff 100755 --- a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh +++ b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-fasttest CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh From 31c5f35e305098ab2a5442571b73777cb4c83985 Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 22 Jul 2022 19:03:23 +0000 Subject: [PATCH 05/14] Update tests --- .../0_stateless/02366_cancel_write_into_file.reference | 2 -- tests/queries/0_stateless/02366_cancel_write_into_file.sh | 5 ++--- .../queries/0_stateless/02367_cancel_write_into_s3.reference | 2 -- tests/queries/0_stateless/02367_cancel_write_into_s3.sh | 4 ++-- .../0_stateless/02368_cancel_write_into_hdfs.reference | 2 -- tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh | 4 ++-- 6 files changed, 6 insertions(+), 13 deletions(-) diff --git a/tests/queries/0_stateless/02366_cancel_write_into_file.reference b/tests/queries/0_stateless/02366_cancel_write_into_file.reference index b2f7f08c170..e69de29bb2d 100644 --- a/tests/queries/0_stateless/02366_cancel_write_into_file.reference +++ b/tests/queries/0_stateless/02366_cancel_write_into_file.reference @@ -1,2 +0,0 @@ -10 -10 diff --git a/tests/queries/0_stateless/02366_cancel_write_into_file.sh b/tests/queries/0_stateless/02366_cancel_write_into_file.sh index 47fa3094948..6f213c5bac4 100755 --- a/tests/queries/0_stateless/02366_cancel_write_into_file.sh +++ b/tests/queries/0_stateless/02366_cancel_write_into_file.sh @@ -13,8 +13,7 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" | grep "finished" -c - +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" > /dev/null for i in $(seq 1 10); do $CLICKHOUSE_CLIENT --query_id="02366_$i" -q "insert into function file('02366_data_$i.jsonl') select range(number % 1000) from numbers(100000) settings output_format_parallel_formatting=0" 2> /dev/null & @@ -22,4 +21,4 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" | grep "finished" -c +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" > /dev/null diff --git a/tests/queries/0_stateless/02367_cancel_write_into_s3.reference b/tests/queries/0_stateless/02367_cancel_write_into_s3.reference index b2f7f08c170..e69de29bb2d 100644 --- a/tests/queries/0_stateless/02367_cancel_write_into_s3.reference +++ b/tests/queries/0_stateless/02367_cancel_write_into_s3.reference @@ -1,2 +0,0 @@ -10 -10 diff --git a/tests/queries/0_stateless/02367_cancel_write_into_s3.sh b/tests/queries/0_stateless/02367_cancel_write_into_s3.sh index d24462bf584..facbd63dda1 100755 --- a/tests/queries/0_stateless/02367_cancel_write_into_s3.sh +++ b/tests/queries/0_stateless/02367_cancel_write_into_s3.sh @@ -13,7 +13,7 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" | grep "finished" -c +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" > /dev/null for i in $(seq 1 10); do @@ -22,4 +22,4 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" | grep "finished" -c +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" > /dev/null diff --git a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.reference b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.reference index b2f7f08c170..e69de29bb2d 100644 --- a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.reference +++ b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.reference @@ -1,2 +0,0 @@ -10 -10 diff --git a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh index 6fda8e9dcff..9a56663a1b6 100755 --- a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh +++ b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh @@ -13,7 +13,7 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" | grep "finished" -c +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" > /dev/null for i in $(seq 1 10); do @@ -22,4 +22,4 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" | grep "finished" -c +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" > /dev/null From 0bf0ac6ee9b1d0498bdc7219196dc09d4fed4769 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 25 Jul 2022 13:33:57 +0000 Subject: [PATCH 06/14] Update tests --- tests/queries/0_stateless/02366_cancel_write_into_file.sh | 8 ++++++-- tests/queries/0_stateless/02367_cancel_write_into_s3.sh | 2 ++ tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh | 2 ++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02366_cancel_write_into_file.sh b/tests/queries/0_stateless/02366_cancel_write_into_file.sh index 6f213c5bac4..9771c5a8744 100755 --- a/tests/queries/0_stateless/02366_cancel_write_into_file.sh +++ b/tests/queries/0_stateless/02366_cancel_write_into_file.sh @@ -13,7 +13,9 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" > /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" 2>&1 /dev/null +$CLICKHOUSE_CLIENT -q "select demangle(addressToSymbol(arrayJoin(trace))) from system.stack_trace where startsWith(query_id, '02366_')" 2>&1 /dev/null + for i in $(seq 1 10); do $CLICKHOUSE_CLIENT --query_id="02366_$i" -q "insert into function file('02366_data_$i.jsonl') select range(number % 1000) from numbers(100000) settings output_format_parallel_formatting=0" 2> /dev/null & @@ -21,4 +23,6 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" > /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" 2>&1 /dev/null +$CLICKHOUSE_CLIENT -q "select demangle(addressToSymbol(arrayJoin(trace))) from system.stack_trace where startsWith(query_id, '02366_')" 2>&1 /dev/null + diff --git a/tests/queries/0_stateless/02367_cancel_write_into_s3.sh b/tests/queries/0_stateless/02367_cancel_write_into_s3.sh index facbd63dda1..a7eddfe0df1 100755 --- a/tests/queries/0_stateless/02367_cancel_write_into_s3.sh +++ b/tests/queries/0_stateless/02367_cancel_write_into_s3.sh @@ -14,6 +14,7 @@ done sleep 2 $CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" > /dev/null +$CLICKHOUSE_CLIENT -q "select demangle(addressToSymbol(arrayJoin(trace))) from system.stack_trace where startsWith(query_id, '02367_')" 2>&1 /dev/null for i in $(seq 1 10); do @@ -23,3 +24,4 @@ done sleep 2 $CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" > /dev/null +$CLICKHOUSE_CLIENT -q "select demangle(addressToSymbol(arrayJoin(trace))) from system.stack_trace where startsWith(query_id, '02367_')" 2>&1 /dev/null diff --git a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh index 9a56663a1b6..9704e7e1451 100755 --- a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh +++ b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh @@ -14,6 +14,7 @@ done sleep 2 $CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" > /dev/null +$CLICKHOUSE_CLIENT -q "select demangle(addressToSymbol(arrayJoin(trace))) from system.stack_trace where startsWith(query_id, '02368_')" 2>&1 /dev/null for i in $(seq 1 10); do @@ -23,3 +24,4 @@ done sleep 2 $CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" > /dev/null +$CLICKHOUSE_CLIENT -q "select demangle(addressToSymbol(arrayJoin(trace))) from system.stack_trace where startsWith(query_id, '02368_')" 2>&1 /dev/null From c6131e93365cfc698d1e60e8542d786d956219b0 Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 25 Jul 2022 15:05:12 +0000 Subject: [PATCH 07/14] Update tests --- tests/queries/0_stateless/02366_cancel_write_into_file.sh | 2 -- tests/queries/0_stateless/02367_cancel_write_into_s3.sh | 6 ++---- tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh | 8 +++----- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/tests/queries/0_stateless/02366_cancel_write_into_file.sh b/tests/queries/0_stateless/02366_cancel_write_into_file.sh index 9771c5a8744..4f999c95690 100755 --- a/tests/queries/0_stateless/02366_cancel_write_into_file.sh +++ b/tests/queries/0_stateless/02366_cancel_write_into_file.sh @@ -14,7 +14,6 @@ done sleep 2 $CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" 2>&1 /dev/null -$CLICKHOUSE_CLIENT -q "select demangle(addressToSymbol(arrayJoin(trace))) from system.stack_trace where startsWith(query_id, '02366_')" 2>&1 /dev/null for i in $(seq 1 10); do @@ -24,5 +23,4 @@ done sleep 2 $CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" 2>&1 /dev/null -$CLICKHOUSE_CLIENT -q "select demangle(addressToSymbol(arrayJoin(trace))) from system.stack_trace where startsWith(query_id, '02366_')" 2>&1 /dev/null diff --git a/tests/queries/0_stateless/02367_cancel_write_into_s3.sh b/tests/queries/0_stateless/02367_cancel_write_into_s3.sh index a7eddfe0df1..ab9f4eaaa69 100755 --- a/tests/queries/0_stateless/02367_cancel_write_into_s3.sh +++ b/tests/queries/0_stateless/02367_cancel_write_into_s3.sh @@ -13,8 +13,7 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" > /dev/null -$CLICKHOUSE_CLIENT -q "select demangle(addressToSymbol(arrayJoin(trace))) from system.stack_trace where startsWith(query_id, '02367_')" 2>&1 /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" 2>&1 /dev/null for i in $(seq 1 10); do @@ -23,5 +22,4 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" > /dev/null -$CLICKHOUSE_CLIENT -q "select demangle(addressToSymbol(arrayJoin(trace))) from system.stack_trace where startsWith(query_id, '02367_')" 2>&1 /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" 2>&1 /dev/null diff --git a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh index 9704e7e1451..ac0d6d35de5 100755 --- a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh +++ b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-fasttest +# Tags: no-fasttest, no-stress CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -13,8 +13,7 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" > /dev/null -$CLICKHOUSE_CLIENT -q "select demangle(addressToSymbol(arrayJoin(trace))) from system.stack_trace where startsWith(query_id, '02368_')" 2>&1 /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" 2>&1 /dev/null for i in $(seq 1 10); do @@ -23,5 +22,4 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" > /dev/null -$CLICKHOUSE_CLIENT -q "select demangle(addressToSymbol(arrayJoin(trace))) from system.stack_trace where startsWith(query_id, '02368_')" 2>&1 /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" 2>&1 /dev/null From 9553b8964ca5d7dfe5e18f19760e27c53a98600a Mon Sep 17 00:00:00 2001 From: avogar Date: Wed, 27 Jul 2022 09:39:57 +0000 Subject: [PATCH 08/14] Fix tests --- tests/queries/0_stateless/02366_cancel_write_into_file.sh | 4 ++-- tests/queries/0_stateless/02367_cancel_write_into_s3.sh | 4 ++-- tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh | 4 ++-- tests/queries/0_stateless/test_ugtxj2/tuples | 0 4 files changed, 6 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/test_ugtxj2/tuples diff --git a/tests/queries/0_stateless/02366_cancel_write_into_file.sh b/tests/queries/0_stateless/02366_cancel_write_into_file.sh index 4f999c95690..f7aca608aab 100755 --- a/tests/queries/0_stateless/02366_cancel_write_into_file.sh +++ b/tests/queries/0_stateless/02366_cancel_write_into_file.sh @@ -13,7 +13,7 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" 2>&1 /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" 2>&1 > /dev/null for i in $(seq 1 10); do @@ -22,5 +22,5 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" 2>&1 /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" 2>&1 > /dev/null diff --git a/tests/queries/0_stateless/02367_cancel_write_into_s3.sh b/tests/queries/0_stateless/02367_cancel_write_into_s3.sh index ab9f4eaaa69..bea990e32e3 100755 --- a/tests/queries/0_stateless/02367_cancel_write_into_s3.sh +++ b/tests/queries/0_stateless/02367_cancel_write_into_s3.sh @@ -13,7 +13,7 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" 2>&1 /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" 2>&1 > /dev/null for i in $(seq 1 10); do @@ -22,4 +22,4 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" 2>&1 /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" 2>&1 > /dev/null diff --git a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh index ac0d6d35de5..405f059b52e 100755 --- a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh +++ b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh @@ -13,7 +13,7 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" 2>&1 /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" 2>&1 > /dev/null for i in $(seq 1 10); do @@ -22,4 +22,4 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" 2>&1 /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" 2>&1 > /dev/null diff --git a/tests/queries/0_stateless/test_ugtxj2/tuples b/tests/queries/0_stateless/test_ugtxj2/tuples new file mode 100644 index 00000000000..e69de29bb2d From 573d9c1b57037e9fc2dac5824bbd64f53a2352b4 Mon Sep 17 00:00:00 2001 From: avogar Date: Fri, 29 Jul 2022 18:30:40 +0000 Subject: [PATCH 09/14] Fix stress test --- docker/test/stress/stress | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/stress/stress b/docker/test/stress/stress index 6d90b9d5437..64cca4beb3a 100755 --- a/docker/test/stress/stress +++ b/docker/test/stress/stress @@ -77,7 +77,7 @@ def run_func_test( pipes = [] for i in range(0, len(output_paths)): f = open(output_paths[i], "w") - full_command = "{} {} {} {} {}".format( + full_command = "{} {} {} {} {} --stress".format( cmd, get_options(i, backward_compatibility_check), global_time_limit_option, From 74f87a95c707f4c28c046511b6da08f781b325ab Mon Sep 17 00:00:00 2001 From: avogar Date: Mon, 1 Aug 2022 10:57:55 +0000 Subject: [PATCH 10/14] Fis style --- tests/queries/0_stateless/02366_cancel_write_into_file.sh | 4 ++-- tests/queries/0_stateless/02367_cancel_write_into_s3.sh | 4 ++-- tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/queries/0_stateless/02366_cancel_write_into_file.sh b/tests/queries/0_stateless/02366_cancel_write_into_file.sh index f7aca608aab..060fe1ec74b 100755 --- a/tests/queries/0_stateless/02366_cancel_write_into_file.sh +++ b/tests/queries/0_stateless/02366_cancel_write_into_file.sh @@ -13,7 +13,7 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" 2>&1 > /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" > /dev/null 2>&1 for i in $(seq 1 10); do @@ -22,5 +22,5 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" 2>&1 > /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02366_') sync" > /dev/null 2>&1 diff --git a/tests/queries/0_stateless/02367_cancel_write_into_s3.sh b/tests/queries/0_stateless/02367_cancel_write_into_s3.sh index bea990e32e3..54f6d2a941b 100755 --- a/tests/queries/0_stateless/02367_cancel_write_into_s3.sh +++ b/tests/queries/0_stateless/02367_cancel_write_into_s3.sh @@ -13,7 +13,7 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" 2>&1 > /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" > /dev/null 2>&1 for i in $(seq 1 10); do @@ -22,4 +22,4 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" 2>&1 > /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" > /dev/null 2>&1 diff --git a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh index 405f059b52e..8262cd7eab5 100755 --- a/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh +++ b/tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh @@ -13,7 +13,7 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" 2>&1 > /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" > /dev/null 2>&1 for i in $(seq 1 10); do @@ -22,4 +22,4 @@ done sleep 2 -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" 2>&1 > /dev/null +$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" > /dev/null 2>&1 From c5eab9c760c8f33752978f4f90bd33f65a60604c Mon Sep 17 00:00:00 2001 From: avogar Date: Tue, 2 Aug 2022 08:38:15 +0000 Subject: [PATCH 11/14] Delete test for s3 --- .../02367_cancel_write_into_s3.reference | 0 .../0_stateless/02367_cancel_write_into_s3.sh | 25 ------------------- 2 files changed, 25 deletions(-) delete mode 100644 tests/queries/0_stateless/02367_cancel_write_into_s3.reference delete mode 100755 tests/queries/0_stateless/02367_cancel_write_into_s3.sh diff --git a/tests/queries/0_stateless/02367_cancel_write_into_s3.reference b/tests/queries/0_stateless/02367_cancel_write_into_s3.reference deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/queries/0_stateless/02367_cancel_write_into_s3.sh b/tests/queries/0_stateless/02367_cancel_write_into_s3.sh deleted file mode 100755 index 54f6d2a941b..00000000000 --- a/tests/queries/0_stateless/02367_cancel_write_into_s3.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# Tags: no-fasttest - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -# shellcheck source=../shell_config.sh -. "$CURDIR"/../shell_config.sh - - -for i in $(seq 1 10); -do - $CLICKHOUSE_CLIENT --query_id="02367_$i" -q "insert into function s3('http://localhost:11111/test/02367_data_$i.jsonl') select range(number % 1000) from numbers(100000) settings s3_truncate_on_insert=1, output_format_parallel_formatting=1" 2> /dev/null & -done - -sleep 2 - -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" > /dev/null 2>&1 - -for i in $(seq 1 10); -do - $CLICKHOUSE_CLIENT --query_id="02367_$i" -q "insert into function s3('http://localhost:11111/test/02367_data_$i.jsonl') select range(number % 1000) from numbers(100000) settings s3_truncate_on_insert=1, output_format_parallel_formatting=0" 2> /dev/null & -done - -sleep 2 - -$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02367_') sync" > /dev/null 2>&1 From 0b82fb78164ea08d5ee4e8f89055bc7285a82b52 Mon Sep 17 00:00:00 2001 From: Nityananda Gohain Date: Wed, 3 Aug 2022 12:52:06 +0530 Subject: [PATCH 12/14] Extra semicolon removed from the TTL example This PR removes an extra semicolon from the TTL example. --- docs/en/sql-reference/statements/alter/ttl.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/alter/ttl.md b/docs/en/sql-reference/statements/alter/ttl.md index 37a171d4969..2682279d1f7 100644 --- a/docs/en/sql-reference/statements/alter/ttl.md +++ b/docs/en/sql-reference/statements/alter/ttl.md @@ -34,7 +34,7 @@ CREATE TABLE table_with_ttl ) ENGINE MergeTree() ORDER BY tuple() -TTL event_time + INTERVAL 3 MONTH; +TTL event_time + INTERVAL 3 MONTH SETTINGS min_bytes_for_wide_part = 0; INSERT INTO table_with_ttl VALUES (now(), 1, 'username1'); From 56a4d26e87b5100f59b54a4cf2bccbcb15fdcebd Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 3 Aug 2022 14:15:45 +0200 Subject: [PATCH 13/14] Better total part size calculation on mutation --- src/Storages/MergeTree/IMergeTreeDataPart.cpp | 3 +-- src/Storages/MergeTree/MutateTask.cpp | 11 ++++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 60941108f00..f1bf78409c7 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -946,7 +946,6 @@ void IMergeTreeDataPart::appendFilesOfPartitionAndMinMaxIndex(Strings & files) c void IMergeTreeDataPart::loadChecksums(bool require) { - //const String path = fs::path(getRelativePath()) / "checksums.txt"; bool exists = metadata_manager->exists("checksums.txt"); if (exists) { @@ -957,7 +956,7 @@ void IMergeTreeDataPart::loadChecksums(bool require) bytes_on_disk = checksums.getTotalSizeOnDisk(); } else - bytes_on_disk = data_part_storage->calculateTotalSizeOnDisk(); //calculateTotalSizeOnDisk(volume->getDisk(), getRelativePath()); + bytes_on_disk = data_part_storage->calculateTotalSizeOnDisk(); } else { diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 3a5aa2f8860..53e437ae7dc 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -589,7 +589,7 @@ void finalizeMutatedPart( { auto out = data_part_storage_builder->writeFile(IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME, 4096, context->getWriteSettings()); DB::writeText(queryToString(codec->getFullCodecDesc()), *out); - } + } /// close fd { /// Write a file with a description of columns. @@ -603,9 +603,14 @@ void finalizeMutatedPart( new_data_part->minmax_idx = source_part->minmax_idx; new_data_part->modification_time = time(nullptr); new_data_part->loadProjections(false, false); - new_data_part->setBytesOnDisk(new_data_part->data_part_storage->calculateTotalSizeOnDisk()); - new_data_part->default_codec = codec; + + /// All information about sizes is stored in checksums. + /// It doesn't make sense to touch filesystem for sizes. + new_data_part->setBytesOnDisk(new_data_part->checksums->getTotalSizeOnDisk()); + /// Also use information from checksums new_data_part->calculateColumnsAndSecondaryIndicesSizesOnDisk(); + + new_data_part->default_codec = codec; } } From f608e62c27bd5c830b87d646758081def912224f Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 3 Aug 2022 14:17:31 +0200 Subject: [PATCH 14/14] Fix call --- src/Storages/MergeTree/MutateTask.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 53e437ae7dc..bf1ee037257 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -606,7 +606,7 @@ void finalizeMutatedPart( /// All information about sizes is stored in checksums. /// It doesn't make sense to touch filesystem for sizes. - new_data_part->setBytesOnDisk(new_data_part->checksums->getTotalSizeOnDisk()); + new_data_part->setBytesOnDisk(new_data_part->checksums.getTotalSizeOnDisk()); /// Also use information from checksums new_data_part->calculateColumnsAndSecondaryIndicesSizesOnDisk();