From 40c774f3876dc39988537319a7b38d918a6d2b1a Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 5 Apr 2023 13:04:53 +0000 Subject: [PATCH 1/3] Enable temporary_data_in_cache in s3 tests in CI --- tests/config/config.d/s3_storage_policy_by_default.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/config/config.d/s3_storage_policy_by_default.xml b/tests/config/config.d/s3_storage_policy_by_default.xml index dd93a317a77..4e3d9636daf 100644 --- a/tests/config/config.d/s3_storage_policy_by_default.xml +++ b/tests/config/config.d/s3_storage_policy_by_default.xml @@ -25,4 +25,5 @@ s3 + cached_s3 From 6a70bfcdb76240e16d61b2f95b1bdf9a7c0ea38c Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 21 Apr 2023 10:51:42 +0000 Subject: [PATCH 2/3] Print profile events in 02402_external_disk_mertrics --- .../02402_external_disk_mertrics.reference | 6 +- .../02402_external_disk_mertrics.sql | 60 +++++++++++-------- 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/tests/queries/0_stateless/02402_external_disk_mertrics.reference b/tests/queries/0_stateless/02402_external_disk_mertrics.reference index e8183f05f5d..7614df8ec46 100644 --- a/tests/queries/0_stateless/02402_external_disk_mertrics.reference +++ b/tests/queries/0_stateless/02402_external_disk_mertrics.reference @@ -1,3 +1,3 @@ -1 -1 -1 +ok +ok +ok diff --git a/tests/queries/0_stateless/02402_external_disk_mertrics.sql b/tests/queries/0_stateless/02402_external_disk_mertrics.sql index e9696eb7122..7237ea19775 100644 --- a/tests/queries/0_stateless/02402_external_disk_mertrics.sql +++ b/tests/queries/0_stateless/02402_external_disk_mertrics.sql @@ -31,40 +31,52 @@ FORMAT Null; SYSTEM FLUSH LOGS; SELECT - any(ProfileEvents['ExternalProcessingFilesTotal']) >= 1 AND - any(ProfileEvents['ExternalProcessingCompressedBytesTotal']) >= 100000 AND - any(ProfileEvents['ExternalProcessingUncompressedBytesTotal']) >= 100000 AND - any(ProfileEvents['ExternalSortWritePart']) >= 1 AND - any(ProfileEvents['ExternalSortMerge']) >= 1 AND - any(ProfileEvents['ExternalSortCompressedBytes']) >= 100000 AND - any(ProfileEvents['ExternalSortUncompressedBytes']) >= 100000 AND - count() == 1 + if( + any(ProfileEvents['ExternalProcessingFilesTotal']) >= 1 AND + any(ProfileEvents['ExternalProcessingCompressedBytesTotal']) >= 100000 AND + any(ProfileEvents['ExternalProcessingUncompressedBytesTotal']) >= 100000 AND + any(ProfileEvents['ExternalSortWritePart']) >= 1 AND + any(ProfileEvents['ExternalSortMerge']) >= 1 AND + any(ProfileEvents['ExternalSortCompressedBytes']) >= 100000 AND + any(ProfileEvents['ExternalSortUncompressedBytes']) >= 100000 AND + count() == 1, + 'ok', + 'fail: ' || toString(count()) || ' ' || toString(any(ProfileEvents)) + ) FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '02402_external_disk_mertrics/sort' AND query ILIKE 'SELECT%2097152%' AND type = 'QueryFinish'; SELECT - any(ProfileEvents['ExternalProcessingFilesTotal']) >= 1 AND - any(ProfileEvents['ExternalProcessingCompressedBytesTotal']) >= 100000 AND - any(ProfileEvents['ExternalProcessingUncompressedBytesTotal']) >= 100000 AND - any(ProfileEvents['ExternalAggregationWritePart']) >= 1 AND - any(ProfileEvents['ExternalAggregationMerge']) >= 1 AND - any(ProfileEvents['ExternalAggregationCompressedBytes']) >= 100000 AND - any(ProfileEvents['ExternalAggregationUncompressedBytes']) >= 100000 AND - count() == 1 + if( + any(ProfileEvents['ExternalProcessingFilesTotal']) >= 1 AND + any(ProfileEvents['ExternalProcessingCompressedBytesTotal']) >= 100000 AND + any(ProfileEvents['ExternalProcessingUncompressedBytesTotal']) >= 100000 AND + any(ProfileEvents['ExternalAggregationWritePart']) >= 1 AND + any(ProfileEvents['ExternalAggregationMerge']) >= 1 AND + any(ProfileEvents['ExternalAggregationCompressedBytes']) >= 100000 AND + any(ProfileEvents['ExternalAggregationUncompressedBytes']) >= 100000 AND + count() == 1, + 'ok', + 'fail: ' || toString(count()) || ' ' || toString(any(ProfileEvents)) + ) FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '02402_external_disk_mertrics/aggregation' AND query ILIKE 'SELECT%2097152%' AND type = 'QueryFinish'; SELECT - any(ProfileEvents['ExternalProcessingFilesTotal']) >= 1 AND - any(ProfileEvents['ExternalProcessingCompressedBytesTotal']) >= 100000 AND - any(ProfileEvents['ExternalProcessingUncompressedBytesTotal']) >= 100000 AND - any(ProfileEvents['ExternalJoinWritePart']) >= 1 AND - any(ProfileEvents['ExternalJoinMerge']) >= 0 AND - any(ProfileEvents['ExternalJoinCompressedBytes']) >= 100000 AND - any(ProfileEvents['ExternalJoinUncompressedBytes']) >= 100000 AND - count() == 1 + if( + any(ProfileEvents['ExternalProcessingFilesTotal']) >= 1 AND + any(ProfileEvents['ExternalProcessingCompressedBytesTotal']) >= 100000 AND + any(ProfileEvents['ExternalProcessingUncompressedBytesTotal']) >= 100000 AND + any(ProfileEvents['ExternalJoinWritePart']) >= 1 AND + any(ProfileEvents['ExternalJoinMerge']) >= 0 AND + any(ProfileEvents['ExternalJoinCompressedBytes']) >= 100000 AND + any(ProfileEvents['ExternalJoinUncompressedBytes']) >= 100000 AND + count() == 1, + 'ok', + 'fail: ' || toString(count()) || ' ' || toString(any(ProfileEvents)) + ) FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '02402_external_disk_mertrics/join' AND query ILIKE 'SELECT%2097152%' AND type = 'QueryFinish'; From ed855cf51350cdd0651fcceb1a13467938f4d30a Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 27 Nov 2023 19:52:17 +0000 Subject: [PATCH 3/3] Fix 02402_external_disk_mertrics with tmp data in cache --- src/Interpreters/TemporaryDataOnDisk.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/TemporaryDataOnDisk.cpp b/src/Interpreters/TemporaryDataOnDisk.cpp index a424ed0ce4e..6589a974087 100644 --- a/src/Interpreters/TemporaryDataOnDisk.cpp +++ b/src/Interpreters/TemporaryDataOnDisk.cpp @@ -14,6 +14,11 @@ #include #include +namespace ProfileEvents +{ + extern const Event ExternalProcessingFilesTotal; +} + namespace DB { @@ -97,6 +102,8 @@ FileSegmentsHolderPtr TemporaryDataOnDisk::createCacheFile(size_t max_file_size) if (!file_cache) throw Exception(ErrorCodes::LOGICAL_ERROR, "TemporaryDataOnDiskScope has no cache"); + ProfileEvents::increment(ProfileEvents::ExternalProcessingFilesTotal); + const auto key = FileSegment::Key::random(); auto holder = file_cache->set(key, 0, std::max(10_MiB, max_file_size), CreateFileSegmentSettings(FileSegmentKind::Temporary, /* unbounded */ true)); fs::create_directories(file_cache->getPathInLocalCache(key)); @@ -120,7 +127,7 @@ TemporaryFileOnDiskHolder TemporaryDataOnDisk::createRegularFile(size_t max_file { disk = volume->getDisk(); } - + /// We do not increment ProfileEvents::ExternalProcessingFilesTotal here because it is incremented in TemporaryFileOnDisk constructor. return std::make_unique(disk, current_metric_scope); }