thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 07:31:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #48425 from ClickHouse/vdimir/enable-tmp-in-cache-ci

Browse Source 
Enable temporary_data_in_cache in s3 tests in CI
This commit is contained in:
Kseniia Sumarokova 2023-11-29 11:59:32 +01:00 committed by GitHub
commit 4d27f87be8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 48 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/Interpreters/TemporaryDataOnDisk.cpp
View File

@ -14,6 +14,11 @@
#include <Core/Defines.h>
#include <Interpreters/Cache/WriteBufferToFileSegment.h>
namespace ProfileEvents
{
extern const Event ExternalProcessingFilesTotal;
}
namespace DB
{
@ -97,6 +102,8 @@ FileSegmentsHolderPtr TemporaryDataOnDisk::createCacheFile(size_t max_file_size)
if (!file_cache)
throw Exception(ErrorCodes::LOGICAL_ERROR, "TemporaryDataOnDiskScope has no cache");
ProfileEvents::increment(ProfileEvents::ExternalProcessingFilesTotal);
const auto key = FileSegment::Key::random();
auto holder = file_cache->set(key, 0, std::max(10_MiB, max_file_size), CreateFileSegmentSettings(FileSegmentKind::Temporary, /* unbounded */ true));
fs::create_directories(file_cache->getPathInLocalCache(key));
@ -120,7 +127,7 @@ TemporaryFileOnDiskHolder TemporaryDataOnDisk::createRegularFile(size_t max_file
{
disk = volume->getDisk();
}
/// We do not increment ProfileEvents::ExternalProcessingFilesTotal here because it is incremented in TemporaryFileOnDisk constructor.
return std::make_unique<TemporaryFileOnDisk>(disk, current_metric_scope);
}

1
tests/config/config.d/s3_storage_policy_by_default.xml
View File

@ -25,4 +25,5 @@
<merge_tree>
<storage_policy>s3</storage_policy>
</merge_tree>
<temporary_data_in_cache>cached_s3</temporary_data_in_cache>
</clickhouse>

6
tests/queries/0_stateless/02402_external_disk_mertrics.reference
View File

@ -1,3 +1,3 @@
1
1
1
ok
ok
ok

60
tests/queries/0_stateless/02402_external_disk_mertrics.sql
View File

@ -31,40 +31,52 @@ FORMAT Null;
SYSTEM FLUSH LOGS;
SELECT
any(ProfileEvents['ExternalProcessingFilesTotal']) >= 1 AND
any(ProfileEvents['ExternalProcessingCompressedBytesTotal']) >= 100000 AND
any(ProfileEvents['ExternalProcessingUncompressedBytesTotal']) >= 100000 AND
any(ProfileEvents['ExternalSortWritePart']) >= 1 AND
any(ProfileEvents['ExternalSortMerge']) >= 1 AND
any(ProfileEvents['ExternalSortCompressedBytes']) >= 100000 AND
any(ProfileEvents['ExternalSortUncompressedBytes']) >= 100000 AND
count() == 1
if(
any(ProfileEvents['ExternalProcessingFilesTotal']) >= 1 AND
any(ProfileEvents['ExternalProcessingCompressedBytesTotal']) >= 100000 AND
any(ProfileEvents['ExternalProcessingUncompressedBytesTotal']) >= 100000 AND
any(ProfileEvents['ExternalSortWritePart']) >= 1 AND
any(ProfileEvents['ExternalSortMerge']) >= 1 AND
any(ProfileEvents['ExternalSortCompressedBytes']) >= 100000 AND
any(ProfileEvents['ExternalSortUncompressedBytes']) >= 100000 AND
count() == 1,
'ok',
'fail: ' || toString(count()) || ' ' || toString(any(ProfileEvents))
)
FROM system.query_log WHERE current_database = currentDatabase()
AND log_comment = '02402_external_disk_mertrics/sort'
AND query ILIKE 'SELECT%2097152%' AND type = 'QueryFinish';
SELECT
any(ProfileEvents['ExternalProcessingFilesTotal']) >= 1 AND
any(ProfileEvents['ExternalProcessingCompressedBytesTotal']) >= 100000 AND
any(ProfileEvents['ExternalProcessingUncompressedBytesTotal']) >= 100000 AND
any(ProfileEvents['ExternalAggregationWritePart']) >= 1 AND
any(ProfileEvents['ExternalAggregationMerge']) >= 1 AND
any(ProfileEvents['ExternalAggregationCompressedBytes']) >= 100000 AND
any(ProfileEvents['ExternalAggregationUncompressedBytes']) >= 100000 AND
count() == 1
if(
any(ProfileEvents['ExternalProcessingFilesTotal']) >= 1 AND
any(ProfileEvents['ExternalProcessingCompressedBytesTotal']) >= 100000 AND
any(ProfileEvents['ExternalProcessingUncompressedBytesTotal']) >= 100000 AND
any(ProfileEvents['ExternalAggregationWritePart']) >= 1 AND
any(ProfileEvents['ExternalAggregationMerge']) >= 1 AND
any(ProfileEvents['ExternalAggregationCompressedBytes']) >= 100000 AND
any(ProfileEvents['ExternalAggregationUncompressedBytes']) >= 100000 AND
count() == 1,
'ok',
'fail: ' || toString(count()) || ' ' || toString(any(ProfileEvents))
)
FROM system.query_log WHERE current_database = currentDatabase()
AND log_comment = '02402_external_disk_mertrics/aggregation'
AND query ILIKE 'SELECT%2097152%' AND type = 'QueryFinish';
SELECT
any(ProfileEvents['ExternalProcessingFilesTotal']) >= 1 AND
any(ProfileEvents['ExternalProcessingCompressedBytesTotal']) >= 100000 AND
any(ProfileEvents['ExternalProcessingUncompressedBytesTotal']) >= 100000 AND
any(ProfileEvents['ExternalJoinWritePart']) >= 1 AND
any(ProfileEvents['ExternalJoinMerge']) >= 0 AND
any(ProfileEvents['ExternalJoinCompressedBytes']) >= 100000 AND
any(ProfileEvents['ExternalJoinUncompressedBytes']) >= 100000 AND
count() == 1
if(
any(ProfileEvents['ExternalProcessingFilesTotal']) >= 1 AND
any(ProfileEvents['ExternalProcessingCompressedBytesTotal']) >= 100000 AND
any(ProfileEvents['ExternalProcessingUncompressedBytesTotal']) >= 100000 AND
any(ProfileEvents['ExternalJoinWritePart']) >= 1 AND
any(ProfileEvents['ExternalJoinMerge']) >= 0 AND
any(ProfileEvents['ExternalJoinCompressedBytes']) >= 100000 AND
any(ProfileEvents['ExternalJoinUncompressedBytes']) >= 100000 AND
count() == 1,
'ok',
'fail: ' || toString(count()) || ' ' || toString(any(ProfileEvents))
)
FROM system.query_log WHERE current_database = currentDatabase()
AND log_comment = '02402_external_disk_mertrics/join'
AND query ILIKE 'SELECT%2097152%' AND type = 'QueryFinish';