mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-10 09:32:06 +00:00
Improve test
This commit is contained in:
parent
34c0690a69
commit
8c1be8950e
@ -303,7 +303,7 @@ MergeTreeData::MergeTreeData(
|
||||
format_version = min_format_version;
|
||||
if (!version_file.second->isReadOnly())
|
||||
{
|
||||
auto buf = version_file.second->writeFile(version_file.first);
|
||||
auto buf = version_file.second->writeFile(version_file.first, DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite, context_->getWriteSettings());
|
||||
writeIntText(format_version.toUnderType(), *buf);
|
||||
if (getContext()->getSettingsRef().fsync_metadata)
|
||||
buf->sync();
|
||||
@ -3699,9 +3699,9 @@ RestoreDataTasks MergeTreeData::restoreDataPartsFromBackup(const BackupPtr & bac
|
||||
continue;
|
||||
|
||||
UInt64 total_size_of_part = 0;
|
||||
Strings filenames = backup->listFiles(data_path_in_backup + part_name + "/", "");
|
||||
Strings filenames = backup->listFiles(fs::path(data_path_in_backup) / part_name / "", "");
|
||||
for (const String & filename : filenames)
|
||||
total_size_of_part += backup->getFileSize(data_path_in_backup + part_name + "/" + filename);
|
||||
total_size_of_part += backup->getFileSize(fs::path(data_path_in_backup) / part_name / filename);
|
||||
|
||||
std::shared_ptr<IReservation> reservation = getStoragePolicy()->reserveAndCheck(total_size_of_part);
|
||||
|
||||
@ -3725,9 +3725,9 @@ RestoreDataTasks MergeTreeData::restoreDataPartsFromBackup(const BackupPtr & bac
|
||||
|
||||
for (const String & filename : filenames)
|
||||
{
|
||||
auto backup_entry = backup->readFile(data_path_in_backup + part_name + "/" + filename);
|
||||
auto backup_entry = backup->readFile(fs::path(data_path_in_backup) / part_name / filename);
|
||||
auto read_buffer = backup_entry->getReadBuffer();
|
||||
auto write_buffer = disk->writeFile(temp_part_dir + "/" + filename);
|
||||
auto write_buffer = disk->writeFile(fs::path(temp_part_dir) / filename);
|
||||
copyData(*read_buffer, *write_buffer);
|
||||
}
|
||||
|
||||
|
@ -1613,7 +1613,7 @@ CheckResults StorageMergeTree::checkData(const ASTPtr & query, ContextPtr local_
|
||||
{
|
||||
auto calculated_checksums = checkDataPart(part, false);
|
||||
calculated_checksums.checkEqual(part->checksums, true);
|
||||
auto out = disk->writeFile(tmp_checksums_path, 4096);
|
||||
auto out = disk->writeFile(tmp_checksums_path, 4096, WriteMode::Rewrite, local_context->getWriteSettings());
|
||||
part->checksums.write(*out);
|
||||
disk->moveFile(tmp_checksums_path, checksums_path);
|
||||
results.emplace_back(part->name, true, "Checksums recounted and written to disk.");
|
||||
|
@ -7,7 +7,9 @@
|
||||
<access_key_id>clickhouse</access_key_id>
|
||||
<secret_access_key>clickhouse</secret_access_key>
|
||||
<data_cache_enabled>1</data_cache_enabled>
|
||||
<cache_enabled>0</cache_enabled>
|
||||
<data_cache_max_size>22548578304</data_cache_max_size>
|
||||
<cache_on_insert>1<cache_on_insert>
|
||||
</s3_cache>
|
||||
</disks>
|
||||
<policies>
|
||||
|
@ -2,44 +2,48 @@
|
||||
|
||||
SET remote_fs_cache_on_insert=1;
|
||||
DROP TABLE IF EXISTS test;
|
||||
CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache';
|
||||
CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3';
|
||||
SYSTEM DROP REMOTE FILESYSTEM CACHE;
|
||||
SELECT file_segment_range, size, state FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.remote_filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path FORMAT Vertical;
|
||||
SELECT file_segment_range, size, state FROM system.remote_filesystem_cache format Vertical;
|
||||
SELECT file_segment_range, size, state FROM (SELECT file_segment_range, size, state, local_path FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.remote_filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path) WHERE endsWith(local_path, 'data.bin') FORMAT Vertical;
|
||||
SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.remote_filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path;
|
||||
0
|
||||
SELECT count() FROM system.remote_filesystem_cache;
|
||||
0
|
||||
INSERT INTO test SELECT number, toString(number) FROM numbers(100);
|
||||
SELECT file_segment_range, size, state FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.remote_filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path FORMAT Vertical;
|
||||
SELECT file_segment_range, size, state FROM (SELECT file_segment_range, size, state, local_path FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.remote_filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path) WHERE endsWith(local_path, 'data.bin') FORMAT Vertical;
|
||||
Row 1:
|
||||
──────
|
||||
file_segment_range: (0,745)
|
||||
size: 746
|
||||
state: DOWNLOADED
|
||||
SELECT file_segment_range, size, state FROM system.remote_filesystem_cache format Vertical;
|
||||
Row 1:
|
||||
──────
|
||||
file_segment_range: (0,745)
|
||||
size: 746
|
||||
state: DOWNLOADED
|
||||
SELECT cache_hits FROM system.remote_filesystem_cache;
|
||||
SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.remote_filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path;
|
||||
7
|
||||
SELECT count() FROM system.remote_filesystem_cache;
|
||||
7
|
||||
SELECT count() FROM system.remote_filesystem_cache WHERE cache_hits > 0;
|
||||
0
|
||||
SELECT * FROM test FORMAT Null;
|
||||
SELECT cache_hits FROM system.remote_filesystem_cache;
|
||||
1
|
||||
SELECT count() FROM system.remote_filesystem_cache WHERE cache_hits > 0;
|
||||
2
|
||||
SELECT * FROM test FORMAT Null;
|
||||
SELECT cache_hits FROM system.remote_filesystem_cache;
|
||||
SELECT count() FROM system.remote_filesystem_cache WHERE cache_hits > 0;
|
||||
2
|
||||
SELECT count() size FROM system.remote_filesystem_cache;
|
||||
1
|
||||
7
|
||||
SYSTEM DROP REMOTE FILESYSTEM CACHE;
|
||||
INSERT INTO test SELECT number, toString(number) FROM numbers(100, 200);
|
||||
SELECT file_segment_range, size, state FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.remote_filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path ORDER BY size FORMAT Vertical;
|
||||
Row 1:
|
||||
──────
|
||||
file_segment_range: (0,1659)
|
||||
size: 1660
|
||||
state: DOWNLOADED
|
||||
SELECT file_segment_range, size, state FROM system.remote_filesystem_cache format Vertical;
|
||||
SELECT file_segment_range, size, state FROM (SELECT file_segment_range, size, state, local_path FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.remote_filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path) WHERE endsWith(local_path, 'data.bin') FORMAT Vertical;
|
||||
Row 1:
|
||||
──────
|
||||
file_segment_range: (0,1659)
|
||||
size: 1660
|
||||
state: DOWNLOADED
|
||||
SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.remote_filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path;
|
||||
7
|
||||
SELECT count() FROM system.remote_filesystem_cache;
|
||||
7
|
||||
SELECT count() FROM system.remote_filesystem_cache;
|
||||
7
|
||||
INSERT INTO test SELECT number, toString(number) FROM numbers(100) SETTINGS remote_fs_cache_on_insert=0; -- still writes cache because now config setting is used
|
||||
SELECT count() FROM system.remote_filesystem_cache;
|
||||
7
|
||||
|
@ -5,25 +5,27 @@
|
||||
SET remote_fs_cache_on_insert=1;
|
||||
|
||||
DROP TABLE IF EXISTS test;
|
||||
CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3_cache';
|
||||
CREATE TABLE test (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='s3';
|
||||
|
||||
SYSTEM DROP REMOTE FILESYSTEM CACHE;
|
||||
|
||||
SELECT file_segment_range, size, state FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.remote_filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path FORMAT Vertical;
|
||||
SELECT file_segment_range, size, state FROM system.remote_filesystem_cache format Vertical;
|
||||
SELECT file_segment_range, size, state FROM (SELECT file_segment_range, size, state, local_path FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.remote_filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path) WHERE endsWith(local_path, 'data.bin') FORMAT Vertical;
|
||||
SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.remote_filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path;
|
||||
SELECT count() FROM system.remote_filesystem_cache;
|
||||
|
||||
INSERT INTO test SELECT number, toString(number) FROM numbers(100);
|
||||
|
||||
SELECT file_segment_range, size, state FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.remote_filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path FORMAT Vertical;
|
||||
SELECT file_segment_range, size, state FROM system.remote_filesystem_cache format Vertical;
|
||||
SELECT file_segment_range, size, state FROM (SELECT file_segment_range, size, state, local_path FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.remote_filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path) WHERE endsWith(local_path, 'data.bin') FORMAT Vertical;
|
||||
SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.remote_filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path;
|
||||
SELECT count() FROM system.remote_filesystem_cache;
|
||||
|
||||
SELECT cache_hits FROM system.remote_filesystem_cache;
|
||||
SELECT count() FROM system.remote_filesystem_cache WHERE cache_hits > 0;
|
||||
|
||||
SELECT * FROM test FORMAT Null;
|
||||
SELECT cache_hits FROM system.remote_filesystem_cache;
|
||||
SELECT count() FROM system.remote_filesystem_cache WHERE cache_hits > 0;
|
||||
|
||||
SELECT * FROM test FORMAT Null;
|
||||
SELECT cache_hits FROM system.remote_filesystem_cache;
|
||||
SELECT count() FROM system.remote_filesystem_cache WHERE cache_hits > 0;
|
||||
|
||||
SELECT count() size FROM system.remote_filesystem_cache;
|
||||
|
||||
@ -31,7 +33,12 @@ SYSTEM DROP REMOTE FILESYSTEM CACHE;
|
||||
|
||||
INSERT INTO test SELECT number, toString(number) FROM numbers(100, 200);
|
||||
|
||||
SELECT file_segment_range, size, state FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.remote_filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path ORDER BY size FORMAT Vertical;
|
||||
SELECT file_segment_range, size, state FROM system.remote_filesystem_cache format Vertical;
|
||||
SELECT file_segment_range, size, state FROM (SELECT file_segment_range, size, state, local_path FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.remote_filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path) WHERE endsWith(local_path, 'data.bin') FORMAT Vertical;
|
||||
SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.remote_filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path;
|
||||
SELECT count() FROM system.remote_filesystem_cache;
|
||||
|
||||
SELECT count() FROM system.remote_filesystem_cache;
|
||||
INSERT INTO test SELECT number, toString(number) FROM numbers(100) SETTINGS remote_fs_cache_on_insert=0; -- still writes cache because now config setting is used
|
||||
SELECT count() FROM system.remote_filesystem_cache;
|
||||
|
||||
|
||||
-- INSERT INTO test SELECT number, toString(number) FROM numbers(100) SETTINGS remote_fs_cache_on_insert=0; -- still writes cache because now config setting is used
|
||||
|
Loading…
Reference in New Issue
Block a user