mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-26 01:22:04 +00:00
Merge pull request #59943 from ClickHouse/Add_azure_storage_policy_to_tests
Add azure_cache as storage policy to tests
This commit is contained in:
SmitaRKulkarni
GitHub
commit
19beb5ecc4
@ -183,13 +183,7 @@ elif [[ "$USE_AZURE_STORAGE_FOR_MERGE_TREE" == "1" ]]; then
|
||||
fi
|
||||
|
||||
if [[ -n "$EXPORT_S3_STORAGE_POLICIES" ]]; then
|
||||
if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
|
||||
echo "Azure configuration will not be added"
|
||||
else
|
||||
echo "Adding azure configuration"
|
||||
ln -sf $SRC_PATH/config.d/azure_storage_conf.xml $DEST_SERVER_PATH/config.d/
|
||||
fi
|
||||
|
||||
ln -sf $SRC_PATH/config.d/azure_storage_conf.xml $DEST_SERVER_PATH/config.d/
|
||||
ln -sf $SRC_PATH/config.d/storage_conf.xml $DEST_SERVER_PATH/config.d/
|
||||
ln -sf $SRC_PATH/config.d/storage_conf_02944.xml $DEST_SERVER_PATH/config.d/
|
||||
ln -sf $SRC_PATH/config.d/storage_conf_02963.xml $DEST_SERVER_PATH/config.d/
|
||||
|
||||
@ -34,3 +34,21 @@ DOWNLOADED 0 79 80
|
||||
DOWNLOADED 0 745 746
|
||||
2
|
||||
Expect no cache
|
||||
Using storage policy: azure_cache
|
||||
0
|
||||
Expect cache
|
||||
DOWNLOADED 0 0 1
|
||||
DOWNLOADED 0 79 80
|
||||
DOWNLOADED 0 745 746
|
||||
3
|
||||
Expect cache
|
||||
DOWNLOADED 0 0 1
|
||||
DOWNLOADED 0 79 80
|
||||
DOWNLOADED 0 745 746
|
||||
3
|
||||
Expect no cache
|
||||
Expect cache
|
||||
DOWNLOADED 0 79 80
|
||||
DOWNLOADED 0 745 746
|
||||
2
|
||||
Expect no cache
|
||||
|
||||
@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
for STORAGE_POLICY in 's3_cache' 'local_cache'; do
|
||||
for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do
|
||||
echo "Using storage policy: $STORAGE_POLICY"
|
||||
${CLICKHOUSE_CLIENT} --query "SYSTEM DROP FILESYSTEM CACHE"
|
||||
${CLICKHOUSE_CLIENT} --query "SYSTEM DROP MARK CACHE"
|
||||
|
||||
@ -218,3 +218,113 @@ SELECT count() FROM test_02241
|
||||
5010500
|
||||
SELECT count() FROM test_02241 WHERE value LIKE '%010%'
|
||||
18816
|
||||
Using storage policy: azure_cache
|
||||
DROP TABLE IF EXISTS test_02241
|
||||
CREATE TABLE test_02241 (key UInt32, value String) Engine=MergeTree() ORDER BY key SETTINGS storage_policy='azure_cache', min_bytes_for_wide_part = 10485760, compress_marks=false, compress_primary_key=false, ratio_of_defaults_for_sparse_serialization = 1
|
||||
SYSTEM STOP MERGES test_02241
|
||||
SYSTEM DROP FILESYSTEM CACHE
|
||||
SELECT file_segment_range_begin, file_segment_range_end, size, state
|
||||
FROM
|
||||
(
|
||||
SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path
|
||||
FROM
|
||||
(
|
||||
SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path
|
||||
FROM system.remote_data_paths
|
||||
) AS data_paths
|
||||
INNER JOIN
|
||||
system.filesystem_cache AS caches
|
||||
ON data_paths.cache_path = caches.cache_path
|
||||
)
|
||||
WHERE endsWith(local_path, 'data.bin')
|
||||
FORMAT Vertical
|
||||
SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path
|
||||
0
|
||||
SELECT count(), sum(size) FROM system.filesystem_cache
|
||||
0 0
|
||||
INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100)
|
||||
SELECT file_segment_range_begin, file_segment_range_end, size, state
|
||||
FROM
|
||||
(
|
||||
SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path
|
||||
FROM
|
||||
(
|
||||
SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path
|
||||
FROM system.remote_data_paths
|
||||
) AS data_paths
|
||||
INNER JOIN
|
||||
system.filesystem_cache AS caches
|
||||
ON data_paths.cache_path = caches.cache_path
|
||||
)
|
||||
WHERE endsWith(local_path, 'data.bin')
|
||||
FORMAT Vertical
|
||||
Row 1:
|
||||
──────
|
||||
file_segment_range_begin: 0
|
||||
file_segment_range_end: 745
|
||||
size: 746
|
||||
state: DOWNLOADED
|
||||
SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path
|
||||
8
|
||||
SELECT count(), sum(size) FROM system.filesystem_cache
|
||||
8 1100
|
||||
SELECT count() FROM system.filesystem_cache WHERE cache_hits > 0
|
||||
0
|
||||
SELECT * FROM test_02241 FORMAT Null
|
||||
SELECT count() FROM system.filesystem_cache WHERE cache_hits > 0
|
||||
2
|
||||
SELECT * FROM test_02241 FORMAT Null
|
||||
SELECT count() FROM system.filesystem_cache WHERE cache_hits > 0
|
||||
2
|
||||
SELECT count(), sum(size) size FROM system.filesystem_cache
|
||||
8 1100
|
||||
SYSTEM DROP FILESYSTEM CACHE
|
||||
INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100, 200)
|
||||
SELECT file_segment_range_begin, file_segment_range_end, size, state
|
||||
FROM
|
||||
(
|
||||
SELECT file_segment_range_begin, file_segment_range_end, size, state, local_path
|
||||
FROM
|
||||
(
|
||||
SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path
|
||||
FROM system.remote_data_paths
|
||||
) AS data_paths
|
||||
INNER JOIN
|
||||
system.filesystem_cache AS caches
|
||||
ON data_paths.cache_path = caches.cache_path
|
||||
)
|
||||
WHERE endsWith(local_path, 'data.bin')
|
||||
FORMAT Vertical;
|
||||
Row 1:
|
||||
──────
|
||||
file_segment_range_begin: 0
|
||||
file_segment_range_end: 1659
|
||||
size: 1660
|
||||
state: DOWNLOADED
|
||||
SELECT count() FROM (SELECT arrayJoin(cache_paths) AS cache_path, local_path, remote_path FROM system.remote_data_paths ) AS data_paths INNER JOIN system.filesystem_cache AS caches ON data_paths.cache_path = caches.cache_path
|
||||
8
|
||||
SELECT count(), sum(size) FROM system.filesystem_cache
|
||||
8 2014
|
||||
SELECT count(), sum(size) FROM system.filesystem_cache
|
||||
8 2014
|
||||
INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100) SETTINGS enable_filesystem_cache_on_write_operations=0
|
||||
SELECT count(), sum(size) FROM system.filesystem_cache
|
||||
8 2014
|
||||
INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(100)
|
||||
INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(300, 10000)
|
||||
SELECT count(), sum(size) FROM system.filesystem_cache
|
||||
24 84045
|
||||
SYSTEM START MERGES test_02241
|
||||
OPTIMIZE TABLE test_02241 FINAL
|
||||
SELECT count(), sum(size) FROM system.filesystem_cache
|
||||
32 167243
|
||||
ALTER TABLE test_02241 UPDATE value = 'kek' WHERE key = 100
|
||||
SELECT count(), sum(size) FROM system.filesystem_cache
|
||||
41 250541
|
||||
INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(5000000)
|
||||
SYSTEM FLUSH LOGS
|
||||
INSERT INTO test_02241 SELECT number, toString(number) FROM numbers(5000000) 0
|
||||
SELECT count() FROM test_02241
|
||||
5010500
|
||||
SELECT count() FROM test_02241 WHERE value LIKE '%010%'
|
||||
18816
|
||||
|
||||
@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
for STORAGE_POLICY in 's3_cache' 'local_cache'; do
|
||||
for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do
|
||||
echo "Using storage policy: $STORAGE_POLICY"
|
||||
|
||||
$CLICKHOUSE_CLIENT --echo --query "DROP TABLE IF EXISTS test_02241"
|
||||
|
||||
@ -6,3 +6,7 @@ Using storage policy: local_cache
|
||||
(0,519) READ_FROM_FS_AND_DOWNLOADED_TO_CACHE
|
||||
(0,808110) READ_FROM_FS_AND_DOWNLOADED_TO_CACHE
|
||||
(0,808110) READ_FROM_CACHE
|
||||
Using storage policy: azure_cache
|
||||
(0,519) READ_FROM_FS_AND_DOWNLOADED_TO_CACHE
|
||||
(0,808110) READ_FROM_FS_AND_DOWNLOADED_TO_CACHE
|
||||
(0,808110) READ_FROM_CACHE
|
||||
|
||||
@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
for STORAGE_POLICY in 's3_cache' 'local_cache'; do
|
||||
for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do
|
||||
echo "Using storage policy: $STORAGE_POLICY"
|
||||
|
||||
$CLICKHOUSE_CLIENT --query "SYSTEM DROP FILESYSTEM CACHE"
|
||||
|
||||
@ -16,3 +16,12 @@ Using storage policy: local_cache
|
||||
1
|
||||
1
|
||||
0
|
||||
Using storage policy: azure_cache
|
||||
0
|
||||
2
|
||||
0
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
0
|
||||
|
||||
@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
for STORAGE_POLICY in 's3_cache' 'local_cache'; do
|
||||
for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do
|
||||
echo "Using storage policy: $STORAGE_POLICY"
|
||||
$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test_02286"
|
||||
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
Using storage policy: s3_cache
|
||||
Using storage policy: local_cache
|
||||
Using storage policy: s3_cache_multi
|
||||
Using storage policy: azure_cache
|
||||
|
||||
@ -8,7 +8,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
|
||||
for STORAGE_POLICY in 's3_cache' 'local_cache' 's3_cache_multi'; do
|
||||
for STORAGE_POLICY in 's3_cache' 'local_cache' 's3_cache_multi' 'azure_cache'; do
|
||||
echo "Using storage policy: $STORAGE_POLICY"
|
||||
$CLICKHOUSE_CLIENT --query "SYSTEM DROP FILESYSTEM CACHE"
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user