From c219a53b4a9497664f0ae4cba26f1c1b5e625cd1 Mon Sep 17 00:00:00 2001 From: alesapin Date: Sun, 30 Jan 2022 17:28:52 +0300 Subject: [PATCH 1/6] Add func tests run with s3 --- .github/workflows/pull_request.yml | 36 +++++++++++++++++++ tests/ci/functional_test_check.py | 2 ++ .../config.d/s3_storage_policy_by_default.xml | 24 +++++++++++++ tests/config/install.sh | 4 +++ 4 files changed, 66 insertions(+) create mode 100644 tests/config/config.d/s3_storage_policy_by_default.xml diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index cd8517de8fe..3d9ebeff2df 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -1212,6 +1212,41 @@ jobs: docker kill "$(docker ps -q)" ||: docker rm -f "$(docker ps -a -q)" ||: sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestReleaseS3: + needs: [BuilderDebRelease] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_s3_storage + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (release, s3 storage, actions) + REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse + KILL_TIMEOUT=10800 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker kill "$(docker ps -q)" ||: + docker rm -f "$(docker ps -a -q)" ||: + sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestAarch64: needs: [BuilderDebAarch64] runs-on: [self-hosted, func-tester-aarch64] @@ -3034,6 +3069,7 @@ jobs: - FunctionalStatefulTestTsan - FunctionalStatefulTestMsan - FunctionalStatefulTestUBsan + - FunctionalStatelessTestReleaseS3 - StressTestDebug - StressTestAsan - StressTestTsan diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 7220b86a482..5bd7857ee2b 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -29,6 +29,8 @@ def get_additional_envs(check_name, run_by_hash_num, run_by_hash_total): result.append("USE_DATABASE_ORDINARY=1") if 'wide parts enabled' in check_name: result.append("USE_POLYMORPHIC_PARTS=1") + if 's3 storage' in check_name: + result.append("USE_S3_STORAGE_FOR_MERGE_TREE=1") if run_by_hash_total != 0: result.append(f"RUN_BY_HASH_NUM={run_by_hash_num}") diff --git a/tests/config/config.d/s3_storage_policy_by_default.xml b/tests/config/config.d/s3_storage_policy_by_default.xml new file mode 100644 index 00000000000..6ce997a2c16 --- /dev/null +++ b/tests/config/config.d/s3_storage_policy_by_default.xml @@ -0,0 +1,24 @@ + + + + + s3 + http://localhost:11111/test/test/ + clickhouse + clickhouse + + + + + +
+ s3 +
+ + + + + + s3 + + diff --git a/tests/config/install.sh b/tests/config/install.sh index 731a4317f44..320a3d0f799 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -107,4 +107,8 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]] sudo chgrp clickhouse /var/lib/clickhouse2 fi +if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then + ln -sf $SRC_PATH/config.d/s3_storage_policy_by_default.xml $DEST_SERVER_PATH/config.d/ +fi + ln -sf $SRC_PATH/client_config.xml $DEST_CLIENT_PATH/config.xml From 9eea8e3183c68543585cb069a23e7dce47cd1c2e Mon Sep 17 00:00:00 2001 From: alesapin Date: Sun, 30 Jan 2022 18:22:35 +0300 Subject: [PATCH 2/6] Fix missed change --- tests/ci/ci_config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 70fdf06d40b..000d3d9a000 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -270,6 +270,9 @@ CI_CONFIG = { "Stateless tests (release, DatabaseReplicated, actions)": { "required_build": "package_release", }, + "Stateless tests (release, s3 storage, actions)": { + "required_build": "package_release", + }, "Stress test (address, actions)": { "required_build": "package_asan", }, From ea8ec87b34077592f15ad5428e742005e8fc2b2a Mon Sep 17 00:00:00 2001 From: alesapin Date: Sun, 30 Jan 2022 21:03:12 +0300 Subject: [PATCH 3/6] Fix stupid bug --- src/Compression/CachedCompressedReadBuffer.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Compression/CachedCompressedReadBuffer.cpp b/src/Compression/CachedCompressedReadBuffer.cpp index f942f81f5e9..6467ff00eab 100644 --- a/src/Compression/CachedCompressedReadBuffer.cpp +++ b/src/Compression/CachedCompressedReadBuffer.cpp @@ -30,6 +30,7 @@ void CachedCompressedReadBuffer::initInput() void CachedCompressedReadBuffer::prefetch() { + initInput(); file_in->prefetch(); } From 5be396852639ecdcae410708f8d46171c49b3770 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 31 Jan 2022 13:40:26 +0300 Subject: [PATCH 4/6] Fix bug in cache compressed read buffer --- src/Compression/CachedCompressedReadBuffer.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Compression/CachedCompressedReadBuffer.h b/src/Compression/CachedCompressedReadBuffer.h index 16770e343cc..8394c478319 100644 --- a/src/Compression/CachedCompressedReadBuffer.h +++ b/src/Compression/CachedCompressedReadBuffer.h @@ -61,14 +61,14 @@ public: void setReadUntilPosition(size_t position) override { - if (file_in) - file_in->setReadUntilPosition(position); + initInput(); + file_in->setReadUntilPosition(position); } void setReadUntilEnd() override { - if (file_in) - file_in->setReadUntilEnd(); + initInput(); + file_in->setReadUntilEnd(); } }; From cbf4790ea413b56ccdd02e268d33dafc31ff83b8 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 31 Jan 2022 13:59:16 +0300 Subject: [PATCH 5/6] More clear error --- src/Disks/S3/registerDiskS3.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Disks/S3/registerDiskS3.cpp b/src/Disks/S3/registerDiskS3.cpp index f6824a1b3af..b59cb6910a3 100644 --- a/src/Disks/S3/registerDiskS3.cpp +++ b/src/Disks/S3/registerDiskS3.cpp @@ -174,6 +174,10 @@ void registerDiskS3(DiskFactory & factory) ContextPtr context, const DisksMap & /*map*/) -> DiskPtr { S3::URI uri(Poco::URI(config.getString(config_prefix + ".endpoint"))); + + if (uri.key.empty()) + throw Exception("Empty S3 path specified in disk configuration", ErrorCodes::BAD_ARGUMENTS); + if (uri.key.back() != '/') throw Exception("S3 path must ends with '/', but '" + uri.key + "' doesn't.", ErrorCodes::BAD_ARGUMENTS); From b6828f1b4a15a44c5b1061dfd410f4fddd93f5ec Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 31 Jan 2022 19:09:25 +0300 Subject: [PATCH 6/6] Supress some tests for S3 --- docker/test/stateless/run.sh | 4 ++++ tests/clickhouse-test | 6 ++++++ .../queries/0_stateless/00502_custom_partitioning_local.sql | 1 + .../00731_long_merge_tree_select_opened_files.sh | 2 +- .../00753_system_columns_and_system_tables_long.sql | 2 +- .../0_stateless/01070_mutations_with_dependencies.sql | 2 +- .../0_stateless/01078_merge_tree_read_one_thread.sql | 2 ++ tests/queries/0_stateless/01221_system_settings.sql | 1 + .../queries/0_stateless/01343_min_bytes_to_use_mmap_io.sql | 1 + .../0_stateless/01344_min_bytes_to_use_mmap_io_index.sql | 1 + tests/queries/0_stateless/01475_read_subcolumns.sql | 1 + tests/queries/0_stateless/01533_multiple_nested.sql | 2 ++ 12 files changed, 22 insertions(+), 3 deletions(-) diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index d6d9f189e89..b06121f6e88 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -85,6 +85,10 @@ function run_tests() # everything in parallel except DatabaseReplicated. See below. fi + if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then + ADDITIONAL_OPTIONS+=('--s3-storage') + fi + if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then ADDITIONAL_OPTIONS+=('--replicated-database') ADDITIONAL_OPTIONS+=('--jobs') diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 4566cabf1e7..111222e90af 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -329,6 +329,7 @@ class FailureReason(enum.Enum): FAST_ONLY = "running fast tests only" NO_LONG = "not running long tests" REPLICATED_DB = "replicated-database" + S3_STORAGE = "s3-storage" BUILD = "not running for current build" # UNKNOWN reasons @@ -463,6 +464,10 @@ class TestCase: elif tags and ('no-replicated-database' in tags) and args.replicated_database: return FailureReason.REPLICATED_DB + elif tags and ('no-s3-storage' in tags) and args.s3_storage: + return FailureReason.S3_STORAGE + + elif tags: for build_flag in args.build_flags: if 'no-' + build_flag in tags: @@ -1369,6 +1374,7 @@ if __name__ == '__main__': parser.add_argument('--client-option', nargs='+', help='Specify additional client argument') parser.add_argument('--print-time', action='store_true', dest='print_time', help='Print test time') parser.add_argument('--check-zookeeper-session', action='store_true', help='Check ZooKeeper session uptime to determine if failed test should be retried') + parser.add_argument('--s3-storage', action='store_true', default=False, help='Run tests over s3 storage') parser.add_argument('--run-by-hash-num', type=int, help='Run tests matching crc32(test_name) % run_by_hash_total == run_by_hash_num') parser.add_argument('--run-by-hash-total', type=int, help='Total test groups for crc32(test_name) % run_by_hash_total == run_by_hash_num') diff --git a/tests/queries/0_stateless/00502_custom_partitioning_local.sql b/tests/queries/0_stateless/00502_custom_partitioning_local.sql index eda267c0445..55c9b559ed0 100644 --- a/tests/queries/0_stateless/00502_custom_partitioning_local.sql +++ b/tests/queries/0_stateless/00502_custom_partitioning_local.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage SELECT '*** Not partitioned ***'; DROP TABLE IF EXISTS not_partitioned; diff --git a/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh b/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh index f9e33645527..216f4bf4502 100755 --- a/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh +++ b/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long +# Tags: long, no-s3-storage set -e diff --git a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql index dc54e4630d6..ca3df7c4cd4 100644 --- a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql +++ b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql @@ -1,4 +1,4 @@ --- Tags: long +-- Tags: long, no-s3-storage DROP TABLE IF EXISTS check_system_tables; diff --git a/tests/queries/0_stateless/01070_mutations_with_dependencies.sql b/tests/queries/0_stateless/01070_mutations_with_dependencies.sql index 6f297e25b95..506fd23904f 100644 --- a/tests/queries/0_stateless/01070_mutations_with_dependencies.sql +++ b/tests/queries/0_stateless/01070_mutations_with_dependencies.sql @@ -1,4 +1,4 @@ --- Tags: no-parallel +-- Tags: no-parallel, no-s3-storage drop table if exists ttl; set mutations_sync = 2; diff --git a/tests/queries/0_stateless/01078_merge_tree_read_one_thread.sql b/tests/queries/0_stateless/01078_merge_tree_read_one_thread.sql index 2d6007c678c..3a05e4507a2 100644 --- a/tests/queries/0_stateless/01078_merge_tree_read_one_thread.sql +++ b/tests/queries/0_stateless/01078_merge_tree_read_one_thread.sql @@ -1,3 +1,5 @@ +-- Tags: no-s3-storage +-- Output slightly different plan drop table if exists t; create table t (a Int, b Int) engine = MergeTree order by (a, b) settings index_granularity = 400; diff --git a/tests/queries/0_stateless/01221_system_settings.sql b/tests/queries/0_stateless/01221_system_settings.sql index 226be55503d..fcffd6c45fe 100644 --- a/tests/queries/0_stateless/01221_system_settings.sql +++ b/tests/queries/0_stateless/01221_system_settings.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage select * from system.settings where name = 'send_timeout'; select * from system.merge_tree_settings order by length(description) limit 1; diff --git a/tests/queries/0_stateless/01343_min_bytes_to_use_mmap_io.sql b/tests/queries/0_stateless/01343_min_bytes_to_use_mmap_io.sql index c8c49ea8c58..614629351ef 100644 --- a/tests/queries/0_stateless/01343_min_bytes_to_use_mmap_io.sql +++ b/tests/queries/0_stateless/01343_min_bytes_to_use_mmap_io.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage DROP TABLE IF EXISTS test_01343; CREATE TABLE test_01343 (x String) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; INSERT INTO test_01343 VALUES ('Hello, world'); diff --git a/tests/queries/0_stateless/01344_min_bytes_to_use_mmap_io_index.sql b/tests/queries/0_stateless/01344_min_bytes_to_use_mmap_io_index.sql index 7805ecea392..2e5ec563641 100644 --- a/tests/queries/0_stateless/01344_min_bytes_to_use_mmap_io_index.sql +++ b/tests/queries/0_stateless/01344_min_bytes_to_use_mmap_io_index.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage DROP TABLE IF EXISTS test_01344; CREATE TABLE test_01344 (x String, INDEX idx (x) TYPE set(10) GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; INSERT INTO test_01344 VALUES ('Hello, world'); diff --git a/tests/queries/0_stateless/01475_read_subcolumns.sql b/tests/queries/0_stateless/01475_read_subcolumns.sql index 6e2c8d458ae..fb26b19ed30 100644 --- a/tests/queries/0_stateless/01475_read_subcolumns.sql +++ b/tests/queries/0_stateless/01475_read_subcolumns.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage SELECT '====array===='; DROP TABLE IF EXISTS t_arr; CREATE TABLE t_arr (a Array(UInt32)) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; diff --git a/tests/queries/0_stateless/01533_multiple_nested.sql b/tests/queries/0_stateless/01533_multiple_nested.sql index 82049243006..40f287b4afd 100644 --- a/tests/queries/0_stateless/01533_multiple_nested.sql +++ b/tests/queries/0_stateless/01533_multiple_nested.sql @@ -1,3 +1,5 @@ +-- Tags: no-s3-storage +-- Temporary supressed DROP TABLE IF EXISTS nested; SET flatten_nested = 0;