From da179c607e3b2c4e44b1a593963c3cb7cc352f41 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 8 Mar 2022 15:19:05 +0000 Subject: [PATCH 01/30] Add bugfux validate check --- .github/workflows/pull_request.yml | 35 ++++++++++++++++++++++++++++++ tests/ci/docker_pull_helper.py | 8 ++++--- tests/ci/functional_test_check.py | 10 ++++++--- 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 4a3880543c4..f9883546c23 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -1733,6 +1733,41 @@ jobs: docker kill "$(docker ps -q)" ||: docker rm -f "$(docker ps -a -q)" ||: sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestBugfixCheck: + needs: [BuilderDebAsan] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_bugfix_asan + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests bugfix validate check (address, actions) + REPO_COPY=${{runner.temp}}/stateless_bugfix_asan/ClickHouse + KILL_TIMEOUT=3600 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" --validate-bugfix + - name: Cleanup + if: always() + run: | + docker kill "$(docker ps -q)" ||: + docker rm -f "$(docker ps -a -q)" ||: + sudo rm -fr "$TEMP_PATH" ############################################################################################## ############################ FUNCTIONAl STATEFUL TESTS ####################################### ############################################################################################## diff --git a/tests/ci/docker_pull_helper.py b/tests/ci/docker_pull_helper.py index 50354da6801..0bf3bcfaeac 100644 --- a/tests/ci/docker_pull_helper.py +++ b/tests/ci/docker_pull_helper.py @@ -17,7 +17,7 @@ class DockerImage: def __str__(self): return f"{self.name}:{self.version}" -def get_images_with_versions(reports_path, required_image, pull=True): +def get_images_with_versions(reports_path, required_image, pull=True, version=None): images_path = None for root, _, files in os.walk(reports_path): for f in files: @@ -43,6 +43,8 @@ def get_images_with_versions(reports_path, required_image, pull=True): docker_image = DockerImage(image_name) if image_name in images: docker_image.version = images[image_name] + if version is not None: + docker_image.version = str(version) docker_images.append(docker_image) if pull: @@ -60,5 +62,5 @@ def get_images_with_versions(reports_path, required_image, pull=True): return docker_images -def get_image_with_version(reports_path, image, pull=True): - return get_images_with_versions(reports_path, [image], pull)[0] +def get_image_with_version(reports_path, image, pull=True, version=None): + return get_images_with_versions(reports_path, [image], pull, version=version)[0] diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index da08ff28a0f..efec6c03278 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -139,10 +139,13 @@ if __name__ == "__main__": check_name = sys.argv[1] kill_timeout = int(sys.argv[2]) + validate_bugix_check = len(sys.argv) >= 4 and sys.argv[3] == "--validate-bugfix" flaky_check = 'flaky' in check_name.lower() + + run_changed_tests = flaky_check or validate_bugix_check gh = Github(get_best_robot_token()) - pr_info = PRInfo(need_changed_files=flaky_check) + pr_info = PRInfo(need_changed_files=run_changed_tests) if 'RUN_BY_HASH_NUM' in os.environ: run_by_hash_num = int(os.getenv('RUN_BY_HASH_NUM')) @@ -162,7 +165,7 @@ if __name__ == "__main__": os.makedirs(temp_path) tests_to_run = [] - if flaky_check: + if run_changed_tests: tests_to_run = get_tests_to_run(pr_info) if not tests_to_run: commit = get_commit(gh, pr_info.sha) @@ -170,7 +173,8 @@ if __name__ == "__main__": sys.exit(0) image_name = get_image_name(check_name) - docker_image = get_image_with_version(reports_path, image_name) + image_ver = 0 if validate_bugix_check else None + docker_image = get_image_with_version(reports_path, image_name, version=image_ver) repo_tests_path = os.path.join(repo_path, "tests") From 73b69805a81eca9f0e8f1604ffc927a4b80c299d Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 8 Mar 2022 15:19:33 +0000 Subject: [PATCH 02/30] add testcase to check bugfix validation --- tests/queries/0_stateless/02232_fail_on_master.reference | 1 + tests/queries/0_stateless/02232_fail_on_master.sql | 1 + tests/queries/0_stateless/02232_success_on_master.reference | 1 + tests/queries/0_stateless/02232_success_on_master.sql | 1 + 4 files changed, 4 insertions(+) create mode 100644 tests/queries/0_stateless/02232_fail_on_master.reference create mode 100644 tests/queries/0_stateless/02232_fail_on_master.sql create mode 100644 tests/queries/0_stateless/02232_success_on_master.reference create mode 100644 tests/queries/0_stateless/02232_success_on_master.sql diff --git a/tests/queries/0_stateless/02232_fail_on_master.reference b/tests/queries/0_stateless/02232_fail_on_master.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02232_fail_on_master.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02232_fail_on_master.sql b/tests/queries/0_stateless/02232_fail_on_master.sql new file mode 100644 index 00000000000..b293a1f7a37 --- /dev/null +++ b/tests/queries/0_stateless/02232_fail_on_master.sql @@ -0,0 +1 @@ +SELECT value != 'master' FROM system.build_options WHERE name = 'GIT_BRANCH'; diff --git a/tests/queries/0_stateless/02232_success_on_master.reference b/tests/queries/0_stateless/02232_success_on_master.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02232_success_on_master.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02232_success_on_master.sql b/tests/queries/0_stateless/02232_success_on_master.sql new file mode 100644 index 00000000000..e0ac49d1ecf --- /dev/null +++ b/tests/queries/0_stateless/02232_success_on_master.sql @@ -0,0 +1 @@ +SELECT 1; From 196645679f7c7995b0a3654713ada04be1d25481 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 9 Mar 2022 11:19:50 +0000 Subject: [PATCH 03/30] Add bugfix validate check to ci_config --- tests/ci/ci_config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index b45a4ce90c6..ef07668f506 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -349,6 +349,9 @@ CI_CONFIG = { "Stateless tests flaky check (address, actions)": { "required_build": "package_asan", }, + "Stateless tests bugfix validate check (address, actions)": { + "required_build": "package_asan", + }, "ClickHouse Keeper Jepsen (actions)": { "required_build": "binary_release", }, From 4947d6db13952b9d73d22a2236ddddec957f584d Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 9 Mar 2022 16:24:58 +0000 Subject: [PATCH 04/30] Use download_previous_release for bugfix validation Should be merged with https://github.com/ClickHouse/ClickHouse/pull/27928 --- .github/workflows/pull_request.yml | 2 +- tests/ci/download_previous_release.py | 126 ++++++++++++++++++++++++++ tests/ci/functional_test_check.py | 9 +- 3 files changed, 133 insertions(+), 4 deletions(-) create mode 100644 tests/ci/download_previous_release.py diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index f9883546c23..ae6d4c5aaef 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -1734,7 +1734,7 @@ jobs: docker rm -f "$(docker ps -a -q)" ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestBugfixCheck: - needs: [BuilderDebAsan] + needs: [] runs-on: [self-hosted, func-tester] steps: - name: Set envs diff --git a/tests/ci/download_previous_release.py b/tests/ci/download_previous_release.py new file mode 100644 index 00000000000..b11a84a36f6 --- /dev/null +++ b/tests/ci/download_previous_release.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 + +########################################################################### +# # +# TODO (@vdimir, @Avogar) # +# Merge with one from https://github.com/ClickHouse/ClickHouse/pull/27928 # +# # +########################################################################### + +import requests +import re +import os + +CLICKHOUSE_TAGS_URL = "https://api.github.com/repos/ClickHouse/ClickHouse/tags" + +CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static_{version}_amd64.deb" +CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static-dbg_{version}_amd64.deb" +CLICKHOUSE_SERVER_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-server_{version}_all.deb" +CLICKHOUSE_CLIENT_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-client_{version}_amd64.deb" + + +CLICKHOUSE_COMMON_STATIC_PACKET_NAME = "clickhouse-common-static_{version}_amd64.deb" +CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME = "clickhouse-common-static-dbg_{version}_amd64.deb" +CLICKHOUSE_SERVER_PACKET_NAME = "clickhouse-server_{version}_all.deb" +CLICKHOUSE_CLIENT_PACKET_NAME = "clickhouse-client_{version}_all.deb" + +PACKETS_DIR = "previous_release_package_folder/" +VERSION_PATTERN = r"((?:\d+\.)?(?:\d+\.)?(?:\d+\.)?\d+-[a-zA-Z]*)" + + +class Version: + def __init__(self, version): + self.version = version + + def __lt__(self, other): + return list(map(int, self.version.split('.'))) < list(map(int, other.version.split('.'))) + + def __str__(self): + return self.version + + +class ReleaseInfo: + def __init__(self, version, release_type): + self.version = version + self.type = release_type + + def __repr__(self): + return f"ReleaseInfo: {self.version}-{self.type}" + +def find_previous_release(server_version, releases): + releases.sort(key=lambda x: x.version, reverse=True) + print(releases) + if server_version is None: + return True, releases[0] + + for release in releases: + if release.version < server_version: + return True, release + + return False, None + + +def get_previous_release(server_version=None): + page = 1 + found = False + while not found: + response = requests.get(CLICKHOUSE_TAGS_URL, {'page': page, 'per_page': 100}) + if not response.ok: + raise Exception('Cannot load the list of tags from github: ' + response.reason) + + releases_str = set(re.findall(VERSION_PATTERN, response.text)) + if len(releases_str) == 0: + raise Exception('Cannot find previous release for ' + str(server_version) + ' server version') + + releases = list(map(lambda x: ReleaseInfo(Version(x.split('-')[0]), x.split('-')[1]), releases_str)) + found, previous_release = find_previous_release(server_version, releases) + page += 1 + + return previous_release + + +def download_packet(url, out_path): + """ + TODO: use dowload_build_with_progress from build_download_helper.py + """ + + response = requests.get(url) + print(url) + if response.ok: + open(out_path, 'wb').write(response.content) + +def download_packets(release, dest_path=PACKETS_DIR): + if not os.path.exists(dest_path): + os.makedirs(dest_path) + + download_packet( + CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL.format(version=release.version, type=release.type), + out_path=os.path.join(dest_path, CLICKHOUSE_COMMON_STATIC_PACKET_NAME.format(version=release.version)), + ) + + download_packet( + CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL.format(version=release.version, type=release.type), + out_path=os.path.join(dest_path, CLICKHOUSE_COMMON_STATIC_DBG_PACKET_NAME.format(version=release.version)), + ) + + download_packet( + CLICKHOUSE_SERVER_DOWNLOAD_URL.format(version=release.version, type=release.type), + out_path=os.path.join(dest_path, CLICKHOUSE_SERVER_PACKET_NAME.format(version=release.version)), + ) + + download_packet( + CLICKHOUSE_CLIENT_DOWNLOAD_URL.format(version=release.version, type=release.type), + out_path=os.path.join(dest_path, CLICKHOUSE_CLIENT_PACKET_NAME.format(version=release.version)), + ) + + +def download_previous_release(dest_path): + current_release = get_previous_release(None) + download_packets(current_release, dest_path=dest_path) + + +if __name__ == '__main__': + server_version = Version(input()) + previous_release = get_previous_release(server_version) + download_packets(previous_release) + diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index efec6c03278..7890673384a 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -13,6 +13,7 @@ from s3_helper import S3Helper from get_robot_token import get_best_robot_token from pr_info import PRInfo from build_download_helper import download_all_deb_packages +from download_previous_release import download_previous_release from upload_result_helper import upload_results from docker_pull_helper import get_image_with_version from commit_status_helper import post_commit_status, get_commit, override_status @@ -173,8 +174,7 @@ if __name__ == "__main__": sys.exit(0) image_name = get_image_name(check_name) - image_ver = 0 if validate_bugix_check else None - docker_image = get_image_with_version(reports_path, image_name, version=image_ver) + docker_image = get_image_with_version(reports_path, image_name) repo_tests_path = os.path.join(repo_path, "tests") @@ -182,7 +182,10 @@ if __name__ == "__main__": if not os.path.exists(packages_path): os.makedirs(packages_path) - download_all_deb_packages(check_name, reports_path, packages_path) + if not validate_bugix_check: + download_all_deb_packages(check_name, reports_path, packages_path) + else: + download_previous_release(packages_path) server_log_path = os.path.join(temp_path, "server_log") if not os.path.exists(server_log_path): From fcb7e9ed36e0a7b99b09413ce8de3b42e1471504 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 9 Mar 2022 16:50:38 +0000 Subject: [PATCH 05/30] fix CLICKHOUSE_CLIENT_DOWNLOAD_URL in download_previous_release.py --- tests/ci/download_previous_release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/download_previous_release.py b/tests/ci/download_previous_release.py index b11a84a36f6..88010cb1ec5 100644 --- a/tests/ci/download_previous_release.py +++ b/tests/ci/download_previous_release.py @@ -16,7 +16,7 @@ CLICKHOUSE_TAGS_URL = "https://api.github.com/repos/ClickHouse/ClickHouse/tags" CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static_{version}_amd64.deb" CLICKHOUSE_COMMON_STATIC_DBG_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static-dbg_{version}_amd64.deb" CLICKHOUSE_SERVER_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-server_{version}_all.deb" -CLICKHOUSE_CLIENT_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-client_{version}_amd64.deb" +CLICKHOUSE_CLIENT_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-client_{version}_all.deb" CLICKHOUSE_COMMON_STATIC_PACKET_NAME = "clickhouse-common-static_{version}_amd64.deb" From 000a31df3b88841498783a934113757b625d059e Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 10 Mar 2022 12:22:24 +0000 Subject: [PATCH 06/30] no-random-settings for bugfix validate --- tests/ci/functional_test_check.py | 3 +++ tests/clickhouse-test | 3 +++ 2 files changed, 6 insertions(+) diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 7890673384a..0ba9532b171 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -198,6 +198,9 @@ if __name__ == "__main__": run_log_path = os.path.join(result_path, "runlog.log") additional_envs = get_additional_envs(check_name, run_by_hash_num, run_by_hash_total) + if validate_bugix_check: + additional_envs.append('GLOBAL_TAGS=no-random-settings') + run_command = get_run_command(packages_path, repo_tests_path, result_path, server_log_path, kill_timeout, additional_envs, docker_image, flaky_check, tests_to_run) logging.info("Going to run func tests: %s", run_command) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 121a283d0e4..6f233550ddd 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -464,6 +464,9 @@ class TestCase: self.case: str = case # case file name self.tags: Set[str] = suite.all_tags[case] if case in suite.all_tags else set() + for tag in os.getenv("GLOBAL_TAGS", "").split(","): + self.tags.add(tag.strip()) + self.case_file: str = os.path.join(suite.suite_path, case) (self.name, self.ext) = os.path.splitext(case) From 355a0d6fefc0e0cdbabfa60b8535f4c09bdfad62 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 10 Mar 2022 12:57:38 +0000 Subject: [PATCH 07/30] upd test for bugfix valiate --- ..._fail_on_master.reference => 02232_err_on_master.reference} | 0 .../{02232_fail_on_master.sql => 02232_err_on_master.sql} | 1 + tests/queries/0_stateless/02232_success_on_master.reference | 2 +- tests/queries/0_stateless/02232_success_on_master.sql | 3 ++- 4 files changed, 4 insertions(+), 2 deletions(-) rename tests/queries/0_stateless/{02232_fail_on_master.reference => 02232_err_on_master.reference} (100%) rename tests/queries/0_stateless/{02232_fail_on_master.sql => 02232_err_on_master.sql} (78%) diff --git a/tests/queries/0_stateless/02232_fail_on_master.reference b/tests/queries/0_stateless/02232_err_on_master.reference similarity index 100% rename from tests/queries/0_stateless/02232_fail_on_master.reference rename to tests/queries/0_stateless/02232_err_on_master.reference diff --git a/tests/queries/0_stateless/02232_fail_on_master.sql b/tests/queries/0_stateless/02232_err_on_master.sql similarity index 78% rename from tests/queries/0_stateless/02232_fail_on_master.sql rename to tests/queries/0_stateless/02232_err_on_master.sql index b293a1f7a37..073afbeb4f8 100644 --- a/tests/queries/0_stateless/02232_fail_on_master.sql +++ b/tests/queries/0_stateless/02232_err_on_master.sql @@ -1 +1,2 @@ +-- Tags: no-fasttest SELECT value != 'master' FROM system.build_options WHERE name = 'GIT_BRANCH'; diff --git a/tests/queries/0_stateless/02232_success_on_master.reference b/tests/queries/0_stateless/02232_success_on_master.reference index d00491fd7e5..1f7391f92b6 100644 --- a/tests/queries/0_stateless/02232_success_on_master.reference +++ b/tests/queries/0_stateless/02232_success_on_master.reference @@ -1 +1 @@ -1 +master diff --git a/tests/queries/0_stateless/02232_success_on_master.sql b/tests/queries/0_stateless/02232_success_on_master.sql index e0ac49d1ecf..d909cb1a780 100644 --- a/tests/queries/0_stateless/02232_success_on_master.sql +++ b/tests/queries/0_stateless/02232_success_on_master.sql @@ -1 +1,2 @@ -SELECT 1; +-- Tags: no-fasttest +SELECT value FROM system.build_options WHERE name = 'GIT_BRANCH'; From b8c7e4657f9b31b37d00276da8d8d36a1057de65 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 10 Mar 2022 13:56:59 +0000 Subject: [PATCH 08/30] invert check in validate bugfix --- tests/ci/download_previous_release.py | 7 +++++-- tests/ci/functional_test_check.py | 9 +++++++++ tests/queries/0_stateless/02232_err_on_master.sql | 2 +- .../0_stateless/02232_success_on_master.reference | 2 +- tests/queries/0_stateless/02232_success_on_master.sql | 2 +- 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/tests/ci/download_previous_release.py b/tests/ci/download_previous_release.py index 88010cb1ec5..d1ae5a32e09 100644 --- a/tests/ci/download_previous_release.py +++ b/tests/ci/download_previous_release.py @@ -10,6 +10,7 @@ import requests import re import os +import logging CLICKHOUSE_TAGS_URL = "https://api.github.com/repos/ClickHouse/ClickHouse/tags" @@ -49,7 +50,7 @@ class ReleaseInfo: def find_previous_release(server_version, releases): releases.sort(key=lambda x: x.version, reverse=True) - print(releases) + if server_version is None: return True, releases[0] @@ -85,7 +86,7 @@ def download_packet(url, out_path): """ response = requests.get(url) - print(url) + logging.info(f'Downloading {url}') if response.ok: open(out_path, 'wb').write(response.content) @@ -93,6 +94,8 @@ def download_packets(release, dest_path=PACKETS_DIR): if not os.path.exists(dest_path): os.makedirs(dest_path) + logging.info(f'Will download {release}') + download_packet( CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL.format(version=release.version, type=release.type), out_path=os.path.join(dest_path, CLICKHOUSE_COMMON_STATIC_PACKET_NAME.format(version=release.version)), diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 0ba9532b171..3d382ab409f 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -49,6 +49,13 @@ def get_image_name(check_name): else: raise Exception(f"Cannot deduce image name based on check name {check_name}") + +def invert_status(status): + if status == 'success': + return 'error' + return 'success' + + def get_run_command(builds_path, repo_tests_path, result_path, server_log_path, kill_timeout, additional_envs, image, flaky_check, tests_to_run): additional_options = ['--hung-check'] additional_options.append('--print-time') @@ -217,6 +224,8 @@ if __name__ == "__main__": state, description, test_results, additional_logs = process_results(result_path, server_log_path) state = override_status(state, check_name) + if validate_bugix_check: + state = invert_status(state) ch_helper = ClickHouseHelper() mark_flaky_tests(ch_helper, check_name, test_results) diff --git a/tests/queries/0_stateless/02232_err_on_master.sql b/tests/queries/0_stateless/02232_err_on_master.sql index 073afbeb4f8..8471a93df8d 100644 --- a/tests/queries/0_stateless/02232_err_on_master.sql +++ b/tests/queries/0_stateless/02232_err_on_master.sql @@ -1,2 +1,2 @@ -- Tags: no-fasttest -SELECT value != 'master' FROM system.build_options WHERE name = 'GIT_BRANCH'; +SELECT value not like '22%' FROM system.build_options WHERE name = 'GIT_BRANCH'; diff --git a/tests/queries/0_stateless/02232_success_on_master.reference b/tests/queries/0_stateless/02232_success_on_master.reference index 1f7391f92b6..d00491fd7e5 100644 --- a/tests/queries/0_stateless/02232_success_on_master.reference +++ b/tests/queries/0_stateless/02232_success_on_master.reference @@ -1 +1 @@ -master +1 diff --git a/tests/queries/0_stateless/02232_success_on_master.sql b/tests/queries/0_stateless/02232_success_on_master.sql index d909cb1a780..01099570b05 100644 --- a/tests/queries/0_stateless/02232_success_on_master.sql +++ b/tests/queries/0_stateless/02232_success_on_master.sql @@ -1,2 +1,2 @@ -- Tags: no-fasttest -SELECT value FROM system.build_options WHERE name = 'GIT_BRANCH'; +SELECT 1; From 6614d6eaaf0456a409fa8eb6ccf48e3c96dafafc Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 10 Mar 2022 16:39:18 +0000 Subject: [PATCH 09/30] bugfix validate integration test --- .github/workflows/pull_request.yml | 34 ++++++++++++++++++++++++++- tests/ci/commit_status_helper.py | 10 ++++++-- tests/ci/download_previous_release.py | 8 ++++--- tests/ci/functional_test_check.py | 14 ++++------- tests/ci/integration_test_check.py | 16 ++++++++++--- tests/ci/pr_info.py | 3 ++- 6 files changed, 66 insertions(+), 19 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index ae6d4c5aaef..e6bc2bff212 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -1734,7 +1734,6 @@ jobs: docker rm -f "$(docker ps -a -q)" ||: sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestBugfixCheck: - needs: [] runs-on: [self-hosted, func-tester] steps: - name: Set envs @@ -2727,6 +2726,39 @@ jobs: docker kill "$(docker ps -q)" ||: docker rm -f "$(docker ps -a -q)" ||: sudo rm -fr "$TEMP_PATH" + IntegrationTestsBugfixCheck: + runs-on: [self-hosted, stress-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/integration_tests_asan_bugfix_check + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Integration tests bugfix validate check (asan, actions) + REPO_COPY=${{runner.temp}}/integration_tests_asan_bugfix_check/ClickHouse + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Integration test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 integration_test_check.py "$CHECK_NAME" --validate-bugfix + - name: Cleanup + if: always() + run: | + docker kill "$(docker ps -q)" ||: + docker rm -f "$(docker ps -a -q)" ||: + sudo rm -fr "$TEMP_PATH" ############################################################################################# #################################### UNIT TESTS ############################################# ############################################################################################# diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py index dd57f742ff1..385567962d5 100644 --- a/tests/ci/commit_status_helper.py +++ b/tests/ci/commit_status_helper.py @@ -7,9 +7,15 @@ from ci_config import CI_CONFIG RETRY = 5 -def override_status(status, check_name): - if CI_CONFIG["tests_config"][check_name].get("force_tests", False): +def override_status(status, check_name, invert=False): + if CI_CONFIG["tests_config"].get(check_name, {}).get("force_tests", False): return "success" + + if invert: + if status == 'success': + return 'error' + return 'success' + return status diff --git a/tests/ci/download_previous_release.py b/tests/ci/download_previous_release.py index d1ae5a32e09..e6dc58adac9 100644 --- a/tests/ci/download_previous_release.py +++ b/tests/ci/download_previous_release.py @@ -7,11 +7,12 @@ # # ########################################################################### -import requests import re import os import logging +import requests + CLICKHOUSE_TAGS_URL = "https://api.github.com/repos/ClickHouse/ClickHouse/tags" CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static_{version}_amd64.deb" @@ -86,7 +87,7 @@ def download_packet(url, out_path): """ response = requests.get(url) - logging.info(f'Downloading {url}') + logging.info('Downloading %s', url) if response.ok: open(out_path, 'wb').write(response.content) @@ -94,7 +95,7 @@ def download_packets(release, dest_path=PACKETS_DIR): if not os.path.exists(dest_path): os.makedirs(dest_path) - logging.info(f'Will download {release}') + logging.info('Will download %s', release) download_packet( CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL.format(version=release.version, type=release.type), @@ -123,6 +124,7 @@ def download_previous_release(dest_path): if __name__ == '__main__': + logging.basicConfig(level=logging.INFO) server_version = Version(input()) previous_release = get_previous_release(server_version) download_packets(previous_release) diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 3d382ab409f..9d04b9cf6a0 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -50,12 +50,6 @@ def get_image_name(check_name): raise Exception(f"Cannot deduce image name based on check name {check_name}") -def invert_status(status): - if status == 'success': - return 'error' - return 'success' - - def get_run_command(builds_path, repo_tests_path, result_path, server_log_path, kill_timeout, additional_envs, image, flaky_check, tests_to_run): additional_options = ['--hung-check'] additional_options.append('--print-time') @@ -155,6 +149,10 @@ if __name__ == "__main__": pr_info = PRInfo(need_changed_files=run_changed_tests) + if validate_bugix_check and 'pr-bugfix' not in pr_info.labels: + logging.info("Skipping %s (no pr-bugfix)", check_name) + sys.exit(0) + if 'RUN_BY_HASH_NUM' in os.environ: run_by_hash_num = int(os.getenv('RUN_BY_HASH_NUM')) run_by_hash_total = int(os.getenv('RUN_BY_HASH_TOTAL')) @@ -223,9 +221,7 @@ if __name__ == "__main__": s3_helper = S3Helper('https://s3.amazonaws.com') state, description, test_results, additional_logs = process_results(result_path, server_log_path) - state = override_status(state, check_name) - if validate_bugix_check: - state = invert_status(state) + state = override_status(state, check_name, validate_bugix_check) ch_helper = ClickHouseHelper() mark_flaky_tests(ch_helper, check_name, test_results) diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py index 786521db418..ed513012296 100644 --- a/tests/ci/integration_test_check.py +++ b/tests/ci/integration_test_check.py @@ -14,9 +14,10 @@ from s3_helper import S3Helper from get_robot_token import get_best_robot_token from pr_info import PRInfo from build_download_helper import download_all_deb_packages +from download_previous_release import download_previous_release from upload_result_helper import upload_results from docker_pull_helper import get_images_with_versions -from commit_status_helper import post_commit_status +from commit_status_helper import post_commit_status, override_status from clickhouse_helper import ClickHouseHelper, mark_flaky_tests, prepare_tests_results_for_clickhouse from stopwatch import Stopwatch from rerun_helper import RerunHelper @@ -112,6 +113,7 @@ if __name__ == "__main__": reports_path = REPORTS_PATH check_name = sys.argv[1] + validate_bugix_check = len(sys.argv) >= 3 and sys.argv[2] == "--validate-bugfix" if 'RUN_BY_HASH_NUM' in os.environ: run_by_hash_num = int(os.getenv('RUN_BY_HASH_NUM')) @@ -126,7 +128,11 @@ if __name__ == "__main__": os.makedirs(temp_path) is_flaky_check = 'flaky' in check_name - pr_info = PRInfo(need_changed_files=is_flaky_check) + pr_info = PRInfo(need_changed_files=is_flaky_check or validate_bugix_check) + + if validate_bugix_check and 'pr-bugfix' not in pr_info.labels: + logging.info("Skipping %s (no pr-bugfix)", check_name) + sys.exit(0) gh = Github(get_best_robot_token()) @@ -149,7 +155,10 @@ if __name__ == "__main__": if not os.path.exists(build_path): os.makedirs(build_path) - download_all_deb_packages(check_name, reports_path, build_path) + if not validate_bugix_check: + download_all_deb_packages(check_name, reports_path, build_path) + else: + download_previous_release(build_path) my_env = get_env_for_runner(build_path, repo_path, result_path, work_path) @@ -172,6 +181,7 @@ if __name__ == "__main__": subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True) state, description, test_results, additional_logs = process_results(result_path) + state = override_status(state, check_name, validate_bugix_check) ch_helper = ClickHouseHelper() mark_flaky_tests(ch_helper, check_name, test_results) diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py index 378804874aa..64e22712059 100644 --- a/tests/ci/pr_info.py +++ b/tests/ci/pr_info.py @@ -78,7 +78,7 @@ class PRInfo: else: github_event = PRInfo.default_event.copy() self.event = github_event - self.changed_files = set([]) + self.changed_files = set() self.body = "" ref = github_event.get("ref", "refs/head/master") if ref and ref.startswith("refs/heads/"): @@ -209,6 +209,7 @@ class PRInfo: else: diff_object = PatchSet(response.text) self.changed_files = {f.path for f in diff_object} + print("Fetched info about %d changed files", len(self.changed_files)) def get_dict(self): return { From f7bcb796cefb73ef78377c7e3b82f39215ef1e88 Mon Sep 17 00:00:00 2001 From: vdimir Date: Thu, 10 Mar 2022 16:43:29 +0000 Subject: [PATCH 10/30] add dummy integration tests --- .../test_dummy_err_on_master/__init__.py | 0 .../test_dummy_err_on_master/test.py | 18 ++++++++++++++++++ .../test_dummy_success_on_master/__init__.py | 0 .../test_dummy_success_on_master/test.py | 18 ++++++++++++++++++ 4 files changed, 36 insertions(+) create mode 100644 tests/integration/test_dummy_err_on_master/__init__.py create mode 100644 tests/integration/test_dummy_err_on_master/test.py create mode 100644 tests/integration/test_dummy_success_on_master/__init__.py create mode 100644 tests/integration/test_dummy_success_on_master/test.py diff --git a/tests/integration/test_dummy_err_on_master/__init__.py b/tests/integration/test_dummy_err_on_master/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_dummy_err_on_master/test.py b/tests/integration/test_dummy_err_on_master/test.py new file mode 100644 index 00000000000..1f2a3595f1d --- /dev/null +++ b/tests/integration/test_dummy_err_on_master/test.py @@ -0,0 +1,18 @@ +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance('node') + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_drop_memory_database(start_cluster): + assert node.query("SELECT value not like '22%' FROM system.build_options WHERE name = 'GIT_BRANCH'").strip() == "1" + diff --git a/tests/integration/test_dummy_success_on_master/__init__.py b/tests/integration/test_dummy_success_on_master/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_dummy_success_on_master/test.py b/tests/integration/test_dummy_success_on_master/test.py new file mode 100644 index 00000000000..81d2e34a675 --- /dev/null +++ b/tests/integration/test_dummy_success_on_master/test.py @@ -0,0 +1,18 @@ +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance('node') + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_drop_memory_database(start_cluster): + assert node.query("SELECT 1").strip() == "1" + From e757837ec09897b1a4af12ffd65ec850c1627cb7 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 11 Mar 2022 13:36:29 +0000 Subject: [PATCH 11/30] Different status colors in report for bugfix validation --- tests/ci/download_previous_release.py | 1 - tests/ci/report.py | 33 ++++++++++++++++++++------- tests/ci/upload_result_helper.py | 10 +++++--- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/tests/ci/download_previous_release.py b/tests/ci/download_previous_release.py index e6dc58adac9..113a57d679d 100644 --- a/tests/ci/download_previous_release.py +++ b/tests/ci/download_previous_release.py @@ -128,4 +128,3 @@ if __name__ == '__main__': server_version = Version(input()) previous_release = get_previous_release(server_version) download_packets(previous_release) - diff --git a/tests/ci/report.py b/tests/ci/report.py index c8ba46da106..f36e38ec2a6 100644 --- a/tests/ci/report.py +++ b/tests/ci/report.py @@ -95,6 +95,17 @@ HTML_TEST_PART = """ BASE_HEADERS = ['Test name', 'Test status'] +class ReportColorTheme: + class ReportColor: + yellow = '#FFB400' + red = '#F00' + green = '#0A0' + blue = '#00B4FF' + + default = (ReportColor.green, ReportColor.red, ReportColor.yellow) + bugfixcheck = (ReportColor.yellow, ReportColor.blue, ReportColor.blue) + + def _format_header(header, branch_name, branch_url=None): result = ' '.join([w.capitalize() for w in header.split(' ')]) result = result.replace("Clickhouse", "ClickHouse") @@ -109,14 +120,20 @@ def _format_header(header, branch_name, branch_url=None): return result -def _get_status_style(status): +def _get_status_style(status, colortheme=None): + ok_statuses = ('OK', 'success', 'PASSED') + fail_statuses = ('FAIL', 'failure', 'error', 'FAILED', 'Timeout') + + if colortheme is None: + colortheme = ReportColorTheme.default + style = "font-weight: bold;" - if status in ('OK', 'success', 'PASSED'): - style += 'color: #0A0;' - elif status in ('FAIL', 'failure', 'error', 'FAILED', 'Timeout'): - style += 'color: #F00;' + if status in ok_statuses: + style += f'color: {colortheme[0]};' + elif status in fail_statuses: + style += f'color: {colortheme[1]};' else: - style += 'color: #FFB400;' + style += f'color: {colortheme[2]};' return style @@ -140,7 +157,7 @@ def _get_html_url(url): return '' -def create_test_html_report(header, test_result, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls=None, with_raw_logs=False): +def create_test_html_report(header, test_result, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls=None, with_raw_logs=False, statuscolors=None): if additional_urls is None: additional_urls = [] @@ -168,7 +185,7 @@ def create_test_html_report(header, test_result, raw_log_url, task_url, branch_u if is_fail and with_raw_logs and test_logs is not None: row = "" row += "" + test_name + "" - style = _get_status_style(test_status) + style = _get_status_style(test_status, colortheme=statuscolors) # Allow to quickly scroll to the first failure. is_fail_id = "" diff --git a/tests/ci/upload_result_helper.py b/tests/ci/upload_result_helper.py index 5a5e8d3f36a..2f4823ada40 100644 --- a/tests/ci/upload_result_helper.py +++ b/tests/ci/upload_result_helper.py @@ -3,7 +3,7 @@ import logging import ast from env_helper import GITHUB_SERVER_URL, GITHUB_REPOSITORY, GITHUB_RUN_ID -from report import create_test_html_report +from report import ReportColorTheme, create_test_html_report def process_logs(s3_client, additional_logs, s3_path_prefix, test_results, with_raw_logs): @@ -39,7 +39,7 @@ def process_logs(s3_client, additional_logs, s3_path_prefix, test_results, with_ return additional_urls -def upload_results(s3_client, pr_number, commit_sha, test_results, additional_files, check_name, with_raw_logs=True): +def upload_results(s3_client, pr_number, commit_sha, test_results, additional_files, check_name, with_raw_logs=True, statuscolors=None): s3_path_prefix = f"{pr_number}/{commit_sha}/" + check_name.lower().replace(' ', '_').replace('(', '_').replace(')', '_').replace(',', '_') additional_urls = process_logs(s3_client, additional_files, s3_path_prefix, test_results, with_raw_logs) @@ -58,7 +58,11 @@ def upload_results(s3_client, pr_number, commit_sha, test_results, additional_fi else: raw_log_url = task_url - html_report = create_test_html_report(check_name, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls, with_raw_logs) + statuscolors = ReportColorTheme.bugfixcheck if 'bugfix validate check' in check_name else None + + html_report = create_test_html_report(check_name, test_results, raw_log_url, + task_url, branch_url, branch_name, commit_url, + additional_urls, with_raw_logs, statuscolors=statuscolors) with open('report.html', 'w', encoding='utf-8') as f: f.write(html_report) From e5c63266c2f6e198df223019f5eb74150ffceb32 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 11 Mar 2022 14:14:16 +0000 Subject: [PATCH 12/30] Integration test bugfix check --- tests/ci/functional_test_check.py | 3 ++- tests/integration/ci-runner.py | 43 +++++++++++++++++++------------ 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 9d04b9cf6a0..60f5d4ff4a5 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -175,7 +175,8 @@ if __name__ == "__main__": tests_to_run = get_tests_to_run(pr_info) if not tests_to_run: commit = get_commit(gh, pr_info.sha) - commit.create_status(context=check_name_with_group, description='Not found changed stateless tests', state='success') + state = override_status('success', check_name, validate_bugix_check) + commit.create_status(context=check_name_with_group, description='Not found changed stateless tests', state=state) sys.exit(0) image_name = get_image_name(check_name) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 8f228d91e9e..238d24ffe0d 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -21,7 +21,7 @@ CLICKHOUSE_BINARY_PATH = "usr/bin/clickhouse" CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH = "usr/bin/clickhouse-odbc-bridge" CLICKHOUSE_LIBRARY_BRIDGE_BINARY_PATH = "usr/bin/clickhouse-library-bridge" -TRIES_COUNT = 10 +FLAKY_TRIES_COUNT = 10 MAX_TIME_SECONDS = 3600 MAX_TIME_IN_SANDBOX = 20 * 60 # 20 minutes @@ -32,8 +32,8 @@ def stringhash(s): return zlib.crc32(s.encode("utf-8")) -def get_tests_to_run(pr_info): - result = set([]) +def get_changed_tests_to_run(pr_info, repo_path): + result = set() changed_files = pr_info["changed_files"] if changed_files is None: @@ -43,7 +43,7 @@ def get_tests_to_run(pr_info): if "tests/integration/test_" in fpath: logging.info("File %s changed and seems like integration test", fpath) result.add(fpath.split("/")[2]) - return list(result) + return filter_existing_tests(result, repo_path) def filter_existing_tests(tests_to_run, repo_path): @@ -207,6 +207,7 @@ class ClickhouseIntegrationTestsRunner: self.image_versions = self.params["docker_images_with_versions"] self.shuffle_groups = self.params["shuffle_test_groups"] self.flaky_check = "flaky check" in self.params["context_name"] + self.bugfix_validate_check = "bugfix validate check" in self.params["context_name"] # if use_tmpfs is not set we assume it to be true, otherwise check self.use_tmpfs = "use_tmpfs" not in self.params or self.params["use_tmpfs"] self.disable_net_host = ( @@ -703,13 +704,14 @@ class ClickhouseIntegrationTestsRunner: return counters, tests_times, log_paths - def run_flaky_check(self, repo_path, build_path): + def run_flaky_check(self, repo_path, build_path, should_fail=False): pr_info = self.params["pr_info"] - # pytest swears, if we require to run some tests which was renamed or deleted - tests_to_run = filter_existing_tests(get_tests_to_run(pr_info), repo_path) + tests_to_run = get_changed_tests_to_run(pr_info, repo_path) if not tests_to_run: logging.info("No tests to run found") + if should_fail: + return "failure", "Nothing to run", [("Nothing to run", "FAIL")], "" return "success", "Nothing to run", [("Nothing to run", "OK")], "" self._install_clickhouse(build_path) @@ -720,26 +722,29 @@ class ClickhouseIntegrationTestsRunner: logging.info("Starting check with retries") final_retry = 0 logs = [] - for i in range(TRIES_COUNT): + tires_num = 1 if should_fail else FLAKY_TRIES_COUNT + for i in range(tires_num): final_retry += 1 logging.info("Running tests for the %s time", i) counters, tests_times, log_paths = self.try_run_test_group( - repo_path, "flaky", tests_to_run, 1, 1 + repo_path, "bugfix" if should_fail else "flaky", tests_to_run, 1, 1 ) logs += log_paths if counters["FAILED"]: logging.info("Found failed tests: %s", " ".join(counters["FAILED"])) - description_prefix = "Flaky tests found: " + description_prefix = "Failed tests found: " result_state = "failure" - break + if not should_fail: + break if counters["ERROR"]: - description_prefix = "Flaky tests found: " + description_prefix = "Failed tests found: " logging.info("Found error tests: %s", " ".join(counters["ERROR"])) # NOTE "error" result state will restart the whole test task, # so we use "failure" here result_state = "failure" - break - assert len(counters["FLAKY"]) == 0 + if not should_fail: + break + assert len(counters["FLAKY"]) == 0 or should_fail logging.info("Try is OK, all tests passed, going to clear env") clear_ip_tables_and_restart_daemons() logging.info("And going to sleep for some time") @@ -771,11 +776,17 @@ class ClickhouseIntegrationTestsRunner: ] ) + if should_fail: + if result_state == "success": + result_state = "failure" + else: + result_state = "success" + return result_state, status_text, test_result, logs def run_impl(self, repo_path, build_path): - if self.flaky_check: - return self.run_flaky_check(repo_path, build_path) + if self.flaky_check or self.bugfix_validate_check: + return self.run_flaky_check(repo_path, build_path, should_fail=self.bugfix_validate_check) self._install_clickhouse(build_path) logging.info( From 23bee2c8435181462ab654ae96e93d7d575653d6 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 11 Mar 2022 14:32:24 +0000 Subject: [PATCH 13/30] Do not invert result of integration bugfix check in ci-runner.py --- tests/integration/ci-runner.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 238d24ffe0d..d45352fa909 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -710,8 +710,6 @@ class ClickhouseIntegrationTestsRunner: tests_to_run = get_changed_tests_to_run(pr_info, repo_path) if not tests_to_run: logging.info("No tests to run found") - if should_fail: - return "failure", "Nothing to run", [("Nothing to run", "FAIL")], "" return "success", "Nothing to run", [("Nothing to run", "OK")], "" self._install_clickhouse(build_path) @@ -776,12 +774,6 @@ class ClickhouseIntegrationTestsRunner: ] ) - if should_fail: - if result_state == "success": - result_state = "failure" - else: - result_state = "success" - return result_state, status_text, test_result, logs def run_impl(self, repo_path, build_path): From 983e8c8bdfeb0bb21f2887647ba335f623dbfe2b Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 11 Mar 2022 14:37:00 +0000 Subject: [PATCH 14/30] Remove tests failed on master --- .../test_dummy_err_on_master/__init__.py | 0 .../test_dummy_err_on_master/test.py | 18 ------------------ .../0_stateless/02232_err_on_master.reference | 1 - .../0_stateless/02232_err_on_master.sql | 2 -- 4 files changed, 21 deletions(-) delete mode 100644 tests/integration/test_dummy_err_on_master/__init__.py delete mode 100644 tests/integration/test_dummy_err_on_master/test.py delete mode 100644 tests/queries/0_stateless/02232_err_on_master.reference delete mode 100644 tests/queries/0_stateless/02232_err_on_master.sql diff --git a/tests/integration/test_dummy_err_on_master/__init__.py b/tests/integration/test_dummy_err_on_master/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/integration/test_dummy_err_on_master/test.py b/tests/integration/test_dummy_err_on_master/test.py deleted file mode 100644 index 1f2a3595f1d..00000000000 --- a/tests/integration/test_dummy_err_on_master/test.py +++ /dev/null @@ -1,18 +0,0 @@ -import pytest -from helpers.cluster import ClickHouseCluster - -cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node') - -@pytest.fixture(scope="module") -def start_cluster(): - try: - cluster.start() - yield cluster - finally: - cluster.shutdown() - - -def test_drop_memory_database(start_cluster): - assert node.query("SELECT value not like '22%' FROM system.build_options WHERE name = 'GIT_BRANCH'").strip() == "1" - diff --git a/tests/queries/0_stateless/02232_err_on_master.reference b/tests/queries/0_stateless/02232_err_on_master.reference deleted file mode 100644 index d00491fd7e5..00000000000 --- a/tests/queries/0_stateless/02232_err_on_master.reference +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/tests/queries/0_stateless/02232_err_on_master.sql b/tests/queries/0_stateless/02232_err_on_master.sql deleted file mode 100644 index 8471a93df8d..00000000000 --- a/tests/queries/0_stateless/02232_err_on_master.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Tags: no-fasttest -SELECT value not like '22%' FROM system.build_options WHERE name = 'GIT_BRANCH'; From f2050b062eba520fa64de9c1b01c8a59788265c5 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 11 Mar 2022 14:42:19 +0000 Subject: [PATCH 15/30] Remove dummy tests ok on master --- .../test_dummy_success_on_master/__init__.py | 0 .../test_dummy_success_on_master/test.py | 18 ------------------ .../02232_success_on_master.reference | 1 - .../0_stateless/02232_success_on_master.sql | 2 -- 4 files changed, 21 deletions(-) delete mode 100644 tests/integration/test_dummy_success_on_master/__init__.py delete mode 100644 tests/integration/test_dummy_success_on_master/test.py delete mode 100644 tests/queries/0_stateless/02232_success_on_master.reference delete mode 100644 tests/queries/0_stateless/02232_success_on_master.sql diff --git a/tests/integration/test_dummy_success_on_master/__init__.py b/tests/integration/test_dummy_success_on_master/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/integration/test_dummy_success_on_master/test.py b/tests/integration/test_dummy_success_on_master/test.py deleted file mode 100644 index 81d2e34a675..00000000000 --- a/tests/integration/test_dummy_success_on_master/test.py +++ /dev/null @@ -1,18 +0,0 @@ -import pytest -from helpers.cluster import ClickHouseCluster - -cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node') - -@pytest.fixture(scope="module") -def start_cluster(): - try: - cluster.start() - yield cluster - finally: - cluster.shutdown() - - -def test_drop_memory_database(start_cluster): - assert node.query("SELECT 1").strip() == "1" - diff --git a/tests/queries/0_stateless/02232_success_on_master.reference b/tests/queries/0_stateless/02232_success_on_master.reference deleted file mode 100644 index d00491fd7e5..00000000000 --- a/tests/queries/0_stateless/02232_success_on_master.reference +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/tests/queries/0_stateless/02232_success_on_master.sql b/tests/queries/0_stateless/02232_success_on_master.sql deleted file mode 100644 index 01099570b05..00000000000 --- a/tests/queries/0_stateless/02232_success_on_master.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Tags: no-fasttest -SELECT 1; From 85f7d90ba2de14fb6532d6180aca28529b03ed46 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 11 Mar 2022 14:49:32 +0000 Subject: [PATCH 16/30] Dummy commit From b8b64b1d15f43406dffe75a9f6763d744d71d2c1 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 14 Mar 2022 13:09:53 +0000 Subject: [PATCH 17/30] Bugfix check requires either functional _or_ stateless test --- .github/workflows/pull_request.yml | 1 + tests/ci/commit_status_helper.py | 21 +++++++++++++++++++++ tests/ci/functional_test_check.py | 15 ++++++++++++--- tests/ci/integration_test_check.py | 14 +++++++++++++- tests/integration/ci-runner.py | 3 ++- 5 files changed, 49 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index e6bc2bff212..de72cb5daa9 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -2728,6 +2728,7 @@ jobs: sudo rm -fr "$TEMP_PATH" IntegrationTestsBugfixCheck: runs-on: [self-hosted, stress-tester] + needs: [FunctionalStatelessTestBugfixCheck] steps: - name: Set envs run: | diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py index 385567962d5..2ec7aba7801 100644 --- a/tests/ci/commit_status_helper.py +++ b/tests/ci/commit_status_helper.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import time +import fnmatch from env_helper import GITHUB_REPOSITORY from ci_config import CI_CONFIG @@ -49,3 +50,23 @@ def post_commit_status(gh, sha, check_name, description, state, report_url): if i == RETRY - 1: raise ex time.sleep(i) + + +def get_post_commit_status(gh, sha, check_name): + MAX_PAGES_NUM = 100 + for i in range(RETRY): + try: + statuses = get_commit(gh, sha, 1).get_statuses() + for num in range(MAX_PAGES_NUM): + page = statuses.get_page(num) + if not page: + break + for status in page: + if fnmatch.fnmatch(status.context, check_name): + return status + num += 1 + return None + except Exception as ex: + if i == RETRY - 1: + raise ex + time.sleep(i) diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 60f5d4ff4a5..82e2e4dbb74 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -22,6 +22,9 @@ from stopwatch import Stopwatch from rerun_helper import RerunHelper from tee_popen import TeePopen +NO_CHANGES_MSG = 'Nothing to run' + + def get_additional_envs(check_name, run_by_hash_num, run_by_hash_total): result = [] if 'DatabaseReplicated' in check_name: @@ -175,8 +178,8 @@ if __name__ == "__main__": tests_to_run = get_tests_to_run(pr_info) if not tests_to_run: commit = get_commit(gh, pr_info.sha) - state = override_status('success', check_name, validate_bugix_check) - commit.create_status(context=check_name_with_group, description='Not found changed stateless tests', state=state) + state = 'success' # mark as success, but integration tests check would fail if it also doesn't have any tests + commit.create_status(context=check_name_with_group, description=NO_CHANGES_MSG, state=state) sys.exit(0) image_name = get_image_name(check_name) @@ -222,7 +225,13 @@ if __name__ == "__main__": s3_helper = S3Helper('https://s3.amazonaws.com') state, description, test_results, additional_logs = process_results(result_path, server_log_path) - state = override_status(state, check_name, validate_bugix_check) + if validate_bugix_check: + if state != 'success': + description = f'Failed tests found: {description}' + # force success, because even if we don't have failed tests here, we need to check if there're intergration tests + state = 'success' + else: + state = override_status(state, check_name) ch_helper = ClickHouseHelper() mark_flaky_tests(ch_helper, check_name, test_results) diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py index ed513012296..a3b5725ecbd 100644 --- a/tests/ci/integration_test_check.py +++ b/tests/ci/integration_test_check.py @@ -17,7 +17,7 @@ from build_download_helper import download_all_deb_packages from download_previous_release import download_previous_release from upload_result_helper import upload_results from docker_pull_helper import get_images_with_versions -from commit_status_helper import post_commit_status, override_status +from commit_status_helper import post_commit_status, override_status, get_post_commit_status from clickhouse_helper import ClickHouseHelper, mark_flaky_tests, prepare_tests_results_for_clickhouse from stopwatch import Stopwatch from rerun_helper import RerunHelper @@ -103,6 +103,14 @@ def process_results(result_folder): return state, description, test_results, additional_files + +def is_stateless_bugfix_check_already_passed(gh, git_sha): + status = get_post_commit_status(gh, git_sha, 'Stateless tests bugfix validate check*') + if status is None: + return False + return status.description.statswith('Failed tests found') + + if __name__ == "__main__": logging.basicConfig(level=logging.INFO) @@ -141,6 +149,10 @@ if __name__ == "__main__": logging.info("Check is already finished according to github status, exiting") sys.exit(0) + if validate_bugix_check and is_stateless_bugfix_check_already_passed(gh, pr_info.sha): + logging.info("There already are stateless tests for this bugfix") + sys.exit(0) + images = get_images_with_versions(reports_path, IMAGES) images_with_versions = {i.name: i.version for i in images} result_path = os.path.join(temp_path, "output_dir") diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index d45352fa909..f67cd902e41 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -12,6 +12,7 @@ import subprocess import time import zlib # for crc32 +NO_CHANGES_MSG = 'Nothing to run' MAX_RETRY = 3 NUM_WORKERS = 5 @@ -710,7 +711,7 @@ class ClickhouseIntegrationTestsRunner: tests_to_run = get_changed_tests_to_run(pr_info, repo_path) if not tests_to_run: logging.info("No tests to run found") - return "success", "Nothing to run", [("Nothing to run", "OK")], "" + return "success", NO_CHANGES_MSG, [(NO_CHANGES_MSG, "OK")], "" self._install_clickhouse(build_path) logging.info("Found '%s' tests to run", " ".join(tests_to_run)) From da1e740b32d8afd715fbf0f3274f6ac92d1b6e6d Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 14 Mar 2022 13:12:32 +0000 Subject: [PATCH 18/30] Add dummy integration tests --- .../test_dummy_err_on_master/__init__.py | 0 .../test_dummy_err_on_master/test.py | 18 ++++++++++++++++++ .../test_dummy_success_on_master/__init__.py | 0 .../test_dummy_success_on_master/test.py | 18 ++++++++++++++++++ 4 files changed, 36 insertions(+) create mode 100644 tests/integration/test_dummy_err_on_master/__init__.py create mode 100644 tests/integration/test_dummy_err_on_master/test.py create mode 100644 tests/integration/test_dummy_success_on_master/__init__.py create mode 100644 tests/integration/test_dummy_success_on_master/test.py diff --git a/tests/integration/test_dummy_err_on_master/__init__.py b/tests/integration/test_dummy_err_on_master/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_dummy_err_on_master/test.py b/tests/integration/test_dummy_err_on_master/test.py new file mode 100644 index 00000000000..1f2a3595f1d --- /dev/null +++ b/tests/integration/test_dummy_err_on_master/test.py @@ -0,0 +1,18 @@ +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance('node') + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_drop_memory_database(start_cluster): + assert node.query("SELECT value not like '22%' FROM system.build_options WHERE name = 'GIT_BRANCH'").strip() == "1" + diff --git a/tests/integration/test_dummy_success_on_master/__init__.py b/tests/integration/test_dummy_success_on_master/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_dummy_success_on_master/test.py b/tests/integration/test_dummy_success_on_master/test.py new file mode 100644 index 00000000000..81d2e34a675 --- /dev/null +++ b/tests/integration/test_dummy_success_on_master/test.py @@ -0,0 +1,18 @@ +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance('node') + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_drop_memory_database(start_cluster): + assert node.query("SELECT 1").strip() == "1" + From 6f1cf419acf750c490147a6cc0bed545fdbd5b48 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 14 Mar 2022 13:24:46 +0000 Subject: [PATCH 19/30] Fix typo --- tests/ci/integration_test_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py index a3b5725ecbd..6f09f92bd41 100644 --- a/tests/ci/integration_test_check.py +++ b/tests/ci/integration_test_check.py @@ -108,7 +108,7 @@ def is_stateless_bugfix_check_already_passed(gh, git_sha): status = get_post_commit_status(gh, git_sha, 'Stateless tests bugfix validate check*') if status is None: return False - return status.description.statswith('Failed tests found') + return status.description.startswith('Failed tests found') if __name__ == "__main__": From 343c4ee6116a5f06a7e3d2d62efad147b9f91eb4 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 14 Mar 2022 14:22:31 +0000 Subject: [PATCH 20/30] Add 02232_err_on_master --- tests/queries/0_stateless/02232_err_on_master.reference | 1 + tests/queries/0_stateless/02232_err_on_master.sql | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 tests/queries/0_stateless/02232_err_on_master.reference create mode 100644 tests/queries/0_stateless/02232_err_on_master.sql diff --git a/tests/queries/0_stateless/02232_err_on_master.reference b/tests/queries/0_stateless/02232_err_on_master.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02232_err_on_master.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02232_err_on_master.sql b/tests/queries/0_stateless/02232_err_on_master.sql new file mode 100644 index 00000000000..8471a93df8d --- /dev/null +++ b/tests/queries/0_stateless/02232_err_on_master.sql @@ -0,0 +1,2 @@ +-- Tags: no-fasttest +SELECT value not like '22%' FROM system.build_options WHERE name = 'GIT_BRANCH'; From 16dbdc30de6898be669a66c577599d972b96f4a0 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 14 Mar 2022 14:23:16 +0000 Subject: [PATCH 21/30] rm test_dummy_err_on_master --- .../test_dummy_err_on_master/__init__.py | 0 .../test_dummy_err_on_master/test.py | 18 ------------------ 2 files changed, 18 deletions(-) delete mode 100644 tests/integration/test_dummy_err_on_master/__init__.py delete mode 100644 tests/integration/test_dummy_err_on_master/test.py diff --git a/tests/integration/test_dummy_err_on_master/__init__.py b/tests/integration/test_dummy_err_on_master/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/integration/test_dummy_err_on_master/test.py b/tests/integration/test_dummy_err_on_master/test.py deleted file mode 100644 index 1f2a3595f1d..00000000000 --- a/tests/integration/test_dummy_err_on_master/test.py +++ /dev/null @@ -1,18 +0,0 @@ -import pytest -from helpers.cluster import ClickHouseCluster - -cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node') - -@pytest.fixture(scope="module") -def start_cluster(): - try: - cluster.start() - yield cluster - finally: - cluster.shutdown() - - -def test_drop_memory_database(start_cluster): - assert node.query("SELECT value not like '22%' FROM system.build_options WHERE name = 'GIT_BRANCH'").strip() == "1" - From 494d5264ddaf48577f323a70cde505e145989784 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 14 Mar 2022 15:42:23 +0000 Subject: [PATCH 22/30] rm dummy tests --- .../test_dummy_success_on_master/__init__.py | 0 .../test_dummy_success_on_master/test.py | 18 ------------------ .../0_stateless/02232_err_on_master.reference | 1 - .../0_stateless/02232_err_on_master.sql | 2 -- 4 files changed, 21 deletions(-) delete mode 100644 tests/integration/test_dummy_success_on_master/__init__.py delete mode 100644 tests/integration/test_dummy_success_on_master/test.py delete mode 100644 tests/queries/0_stateless/02232_err_on_master.reference delete mode 100644 tests/queries/0_stateless/02232_err_on_master.sql diff --git a/tests/integration/test_dummy_success_on_master/__init__.py b/tests/integration/test_dummy_success_on_master/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tests/integration/test_dummy_success_on_master/test.py b/tests/integration/test_dummy_success_on_master/test.py deleted file mode 100644 index 81d2e34a675..00000000000 --- a/tests/integration/test_dummy_success_on_master/test.py +++ /dev/null @@ -1,18 +0,0 @@ -import pytest -from helpers.cluster import ClickHouseCluster - -cluster = ClickHouseCluster(__file__) -node = cluster.add_instance('node') - -@pytest.fixture(scope="module") -def start_cluster(): - try: - cluster.start() - yield cluster - finally: - cluster.shutdown() - - -def test_drop_memory_database(start_cluster): - assert node.query("SELECT 1").strip() == "1" - diff --git a/tests/queries/0_stateless/02232_err_on_master.reference b/tests/queries/0_stateless/02232_err_on_master.reference deleted file mode 100644 index d00491fd7e5..00000000000 --- a/tests/queries/0_stateless/02232_err_on_master.reference +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/tests/queries/0_stateless/02232_err_on_master.sql b/tests/queries/0_stateless/02232_err_on_master.sql deleted file mode 100644 index 8471a93df8d..00000000000 --- a/tests/queries/0_stateless/02232_err_on_master.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Tags: no-fasttest -SELECT value not like '22%' FROM system.build_options WHERE name = 'GIT_BRANCH'; From cf70b79f742549e119afe2e4d0b15dc587a34330 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 14 Mar 2022 15:43:15 +0000 Subject: [PATCH 23/30] Revert "Bugfix check requires either functional _or_ stateless test" This reverts commit b8b64b1d15f43406dffe75a9f6763d744d71d2c1. --- .github/workflows/pull_request.yml | 1 - tests/ci/commit_status_helper.py | 21 --------------------- tests/ci/functional_test_check.py | 15 +++------------ tests/ci/integration_test_check.py | 13 +------------ tests/integration/ci-runner.py | 3 +-- 5 files changed, 5 insertions(+), 48 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index de72cb5daa9..e6bc2bff212 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -2728,7 +2728,6 @@ jobs: sudo rm -fr "$TEMP_PATH" IntegrationTestsBugfixCheck: runs-on: [self-hosted, stress-tester] - needs: [FunctionalStatelessTestBugfixCheck] steps: - name: Set envs run: | diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py index 2ec7aba7801..385567962d5 100644 --- a/tests/ci/commit_status_helper.py +++ b/tests/ci/commit_status_helper.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 import time -import fnmatch from env_helper import GITHUB_REPOSITORY from ci_config import CI_CONFIG @@ -50,23 +49,3 @@ def post_commit_status(gh, sha, check_name, description, state, report_url): if i == RETRY - 1: raise ex time.sleep(i) - - -def get_post_commit_status(gh, sha, check_name): - MAX_PAGES_NUM = 100 - for i in range(RETRY): - try: - statuses = get_commit(gh, sha, 1).get_statuses() - for num in range(MAX_PAGES_NUM): - page = statuses.get_page(num) - if not page: - break - for status in page: - if fnmatch.fnmatch(status.context, check_name): - return status - num += 1 - return None - except Exception as ex: - if i == RETRY - 1: - raise ex - time.sleep(i) diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 82e2e4dbb74..60f5d4ff4a5 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -22,9 +22,6 @@ from stopwatch import Stopwatch from rerun_helper import RerunHelper from tee_popen import TeePopen -NO_CHANGES_MSG = 'Nothing to run' - - def get_additional_envs(check_name, run_by_hash_num, run_by_hash_total): result = [] if 'DatabaseReplicated' in check_name: @@ -178,8 +175,8 @@ if __name__ == "__main__": tests_to_run = get_tests_to_run(pr_info) if not tests_to_run: commit = get_commit(gh, pr_info.sha) - state = 'success' # mark as success, but integration tests check would fail if it also doesn't have any tests - commit.create_status(context=check_name_with_group, description=NO_CHANGES_MSG, state=state) + state = override_status('success', check_name, validate_bugix_check) + commit.create_status(context=check_name_with_group, description='Not found changed stateless tests', state=state) sys.exit(0) image_name = get_image_name(check_name) @@ -225,13 +222,7 @@ if __name__ == "__main__": s3_helper = S3Helper('https://s3.amazonaws.com') state, description, test_results, additional_logs = process_results(result_path, server_log_path) - if validate_bugix_check: - if state != 'success': - description = f'Failed tests found: {description}' - # force success, because even if we don't have failed tests here, we need to check if there're intergration tests - state = 'success' - else: - state = override_status(state, check_name) + state = override_status(state, check_name, validate_bugix_check) ch_helper = ClickHouseHelper() mark_flaky_tests(ch_helper, check_name, test_results) diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py index 6f09f92bd41..02007d42101 100644 --- a/tests/ci/integration_test_check.py +++ b/tests/ci/integration_test_check.py @@ -17,7 +17,7 @@ from build_download_helper import download_all_deb_packages from download_previous_release import download_previous_release from upload_result_helper import upload_results from docker_pull_helper import get_images_with_versions -from commit_status_helper import post_commit_status, override_status, get_post_commit_status +from commit_status_helper import post_commit_status, override_status from clickhouse_helper import ClickHouseHelper, mark_flaky_tests, prepare_tests_results_for_clickhouse from stopwatch import Stopwatch from rerun_helper import RerunHelper @@ -104,13 +104,6 @@ def process_results(result_folder): return state, description, test_results, additional_files -def is_stateless_bugfix_check_already_passed(gh, git_sha): - status = get_post_commit_status(gh, git_sha, 'Stateless tests bugfix validate check*') - if status is None: - return False - return status.description.startswith('Failed tests found') - - if __name__ == "__main__": logging.basicConfig(level=logging.INFO) @@ -149,10 +142,6 @@ if __name__ == "__main__": logging.info("Check is already finished according to github status, exiting") sys.exit(0) - if validate_bugix_check and is_stateless_bugfix_check_already_passed(gh, pr_info.sha): - logging.info("There already are stateless tests for this bugfix") - sys.exit(0) - images = get_images_with_versions(reports_path, IMAGES) images_with_versions = {i.name: i.version for i in images} result_path = os.path.join(temp_path, "output_dir") diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index f67cd902e41..d45352fa909 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -12,7 +12,6 @@ import subprocess import time import zlib # for crc32 -NO_CHANGES_MSG = 'Nothing to run' MAX_RETRY = 3 NUM_WORKERS = 5 @@ -711,7 +710,7 @@ class ClickhouseIntegrationTestsRunner: tests_to_run = get_changed_tests_to_run(pr_info, repo_path) if not tests_to_run: logging.info("No tests to run found") - return "success", NO_CHANGES_MSG, [(NO_CHANGES_MSG, "OK")], "" + return "success", "Nothing to run", [("Nothing to run", "OK")], "" self._install_clickhouse(build_path) logging.info("Found '%s' tests to run", " ".join(tests_to_run)) From 9f2feb4f0b9ffa67a981e96746e4e2336ec2aa78 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 14 Mar 2022 15:52:46 +0000 Subject: [PATCH 24/30] Use argparse in functional/integration_test_check.py --- tests/ci/functional_test_check.py | 22 ++++++++++++++++------ tests/ci/integration_test_check.py | 19 ++++++++++++++----- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 60f5d4ff4a5..fca9ff4f355 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -8,6 +8,7 @@ import sys from github import Github +import argparse from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH from s3_helper import S3Helper from get_robot_token import get_best_robot_token @@ -129,6 +130,14 @@ def process_results(result_folder, server_log_path): return state, description, test_results, additional_files +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("check_name") + parser.add_argument("kill_timeout", type=int) + parser.add_argument("--validate-bugfix", action='store_true', help="Check that added tests failed on latest stable") + return parser.parse_args() + + if __name__ == "__main__": logging.basicConfig(level=logging.INFO) @@ -138,10 +147,11 @@ if __name__ == "__main__": repo_path = REPO_COPY reports_path = REPORTS_PATH - check_name = sys.argv[1] - kill_timeout = int(sys.argv[2]) + args = parse_args() + check_name = args.check_name + kill_timeout = args.kill_timeout + validate_bugix_check = args.validate_bugfix - validate_bugix_check = len(sys.argv) >= 4 and sys.argv[3] == "--validate-bugfix" flaky_check = 'flaky' in check_name.lower() run_changed_tests = flaky_check or validate_bugix_check @@ -188,10 +198,10 @@ if __name__ == "__main__": if not os.path.exists(packages_path): os.makedirs(packages_path) - if not validate_bugix_check: - download_all_deb_packages(check_name, reports_path, packages_path) - else: + if validate_bugix_check: download_previous_release(packages_path) + else: + download_all_deb_packages(check_name, reports_path, packages_path) server_log_path = os.path.join(temp_path, "server_log") if not os.path.exists(server_log_path): diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py index 02007d42101..23327d078f7 100644 --- a/tests/ci/integration_test_check.py +++ b/tests/ci/integration_test_check.py @@ -9,6 +9,7 @@ import csv from github import Github +import argparse from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH from s3_helper import S3Helper from get_robot_token import get_best_robot_token @@ -104,6 +105,13 @@ def process_results(result_folder): return state, description, test_results, additional_files +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("check_name") + parser.add_argument("--validate-bugfix", action='store_true', help="Check that added tests failed on latest stable") + return parser.parse_args() + + if __name__ == "__main__": logging.basicConfig(level=logging.INFO) @@ -113,8 +121,9 @@ if __name__ == "__main__": repo_path = REPO_COPY reports_path = REPORTS_PATH - check_name = sys.argv[1] - validate_bugix_check = len(sys.argv) >= 3 and sys.argv[2] == "--validate-bugfix" + args = parse_args() + check_name = args.check_name + validate_bugix_check = args.validate_bugfix if 'RUN_BY_HASH_NUM' in os.environ: run_by_hash_num = int(os.getenv('RUN_BY_HASH_NUM')) @@ -156,10 +165,10 @@ if __name__ == "__main__": if not os.path.exists(build_path): os.makedirs(build_path) - if not validate_bugix_check: - download_all_deb_packages(check_name, reports_path, build_path) - else: + if validate_bugix_check: download_previous_release(build_path) + else: + download_all_deb_packages(check_name, reports_path, build_path) my_env = get_env_for_runner(build_path, repo_path, result_path, work_path) From a4c410a1f5830bdc749a18c4951f65a3596c094c Mon Sep 17 00:00:00 2001 From: Vladimir C Date: Mon, 14 Mar 2022 16:54:21 +0100 Subject: [PATCH 25/30] Update tests/ci/docker_pull_helper.py Co-authored-by: Mikhail f. Shiryaev --- tests/ci/docker_pull_helper.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/ci/docker_pull_helper.py b/tests/ci/docker_pull_helper.py index 0bf3bcfaeac..b6f63299d4a 100644 --- a/tests/ci/docker_pull_helper.py +++ b/tests/ci/docker_pull_helper.py @@ -6,8 +6,10 @@ import time import subprocess import logging +from typing import Optional + class DockerImage: - def __init__(self, name, version=None): + def __init__(self, name, version : Optional[str] = None): self.name = name if version is None: self.version = 'latest' @@ -17,7 +19,7 @@ class DockerImage: def __str__(self): return f"{self.name}:{self.version}" -def get_images_with_versions(reports_path, required_image, pull=True, version=None): +def get_images_with_versions(reports_path, required_image, pull=True, version : Optional[str] = None): images_path = None for root, _, files in os.walk(reports_path): for f in files: @@ -40,11 +42,9 @@ def get_images_with_versions(reports_path, required_image, pull=True, version=No docker_images = [] for image_name in required_image: - docker_image = DockerImage(image_name) + docker_image = DockerImage(image_name, version) if image_name in images: docker_image.version = images[image_name] - if version is not None: - docker_image.version = str(version) docker_images.append(docker_image) if pull: From 2551adc4168ed4bbbe82429370160d63a0748a87 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 18 Mar 2022 13:36:45 +0100 Subject: [PATCH 26/30] Bugfix validate meta check --- .github/workflows/pull_request.yml | 58 ++++++++++-------------------- tests/ci/bugfix_validate_check.py | 46 ++++++++++++++++++++++++ tests/ci/commit_status_helper.py | 10 ++++++ tests/ci/functional_test_check.py | 21 ++++++++--- tests/ci/integration_test_check.py | 13 +++++-- tests/integration/ci-runner.py | 4 ++- 6 files changed, 104 insertions(+), 48 deletions(-) create mode 100644 tests/ci/bugfix_validate_check.py diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index e6bc2bff212..d50a2151f2f 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -1733,17 +1733,17 @@ jobs: docker kill "$(docker ps -q)" ||: docker rm -f "$(docker ps -a -q)" ||: sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestBugfixCheck: - runs-on: [self-hosted, func-tester] + TestsBugfixCheck: + runs-on: [self-hosted, stress-tester] steps: - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_bugfix_asan + TEMP_PATH=${{runner.temp}}/tests_bugfix_check REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests bugfix validate check (address, actions) - REPO_COPY=${{runner.temp}}/stateless_bugfix_asan/ClickHouse + CHECK_NAME=Tests bugfix validate check (actions) KILL_TIMEOUT=3600 + REPO_COPY=${{runner.temp}}/tests_bugfix_check/ClickHouse EOF - name: Download json reports uses: actions/download-artifact@v2 @@ -1754,13 +1754,24 @@ jobs: sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - name: Check out repository code uses: actions/checkout@v2 - - name: Functional test + - name: Bugfix test run: | sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" --validate-bugfix + + TEMP_PATH="${TEMP_PATH}/integration" \ + REPORTS_PATH="${REPORTS_PATH}/integration" \ + python3 integration_test_check.py "Integration tests bugfix validate check" \ + --validate-bugfix --post-commit-status=file || echo 'ignore exit code' + + TEMP_PATH="${TEMP_PATH}/stateless" \ + REPORTS_PATH="${REPORTS_PATH}/stateless" \ + python3 functional_test_check.py "Stateless tests bugfix validate check" "$KILL_TIMEOUT" \ + --validate-bugfix --post-commit-status=file || echo 'ignore exit code' + + python3 bugfix_validate_check.py "${TEMP_PATH}/stateless/post_commit_status.tsv" "${TEMP_PATH}/integration/post_commit_status.tsv" - name: Cleanup if: always() run: | @@ -2726,39 +2737,6 @@ jobs: docker kill "$(docker ps -q)" ||: docker rm -f "$(docker ps -a -q)" ||: sudo rm -fr "$TEMP_PATH" - IntegrationTestsBugfixCheck: - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan_bugfix_check - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests bugfix validate check (asan, actions) - REPO_COPY=${{runner.temp}}/integration_tests_asan_bugfix_check/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" --validate-bugfix - - name: Cleanup - if: always() - run: | - docker kill "$(docker ps -q)" ||: - docker rm -f "$(docker ps -a -q)" ||: - sudo rm -fr "$TEMP_PATH" ############################################################################################# #################################### UNIT TESTS ############################################# ############################################################################################# diff --git a/tests/ci/bugfix_validate_check.py b/tests/ci/bugfix_validate_check.py new file mode 100644 index 00000000000..ed579006192 --- /dev/null +++ b/tests/ci/bugfix_validate_check.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +import argparse +import csv +import itertools +import os +import sys + +NO_CHANGES_MSG = 'Nothing to run' + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('report1') + parser.add_argument('report2') + return parser.parse_args() + + +def post_commit_status_from_file(file_path): + res = [] + with open(file_path, 'r', encoding='utf-8') as f: + fin = csv.reader(f, delimiter='\t') + res = list(itertools.islice(fin, 1)) + if len(res) < 1: + raise Exception(f'Can\'t read from "{file_path}"') + if len(res[0]) != 3: + raise Exception(f'Can\'t read from "{file_path}"') + return res[0] + + +def process_results(file_path): + state, report_url, description = post_commit_status_from_file(file_path) + prefix = os.path.basename(os.path.dirname(file_path)) + print(f'::notice:: bugfix check: {prefix} - {state}: {description} Report url: {report_url}') + return state == 'success' + + +def main(args): + is_ok = False + is_ok = process_results(args.report1) or is_ok + is_ok = process_results(args.report2) or is_ok + sys.exit(0 if is_ok else 1) + + +if __name__ == '__main__': + main(parse_args()) diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py index 385567962d5..08f14261fa8 100644 --- a/tests/ci/commit_status_helper.py +++ b/tests/ci/commit_status_helper.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import time +import os +import csv from env_helper import GITHUB_REPOSITORY from ci_config import CI_CONFIG @@ -49,3 +51,11 @@ def post_commit_status(gh, sha, check_name, description, state, report_url): if i == RETRY - 1: raise ex time.sleep(i) + + +def post_commit_status_to_file(file_path, description, state, report_url): + if os.path.exists(file_path): + raise Exception(f'File "{file_path}" already exists!') + with open(file_path, 'w', encoding='utf-8') as f: + out = csv.writer(f, delimiter='\t') + out.writerow([state, report_url, description]) diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index fca9ff4f355..16aa6d3f6d1 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -17,12 +17,15 @@ from build_download_helper import download_all_deb_packages from download_previous_release import download_previous_release from upload_result_helper import upload_results from docker_pull_helper import get_image_with_version -from commit_status_helper import post_commit_status, get_commit, override_status +from commit_status_helper import post_commit_status, get_commit, override_status, post_commit_status_to_file from clickhouse_helper import ClickHouseHelper, mark_flaky_tests, prepare_tests_results_for_clickhouse from stopwatch import Stopwatch from rerun_helper import RerunHelper from tee_popen import TeePopen +NO_CHANGES_MSG = 'Nothing to run' + + def get_additional_envs(check_name, run_by_hash_num, run_by_hash_total): result = [] if 'DatabaseReplicated' in check_name: @@ -135,6 +138,7 @@ def parse_args(): parser.add_argument("check_name") parser.add_argument("kill_timeout", type=int) parser.add_argument("--validate-bugfix", action='store_true', help="Check that added tests failed on latest stable") + parser.add_argument("--post-commit-status", default='commit_status', choices=['commit_status', 'file'], help="Where to public post commit status") return parser.parse_args() @@ -186,7 +190,11 @@ if __name__ == "__main__": if not tests_to_run: commit = get_commit(gh, pr_info.sha) state = override_status('success', check_name, validate_bugix_check) - commit.create_status(context=check_name_with_group, description='Not found changed stateless tests', state=state) + if args.post_commit_status == 'commit_status': + commit.create_status(context=check_name_with_group, description=NO_CHANGES_MSG, state=state) + elif args.post_commit_status == 'file': + fpath = os.path.join(temp_path, "post_commit_status.tsv") + post_commit_status_to_file(fpath, description=NO_CHANGES_MSG, state=state, report_url='null') sys.exit(0) image_name = get_image_name(check_name) @@ -239,8 +247,13 @@ if __name__ == "__main__": report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [run_log_path] + additional_logs, check_name_with_group) - print(f"::notice ::Report url: {report_url}") - post_commit_status(gh, pr_info.sha, check_name_with_group, description, state, report_url) + print(f"::notice:: {check_name} Report url: {report_url}") + if args.post_commit_status == 'commit_status': + post_commit_status(gh, pr_info.sha, check_name_with_group, description, state, report_url) + elif args.post_commit_status == 'file': + post_commit_status_to_file(os.path.join(temp_path, "post_commit_status.tsv"), description, state, report_url) + else: + raise Exception(f'Unknown post_commit_status option "{args.post_commit_status}"') prepared_events = prepare_tests_results_for_clickhouse(pr_info, test_results, state, stopwatch.duration_seconds, stopwatch.start_time_str, report_url, check_name_with_group) ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events) diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py index 23327d078f7..dee075a9f03 100644 --- a/tests/ci/integration_test_check.py +++ b/tests/ci/integration_test_check.py @@ -18,7 +18,7 @@ from build_download_helper import download_all_deb_packages from download_previous_release import download_previous_release from upload_result_helper import upload_results from docker_pull_helper import get_images_with_versions -from commit_status_helper import post_commit_status, override_status +from commit_status_helper import post_commit_status, override_status, post_commit_status_to_file from clickhouse_helper import ClickHouseHelper, mark_flaky_tests, prepare_tests_results_for_clickhouse from stopwatch import Stopwatch from rerun_helper import RerunHelper @@ -109,6 +109,7 @@ def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("check_name") parser.add_argument("--validate-bugfix", action='store_true', help="Check that added tests failed on latest stable") + parser.add_argument("--post-commit-status", default='commit_status', choices=['commit_status', 'file'], help="Where to public post commit status") return parser.parse_args() @@ -198,8 +199,14 @@ if __name__ == "__main__": s3_helper = S3Helper('https://s3.amazonaws.com') report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [output_path_log] + additional_logs, check_name_with_group, False) - print(f"::notice ::Report url: {report_url}") - post_commit_status(gh, pr_info.sha, check_name_with_group, description, state, report_url) + + print(f"::notice:: {check_name} Report url: {report_url}") + if args.post_commit_status == 'commit_status': + post_commit_status(gh, pr_info.sha, check_name_with_group, description, state, report_url) + elif args.post_commit_status == 'file': + post_commit_status_to_file(os.path.join(temp_path, "post_commit_status.tsv"), description, state, report_url) + else: + raise Exception(f'Unknown post_commit_status option "{args.post_commit_status}"') prepared_events = prepare_tests_results_for_clickhouse(pr_info, test_results, state, stopwatch.duration_seconds, stopwatch.start_time_str, report_url, check_name_with_group) ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index d45352fa909..8a42510570a 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -27,6 +27,8 @@ MAX_TIME_SECONDS = 3600 MAX_TIME_IN_SANDBOX = 20 * 60 # 20 minutes TASK_TIMEOUT = 8 * 60 * 60 # 8 hours +NO_CHANGES_MSG = 'Nothing to run' + def stringhash(s): return zlib.crc32(s.encode("utf-8")) @@ -710,7 +712,7 @@ class ClickhouseIntegrationTestsRunner: tests_to_run = get_changed_tests_to_run(pr_info, repo_path) if not tests_to_run: logging.info("No tests to run found") - return "success", "Nothing to run", [("Nothing to run", "OK")], "" + return "success", NO_CHANGES_MSG, [(NO_CHANGES_MSG, "OK")], "" self._install_clickhouse(build_path) logging.info("Found '%s' tests to run", " ".join(tests_to_run)) From a809c16b88b7e19c8d24442f38a310812d9a1e67 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 14 Mar 2022 14:22:31 +0000 Subject: [PATCH 27/30] Add 02232_err_on_master --- tests/queries/0_stateless/02232_err_on_master.reference | 1 + tests/queries/0_stateless/02232_err_on_master.sql | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 tests/queries/0_stateless/02232_err_on_master.reference create mode 100644 tests/queries/0_stateless/02232_err_on_master.sql diff --git a/tests/queries/0_stateless/02232_err_on_master.reference b/tests/queries/0_stateless/02232_err_on_master.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02232_err_on_master.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02232_err_on_master.sql b/tests/queries/0_stateless/02232_err_on_master.sql new file mode 100644 index 00000000000..8471a93df8d --- /dev/null +++ b/tests/queries/0_stateless/02232_err_on_master.sql @@ -0,0 +1,2 @@ +-- Tags: no-fasttest +SELECT value not like '22%' FROM system.build_options WHERE name = 'GIT_BRANCH'; From daa716f6823d0eceb6e4f75ee301fde6f73fc910 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 18 Mar 2022 21:32:59 +0100 Subject: [PATCH 28/30] fix style --- tests/ci/functional_test_check.py | 4 ++-- tests/ci/integration_test_check.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 16aa6d3f6d1..56adc565b63 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -1,14 +1,14 @@ #!/usr/bin/env python3 +import argparse import csv import logging -import subprocess import os +import subprocess import sys from github import Github -import argparse from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH from s3_helper import S3Helper from get_robot_token import get_best_robot_token diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py index dee075a9f03..e749a37b3e4 100644 --- a/tests/ci/integration_test_check.py +++ b/tests/ci/integration_test_check.py @@ -1,15 +1,15 @@ #!/usr/bin/env python3 -import os -import logging -import sys -import json -import subprocess +import argparse import csv +import json +import logging +import os +import subprocess +import sys from github import Github -import argparse from env_helper import TEMP_PATH, REPO_COPY, REPORTS_PATH from s3_helper import S3Helper from get_robot_token import get_best_robot_token From 0172ad62390a71fce712ee1628c363d5c62b79db Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 18 Mar 2022 21:33:18 +0100 Subject: [PATCH 29/30] Revert "Add 02232_err_on_master" This reverts commit a809c16b88b7e19c8d24442f38a310812d9a1e67. --- tests/queries/0_stateless/02232_err_on_master.reference | 1 - tests/queries/0_stateless/02232_err_on_master.sql | 2 -- 2 files changed, 3 deletions(-) delete mode 100644 tests/queries/0_stateless/02232_err_on_master.reference delete mode 100644 tests/queries/0_stateless/02232_err_on_master.sql diff --git a/tests/queries/0_stateless/02232_err_on_master.reference b/tests/queries/0_stateless/02232_err_on_master.reference deleted file mode 100644 index d00491fd7e5..00000000000 --- a/tests/queries/0_stateless/02232_err_on_master.reference +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/tests/queries/0_stateless/02232_err_on_master.sql b/tests/queries/0_stateless/02232_err_on_master.sql deleted file mode 100644 index 8471a93df8d..00000000000 --- a/tests/queries/0_stateless/02232_err_on_master.sql +++ /dev/null @@ -1,2 +0,0 @@ --- Tags: no-fasttest -SELECT value not like '22%' FROM system.build_options WHERE name = 'GIT_BRANCH'; From 41c32e24fe4a125d31e0a93a412ee23aec1112b4 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 21 Mar 2022 13:02:51 +0100 Subject: [PATCH 30/30] Skip bugfix_validate_check when no pr-bugfix --- tests/ci/functional_test_check.py | 10 ++++++---- tests/ci/integration_test_check.py | 4 +++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 56adc565b63..2e19f94b9c0 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -163,8 +163,13 @@ if __name__ == "__main__": pr_info = PRInfo(need_changed_files=run_changed_tests) + if not os.path.exists(temp_path): + os.makedirs(temp_path) + if validate_bugix_check and 'pr-bugfix' not in pr_info.labels: - logging.info("Skipping %s (no pr-bugfix)", check_name) + if args.post_commit_status == 'file': + post_commit_status_to_file(os.path.join(temp_path, "post_commit_status.tsv"), 'Skipped (no pr-bugfix)', 'success', 'null') + logging.info("Skipping '%s' (no pr-bugfix)", check_name) sys.exit(0) if 'RUN_BY_HASH_NUM' in os.environ: @@ -181,9 +186,6 @@ if __name__ == "__main__": logging.info("Check is already finished according to github status, exiting") sys.exit(0) - if not os.path.exists(temp_path): - os.makedirs(temp_path) - tests_to_run = [] if run_changed_tests: tests_to_run = get_tests_to_run(pr_info) diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py index e749a37b3e4..17edde49b63 100644 --- a/tests/ci/integration_test_check.py +++ b/tests/ci/integration_test_check.py @@ -142,7 +142,9 @@ if __name__ == "__main__": pr_info = PRInfo(need_changed_files=is_flaky_check or validate_bugix_check) if validate_bugix_check and 'pr-bugfix' not in pr_info.labels: - logging.info("Skipping %s (no pr-bugfix)", check_name) + if args.post_commit_status == 'file': + post_commit_status_to_file(os.path.join(temp_path, "post_commit_status.tsv"), 'Skipped (no pr-bugfix)', 'success', 'null') + logging.info("Skipping '%s' (no pr-bugfix)", check_name) sys.exit(0) gh = Github(get_best_robot_token())