From 2551adc4168ed4bbbe82429370160d63a0748a87 Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 18 Mar 2022 13:36:45 +0100 Subject: [PATCH] Bugfix validate meta check --- .github/workflows/pull_request.yml | 58 ++++++++++-------------------- tests/ci/bugfix_validate_check.py | 46 ++++++++++++++++++++++++ tests/ci/commit_status_helper.py | 10 ++++++ tests/ci/functional_test_check.py | 21 ++++++++--- tests/ci/integration_test_check.py | 13 +++++-- tests/integration/ci-runner.py | 4 ++- 6 files changed, 104 insertions(+), 48 deletions(-) create mode 100644 tests/ci/bugfix_validate_check.py diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index e6bc2bff212..d50a2151f2f 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -1733,17 +1733,17 @@ jobs: docker kill "$(docker ps -q)" ||: docker rm -f "$(docker ps -a -q)" ||: sudo rm -fr "$TEMP_PATH" - FunctionalStatelessTestBugfixCheck: - runs-on: [self-hosted, func-tester] + TestsBugfixCheck: + runs-on: [self-hosted, stress-tester] steps: - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/stateless_bugfix_asan + TEMP_PATH=${{runner.temp}}/tests_bugfix_check REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Stateless tests bugfix validate check (address, actions) - REPO_COPY=${{runner.temp}}/stateless_bugfix_asan/ClickHouse + CHECK_NAME=Tests bugfix validate check (actions) KILL_TIMEOUT=3600 + REPO_COPY=${{runner.temp}}/tests_bugfix_check/ClickHouse EOF - name: Download json reports uses: actions/download-artifact@v2 @@ -1754,13 +1754,24 @@ jobs: sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - name: Check out repository code uses: actions/checkout@v2 - - name: Functional test + - name: Bugfix test run: | sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" cd "$REPO_COPY/tests/ci" - python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" --validate-bugfix + + TEMP_PATH="${TEMP_PATH}/integration" \ + REPORTS_PATH="${REPORTS_PATH}/integration" \ + python3 integration_test_check.py "Integration tests bugfix validate check" \ + --validate-bugfix --post-commit-status=file || echo 'ignore exit code' + + TEMP_PATH="${TEMP_PATH}/stateless" \ + REPORTS_PATH="${REPORTS_PATH}/stateless" \ + python3 functional_test_check.py "Stateless tests bugfix validate check" "$KILL_TIMEOUT" \ + --validate-bugfix --post-commit-status=file || echo 'ignore exit code' + + python3 bugfix_validate_check.py "${TEMP_PATH}/stateless/post_commit_status.tsv" "${TEMP_PATH}/integration/post_commit_status.tsv" - name: Cleanup if: always() run: | @@ -2726,39 +2737,6 @@ jobs: docker kill "$(docker ps -q)" ||: docker rm -f "$(docker ps -a -q)" ||: sudo rm -fr "$TEMP_PATH" - IntegrationTestsBugfixCheck: - runs-on: [self-hosted, stress-tester] - steps: - - name: Set envs - run: | - cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/integration_tests_asan_bugfix_check - REPORTS_PATH=${{runner.temp}}/reports_dir - CHECK_NAME=Integration tests bugfix validate check (asan, actions) - REPO_COPY=${{runner.temp}}/integration_tests_asan_bugfix_check/ClickHouse - EOF - - name: Download json reports - uses: actions/download-artifact@v2 - with: - path: ${{ env.REPORTS_PATH }} - - name: Clear repository - run: | - sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" - - name: Check out repository code - uses: actions/checkout@v2 - - name: Integration test - run: | - sudo rm -fr "$TEMP_PATH" - mkdir -p "$TEMP_PATH" - cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" - python3 integration_test_check.py "$CHECK_NAME" --validate-bugfix - - name: Cleanup - if: always() - run: | - docker kill "$(docker ps -q)" ||: - docker rm -f "$(docker ps -a -q)" ||: - sudo rm -fr "$TEMP_PATH" ############################################################################################# #################################### UNIT TESTS ############################################# ############################################################################################# diff --git a/tests/ci/bugfix_validate_check.py b/tests/ci/bugfix_validate_check.py new file mode 100644 index 00000000000..ed579006192 --- /dev/null +++ b/tests/ci/bugfix_validate_check.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 + +import argparse +import csv +import itertools +import os +import sys + +NO_CHANGES_MSG = 'Nothing to run' + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('report1') + parser.add_argument('report2') + return parser.parse_args() + + +def post_commit_status_from_file(file_path): + res = [] + with open(file_path, 'r', encoding='utf-8') as f: + fin = csv.reader(f, delimiter='\t') + res = list(itertools.islice(fin, 1)) + if len(res) < 1: + raise Exception(f'Can\'t read from "{file_path}"') + if len(res[0]) != 3: + raise Exception(f'Can\'t read from "{file_path}"') + return res[0] + + +def process_results(file_path): + state, report_url, description = post_commit_status_from_file(file_path) + prefix = os.path.basename(os.path.dirname(file_path)) + print(f'::notice:: bugfix check: {prefix} - {state}: {description} Report url: {report_url}') + return state == 'success' + + +def main(args): + is_ok = False + is_ok = process_results(args.report1) or is_ok + is_ok = process_results(args.report2) or is_ok + sys.exit(0 if is_ok else 1) + + +if __name__ == '__main__': + main(parse_args()) diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py index 385567962d5..08f14261fa8 100644 --- a/tests/ci/commit_status_helper.py +++ b/tests/ci/commit_status_helper.py @@ -1,6 +1,8 @@ #!/usr/bin/env python3 import time +import os +import csv from env_helper import GITHUB_REPOSITORY from ci_config import CI_CONFIG @@ -49,3 +51,11 @@ def post_commit_status(gh, sha, check_name, description, state, report_url): if i == RETRY - 1: raise ex time.sleep(i) + + +def post_commit_status_to_file(file_path, description, state, report_url): + if os.path.exists(file_path): + raise Exception(f'File "{file_path}" already exists!') + with open(file_path, 'w', encoding='utf-8') as f: + out = csv.writer(f, delimiter='\t') + out.writerow([state, report_url, description]) diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index fca9ff4f355..16aa6d3f6d1 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -17,12 +17,15 @@ from build_download_helper import download_all_deb_packages from download_previous_release import download_previous_release from upload_result_helper import upload_results from docker_pull_helper import get_image_with_version -from commit_status_helper import post_commit_status, get_commit, override_status +from commit_status_helper import post_commit_status, get_commit, override_status, post_commit_status_to_file from clickhouse_helper import ClickHouseHelper, mark_flaky_tests, prepare_tests_results_for_clickhouse from stopwatch import Stopwatch from rerun_helper import RerunHelper from tee_popen import TeePopen +NO_CHANGES_MSG = 'Nothing to run' + + def get_additional_envs(check_name, run_by_hash_num, run_by_hash_total): result = [] if 'DatabaseReplicated' in check_name: @@ -135,6 +138,7 @@ def parse_args(): parser.add_argument("check_name") parser.add_argument("kill_timeout", type=int) parser.add_argument("--validate-bugfix", action='store_true', help="Check that added tests failed on latest stable") + parser.add_argument("--post-commit-status", default='commit_status', choices=['commit_status', 'file'], help="Where to public post commit status") return parser.parse_args() @@ -186,7 +190,11 @@ if __name__ == "__main__": if not tests_to_run: commit = get_commit(gh, pr_info.sha) state = override_status('success', check_name, validate_bugix_check) - commit.create_status(context=check_name_with_group, description='Not found changed stateless tests', state=state) + if args.post_commit_status == 'commit_status': + commit.create_status(context=check_name_with_group, description=NO_CHANGES_MSG, state=state) + elif args.post_commit_status == 'file': + fpath = os.path.join(temp_path, "post_commit_status.tsv") + post_commit_status_to_file(fpath, description=NO_CHANGES_MSG, state=state, report_url='null') sys.exit(0) image_name = get_image_name(check_name) @@ -239,8 +247,13 @@ if __name__ == "__main__": report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [run_log_path] + additional_logs, check_name_with_group) - print(f"::notice ::Report url: {report_url}") - post_commit_status(gh, pr_info.sha, check_name_with_group, description, state, report_url) + print(f"::notice:: {check_name} Report url: {report_url}") + if args.post_commit_status == 'commit_status': + post_commit_status(gh, pr_info.sha, check_name_with_group, description, state, report_url) + elif args.post_commit_status == 'file': + post_commit_status_to_file(os.path.join(temp_path, "post_commit_status.tsv"), description, state, report_url) + else: + raise Exception(f'Unknown post_commit_status option "{args.post_commit_status}"') prepared_events = prepare_tests_results_for_clickhouse(pr_info, test_results, state, stopwatch.duration_seconds, stopwatch.start_time_str, report_url, check_name_with_group) ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events) diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py index 23327d078f7..dee075a9f03 100644 --- a/tests/ci/integration_test_check.py +++ b/tests/ci/integration_test_check.py @@ -18,7 +18,7 @@ from build_download_helper import download_all_deb_packages from download_previous_release import download_previous_release from upload_result_helper import upload_results from docker_pull_helper import get_images_with_versions -from commit_status_helper import post_commit_status, override_status +from commit_status_helper import post_commit_status, override_status, post_commit_status_to_file from clickhouse_helper import ClickHouseHelper, mark_flaky_tests, prepare_tests_results_for_clickhouse from stopwatch import Stopwatch from rerun_helper import RerunHelper @@ -109,6 +109,7 @@ def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("check_name") parser.add_argument("--validate-bugfix", action='store_true', help="Check that added tests failed on latest stable") + parser.add_argument("--post-commit-status", default='commit_status', choices=['commit_status', 'file'], help="Where to public post commit status") return parser.parse_args() @@ -198,8 +199,14 @@ if __name__ == "__main__": s3_helper = S3Helper('https://s3.amazonaws.com') report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [output_path_log] + additional_logs, check_name_with_group, False) - print(f"::notice ::Report url: {report_url}") - post_commit_status(gh, pr_info.sha, check_name_with_group, description, state, report_url) + + print(f"::notice:: {check_name} Report url: {report_url}") + if args.post_commit_status == 'commit_status': + post_commit_status(gh, pr_info.sha, check_name_with_group, description, state, report_url) + elif args.post_commit_status == 'file': + post_commit_status_to_file(os.path.join(temp_path, "post_commit_status.tsv"), description, state, report_url) + else: + raise Exception(f'Unknown post_commit_status option "{args.post_commit_status}"') prepared_events = prepare_tests_results_for_clickhouse(pr_info, test_results, state, stopwatch.duration_seconds, stopwatch.start_time_str, report_url, check_name_with_group) ch_helper.insert_events_into(db="gh-data", table="checks", events=prepared_events) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index d45352fa909..8a42510570a 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -27,6 +27,8 @@ MAX_TIME_SECONDS = 3600 MAX_TIME_IN_SANDBOX = 20 * 60 # 20 minutes TASK_TIMEOUT = 8 * 60 * 60 # 8 hours +NO_CHANGES_MSG = 'Nothing to run' + def stringhash(s): return zlib.crc32(s.encode("utf-8")) @@ -710,7 +712,7 @@ class ClickhouseIntegrationTestsRunner: tests_to_run = get_changed_tests_to_run(pr_info, repo_path) if not tests_to_run: logging.info("No tests to run found") - return "success", "Nothing to run", [("Nothing to run", "OK")], "" + return "success", NO_CHANGES_MSG, [(NO_CHANGES_MSG, "OK")], "" self._install_clickhouse(build_path) logging.info("Found '%s' tests to run", " ".join(tests_to_run))