From 081bfdfe82ba5ce9ca087de3e87f098e9eec34ed Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 26 Nov 2021 13:57:36 +0300 Subject: [PATCH] Get rid of build numbers and simplify builds paths in S3 --- .github/workflows/main.yml | 34 +- .github/workflows/master.yml | 36 +- .github/workflows/release_branches.yml | 26 +- tests/ci/ast_fuzzer_check.py | 11 +- tests/ci/build_check.py | 20 +- tests/ci/build_download_helper.py | 17 +- tests/ci/ci_config.py | 539 +++---------------------- tests/ci/metrics_lambda/app.py | 2 +- tests/integration/ci-runner.py | 3 +- 9 files changed, 135 insertions(+), 553 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d0a6189313d..24b3e178651 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -211,12 +211,12 @@ jobs: REPO_COPY: ${{runner.temp}}/build_check/ClickHouse CACHES_PATH: ${{runner.temp}}/../ccaches CHECK_NAME: 'ClickHouse build check (actions)' - BUILD_NUMBER: 0 + BUILD_NAME: 'package_release' run: | sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER + cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME - name: Upload build URLs to artifacts uses: actions/upload-artifact@v2 with: @@ -250,12 +250,12 @@ jobs: REPO_COPY: ${{runner.temp}}/build_check/ClickHouse CACHES_PATH: ${{runner.temp}}/../ccaches CHECK_NAME: 'ClickHouse build check (actions)' - BUILD_NUMBER: 8 + BUILD_NAME: 'binary_release' run: | sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER + cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME - name: Upload build URLs to artifacts uses: actions/upload-artifact@v2 with: @@ -289,12 +289,12 @@ jobs: REPO_COPY: ${{runner.temp}}/build_check/ClickHouse CACHES_PATH: ${{runner.temp}}/../ccaches CHECK_NAME: 'ClickHouse build check (actions)' - BUILD_NUMBER: 3 + BUILD_NAME: 'package_asan' run: | sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER + cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME - name: Upload build URLs to artifacts uses: actions/upload-artifact@v2 with: @@ -328,12 +328,12 @@ jobs: REPO_COPY: ${{runner.temp}}/build_check/ClickHouse CACHES_PATH: ${{runner.temp}}/../ccaches CHECK_NAME: 'ClickHouse build check (actions)' - BUILD_NUMBER: 4 + BUILD_NAME: 'package_ubsan' run: | sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER + cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME - name: Upload build URLs to artifacts uses: actions/upload-artifact@v2 with: @@ -367,12 +367,12 @@ jobs: REPO_COPY: ${{runner.temp}}/build_check/ClickHouse CACHES_PATH: ${{runner.temp}}/../ccaches CHECK_NAME: 'ClickHouse build check (actions)' - BUILD_NUMBER: 5 + BUILD_NAME: 'package_tsan' run: | sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER + cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME - name: Upload build URLs to artifacts uses: actions/upload-artifact@v2 with: @@ -406,12 +406,12 @@ jobs: REPO_COPY: ${{runner.temp}}/build_check/ClickHouse CACHES_PATH: ${{runner.temp}}/../ccaches CHECK_NAME: 'ClickHouse build check (actions)' - BUILD_NUMBER: 6 + BUILD_NAME: 'package_msan' run: | sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER + cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME - name: Upload build URLs to artifacts uses: actions/upload-artifact@v2 with: @@ -445,12 +445,12 @@ jobs: REPO_COPY: ${{runner.temp}}/build_check/ClickHouse CACHES_PATH: ${{runner.temp}}/../ccaches CHECK_NAME: 'ClickHouse build check (actions)' - BUILD_NUMBER: 7 + BUILD_NAME: 'package_debug' run: | sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER + cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME - name: Upload build URLs to artifacts uses: actions/upload-artifact@v2 with: @@ -486,13 +486,13 @@ jobs: IMAGES_PATH: ${{runner.temp}}/images_path REPO_COPY: ${{runner.temp}}/build_check/ClickHouse CACHES_PATH: ${{runner.temp}}/../ccaches - CHECK_NAME: 'ClickHouse special build check (actions)' - BUILD_NUMBER: 1 + CHECK_NAME: 'ClickHouse build check (actions)' + BUILD_NAME: 'binary_splitted' run: | sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER + cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME - name: Upload build URLs to artifacts uses: actions/upload-artifact@v2 with: diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 9a0da9c3172..18969eeedd1 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -84,7 +84,7 @@ jobs: sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 compatibility_check.py 0 + cd $REPO_COPY/tests/ci && python3 compatibility_check.py - name: Cleanup if: always() run: | @@ -142,12 +142,12 @@ jobs: REPO_COPY: ${{runner.temp}}/build_check/ClickHouse CACHES_PATH: ${{runner.temp}}/../ccaches CHECK_NAME: 'ClickHouse build check (actions)' - BUILD_NUMBER: 0 + BUILD_NAME: 'package_release' run: | sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER + cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME - name: Upload build URLs to artifacts uses: actions/upload-artifact@v2 with: @@ -181,12 +181,12 @@ jobs: REPO_COPY: ${{runner.temp}}/build_check/ClickHouse CACHES_PATH: ${{runner.temp}}/../ccaches CHECK_NAME: 'ClickHouse build check (actions)' - BUILD_NUMBER: 8 + BUILD_NAME: 'binary_release' run: | sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER + cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME - name: Upload build URLs to artifacts uses: actions/upload-artifact@v2 with: @@ -219,12 +219,12 @@ jobs: REPO_COPY: ${{runner.temp}}/build_check/ClickHouse CACHES_PATH: ${{runner.temp}}/../ccaches CHECK_NAME: 'ClickHouse build check (actions)' - BUILD_NUMBER: 3 + BUILD_NAME: 'package_asan' run: | sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER + cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME - name: Upload build URLs to artifacts uses: actions/upload-artifact@v2 with: @@ -257,12 +257,12 @@ jobs: REPO_COPY: ${{runner.temp}}/build_check/ClickHouse CACHES_PATH: ${{runner.temp}}/../ccaches CHECK_NAME: 'ClickHouse build check (actions)' - BUILD_NUMBER: 4 + BUILD_NAME: 'package_ubsan' run: | sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER + cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME - name: Upload build URLs to artifacts uses: actions/upload-artifact@v2 with: @@ -295,12 +295,12 @@ jobs: REPO_COPY: ${{runner.temp}}/build_check/ClickHouse CACHES_PATH: ${{runner.temp}}/../ccaches CHECK_NAME: 'ClickHouse build check (actions)' - BUILD_NUMBER: 5 + BUILD_NAME: 'package_tsan' run: | sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER + cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME - name: Upload build URLs to artifacts uses: actions/upload-artifact@v2 with: @@ -333,12 +333,12 @@ jobs: REPO_COPY: ${{runner.temp}}/build_check/ClickHouse CACHES_PATH: ${{runner.temp}}/../ccaches CHECK_NAME: 'ClickHouse build check (actions)' - BUILD_NUMBER: 6 + BUILD_NAME: 'package_msan' run: | sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER + cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME - name: Upload build URLs to artifacts uses: actions/upload-artifact@v2 with: @@ -371,12 +371,12 @@ jobs: REPO_COPY: ${{runner.temp}}/build_check/ClickHouse CACHES_PATH: ${{runner.temp}}/../ccaches CHECK_NAME: 'ClickHouse build check (actions)' - BUILD_NUMBER: 7 + BUILD_NAME: 'package_debug' run: | sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER + cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME - name: Upload build URLs to artifacts uses: actions/upload-artifact@v2 with: @@ -412,13 +412,13 @@ jobs: IMAGES_PATH: ${{runner.temp}}/images_path REPO_COPY: ${{runner.temp}}/build_check/ClickHouse CACHES_PATH: ${{runner.temp}}/../ccaches - CHECK_NAME: 'ClickHouse special build check (actions)' - BUILD_NUMBER: 1 + CHECK_NAME: 'ClickHouse build check (actions)' + BUILD_NAME: 'binary_splitted' run: | sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER + cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME - name: Upload build URLs to artifacts uses: actions/upload-artifact@v2 with: diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index e279ae91588..988db77e62a 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -41,7 +41,7 @@ jobs: sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 compatibility_check.py 0 + cd $REPO_COPY/tests/ci && python3 compatibility_check.py - name: Cleanup if: always() run: | @@ -72,12 +72,12 @@ jobs: REPO_COPY: ${{runner.temp}}/build_check/ClickHouse CACHES_PATH: ${{runner.temp}}/../ccaches CHECK_NAME: 'ClickHouse build check (actions)' - BUILD_NUMBER: 0 + BUILD_NAME: 'package_release' run: | sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER + cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME - name: Upload build URLs to artifacts uses: actions/upload-artifact@v2 with: @@ -110,12 +110,12 @@ jobs: REPO_COPY: ${{runner.temp}}/build_check/ClickHouse CACHES_PATH: ${{runner.temp}}/../ccaches CHECK_NAME: 'ClickHouse build check (actions)' - BUILD_NUMBER: 3 + BUILD_NAME: 'package_asan' run: | sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER + cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME - name: Upload build URLs to artifacts uses: actions/upload-artifact@v2 with: @@ -148,12 +148,12 @@ jobs: REPO_COPY: ${{runner.temp}}/build_check/ClickHouse CACHES_PATH: ${{runner.temp}}/../ccaches CHECK_NAME: 'ClickHouse build check (actions)' - BUILD_NUMBER: 4 + BUILD_NAME: 'package_ubsan' run: | sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER + cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME - name: Upload build URLs to artifacts uses: actions/upload-artifact@v2 with: @@ -186,12 +186,12 @@ jobs: REPO_COPY: ${{runner.temp}}/build_check/ClickHouse CACHES_PATH: ${{runner.temp}}/../ccaches CHECK_NAME: 'ClickHouse build check (actions)' - BUILD_NUMBER: 5 + BUILD_NAME: 'package_tsan' run: | sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER + cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME - name: Upload build URLs to artifacts uses: actions/upload-artifact@v2 with: @@ -224,12 +224,12 @@ jobs: REPO_COPY: ${{runner.temp}}/build_check/ClickHouse CACHES_PATH: ${{runner.temp}}/../ccaches CHECK_NAME: 'ClickHouse build check (actions)' - BUILD_NUMBER: 6 + BUILD_NAME: 'package_msan' run: | sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER + cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME - name: Upload build URLs to artifacts uses: actions/upload-artifact@v2 with: @@ -262,12 +262,12 @@ jobs: REPO_COPY: ${{runner.temp}}/build_check/ClickHouse CACHES_PATH: ${{runner.temp}}/../ccaches CHECK_NAME: 'ClickHouse build check (actions)' - BUILD_NUMBER: 7 + BUILD_NAME: 'package_debug' run: | sudo rm -fr $TEMP_PATH mkdir -p $TEMP_PATH cp -r $GITHUB_WORKSPACE $TEMP_PATH - cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER + cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME - name: Upload build URLs to artifacts uses: actions/upload-artifact@v2 with: diff --git a/tests/ci/ast_fuzzer_check.py b/tests/ci/ast_fuzzer_check.py index d842d484841..02c81a4db31 100644 --- a/tests/ci/ast_fuzzer_check.py +++ b/tests/ci/ast_fuzzer_check.py @@ -11,8 +11,7 @@ from github import Github from s3_helper import S3Helper from get_robot_token import get_best_robot_token from pr_info import PRInfo -from ci_config import build_config_to_string -from build_download_helper import get_build_config_for_check, get_build_urls +from build_download_helper import get_build_name_for_check, get_build_urls from docker_pull_helper import get_image_with_version from commit_status_helper import post_commit_status from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse @@ -54,11 +53,9 @@ if __name__ == "__main__": docker_image = get_image_with_version(temp_path, IMAGE_NAME) - build_config = get_build_config_for_check(check_name) - print(build_config) - build_config_str = build_config_to_string(build_config) - print(build_config_str) - urls = get_build_urls(build_config_str, reports_path) + build_name = get_build_name_for_check(check_name) + print(build_name) + urls = get_build_urls(build_name, reports_path) if not urls: raise Exception("No build URLs found") diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 573a5cf82b4..a65f0de6827 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -12,19 +12,17 @@ from pr_info import PRInfo from get_robot_token import get_best_robot_token from version_helper import get_version_from_repo, update_version_local from ccache_utils import get_ccache_if_not_exists, upload_ccache -from ci_config import build_config_to_string, CI_CONFIG +from ci_config import CI_CONFIG from docker_pull_helper import get_image_with_version -def get_build_config(build_check_name, build_number): +def get_build_config(build_check_name, build_name): if build_check_name == 'ClickHouse build check (actions)': build_config_name = 'build_config' - elif build_check_name == 'ClickHouse special build check (actions)': - build_config_name = 'special_build_config' else: raise Exception(f"Unknown build check name {build_check_name}") - return CI_CONFIG[build_config_name][build_number] + return CI_CONFIG[build_config_name][build_name] def _can_export_binaries(build_config): @@ -94,9 +92,9 @@ if __name__ == "__main__": caches_path = os.getenv("CACHES_PATH", temp_path) build_check_name = sys.argv[1] - build_number = int(sys.argv[2]) + build_name = sys.argv[2] - build_config = get_build_config(build_check_name, build_number) + build_config = get_build_config(build_check_name, build_name) if not os.path.exists(temp_path): os.makedirs(temp_path) @@ -125,7 +123,6 @@ if __name__ == "__main__": logging.info("Updated local files with version") - build_name = build_config_to_string(build_config) logging.info("Build short name %s", build_name) subprocess.check_call(f"echo 'BUILD_NAME=build_urls_{build_name}' >> $GITHUB_ENV", shell=True) @@ -161,7 +158,12 @@ if __name__ == "__main__": logging.info("Will upload cache") upload_ccache(ccache_path, s3_helper, pr_info.number, temp_path) - s3_path_prefix = str(pr_info.number) + "/" + pr_info.sha + "/" + build_check_name.lower().replace(' ', '_') + "/" + build_name + # for release pull requests we use fixed names + if 'release' in pr_info.labels or 'release-lts' in pr_info.labels: + s3_path_prefix = pr_info.head_ref + "/" + pr_info.sha + "/" + build_name + else: + s3_path_prefix = str(pr_info.number) + "/" + pr_info.sha + "/" + build_name + if os.path.exists(log_path): log_url = s3_helper.upload_build_file_to_s3(log_path, s3_path_prefix + "/" + os.path.basename(log_path)) logging.info("Log url %s", log_url) diff --git a/tests/ci/build_download_helper.py b/tests/ci/build_download_helper.py index 2770b737041..5ce54423e19 100644 --- a/tests/ci/build_download_helper.py +++ b/tests/ci/build_download_helper.py @@ -8,17 +8,17 @@ import time import requests -from ci_config import CI_CONFIG, build_config_to_string +from ci_config import CI_CONFIG DOWNLOAD_RETRIES_COUNT = 5 -def get_build_config_for_check(check_name): - return CI_CONFIG["tests_config"][check_name]['required_build_properties'] +def get_build_name_for_check(check_name): + return CI_CONFIG['tests_config'][check_name]['required_build'] -def get_build_urls(build_config_str, reports_path): +def get_build_urls(build_name, reports_path): for root, _, files in os.walk(reports_path): for f in files: - if build_config_str in f : + if build_name in f : logging.info("Found build report json %s", f) with open(os.path.join(root, f), 'r', encoding='utf-8') as file_handler: build_report = json.load(file_handler) @@ -72,11 +72,8 @@ def download_builds(result_path, build_urls, filter_fn): dowload_build_with_progress(url, os.path.join(result_path, fname)) def download_builds_filter(check_name, reports_path, result_path, filter_fn=lambda _: True): - build_config = get_build_config_for_check(check_name) - print(build_config) - build_config_str = build_config_to_string(build_config) - print(build_config_str) - urls = get_build_urls(build_config_str, reports_path) + build_name = get_build_name_for_check(check_name) + urls = get_build_urls(build_name, reports_path) print(urls) if not urls: diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 161f50fed20..30f604a6efc 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 CI_CONFIG = { - "build_config": [ - { + "build_config": { + "package_release": { "compiler": "clang-13", "build_type": "", "sanitizer": "", @@ -13,7 +13,7 @@ CI_CONFIG = { "tidy": "disable", "with_coverage": False }, - { + "performance": { "compiler": "clang-13", "build_type": "", "sanitizer": "", @@ -23,7 +23,7 @@ CI_CONFIG = { "tidy": "disable", "with_coverage": False }, - { + "binary_gcc": { "compiler": "gcc-11", "build_type": "", "sanitizer": "", @@ -33,7 +33,7 @@ CI_CONFIG = { "tidy": "disable", "with_coverage": False }, - { + "package_asan": { "compiler": "clang-13", "build_type": "", "sanitizer": "address", @@ -43,7 +43,7 @@ CI_CONFIG = { "tidy": "disable", "with_coverage": False }, - { + "package_ubsan": { "compiler": "clang-13", "build_type": "", "sanitizer": "undefined", @@ -53,7 +53,7 @@ CI_CONFIG = { "tidy": "disable", "with_coverage": False }, - { + "package_tsan": { "compiler": "clang-13", "build_type": "", "sanitizer": "thread", @@ -63,7 +63,7 @@ CI_CONFIG = { "tidy": "disable", "with_coverage": False }, - { + "package_msan": { "compiler": "clang-13", "build_type": "", "sanitizer": "memory", @@ -73,7 +73,7 @@ CI_CONFIG = { "tidy": "disable", "with_coverage": False }, - { + "package_debug": { "compiler": "clang-13", "build_type": "debug", "sanitizer": "", @@ -83,7 +83,7 @@ CI_CONFIG = { "tidy": "disable", "with_coverage": False }, - { + "binary_release": { "compiler": "clang-13", "build_type": "", "sanitizer": "", @@ -92,10 +92,8 @@ CI_CONFIG = { "splitted": "unsplitted", "tidy": "disable", "with_coverage": False - } - ], - "special_build_config": [ - { + }, + "package_tidy": { "compiler": "clang-13", "build_type": "debug", "sanitizer": "", @@ -105,7 +103,7 @@ CI_CONFIG = { "tidy": "enable", "with_coverage": False }, - { + "binary_splitted": { "compiler": "clang-13", "build_type": "", "sanitizer": "", @@ -115,7 +113,7 @@ CI_CONFIG = { "tidy": "disable", "with_coverage": False }, - { + "binary_darwin": { "compiler": "clang-13-darwin", "build_type": "", "sanitizer": "", @@ -125,7 +123,7 @@ CI_CONFIG = { "tidy": "disable", "with_coverage": False }, - { + "binary_aarch64": { "compiler": "clang-13-aarch64", "build_type": "", "sanitizer": "", @@ -135,7 +133,7 @@ CI_CONFIG = { "tidy": "disable", "with_coverage": False }, - { + "binary_freebsd": { "compiler": "clang-13-freebsd", "build_type": "", "sanitizer": "", @@ -145,7 +143,7 @@ CI_CONFIG = { "tidy": "disable", "with_coverage": False }, - { + "binary_darwin_aarch64": { "compiler": "clang-13-darwin-aarch64", "build_type": "", "sanitizer": "", @@ -155,7 +153,7 @@ CI_CONFIG = { "tidy": "disable", "with_coverage": False }, - { + "binary_ppc64le": { "compiler": "clang-13-ppc64le", "build_type": "", "sanitizer": "", @@ -165,550 +163,139 @@ CI_CONFIG = { "tidy": "disable", "with_coverage": False } - ], + }, "tests_config": { "Stateful tests (address, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "address", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_asan", }, "Stateful tests (thread, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "thread", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_tsan", }, "Stateful tests (memory, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "memory", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_msan", }, "Stateful tests (ubsan, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "undefined", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_ubsan", }, "Stateful tests (debug, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "debug", - "sanitizer": "none", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_debug", }, "Stateful tests (release, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "none", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_release", }, "Stateful tests (release, DatabaseOrdinary, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "none", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_release", }, "Stateful tests (release, DatabaseReplicated, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "none", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_release", }, "Stateless tests (address, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "address", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_asan", }, "Stateless tests (thread, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "thread", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_tsan", }, "Stateless tests (memory, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "memory", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_msan", }, "Stateless tests (ubsan, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "undefined", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_ubsan", }, "Stateless tests (debug, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "debug", - "sanitizer": "none", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_debug", }, "Stateless tests (release, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "none", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_release", }, "Stateless tests (release, wide parts enabled, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "none", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_release", }, "Stateless tests (release, DatabaseOrdinary, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "none", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_release", }, "Stateless tests (release, DatabaseReplicated, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "none", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_release", }, "Stress test (address, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "address", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_asan", }, "Stress test (thread, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "thread", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_tsan", }, "Stress test (undefined, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "undefined", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_ubsan", }, "Stress test (memory, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "memory", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_msan", }, "Stress test (debug, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "debug", - "sanitizer": "none", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_debug", }, "Integration tests (asan, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "address", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_asan", }, "Integration tests (thread, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "thread", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_tsan", }, "Integration tests (release, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "none", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_release", }, "Integration tests (memory, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "memory", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_msan", }, "Integration tests flaky check (asan, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "address", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_asan", }, "Compatibility check (actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "none", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_release", }, "Split build smoke test (actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "binary", - "build_type": "relwithdebuginfo", - "sanitizer": "none", - "bundled": "bundled", - "splitted": "splitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "binary_splitted", }, "Testflows check (actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "none", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_release", }, "Unit tests (release-gcc, actions)": { - "required_build_properties": { - "compiler": "gcc-11", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "none", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "binary_gcc", }, "Unit tests (release-clang, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "binary", - "build_type": "relwithdebuginfo", - "sanitizer": "none", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_release", }, "Unit tests (asan, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "address", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_asan", }, "Unit tests (msan, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "memory", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_msan", }, "Unit tests (tsan, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "thread", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_tsan", }, "Unit tests (ubsan, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "undefined", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_ubsan", }, "AST fuzzer (debug, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "debug", - "sanitizer": "none", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_debug", }, "AST fuzzer (ASan, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "address", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_asan", }, "AST fuzzer (MSan, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "memory", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_msan", }, "AST fuzzer (TSan, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "thread", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_tsan", }, "AST fuzzer (UBSan, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "undefined", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_ubsan", }, "Release (actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "none", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_release", }, "Stateless tests flaky check (address, actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "deb", - "build_type": "relwithdebuginfo", - "sanitizer": "address", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "package_asan", }, "ClickHouse Keeper Jepsen (actions)": { - "required_build_properties": { - "compiler": "clang-13", - "package_type": "binary", - "build_type": "relwithdebuginfo", - "sanitizer": "none", - "bundled": "bundled", - "splitted": "unsplitted", - "clang_tidy": "disable", - "with_coverage": False - } + "required_build": "binary_release", } } } - -def build_config_to_string(build_config): - if build_config["package_type"] == "performance": - return "performance" - - return "_".join([ - build_config['compiler'], - build_config['build_type'] if build_config['build_type'] else "relwithdebuginfo", - build_config['sanitizer'] if build_config['sanitizer'] else "none", - build_config['bundled'], - build_config['splitted'], - 'tidy' if 'tidy' in build_config and build_config['tidy'] == 'enable' else 'notidy', - 'with_coverage' if 'with_coverage' in build_config and build_config['with_coverage'] else 'without_coverage', - build_config['package_type'], - ]) diff --git a/tests/ci/metrics_lambda/app.py b/tests/ci/metrics_lambda/app.py index 4bf967a51e1..d776aa2be49 100644 --- a/tests/ci/metrics_lambda/app.py +++ b/tests/ci/metrics_lambda/app.py @@ -163,7 +163,7 @@ def main(github_secret_key, github_app_id, push_to_cloudwatch, delete_offline_ru if delete_offline_runners: print("Going to delete offline runners") for runner in runners: - if runner.offline: + if runner.offline and not runner.busy: print("Deleting runner", runner) delete_runner(access_token, runner) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 25d09a8c4c5..f23ed0c19f0 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -13,8 +13,7 @@ import csv MAX_RETRY = 3 -NUM_WORKERS = 5 -SLEEP_BETWEEN_RETRIES = 5 +NUM_WORKERS = 8 PARALLEL_GROUP_SIZE = 100 CLICKHOUSE_BINARY_PATH = "usr/bin/clickhouse" CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH = "usr/bin/clickhouse-odbc-bridge"