thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 23:21:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Get rid of build numbers and simplify builds paths in S3

Browse Source
This commit is contained in:
alesapin 2021-11-26 13:57:36 +03:00
parent c786c0a52a
commit 081bfdfe82
9 changed files with 135 additions and 553 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
.github/workflows/main.yml vendored
View File

@ -211,12 +211,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 0
BUILD_NAME: 'package_release'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -250,12 +250,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 8
BUILD_NAME: 'binary_release'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -289,12 +289,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 3
BUILD_NAME: 'package_asan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -328,12 +328,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 4
BUILD_NAME: 'package_ubsan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -367,12 +367,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 5
BUILD_NAME: 'package_tsan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -406,12 +406,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 6
BUILD_NAME: 'package_msan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -445,12 +445,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 7
BUILD_NAME: 'package_debug'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -486,13 +486,13 @@ jobs:
IMAGES_PATH: ${{runner.temp}}/images_path
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse special build check (actions)'
BUILD_NUMBER: 1
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NAME: 'binary_splitted'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:

36
.github/workflows/master.yml vendored
View File

@ -84,7 +84,7 @@ jobs:
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 compatibility_check.py 0
cd $REPO_COPY/tests/ci && python3 compatibility_check.py
- name: Cleanup
if: always()
run: |
@ -142,12 +142,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 0
BUILD_NAME: 'package_release'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -181,12 +181,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 8
BUILD_NAME: 'binary_release'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -219,12 +219,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 3
BUILD_NAME: 'package_asan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -257,12 +257,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 4
BUILD_NAME: 'package_ubsan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -295,12 +295,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 5
BUILD_NAME: 'package_tsan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -333,12 +333,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 6
BUILD_NAME: 'package_msan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -371,12 +371,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 7
BUILD_NAME: 'package_debug'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -412,13 +412,13 @@ jobs:
IMAGES_PATH: ${{runner.temp}}/images_path
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse special build check (actions)'
BUILD_NUMBER: 1
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NAME: 'binary_splitted'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:

26
.github/workflows/release_branches.yml vendored
View File

@ -41,7 +41,7 @@ jobs:
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 compatibility_check.py 0
cd $REPO_COPY/tests/ci && python3 compatibility_check.py
- name: Cleanup
if: always()
run: |
@ -72,12 +72,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 0
BUILD_NAME: 'package_release'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -110,12 +110,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 3
BUILD_NAME: 'package_asan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -148,12 +148,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 4
BUILD_NAME: 'package_ubsan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -186,12 +186,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 5
BUILD_NAME: 'package_tsan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -224,12 +224,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 6
BUILD_NAME: 'package_msan'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
@ -262,12 +262,12 @@ jobs:
REPO_COPY: ${{runner.temp}}/build_check/ClickHouse
CACHES_PATH: ${{runner.temp}}/../ccaches
CHECK_NAME: 'ClickHouse build check (actions)'
BUILD_NUMBER: 7
BUILD_NAME: 'package_debug'
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:

11
tests/ci/ast_fuzzer_check.py
View File

@ -11,8 +11,7 @@ from github import Github
from s3_helper import S3Helper
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from ci_config import build_config_to_string
from build_download_helper import get_build_config_for_check, get_build_urls
from build_download_helper import get_build_name_for_check, get_build_urls
from docker_pull_helper import get_image_with_version
from commit_status_helper import post_commit_status
from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse
@ -54,11 +53,9 @@ if __name__ == "__main__":
docker_image = get_image_with_version(temp_path, IMAGE_NAME)
build_config = get_build_config_for_check(check_name)
print(build_config)
build_config_str = build_config_to_string(build_config)
print(build_config_str)
urls = get_build_urls(build_config_str, reports_path)
build_name = get_build_name_for_check(check_name)
print(build_name)
urls = get_build_urls(build_name, reports_path)
if not urls:
raise Exception("No build URLs found")

20
tests/ci/build_check.py
View File

@ -12,19 +12,17 @@ from pr_info import PRInfo
from get_robot_token import get_best_robot_token
from version_helper import get_version_from_repo, update_version_local
from ccache_utils import get_ccache_if_not_exists, upload_ccache
from ci_config import build_config_to_string, CI_CONFIG
from ci_config import CI_CONFIG
from docker_pull_helper import get_image_with_version
def get_build_config(build_check_name, build_number):
def get_build_config(build_check_name, build_name):
if build_check_name == 'ClickHouse build check (actions)':
build_config_name = 'build_config'
elif build_check_name == 'ClickHouse special build check (actions)':
build_config_name = 'special_build_config'
else:
raise Exception(f"Unknown build check name {build_check_name}")
return CI_CONFIG[build_config_name][build_number]
return CI_CONFIG[build_config_name][build_name]
def _can_export_binaries(build_config):
@ -94,9 +92,9 @@ if __name__ == "__main__":
caches_path = os.getenv("CACHES_PATH", temp_path)
build_check_name = sys.argv[1]
build_number = int(sys.argv[2])
build_name = sys.argv[2]
build_config = get_build_config(build_check_name, build_number)
build_config = get_build_config(build_check_name, build_name)
if not os.path.exists(temp_path):
os.makedirs(temp_path)
@ -125,7 +123,6 @@ if __name__ == "__main__":
logging.info("Updated local files with version")
build_name = build_config_to_string(build_config)
logging.info("Build short name %s", build_name)
subprocess.check_call(f"echo 'BUILD_NAME=build_urls_{build_name}' >> $GITHUB_ENV", shell=True)
@ -161,7 +158,12 @@ if __name__ == "__main__":
logging.info("Will upload cache")
upload_ccache(ccache_path, s3_helper, pr_info.number, temp_path)
s3_path_prefix = str(pr_info.number) + "/" + pr_info.sha + "/" + build_check_name.lower().replace(' ', '_') + "/" + build_name
# for release pull requests we use fixed names
if 'release' in pr_info.labels or 'release-lts' in pr_info.labels:
s3_path_prefix = pr_info.head_ref + "/" + pr_info.sha + "/" + build_name
else:
s3_path_prefix = str(pr_info.number) + "/" + pr_info.sha + "/" + build_name
if os.path.exists(log_path):
log_url = s3_helper.upload_build_file_to_s3(log_path, s3_path_prefix + "/" + os.path.basename(log_path))
logging.info("Log url %s", log_url)

17
tests/ci/build_download_helper.py
View File

@ -8,17 +8,17 @@ import time
import requests
from ci_config import CI_CONFIG, build_config_to_string
from ci_config import CI_CONFIG
DOWNLOAD_RETRIES_COUNT = 5
def get_build_config_for_check(check_name):
return CI_CONFIG["tests_config"][check_name]['required_build_properties']
def get_build_name_for_check(check_name):
return CI_CONFIG['tests_config'][check_name]['required_build']
def get_build_urls(build_config_str, reports_path):
def get_build_urls(build_name, reports_path):
for root, _, files in os.walk(reports_path):
for f in files:
if build_config_str in f :
if build_name in f :
logging.info("Found build report json %s", f)
with open(os.path.join(root, f), 'r', encoding='utf-8') as file_handler:
build_report = json.load(file_handler)
@ -72,11 +72,8 @@ def download_builds(result_path, build_urls, filter_fn):
dowload_build_with_progress(url, os.path.join(result_path, fname))
def download_builds_filter(check_name, reports_path, result_path, filter_fn=lambda _: True):
build_config = get_build_config_for_check(check_name)
print(build_config)
build_config_str = build_config_to_string(build_config)
print(build_config_str)
urls = get_build_urls(build_config_str, reports_path)
build_name = get_build_name_for_check(check_name)
urls = get_build_urls(build_name, reports_path)
print(urls)
if not urls:

539
tests/ci/ci_config.py
View File

@ -1,8 +1,8 @@
#!/usr/bin/env python3
CI_CONFIG = {
"build_config": [
{
"build_config": {
"package_release": {
"compiler": "clang-13",
"build_type": "",
"sanitizer": "",
@ -13,7 +13,7 @@ CI_CONFIG = {
"tidy": "disable",
"with_coverage": False
},
{
"performance": {
"compiler": "clang-13",
"build_type": "",
"sanitizer": "",
@ -23,7 +23,7 @@ CI_CONFIG = {
"tidy": "disable",
"with_coverage": False
},
{
"binary_gcc": {
"compiler": "gcc-11",
"build_type": "",
"sanitizer": "",
@ -33,7 +33,7 @@ CI_CONFIG = {
"tidy": "disable",
"with_coverage": False
},
{
"package_asan": {
"compiler": "clang-13",
"build_type": "",
"sanitizer": "address",
@ -43,7 +43,7 @@ CI_CONFIG = {
"tidy": "disable",
"with_coverage": False
},
{
"package_ubsan": {
"compiler": "clang-13",
"build_type": "",
"sanitizer": "undefined",
@ -53,7 +53,7 @@ CI_CONFIG = {
"tidy": "disable",
"with_coverage": False
},
{
"package_tsan": {
"compiler": "clang-13",
"build_type": "",
"sanitizer": "thread",
@ -63,7 +63,7 @@ CI_CONFIG = {
"tidy": "disable",
"with_coverage": False
},
{
"package_msan": {
"compiler": "clang-13",
"build_type": "",
"sanitizer": "memory",
@ -73,7 +73,7 @@ CI_CONFIG = {
"tidy": "disable",
"with_coverage": False
},
{
"package_debug": {
"compiler": "clang-13",
"build_type": "debug",
"sanitizer": "",
@ -83,7 +83,7 @@ CI_CONFIG = {
"tidy": "disable",
"with_coverage": False
},
{
"binary_release": {
"compiler": "clang-13",
"build_type": "",
"sanitizer": "",
@ -92,10 +92,8 @@ CI_CONFIG = {
"splitted": "unsplitted",
"tidy": "disable",
"with_coverage": False
}
],
"special_build_config": [
{
},
"package_tidy": {
"compiler": "clang-13",
"build_type": "debug",
"sanitizer": "",
@ -105,7 +103,7 @@ CI_CONFIG = {
"tidy": "enable",
"with_coverage": False
},
{
"binary_splitted": {
"compiler": "clang-13",
"build_type": "",
"sanitizer": "",
@ -115,7 +113,7 @@ CI_CONFIG = {
"tidy": "disable",
"with_coverage": False
},
{
"binary_darwin": {
"compiler": "clang-13-darwin",
"build_type": "",
"sanitizer": "",
@ -125,7 +123,7 @@ CI_CONFIG = {
"tidy": "disable",
"with_coverage": False
},
{
"binary_aarch64": {
"compiler": "clang-13-aarch64",
"build_type": "",
"sanitizer": "",
@ -135,7 +133,7 @@ CI_CONFIG = {
"tidy": "disable",
"with_coverage": False
},
{
"binary_freebsd": {
"compiler": "clang-13-freebsd",
"build_type": "",
"sanitizer": "",
@ -145,7 +143,7 @@ CI_CONFIG = {
"tidy": "disable",
"with_coverage": False
},
{
"binary_darwin_aarch64": {
"compiler": "clang-13-darwin-aarch64",
"build_type": "",
"sanitizer": "",
@ -155,7 +153,7 @@ CI_CONFIG = {
"tidy": "disable",
"with_coverage": False
},
{
"binary_ppc64le": {
"compiler": "clang-13-ppc64le",
"build_type": "",
"sanitizer": "",
@ -165,550 +163,139 @@ CI_CONFIG = {
"tidy": "disable",
"with_coverage": False
}
],
},
"tests_config": {
"Stateful tests (address, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "address",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_asan",
},
"Stateful tests (thread, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "thread",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_tsan",
},
"Stateful tests (memory, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "memory",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_msan",
},
"Stateful tests (ubsan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "undefined",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_ubsan",
},
"Stateful tests (debug, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "debug",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_debug",
},
"Stateful tests (release, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_release",
},
"Stateful tests (release, DatabaseOrdinary, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_release",
},
"Stateful tests (release, DatabaseReplicated, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_release",
},
"Stateless tests (address, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "address",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_asan",
},
"Stateless tests (thread, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "thread",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_tsan",
},
"Stateless tests (memory, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "memory",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_msan",
},
"Stateless tests (ubsan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "undefined",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_ubsan",
},
"Stateless tests (debug, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "debug",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_debug",
},
"Stateless tests (release, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_release",
},
"Stateless tests (release, wide parts enabled, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_release",
},
"Stateless tests (release, DatabaseOrdinary, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_release",
},
"Stateless tests (release, DatabaseReplicated, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_release",
},
"Stress test (address, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "address",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_asan",
},
"Stress test (thread, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "thread",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_tsan",
},
"Stress test (undefined, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "undefined",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_ubsan",
},
"Stress test (memory, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "memory",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_msan",
},
"Stress test (debug, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "debug",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_debug",
},
"Integration tests (asan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "address",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_asan",
},
"Integration tests (thread, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "thread",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_tsan",
},
"Integration tests (release, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_release",
},
"Integration tests (memory, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "memory",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_msan",
},
"Integration tests flaky check (asan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "address",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_asan",
},
"Compatibility check (actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_release",
},
"Split build smoke test (actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "binary",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "splitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "binary_splitted",
},
"Testflows check (actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_release",
},
"Unit tests (release-gcc, actions)": {
"required_build_properties": {
"compiler": "gcc-11",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "binary_gcc",
},
"Unit tests (release-clang, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "binary",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_release",
},
"Unit tests (asan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "address",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_asan",
},
"Unit tests (msan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "memory",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_msan",
},
"Unit tests (tsan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "thread",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_tsan",
},
"Unit tests (ubsan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "undefined",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_ubsan",
},
"AST fuzzer (debug, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "debug",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_debug",
},
"AST fuzzer (ASan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "address",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_asan",
},
"AST fuzzer (MSan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "memory",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_msan",
},
"AST fuzzer (TSan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "thread",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_tsan",
},
"AST fuzzer (UBSan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "undefined",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_ubsan",
},
"Release (actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_release",
},
"Stateless tests flaky check (address, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "address",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "package_asan",
},
"ClickHouse Keeper Jepsen (actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "binary",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
"required_build": "binary_release",
}
}
}
def build_config_to_string(build_config):
if build_config["package_type"] == "performance":
return "performance"
return "_".join([
build_config['compiler'],
build_config['build_type'] if build_config['build_type'] else "relwithdebuginfo",
build_config['sanitizer'] if build_config['sanitizer'] else "none",
build_config['bundled'],
build_config['splitted'],
'tidy' if 'tidy' in build_config and build_config['tidy'] == 'enable' else 'notidy',
'with_coverage' if 'with_coverage' in build_config and build_config['with_coverage'] else 'without_coverage',
build_config['package_type'],
])

2
tests/ci/metrics_lambda/app.py
View File

@ -163,7 +163,7 @@ def main(github_secret_key, github_app_id, push_to_cloudwatch, delete_offline_ru
if delete_offline_runners:
print("Going to delete offline runners")
for runner in runners:
if runner.offline:
if runner.offline and not runner.busy:
print("Deleting runner", runner)
delete_runner(access_token, runner)

3
tests/integration/ci-runner.py
View File

@ -13,8 +13,7 @@ import csv
MAX_RETRY = 3
NUM_WORKERS = 5
SLEEP_BETWEEN_RETRIES = 5
NUM_WORKERS = 8
PARALLEL_GROUP_SIZE = 100
CLICKHOUSE_BINARY_PATH = "usr/bin/clickhouse"
CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH = "usr/bin/clickhouse-odbc-bridge"