thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 15:42:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #31340 from ClickHouse/get_rid_of_build_number

Browse Source 
Remove some copypaste from CI
This commit is contained in:
alesapin 2021-11-15 12:46:07 +03:00 committed by GitHub
commit fe0cbd6036
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 1095 additions and 1491 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
.github/workflows/main.yml vendored
View File

@ -280,14 +280,13 @@ jobs:
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Stateless tests (debug, actions)'
REPO_COPY: ${{runner.temp}}/stateless_debug/ClickHouse
REQUIRED_BUILD_NUMBER: 7
KILL_TIMEOUT: 10800
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 functional_test_check.py "$CHECK_NAME" $REQUIRED_BUILD_NUMBER $KILL_TIMEOUT
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
- name: Cleanup
if: always()
run: |
@ -310,14 +309,13 @@ jobs:
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Stateful tests (debug, actions)'
REPO_COPY: ${{runner.temp}}/stateful_debug/ClickHouse
REQUIRED_BUILD_NUMBER: 7
KILL_TIMEOUT: 3600
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 functional_test_check.py "$CHECK_NAME" $REQUIRED_BUILD_NUMBER $KILL_TIMEOUT
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
- name: Cleanup
if: always()
run: |
@ -340,14 +338,13 @@ jobs:
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Stateless tests flaky check (address, actions)'
REPO_COPY: ${{runner.temp}}/stateless_flaky_asan/ClickHouse
REQUIRED_BUILD_NUMBER: 3
KILL_TIMEOUT: 3600
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 functional_test_check.py "$CHECK_NAME" $REQUIRED_BUILD_NUMBER $KILL_TIMEOUT
python3 functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT
- name: Cleanup
if: always()
run: |
@ -368,15 +365,14 @@ jobs:
env:
TEMP_PATH: ${{runner.temp}}/stress_debug
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Stress tests (debug, actions)'
CHECK_NAME: 'Stress test (debug, actions)'
REPO_COPY: ${{runner.temp}}/stress_debug/ClickHouse
REQUIRED_BUILD_NUMBER: 7
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 stress_check.py "$CHECK_NAME" $REQUIRED_BUILD_NUMBER
python3 stress_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
@ -399,13 +395,12 @@ jobs:
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'AST fuzzer (debug, actions)'
REPO_COPY: ${{runner.temp}}/ast_fuzzer_debug/ClickHouse
REQUIRED_BUILD_NUMBER: 7
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 ast_fuzzer_check.py "$CHECK_NAME" $REQUIRED_BUILD_NUMBER
python3 ast_fuzzer_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
@ -428,13 +423,12 @@ jobs:
# REPORTS_PATH: ${{runner.temp}}/reports_dir
# CHECK_NAME: 'Integration tests (asan, actions)'
# REPO_COPY: ${{runner.temp}}/integration_tests_asan/ClickHouse
# REQUIRED_BUILD_NUMBER: 3
# run: |
# sudo rm -fr $TEMP_PATH
# mkdir -p $TEMP_PATH
# cp -r $GITHUB_WORKSPACE $TEMP_PATH
# cd $REPO_COPY/tests/ci
# python3 integration_test_check.py "$CHECK_NAME" $REQUIRED_BUILD_NUMBER
# python3 integration_test_check.py "$CHECK_NAME"
# - name: Cleanup
# if: always()
# run: |
@ -457,13 +451,12 @@ jobs:
REPORTS_PATH: ${{runner.temp}}/reports_dir
CHECK_NAME: 'Unit tests (asan, actions)'
REPO_COPY: ${{runner.temp}}/unit_tests_asan/ClickHouse
REQUIRED_BUILD_NUMBER: 3
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 unit_tests_check.py "$CHECK_NAME" $REQUIRED_BUILD_NUMBER
python3 unit_tests_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
@ -560,12 +553,11 @@ jobs:
TEMP_PATH: ${{runner.temp}}/split_build_check
REPO_COPY: ${{runner.temp}}/split_build_check/ClickHouse
REPORTS_PATH: ${{runner.temp}}/reports_dir
SPECIAL_BUILD_NUMBER: 1
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 split_build_smoke_check.py $SPECIAL_BUILD_NUMBER
cd $REPO_COPY/tests/ci && python3 split_build_smoke_check.py
- name: Cleanup
if: always()
run: |

69
tests/ci/ast_fuzzer_check.py
View File

@ -4,7 +4,6 @@ import logging
import subprocess
import os
import json
import time
import sys
from github import Github
@ -12,43 +11,13 @@ from github import Github
from s3_helper import S3Helper
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from ci_config import build_config_to_string
from build_download_helper import get_build_config_for_check, get_build_urls
from docker_pull_helper import get_image_with_version
from commit_status_helper import post_commit_status
DOWNLOAD_RETRIES_COUNT = 5
IMAGE_NAME = 'clickhouse/fuzzer'
def get_build_urls(build_config_str, reports_path):
for root, _, files in os.walk(reports_path):
for f in files:
if build_config_str in f :
logging.info("Found build report json %s", f)
with open(os.path.join(root, f), 'r', encoding='utf-8') as file_handler:
build_report = json.load(file_handler)
return build_report['build_urls']
return []
def get_build_config(build_number, repo_path):
ci_config_path = os.path.join(repo_path, "tests/ci/ci_config.json")
with open(ci_config_path, 'r', encoding='utf-8') as ci_config:
config_dict = json.load(ci_config)
return config_dict['build_config'][build_number]
def build_config_to_string(build_config):
if build_config["package-type"] == "performance":
return "performance"
return "_".join([
build_config['compiler'],
build_config['build-type'] if build_config['build-type'] else "relwithdebuginfo",
build_config['sanitizer'] if build_config['sanitizer'] else "none",
build_config['bundled'],
build_config['splitted'],
"tidy" if build_config['tidy'] == "enable" else "notidy",
"with_coverage" if build_config['with_coverage'] else "without_coverage",
build_config['package-type'],
])
def get_run_command(pr_number, sha, download_url, workspace_path, image):
return f'docker run --network=host --volume={workspace_path}:/workspace ' \
'--cap-add syslog --cap-add sys_admin ' \
@ -67,7 +36,6 @@ if __name__ == "__main__":
reports_path = os.getenv("REPORTS_PATH", "./reports")
check_name = sys.argv[1]
build_number = int(sys.argv[2])
if not os.path.exists(temp_path):
os.makedirs(temp_path)
@ -79,30 +47,12 @@ if __name__ == "__main__":
gh = Github(get_best_robot_token())
images_path = os.path.join(temp_path, 'changed_images.json')
docker_image = get_image_with_version(temp_path, IMAGE_NAME)
docker_image = IMAGE_NAME
if os.path.exists(images_path):
logging.info("Images file exists")
with open(images_path, 'r', encoding='utf-8') as images_fd:
images = json.load(images_fd)
logging.info("Got images %s", images)
if IMAGE_NAME in images:
docker_image += ':' + images[IMAGE_NAME]
for i in range(10):
try:
logging.info("Pulling image %s", docker_image)
subprocess.check_output(f"docker pull {docker_image}", stderr=subprocess.STDOUT, shell=True)
break
except Exception as ex:
time.sleep(i * 3)
logging.info("Got execption pulling docker %s", ex)
else:
raise Exception(f"Cannot pull dockerhub for image docker pull {docker_image}")
build_config = get_build_config(build_number, repo_path)
build_config = get_build_config_for_check(check_name)
print(build_config)
build_config_str = build_config_to_string(build_config)
print(build_config_str)
urls = get_build_urls(build_config_str, reports_path)
if not urls:
raise Exception("No build URLs found")
@ -177,5 +127,4 @@ if __name__ == "__main__":
logging.info("Result: '%s', '%s', '%s'", status, description, report_url)
print(f"::notice ::Report url: {report_url}")
commit = get_commit(gh, pr_info.sha)
commit.create_status(context=check_name, description=description, state=status, target_url=report_url)
post_commit_status(gh, pr_info.sha, check_name, description, status, report_url)

60
tests/ci/build_check.py
View File

@ -12,9 +12,11 @@ from pr_info import PRInfo
from get_robot_token import get_best_robot_token
from version_helper import get_version_from_repo, update_version_local
from ccache_utils import get_ccache_if_not_exists, upload_ccache
from ci_config import build_config_to_string, CI_CONFIG
from docker_pull_helper import get_image_with_version
def get_build_config(build_check_name, build_number, repo_path):
def get_build_config(build_check_name, build_number):
if build_check_name == 'ClickHouse build check (actions)':
build_config_name = 'build_config'
elif build_check_name == 'ClickHouse special build check (actions)':
@ -22,14 +24,11 @@ def get_build_config(build_check_name, build_number, repo_path):
else:
raise Exception(f"Unknown build check name {build_check_name}")
ci_config_path = os.path.join(repo_path, "tests/ci/ci_config.json")
with open(ci_config_path, 'r') as ci_config:
config_dict = json.load(ci_config)
return config_dict[build_config_name][build_number]
return CI_CONFIG[build_config_name][build_number]
def _can_export_binaries(build_config):
if build_config['package-type'] != 'deb':
if build_config['package_type'] != 'deb':
return False
if build_config['bundled'] != "bundled":
return False
@ -37,18 +36,18 @@ def _can_export_binaries(build_config):
return False
if build_config['sanitizer'] != '':
return True
if build_config['build-type'] != '':
if build_config['build_type'] != '':
return True
return False
def get_packager_cmd(build_config, packager_path, output_path, build_version, image_version, ccache_path, pr_info):
package_type = build_config['package-type']
package_type = build_config['package_type']
comp = build_config['compiler']
cmd = f"cd {packager_path} && ./packager --output-dir={output_path} --package-type={package_type} --compiler={comp}"
if build_config['build-type']:
cmd += ' --build-type={}'.format(build_config['build-type'])
if build_config['build_type']:
cmd += ' --build-type={}'.format(build_config['build_type'])
if build_config['sanitizer']:
cmd += ' --sanitizer={}'.format(build_config['sanitizer'])
if build_config['bundled'] == 'unbundled':
@ -76,7 +75,7 @@ def get_packager_cmd(build_config, packager_path, output_path, build_version, im
def get_image_name(build_config):
if build_config['bundled'] != 'bundled':
return 'clickhouse/unbundled-builder'
elif build_config['package-type'] != 'deb':
elif build_config['package_type'] != 'deb':
return 'clickhouse/binary-builder'
else:
return 'clickhouse/deb-builder'
@ -92,21 +91,6 @@ def build_clickhouse(packager_cmd, logs_path):
logging.info("Build failed")
return build_log_path, retcode == 0
def build_config_to_string(build_config):
if build_config["package-type"] == "performance":
return "performance"
return "_".join([
build_config['compiler'],
build_config['build-type'] if build_config['build-type'] else "relwithdebuginfo",
build_config['sanitizer'] if build_config['sanitizer'] else "none",
build_config['bundled'],
build_config['splitted'],
"tidy" if build_config['tidy'] == "enable" else "notidy",
"with_coverage" if build_config['with_coverage'] else "without_coverage",
build_config['package-type'],
])
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
repo_path = os.getenv("REPO_COPY", os.path.abspath("../../"))
@ -116,7 +100,7 @@ if __name__ == "__main__":
build_check_name = sys.argv[1]
build_number = int(sys.argv[2])
build_config = get_build_config(build_check_name, build_number, repo_path)
build_config = get_build_config(build_check_name, build_number)
if not os.path.exists(temp_path):
os.makedirs(temp_path)
@ -130,27 +114,9 @@ if __name__ == "__main__":
gh = Github(get_best_robot_token())
images_path = os.path.join(os.getenv("IMAGES_PATH", temp_path), 'changed_images.json')
image_name = get_image_name(build_config)
image_version = 'latest'
if os.path.exists(images_path):
logging.info("Images file exists")
with open(images_path, 'r') as images_fd:
images = json.load(images_fd)
logging.info("Got images %s", images)
if image_name in images:
image_version = images[image_name]
for i in range(10):
try:
logging.info("Pulling image %s:%s", image_name, image_version)
subprocess.check_output(f"docker pull {image_name}:{image_version}", stderr=subprocess.STDOUT, shell=True)
break
except Exception as ex:
time.sleep(i * 3)
logging.info("Got execption pulling docker %s", ex)
else:
raise Exception(f"Cannot pull dockerhub for image docker pull {image_name}:{image_version}")
docker_image = get_image_with_version(os.getenv("IMAGES_PATH"), image_name)
image_version = docker_image.version
version = get_version_from_repo(repo_path)
version.tweak_update()

97
tests/ci/build_download_helper.py Normal file
View File

@ -0,0 +1,97 @@
#!/usr/bin/env python3
import os
import json
import logging
import sys
import time
import requests
from ci_config import CI_CONFIG, build_config_to_string
DOWNLOAD_RETRIES_COUNT = 5
def get_build_config_for_check(check_name):
return CI_CONFIG["tests_config"][check_name]['required_build_properties']
def get_build_urls(build_config_str, reports_path):
for root, _, files in os.walk(reports_path):
for f in files:
if build_config_str in f :
logging.info("Found build report json %s", f)
with open(os.path.join(root, f), 'r', encoding='utf-8') as file_handler:
build_report = json.load(file_handler)
return build_report['build_urls']
return []
def dowload_build_with_progress(url, path):
logging.info("Downloading from %s to temp path %s", url, path)
for i in range(DOWNLOAD_RETRIES_COUNT):
try:
with open(path, 'wb') as f:
response = requests.get(url, stream=True)
response.raise_for_status()
total_length = response.headers.get('content-length')
if total_length is None or int(total_length) == 0:
logging.info("No content-length, will download file without progress")
f.write(response.content)
else:
dl = 0
total_length = int(total_length)
logging.info("Content length is %ld bytes", total_length)
for data in response.iter_content(chunk_size=4096):
dl += len(data)
f.write(data)
if sys.stdout.isatty():
done = int(50 * dl / total_length)
percent = int(100 * float(dl) / total_length)
eq_str = '=' * done
space_str = ' ' * (50 - done)
sys.stdout.write(f"\r[{eq_str}{space_str}] {percent}%")
sys.stdout.flush()
break
except Exception as ex:
sys.stdout.write("\n")
time.sleep(3)
logging.info("Exception while downloading %s, retry %s", ex, i + 1)
if os.path.exists(path):
os.remove(path)
else:
raise Exception(f"Cannot download dataset from {url}, all retries exceeded")
sys.stdout.write("\n")
logging.info("Downloading finished")
def download_builds(result_path, build_urls, filter_fn):
for url in build_urls:
if filter_fn(url):
fname = os.path.basename(url.replace('%2B', '+').replace('%20', ' '))
logging.info("Will download %s to %s", fname, result_path)
dowload_build_with_progress(url, os.path.join(result_path, fname))
def download_builds_filter(check_name, reports_path, result_path, filter_fn=lambda _: True):
build_config = get_build_config_for_check(check_name)
print(build_config)
build_config_str = build_config_to_string(build_config)
print(build_config_str)
urls = get_build_urls(build_config_str, reports_path)
print(urls)
if not urls:
raise Exception("No build URLs found")
download_builds(result_path, urls, filter_fn)
def download_all_deb_packages(check_name, reports_path, result_path):
download_builds_filter(check_name, reports_path, result_path, lambda x: x.endswith('deb'))
def download_shared_build(check_name, reports_path, result_path):
download_builds_filter(check_name, reports_path, result_path, lambda x: x.endswith('shared_build.tgz'))
def download_unit_tests(check_name, reports_path, result_path):
download_builds_filter(check_name, reports_path, result_path, lambda x: x.endswith('unit_tests_dbms'))
def download_clickhouse_binary(check_name, reports_path, result_path):
download_builds_filter(check_name, reports_path, result_path, lambda x: x.endswith('clickhouse'))

8
tests/ci/build_report_check.py
View File

@ -9,6 +9,7 @@ from report import create_build_html_report
from s3_helper import S3Helper
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from commit_status_helper import get_commit
class BuildResult():
def __init__(self, compiler, build_type, sanitizer, bundled, splitted, status, elapsed_seconds, with_coverage):
@ -36,16 +37,11 @@ def group_by_artifacts(build_urls):
groups['binary'].append(url)
return groups
def get_commit(gh, commit_sha):
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
commit = repo.get_commit(commit_sha)
return commit
def process_report(build_report):
build_config = build_report['build_config']
build_result = BuildResult(
compiler=build_config['compiler'],
build_type=build_config['build-type'],
build_type=build_config['build_type'],
sanitizer=build_config['sanitizer'],
bundled=build_config['bundled'],
splitted=build_config['splitted'],

736
tests/ci/ci_config.py Normal file
View File

@ -0,0 +1,736 @@
#!/usr/bin/env python3
CI_CONFIG = {
"build_config": [
{
"compiler": "clang-13",
"build_type": "",
"sanitizer": "",
"package_type": "deb",
"bundled": "bundled",
"splitted": "unsplitted",
"alien_pkgs": True,
"tidy": "disable",
"with_coverage": False
},
{
"compiler": "clang-13",
"build_type": "",
"sanitizer": "",
"package_type": "performance",
"bundled": "bundled",
"splitted": "unsplitted",
"tidy": "disable",
"with_coverage": False
},
{
"compiler": "gcc-11",
"build_type": "",
"sanitizer": "",
"package_type": "binary",
"bundled": "bundled",
"splitted": "unsplitted",
"tidy": "disable",
"with_coverage": False
},
{
"compiler": "clang-13",
"build_type": "",
"sanitizer": "address",
"package_type": "deb",
"bundled": "bundled",
"splitted": "unsplitted",
"tidy": "disable",
"with_coverage": False
},
{
"compiler": "clang-13",
"build_type": "",
"sanitizer": "undefined",
"package_type": "deb",
"bundled": "bundled",
"splitted": "unsplitted",
"tidy": "disable",
"with_coverage": False
},
{
"compiler": "clang-13",
"build_type": "",
"sanitizer": "thread",
"package_type": "deb",
"bundled": "bundled",
"splitted": "unsplitted",
"tidy": "disable",
"with_coverage": False
},
{
"compiler": "clang-13",
"build_type": "",
"sanitizer": "memory",
"package_type": "deb",
"bundled": "bundled",
"splitted": "unsplitted",
"tidy": "disable",
"with_coverage": False
},
{
"compiler": "clang-13",
"build_type": "debug",
"sanitizer": "",
"package_type": "deb",
"bundled": "bundled",
"splitted": "unsplitted",
"tidy": "disable",
"with_coverage": False
},
{
"compiler": "gcc-11",
"build_type": "",
"sanitizer": "",
"package_type": "deb",
"bundled": "unbundled",
"splitted": "unsplitted",
"tidy": "disable",
"with_coverage": False
},
{
"compiler": "clang-13",
"build_type": "",
"sanitizer": "",
"package_type": "binary",
"bundled": "bundled",
"splitted": "unsplitted",
"tidy": "disable",
"with_coverage": False
}
],
"special_build_config": [
{
"compiler": "clang-13",
"build_type": "debug",
"sanitizer": "",
"package_type": "deb",
"bundled": "bundled",
"splitted": "unsplitted",
"tidy": "enable",
"with_coverage": False
},
{
"compiler": "clang-13",
"build_type": "",
"sanitizer": "",
"package_type": "binary",
"bundled": "bundled",
"splitted": "splitted",
"tidy": "disable",
"with_coverage": False
},
{
"compiler": "clang-13-darwin",
"build_type": "",
"sanitizer": "",
"package_type": "binary",
"bundled": "bundled",
"splitted": "unsplitted",
"tidy": "disable",
"with_coverage": False
},
{
"compiler": "clang-13-aarch64",
"build_type": "",
"sanitizer": "",
"package_type": "binary",
"bundled": "bundled",
"splitted": "unsplitted",
"tidy": "disable",
"with_coverage": False
},
{
"compiler": "clang-13-freebsd",
"build_type": "",
"sanitizer": "",
"package_type": "binary",
"bundled": "bundled",
"splitted": "unsplitted",
"tidy": "disable",
"with_coverage": False
},
{
"compiler": "clang-13-darwin-aarch64",
"build_type": "",
"sanitizer": "",
"package_type": "binary",
"bundled": "bundled",
"splitted": "unsplitted",
"tidy": "disable",
"with_coverage": False
},
{
"compiler": "clang-13-ppc64le",
"build_type": "",
"sanitizer": "",
"package_type": "binary",
"bundled": "bundled",
"splitted": "unsplitted",
"tidy": "disable",
"with_coverage": False
}
],
"tests_config": {
"Stateful tests (address, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "address",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Stateful tests (thread, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "thread",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Stateful tests (memory, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "memory",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Stateful tests (ubsan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "undefined",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Stateful tests (debug, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "debug",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Stateful tests (release, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Stateful tests (release, DatabaseOrdinary, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Stateful tests (release, DatabaseReplicated, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Stateless tests (address, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "address",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Stateless tests (thread, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "thread",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Stateless tests (memory, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "memory",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Stateless tests (ubsan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "undefined",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Stateless tests (debug, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "debug",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Stateless tests (release, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Stateless tests (unbundled, actions)": {
"required_build_properties": {
"compiler": "gcc-11",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "unbundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Stateless tests (release, wide parts enabled, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Stateless tests (release, DatabaseOrdinary, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Stateless tests (release, DatabaseReplicated, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Stress test (address, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "address",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Stress test (thread, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "thread",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Stress test (undefined, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "undefined",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Stress test (memory, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "memory",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Stress test (debug, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "debug",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Integration tests (asan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "address",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Integration tests (thread, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "thread",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Integration tests (release, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Integration tests (memory, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "memory",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Integration tests flaky check (asan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "address",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Compatibility check (actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Split build smoke test (actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "binary",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "splitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Testflows check (actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Unit tests (release-gcc, actions)": {
"required_build_properties": {
"compiler": "gcc-11",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Unit tests (release-clang, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Unit tests (asan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "address",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Unit tests (msan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "memory",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Unit tests (tsan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "thread",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Unit tests (ubsan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "thread",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"AST fuzzer (debug, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "debug",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"AST fuzzer (ASan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "address",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"AST fuzzer (MSan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "memory",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"AST fuzzer (TSan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "thread",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"AST fuzzer (UBSan, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "undefined",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Release (actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"Stateless tests flaky check (address, actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "deb",
"build_type": "relwithdebuginfo",
"sanitizer": "address",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
},
"ClickHouse Keeper Jepsen (actions)": {
"required_build_properties": {
"compiler": "clang-13",
"package_type": "binary",
"build_type": "relwithdebuginfo",
"sanitizer": "none",
"bundled": "bundled",
"splitted": "unsplitted",
"clang_tidy": "disable",
"with_coverage": False
}
}
}
}
def build_config_to_string(build_config):
if build_config["package_type"] == "performance":
return "performance"
return "_".join([
build_config['compiler'],
build_config['build_type'] if build_config['build_type'] else "relwithdebuginfo",
build_config['sanitizer'] if build_config['sanitizer'] else "none",
build_config['bundled'],
build_config['splitted'],
'tidy' if 'tidy' in build_config and build_config['tidy'] == 'enable' else 'notidy',
'with_coverage' if 'with_coverage' in build_config and build_config['with_coverage'] else 'without_coverage',
build_config['package_type'],
])

12
tests/ci/commit_status_helper.py Normal file
View File

@ -0,0 +1,12 @@
#!/usr/bin/env python3
import os
def get_commit(gh, commit_sha):
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
commit = repo.get_commit(commit_sha)
return commit
def post_commit_status(gh, sha, check_name, description, state, report_url):
commit = get_commit(gh, sha)
commit.create_status(context=check_name, description=description, state=state, target_url=report_url)

174
tests/ci/compatibility_check.py
View File

@ -2,20 +2,19 @@
from distutils.version import StrictVersion
import logging
import sys
import os
import json
import subprocess
import time
from github import Github
import requests
from report import create_test_html_report
from s3_helper import S3Helper
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from build_download_helper import download_builds_filter
from upload_result_helper import upload_results
from docker_pull_helper import get_images_with_versions
from commit_status_helper import post_commit_status
IMAGE_UBUNTU = "clickhouse/test-old-ubuntu"
IMAGE_CENTOS = "clickhouse/test-old-centos"
@ -82,82 +81,6 @@ def process_result(result_folder, server_log_folder):
return status, description, summary, result_logs
def dowload_build_with_progress(url, path):
logging.info("Downloading from %s to temp path %s", url, path)
for i in range(DOWNLOAD_RETRIES_COUNT):
try:
with open(path, 'wb') as f:
response = requests.get(url, stream=True)
response.raise_for_status()
total_length = response.headers.get('content-length')
if total_length is None or int(total_length) == 0:
logging.info("No content-length, will download file without progress")
f.write(response.content)
else:
dl = 0
total_length = int(total_length)
logging.info("Content length is %ld bytes", total_length)
for data in response.iter_content(chunk_size=4096):
dl += len(data)
f.write(data)
if sys.stdout.isatty():
done = int(50 * dl / total_length)
percent = int(100 * float(dl) / total_length)
eq_str = '=' * done
space_str = ' ' * (50 - done)
sys.stdout.write(f"\r[{eq_str}{space_str}] {percent}%")
sys.stdout.flush()
break
except Exception as ex:
sys.stdout.write("\n")
time.sleep(3)
logging.info("Exception while downloading %s, retry %s", ex, i + 1)
if os.path.exists(path):
os.remove(path)
else:
raise Exception(f"Cannot download dataset from {url}, all retries exceeded")
sys.stdout.write("\n")
logging.info("Downloading finished")
def download_builds(result_path, build_urls):
for url in build_urls:
if url.endswith('.deb') and ('clickhouse-common-static_' in url or 'clickhouse-server_' in url):
fname = os.path.basename(url)
logging.info("Will download %s to %s", fname, result_path)
dowload_build_with_progress(url, os.path.join(result_path, fname))
def get_build_config(build_number, repo_path):
ci_config_path = os.path.join(repo_path, "tests/ci/ci_config.json")
with open(ci_config_path, 'r', encoding='utf-8') as ci_config:
config_dict = json.load(ci_config)
return config_dict['build_config'][build_number]
def get_build_urls(build_config_str, reports_path):
for root, _, files in os.walk(reports_path):
for f in files:
if build_config_str in f :
logging.info("Found build report json %s", f)
with open(os.path.join(root, f), 'r', encoding='utf-8') as file_handler:
build_report = json.load(file_handler)
return build_report['build_urls']
return []
def build_config_to_string(build_config):
if build_config["package-type"] == "performance":
return "performance"
return "_".join([
build_config['compiler'],
build_config['build-type'] if build_config['build-type'] else "relwithdebuginfo",
build_config['sanitizer'] if build_config['sanitizer'] else "none",
build_config['bundled'],
build_config['splitted'],
"tidy" if build_config['tidy'] == "enable" else "notidy",
"with_coverage" if build_config['with_coverage'] else "without_coverage",
build_config['package-type'],
])
def get_run_commands(build_path, result_folder, server_log_folder, image_centos, image_ubuntu):
return [
@ -171,45 +94,6 @@ def get_run_commands(build_path, result_folder, server_log_folder, image_centos,
f"--volume={server_log_folder}:/var/log/clickhouse-server {image_centos} > {result_folder}/centos:5",
]
def process_logs(s3_client, additional_logs, s3_path_prefix):
additional_urls = []
for log_path in additional_logs:
if log_path:
additional_urls.append(
s3_client.upload_test_report_to_s3(
log_path,
s3_path_prefix + "/" + os.path.basename(log_path)))
return additional_urls
def upload_results(s3_client, pr_number, commit_sha, test_results, additional_files, check_name):
s3_path_prefix = f"{pr_number}/{commit_sha}/" + check_name.lower().replace(' ', '_').replace('(', '_').replace(')', '_').replace(',', '_')
additional_urls = process_logs(s3_client, additional_files, s3_path_prefix)
branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
branch_name = "master"
if pr_number != 0:
branch_name = f"PR #{pr_number}"
branch_url = f"https://github.com/ClickHouse/ClickHouse/pull/{pr_number}"
commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
raw_log_url = additional_urls[0]
additional_urls.pop(0)
html_report = create_test_html_report(check_name, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls, True)
with open('report.html', 'w', encoding='utf-8') as f:
f.write(html_report)
url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
logging.info("Search result in url %s", url)
return url
def get_commit(gh, commit_sha):
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
commit = repo.get_commit(commit_sha)
return commit
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
@ -217,8 +101,6 @@ if __name__ == "__main__":
repo_path = os.getenv("REPO_COPY", os.path.abspath("../../"))
reports_path = os.getenv("REPORTS_PATH", "./reports")
build_number = int(sys.argv[1])
with open(os.getenv('GITHUB_EVENT_PATH'), 'r', encoding='utf-8') as event_file:
event = json.load(event_file)
@ -226,50 +108,17 @@ if __name__ == "__main__":
gh = Github(get_best_robot_token())
for root, _, files in os.walk(reports_path):
for f in files:
if f == 'changed_images.json':
images_path = os.path.join(root, 'changed_images.json')
break
docker_images = []
if images_path and os.path.exists(images_path):
for image_name in [IMAGE_CENTOS, IMAGE_UBUNTU]:
docker_image = image_name
logging.info("Images file exists")
with open(images_path, 'r', encoding='utf-8') as images_fd:
images = json.load(images_fd)
logging.info("Got images %s", images)
if image_name in images:
docker_image += ':' + images[image_name]
docker_images.append(docker_image)
else:
logging.info("Images file not found")
for docker_image in docker_images:
for i in range(10):
try:
logging.info("Pulling image %s", docker_image)
subprocess.check_output(f"docker pull {docker_image}", stderr=subprocess.STDOUT, shell=True)
break
except Exception as ex:
time.sleep(i * 3)
logging.info("Got execption pulling docker %s", ex)
else:
raise Exception(f"Cannot pull dockerhub for image docker pull {docker_image}")
build_config = get_build_config(build_number, repo_path)
build_config_str = build_config_to_string(build_config)
urls = get_build_urls(build_config_str, reports_path)
if not urls:
raise Exception("No build URLs found")
docker_images = get_images_with_versions(reports_path, [IMAGE_CENTOS, IMAGE_UBUNTU])
packages_path = os.path.join(temp_path, "packages")
if not os.path.exists(packages_path):
os.makedirs(packages_path)
download_builds(packages_path, urls)
def url_filter(url):
return url.endswith('.deb') and ('clickhouse-common-static_' in url or 'clickhouse-server_' in url)
download_builds_filter(CHECK_NAME, reports_path, packages_path, url_filter)
for f in os.listdir(packages_path):
if '.deb' in f:
full_path = os.path.join(packages_path, f)
@ -300,5 +149,4 @@ if __name__ == "__main__":
state, description, test_results, additional_logs = process_result(result_path, server_log_path)
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, additional_logs, CHECK_NAME)
print(f"::notice ::Report url: {report_url}")
commit = get_commit(gh, pr_info.sha)
commit.create_status(context=CHECK_NAME, description=description, state=state, target_url=report_url)
post_commit_status(gh, pr_info.sha, CHECK_NAME, description, state, report_url)

38
tests/ci/docker_images_check.py
View File

@ -6,10 +6,11 @@ import os
import time
import shutil
from github import Github
from report import create_test_html_report
from s3_helper import S3Helper
from pr_info import PRInfo
from get_robot_token import get_best_robot_token, get_parameter_from_ssm
from upload_result_helper import upload_results
from commit_status_helper import get_commit
NAME = "Push to Dockerhub (actions)"
@ -149,31 +150,6 @@ def process_test_results(s3_client, test_results, s3_path_prefix):
processed_test_results.append((test_name, status))
return overall_status, processed_test_results
def upload_results(s3_client, pr_number, commit_sha, test_results):
s3_path_prefix = f"{pr_number}/{commit_sha}/" + NAME.lower().replace(' ', '_')
branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
branch_name = "master"
if pr_number != 0:
branch_name = "PR #{}".format(pr_number)
branch_url = "https://github.com/ClickHouse/ClickHouse/pull/" + str(pr_number)
commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
html_report = create_test_html_report(NAME, test_results, "https://hub.docker.com/u/clickhouse", task_url, branch_url, branch_name, commit_url)
with open('report.html', 'w') as f:
f.write(html_report)
url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
logging.info("Search result in url %s", url)
return url
def get_commit(gh, commit_sha):
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
commit = repo.get_commit(commit_sha)
return commit
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
repo_path = os.getenv("GITHUB_WORKSPACE", os.path.abspath("../../"))
@ -211,7 +187,6 @@ if __name__ == "__main__":
else:
description = "Nothing to update"
if len(description) >= 140:
description = description[:136] + "..."
@ -220,14 +195,13 @@ if __name__ == "__main__":
s3_path_prefix = str(pr_info.number) + "/" + pr_info.sha + "/" + NAME.lower().replace(' ', '_')
status, test_results = process_test_results(s3_helper, images_processing_result, s3_path_prefix)
url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results)
gh = Github(get_best_robot_token())
commit = get_commit(gh, pr_info.sha)
commit.create_status(context=NAME, description=description, state=status, target_url=url)
url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [], NAME)
with open(os.path.join(temp_path, 'changed_images.json'), 'w') as images_file:
json.dump(result_images, images_file)
print("::notice ::Report url: {}".format(url))
print("::set-output name=url_output::\"{}\"".format(url))
gh = Github(get_best_robot_token())
commit = get_commit(gh, pr_info.sha)
commit.create_status(context=NAME, description=description, state=status, target_url=url)

59
tests/ci/docker_pull_helper.py Normal file
View File

@ -0,0 +1,59 @@
#!/usr/bin/env python3
import os
import json
import time
import subprocess
import logging
class DockerImage:
def __init__(self, name, version=None):
self.name = name
if version is None:
self.version = 'latest'
else:
self.version = version
def __str__(self):
return f"{self.name}:{self.version}"
def get_images_with_versions(reports_path, required_image, pull=True):
images_path = None
for root, _, files in os.walk(reports_path):
for f in files:
if f == 'changed_images.json':
images_path = os.path.join(root, 'changed_images.json')
break
if images_path is not None and os.path.exists(images_path):
logging.info("Images file exists")
with open(images_path, 'r', encoding='utf-8') as images_fd:
images = json.load(images_fd)
logging.info("Got images %s", images)
else:
images = {}
docker_images = []
for image_name in required_image:
docker_image = DockerImage(image_name)
if image_name in images:
docker_image.version = images[image_name]
docker_images.append(docker_image)
if pull:
for docker_image in docker_images:
for i in range(10):
try:
logging.info("Pulling image %s", docker_image)
latest_error = subprocess.check_output(f"docker pull {docker_image}", stderr=subprocess.STDOUT, shell=True)
break
except Exception as ex:
time.sleep(i * 3)
logging.info("Got execption pulling docker %s", ex)
else:
raise Exception(f"Cannot pull dockerhub for image docker pull {docker_image} because of {latest_error}")
return docker_images
def get_image_with_version(reports_path, image, pull=True):
return get_images_with_versions(reports_path, [image], pull)[0]

73
tests/ci/docs_check.py
View File

@ -2,57 +2,19 @@
import logging
import subprocess
import os
import time
import json
import sys
from github import Github
from report import create_test_html_report
from s3_helper import S3Helper
from pr_info import PRInfo
from get_robot_token import get_best_robot_token
from upload_result_helper import upload_results
from docker_pull_helper import get_image_with_version
from commit_status_helper import post_commit_status, get_commit
NAME = "Docs Check (actions)"
def process_logs(s3_client, additional_logs, s3_path_prefix):
additional_urls = []
for log_path in additional_logs:
if log_path:
additional_urls.append(
s3_client.upload_test_report_to_s3(
log_path,
s3_path_prefix + "/" + os.path.basename(log_path)))
return additional_urls
def upload_results(s3_client, pr_number, commit_sha, test_results, additional_files):
s3_path_prefix = f"{pr_number}/{commit_sha}/docs_check"
additional_urls = process_logs(s3_client, additional_files, s3_path_prefix)
branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
branch_name = "master"
if pr_number != 0:
branch_name = f"PR #{pr_number}"
branch_url = f"https://github.com/ClickHouse/ClickHouse/pull/{pr_number}"
commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
raw_log_url = additional_urls[0]
additional_urls.pop(0)
html_report = create_test_html_report(NAME, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls)
with open('report.html', 'w', encoding='utf-8') as f:
f.write(html_report)
url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
logging.info("Search result in url %s", url)
return url
def get_commit(gh, commit_sha):
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
commit = repo.get_commit(commit_sha)
return commit
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
@ -76,27 +38,7 @@ if __name__ == "__main__":
if not os.path.exists(temp_path):
os.makedirs(temp_path)
images_path = os.path.join(temp_path, 'changed_images.json')
docker_image = 'clickhouse/docs-check'
if os.path.exists(images_path):
logging.info("Images file exists")
with open(images_path, 'r', encoding='utf-8') as images_fd:
images = json.load(images_fd)
logging.info("Got images %s", images)
if 'clickhouse/docs-check' in images:
docker_image += ':' + images['clickhouse/docs-check']
logging.info("Got docker image %s", docker_image)
for i in range(10):
try:
subprocess.check_output(f"docker pull {docker_image}", shell=True)
break
except Exception as ex:
time.sleep(i * 3)
logging.info("Got execption pulling docker %s", ex)
else:
raise Exception(f"Cannot pull dockerhub for image {docker_image}")
docker_image = get_image_with_version(temp_path, 'clickhouse/docs-check')
test_output = os.path.join(temp_path, 'docs_check_log')
if not os.path.exists(test_output):
@ -144,7 +86,6 @@ if __name__ == "__main__":
s3_helper = S3Helper('https://s3.amazonaws.com')
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, lines, additional_files)
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, lines, additional_files, NAME)
print("::notice ::Report url: {report_url}")
commit = get_commit(gh, pr_info.sha)
commit.create_status(context=NAME, description=description, state=status, target_url=report_url)
post_commit_status(gh, pr_info.sha, NAME, description, status, report_url)

73
tests/ci/docs_release.py
View File

@ -1,61 +1,22 @@
#!/usr/bin/env python3
#!/usr/bin/env python3
import logging
import subprocess
import os
import time
import json
import sys
from github import Github
from report import create_test_html_report
from s3_helper import S3Helper
from pr_info import PRInfo
from get_robot_token import get_best_robot_token
from ssh import SSHKey
from upload_result_helper import upload_results
from docker_pull_helper import get_image_with_version
from commit_status_helper import get_commit
NAME = "Docs Release (actions)"
def process_logs(s3_client, additional_logs, s3_path_prefix):
additional_urls = []
for log_path in additional_logs:
if log_path:
additional_urls.append(
s3_client.upload_test_report_to_s3(
log_path,
s3_path_prefix + "/" + os.path.basename(log_path)))
return additional_urls
def upload_results(s3_client, pr_number, commit_sha, test_results, additional_files):
s3_path_prefix = f"{pr_number}/{commit_sha}/docs_release"
additional_urls = process_logs(s3_client, additional_files, s3_path_prefix)
branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
branch_name = "master"
if pr_number != 0:
branch_name = f"PR #{pr_number}"
branch_url = f"https://github.com/ClickHouse/ClickHouse/pull/{pr_number}"
commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
raw_log_url = additional_urls[0]
additional_urls.pop(0)
html_report = create_test_html_report(NAME, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls)
with open('report.html', 'w', encoding='utf-8') as f:
f.write(html_report)
url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
logging.info("Search result in url %s", url)
return url
def get_commit(gh, commit_sha):
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
commit = repo.get_commit(commit_sha)
return commit
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
@ -79,27 +40,7 @@ if __name__ == "__main__":
if not os.path.exists(temp_path):
os.makedirs(temp_path)
images_path = os.path.join(temp_path, 'changed_images.json')
docker_image = 'clickhouse/docs-release'
if os.path.exists(images_path):
logging.info("Images file exists")
with open(images_path, 'r', encoding='utf-8') as images_fd:
images = json.load(images_fd)
logging.info("Got images %s", images)
if 'clickhouse/docs-release' in images:
docker_image += ':' + images['clickhouse/docs-release']
logging.info("Got docker image %s", docker_image)
for i in range(10):
try:
subprocess.check_output(f"docker pull {docker_image}", shell=True)
break
except Exception as ex:
time.sleep(i * 3)
logging.info("Got execption pulling docker %s", ex)
else:
raise Exception(f"Cannot pull dockerhub for image {docker_image}")
docker_image = get_image_with_version(temp_path, 'clickhouse/docs-release')
test_output = os.path.join(temp_path, 'docs_release_log')
if not os.path.exists(test_output):
@ -149,7 +90,7 @@ if __name__ == "__main__":
s3_helper = S3Helper('https://s3.amazonaws.com')
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, lines, additional_files)
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, lines, additional_files, NAME)
print("::notice ::Report url: {report_url}")
commit = get_commit(gh, pr_info.sha)
commit.create_status(context=NAME, description=description, state=status, target_url=report_url)

74
tests/ci/fast_test_check.py
View File

@ -4,13 +4,14 @@ import logging
import subprocess
import os
import json
import time
import csv
from github import Github
from pr_info import PRInfo
from report import create_test_html_report
from s3_helper import S3Helper
from get_robot_token import get_best_robot_token
from upload_result_helper import upload_results
from docker_pull_helper import get_image_with_version
from commit_status_helper import post_commit_status
NAME = 'Fast test (actions)'
@ -47,48 +48,6 @@ def process_results(result_folder):
return state, description, test_results, additional_files
def process_logs(s3_client, additional_logs, s3_path_prefix):
additional_urls = []
for log_path in additional_logs:
if log_path:
additional_urls.append(
s3_client.upload_test_report_to_s3(
log_path,
s3_path_prefix + "/" + os.path.basename(log_path)))
return additional_urls
def upload_results(s3_client, pr_number, commit_sha, test_results, raw_log, additional_files):
additional_files = [raw_log] + additional_files
s3_path_prefix = f"{pr_number}/{commit_sha}/fasttest"
additional_urls = process_logs(s3_client, additional_files, s3_path_prefix)
branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
branch_name = "master"
if pr_number != 0:
branch_name = "PR #{}".format(pr_number)
branch_url = "https://github.com/ClickHouse/ClickHouse/pull/" + str(pr_number)
commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
raw_log_url = additional_urls[0]
additional_urls.pop(0)
html_report = create_test_html_report(NAME, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls, True)
with open('report.html', 'w') as f:
f.write(html_report)
url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
logging.info("Search result in url %s", url)
return url
def get_commit(gh, commit_sha):
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
commit = repo.get_commit(commit_sha)
return commit
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
@ -105,27 +64,7 @@ if __name__ == "__main__":
gh = Github(get_best_robot_token())
images_path = os.path.join(temp_path, 'changed_images.json')
docker_image = 'clickhouse/fasttest'
if os.path.exists(images_path):
logging.info("Images file exists")
with open(images_path, 'r') as images_fd:
images = json.load(images_fd)
logging.info("Got images %s", images)
if 'clickhouse/fasttest' in images:
docker_image += ':' + images['clickhouse/fasttest']
logging.info("Got docker image %s", docker_image)
for i in range(10):
try:
subprocess.check_output(f"docker pull {docker_image}", shell=True)
break
except Exception as ex:
time.sleep(i * 3)
logging.info("Got execption pulling docker %s", ex)
else:
raise Exception(f"Cannot pull dockerhub for image {docker_image}")
docker_image = get_image_with_version(temp_path, 'clickhouse/fasttest')
s3_helper = S3Helper('https://s3.amazonaws.com')
@ -189,7 +128,6 @@ if __name__ == "__main__":
else:
state, description, test_results, additional_logs = process_results(output_path)
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, run_log_path, additional_logs)
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [run_log_path] + additional_logs, NAME, True)
print("::notice ::Report url: {}".format(report_url))
commit = get_commit(gh, pr_info.sha)
commit.create_status(context=NAME, description=description, state=state, target_url=report_url)
post_commit_status(gh, pr_info.sha, NAME, description, state, report_url)

7
tests/ci/finish_check.py
View File

@ -5,6 +5,7 @@ import os
from github import Github
from pr_info import PRInfo
from get_robot_token import get_best_robot_token
from commit_status_helper import get_commit
NAME = 'Run Check (actions)'
@ -23,12 +24,6 @@ def filter_statuses(statuses):
filt[status.context] = status
return filt
def get_commit(gh, commit_sha):
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
commit = repo.get_commit(commit_sha)
return commit
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
with open(os.getenv('GITHUB_EVENT_PATH'), 'r') as event_file:

175
tests/ci/functional_test_check.py
View File

@ -5,60 +5,18 @@ import logging
import subprocess
import os
import json
import time
import sys
from github import Github
import requests
from report import create_test_html_report
from s3_helper import S3Helper
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from build_download_helper import download_all_deb_packages
from upload_result_helper import upload_results
from docker_pull_helper import get_image_with_version
from commit_status_helper import post_commit_status, get_commit
DOWNLOAD_RETRIES_COUNT = 5
def process_logs(s3_client, additional_logs, s3_path_prefix):
additional_urls = []
for log_path in additional_logs:
if log_path:
additional_urls.append(
s3_client.upload_test_report_to_s3(
log_path,
s3_path_prefix + "/" + os.path.basename(log_path)))
return additional_urls
def upload_results(s3_client, pr_number, commit_sha, test_results, raw_log, additional_files, check_name):
additional_files = [raw_log] + additional_files
s3_path_prefix = f"{pr_number}/{commit_sha}/" + check_name.lower().replace(' ', '_').replace('(', '_').replace(')', '_').replace(',', '_')
additional_urls = process_logs(s3_client, additional_files, s3_path_prefix)
branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
branch_name = "master"
if pr_number != 0:
branch_name = f"PR #{pr_number}"
branch_url = f"https://github.com/ClickHouse/ClickHouse/pull/{pr_number}"
commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
raw_log_url = additional_urls[0]
additional_urls.pop(0)
html_report = create_test_html_report(check_name, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls, True)
with open('report.html', 'w', encoding='utf-8') as f:
f.write(html_report)
url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
logging.info("Search result in url %s", url)
return url
def get_commit(gh, commit_sha):
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
commit = repo.get_commit(commit_sha)
return commit
def get_image_name(check_name):
if 'stateless' in check_name.lower():
@ -68,85 +26,6 @@ def get_image_name(check_name):
else:
raise Exception(f"Cannot deduce image name based on check name {check_name}")
def dowload_build_with_progress(url, path):
logging.info("Downloading from %s to temp path %s", url, path)
for i in range(DOWNLOAD_RETRIES_COUNT):
try:
with open(path, 'wb') as f:
response = requests.get(url, stream=True)
response.raise_for_status()
total_length = response.headers.get('content-length')
if total_length is None or int(total_length) == 0:
logging.info("No content-length, will download file without progress")
f.write(response.content)
else:
dl = 0
total_length = int(total_length)
logging.info("Content length is %ld bytes", total_length)
for data in response.iter_content(chunk_size=4096):
dl += len(data)
f.write(data)
if sys.stdout.isatty():
done = int(50 * dl / total_length)
percent = int(100 * float(dl) / total_length)
eq_str = '=' * done
space_str = ' ' * (50 - done)
sys.stdout.write(f"\r[{eq_str}{space_str}] {percent}%")
sys.stdout.flush()
break
except Exception as ex:
sys.stdout.write("\n")
time.sleep(3)
logging.info("Exception while downloading %s, retry %s", ex, i + 1)
if os.path.exists(path):
os.remove(path)
else:
raise Exception(f"Cannot download dataset from {url}, all retries exceeded")
sys.stdout.write("\n")
logging.info("Downloading finished")
def download_builds(result_path, build_urls):
for url in build_urls:
if url.endswith('.deb'):
fname = os.path.basename(url)
logging.info("Will download %s to %s", fname, result_path)
dowload_build_with_progress(url, os.path.join(result_path, fname))
def get_build_config(build_number, repo_path):
ci_config_path = os.path.join(repo_path, "tests/ci/ci_config.json")
with open(ci_config_path, 'r', encoding='utf-8') as ci_config:
config_dict = json.load(ci_config)
return config_dict['build_config'][build_number]
def get_build_urls(build_config_str, reports_path):
for root, _, files in os.walk(reports_path):
for f in files:
if build_config_str in f :
logging.info("Found build report json %s", f)
with open(os.path.join(root, f), 'r', encoding='utf-8') as file_handler:
build_report = json.load(file_handler)
return build_report['build_urls']
return []
def build_config_to_string(build_config):
if build_config["package-type"] == "performance":
return "performance"
return "_".join([
build_config['compiler'],
build_config['build-type'] if build_config['build-type'] else "relwithdebuginfo",
build_config['sanitizer'] if build_config['sanitizer'] else "none",
build_config['bundled'],
build_config['splitted'],
"tidy" if build_config['tidy'] == "enable" else "notidy",
"with_coverage" if build_config['with_coverage'] else "without_coverage",
build_config['package-type'],
])
def get_run_command(builds_path, result_path, server_log_path, kill_timeout, additional_envs, image, flaky_check, tests_to_run):
additional_options = ['--hung-check']
additional_options.append('--print-time')
@ -222,8 +101,7 @@ if __name__ == "__main__":
reports_path = os.getenv("REPORTS_PATH", "./reports")
check_name = sys.argv[1]
build_number = int(sys.argv[2])
kill_timeout = int(sys.argv[3])
kill_timeout = int(sys.argv[2])
flaky_check = 'flaky' in check_name.lower()
if not os.path.exists(temp_path):
@ -242,46 +120,16 @@ if __name__ == "__main__":
commit.create_status(context=check_name, description='Not found changed stateless tests', state='success')
sys.exit(0)
for root, _, files in os.walk(reports_path):
for f in files:
if f == 'changed_images.json':
images_path = os.path.join(root, 'changed_images.json')
break
image_name = get_image_name(check_name)
docker_image = image_name
if images_path and os.path.exists(images_path):
logging.info("Images file exists")
with open(images_path, 'r', encoding='utf-8') as images_fd:
images = json.load(images_fd)
logging.info("Got images %s", images)
if image_name in images:
docker_image += ':' + images[image_name]
else:
logging.info("Images file not found")
for i in range(10):
try:
logging.info("Pulling image %s", docker_image)
subprocess.check_output(f"docker pull {docker_image}", stderr=subprocess.STDOUT, shell=True)
break
except Exception as ex:
time.sleep(i * 3)
logging.info("Got execption pulling docker %s", ex)
else:
raise Exception(f"Cannot pull dockerhub for image docker pull {docker_image}")
build_config = get_build_config(build_number, repo_path)
build_config_str = build_config_to_string(build_config)
urls = get_build_urls(build_config_str, reports_path)
if not urls:
raise Exception("No build URLs found")
docker_image = get_image_with_version(reports_path, image_name)
packages_path = os.path.join(temp_path, "packages")
if not os.path.exists(packages_path):
os.makedirs(packages_path)
download_all_deb_packages(check_name, reports_path, packages_path)
server_log_path = os.path.join(temp_path, "server_log")
if not os.path.exists(server_log_path):
os.makedirs(server_log_path)
@ -292,7 +140,6 @@ if __name__ == "__main__":
run_log_path = os.path.join(result_path, "runlog.log")
download_builds(packages_path, urls)
run_command = get_run_command(packages_path, result_path, server_log_path, kill_timeout, [], docker_image, flaky_check, tests_to_run)
logging.info("Going to run func tests: %s", run_command)
@ -308,7 +155,7 @@ if __name__ == "__main__":
s3_helper = S3Helper('https://s3.amazonaws.com')
state, description, test_results, additional_logs = process_results(result_path, server_log_path)
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, run_log_path, additional_logs, check_name)
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [run_log_path] + additional_logs, check_name)
print(f"::notice ::Report url: {report_url}")
commit = get_commit(gh, pr_info.sha)
commit.create_status(context=check_name, description=description, state=state, target_url=report_url)
post_commit_status(gh, pr_info.sha, check_name, description, state, report_url)

197
tests/ci/integration_test_check.py
View File

@ -4,18 +4,18 @@ import os
import logging
import sys
import json
import time
import subprocess
import csv
import ast
from github import Github
import requests
from report import create_test_html_report
from s3_helper import S3Helper
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from build_download_helper import download_all_deb_packages
from upload_result_helper import upload_results
from docker_pull_helper import get_images_with_versions
from commit_status_helper import post_commit_status
DOWNLOAD_RETRIES_COUNT = 5
@ -60,136 +60,6 @@ def get_env_for_runner(build_path, repo_path, result_path, work_path):
return my_env
def dowload_build_with_progress(url, path):
logging.info("Downloading from %s to temp path %s", url, path)
for i in range(DOWNLOAD_RETRIES_COUNT):
try:
with open(path, 'wb') as f:
response = requests.get(url, stream=True)
response.raise_for_status()
total_length = response.headers.get('content-length')
if total_length is None or int(total_length) == 0:
logging.info("No content-length, will download file without progress")
f.write(response.content)
else:
dl = 0
total_length = int(total_length)
logging.info("Content length is %ld bytes", total_length)
for data in response.iter_content(chunk_size=4096):
dl += len(data)
f.write(data)
if sys.stdout.isatty():
done = int(50 * dl / total_length)
percent = int(100 * float(dl) / total_length)
eq_str = '=' * done
space_str = ' ' * (50 - done)
sys.stdout.write(f"\r[{eq_str}{space_str}] {percent}%")
sys.stdout.flush()
break
except Exception as ex:
sys.stdout.write("\n")
time.sleep(3)
logging.info("Exception while downloading %s, retry %s", ex, i + 1)
if os.path.exists(path):
os.remove(path)
else:
raise Exception(f"Cannot download dataset from {url}, all retries exceeded")
sys.stdout.write("\n")
logging.info("Downloading finished")
def get_build_urls(build_config_str, reports_path):
for root, _, files in os.walk(reports_path):
for f in files:
if build_config_str in f :
logging.info("Found build report json %s", f)
with open(os.path.join(root, f), 'r', encoding='utf-8') as file_handler:
build_report = json.load(file_handler)
return build_report['build_urls']
return []
def get_build_config(build_number, repo_path):
ci_config_path = os.path.join(repo_path, "tests/ci/ci_config.json")
with open(ci_config_path, 'r', encoding='utf-8') as ci_config:
config_dict = json.load(ci_config)
return config_dict['build_config'][build_number]
def build_config_to_string(build_config):
if build_config["package-type"] == "performance":
return "performance"
return "_".join([
build_config['compiler'],
build_config['build-type'] if build_config['build-type'] else "relwithdebuginfo",
build_config['sanitizer'] if build_config['sanitizer'] else "none",
build_config['bundled'],
build_config['splitted'],
"tidy" if build_config['tidy'] == "enable" else "notidy",
"with_coverage" if build_config['with_coverage'] else "without_coverage",
build_config['package-type'],
])
def get_images_with_versions(images_path):
if os.path.exists(images_path):
result = {}
logging.info("Images file exists")
with open(images_path, 'r', encoding='utf-8') as images_fd:
images = json.load(images_fd)
logging.info("Got images %s", images)
for required_image in IMAGES:
if required_image in images:
result[required_image] = images[required_image]
else:
result[required_image] = 'latest'
return result
else:
return {image: 'latest' for image in IMAGES}
def download_builds(result_path, build_urls):
for url in build_urls:
if url.endswith('.deb'):
fname = os.path.basename(url.replace('%2B', '+').replace('%20', ' '))
logging.info("Will download %s to %s", fname, result_path)
dowload_build_with_progress(url, os.path.join(result_path, fname))
def get_commit(gh, commit_sha):
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
commit = repo.get_commit(commit_sha)
return commit
def process_logs(s3_client, additional_logs, s3_path_prefix, test_results):
proccessed_logs = {}
# Firstly convert paths of logs from test_results to urls to s3.
for test_result in test_results:
if len(test_result) <= 3:
continue
# Convert from string repr of list to list.
test_log_paths = ast.literal_eval(test_result[3])
test_log_urls = []
for log_path in test_log_paths:
if log_path in proccessed_logs:
test_log_urls.append(proccessed_logs[log_path])
elif log_path:
url = s3_client.upload_test_report_to_s3(
log_path,
s3_path_prefix + "/" + os.path.basename(log_path))
test_log_urls.append(url)
proccessed_logs[log_path] = url
test_result[3] = test_log_urls
additional_urls = []
for log_path in additional_logs:
if log_path:
additional_urls.append(
s3_client.upload_test_report_to_s3(
log_path,
s3_path_prefix + "/" + os.path.basename(log_path)))
return additional_urls
def process_results(result_folder):
test_results = []
additional_files = []
@ -219,32 +89,6 @@ def process_results(result_folder):
return state, description, test_results, additional_files
def upload_results(s3_client, pr_number, commit_sha, test_results, raw_log, additional_files, check_name):
additional_files = [raw_log] + additional_files
s3_path_prefix = f"{pr_number}/{commit_sha}/" + check_name.lower().replace(' ', '_').replace('(', '_').replace(')', '_').replace(',', '_')
additional_urls = process_logs(s3_client, additional_files, s3_path_prefix, test_results)
branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
branch_name = "master"
if pr_number != 0:
branch_name = f"PR #{pr_number}"
branch_url = f"https://github.com/ClickHouse/ClickHouse/pull/{pr_number}"
commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
raw_log_url = additional_urls[0]
additional_urls.pop(0)
html_report = create_test_html_report(check_name, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls, False)
with open('report.html', 'w', encoding='utf-8') as f:
f.write(html_report)
url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
logging.info("Search result in url %s", url)
return url
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
temp_path = os.getenv("TEMP_PATH", os.path.abspath("."))
@ -252,7 +96,6 @@ if __name__ == "__main__":
reports_path = os.getenv("REPORTS_PATH", "./reports")
check_name = sys.argv[1]
build_number = int(sys.argv[2])
if not os.path.exists(temp_path):
os.makedirs(temp_path)
@ -264,21 +107,8 @@ if __name__ == "__main__":
gh = Github(get_best_robot_token())
images_path = os.path.join(temp_path, 'changed_images.json')
images_with_version = get_images_with_versions(images_path)
for image, version in images_with_version.items():
docker_image = image + ':' + version
for i in range(10):
try:
logging.info("Pulling image %s", docker_image)
subprocess.check_output(f"docker pull {docker_image}", stderr=subprocess.STDOUT, shell=True)
break
except Exception as ex:
time.sleep(i * 3)
logging.info("Got execption pulling docker %s", ex)
else:
raise Exception(f"Cannot pull dockerhub for image docker pull {docker_image}")
images = get_images_with_versions(temp_path, IMAGES)
images_with_versions = {i.name: i.version for i in images}
result_path = os.path.join(temp_path, "output_dir")
if not os.path.exists(result_path):
os.makedirs(result_path)
@ -287,23 +117,17 @@ if __name__ == "__main__":
if not os.path.exists(work_path):
os.makedirs(work_path)
build_config = get_build_config(build_number, repo_path)
build_config_str = build_config_to_string(build_config)
urls = get_build_urls(build_config_str, reports_path)
if not urls:
raise Exception("No build URLs found")
build_path = os.path.join(temp_path, "build")
if not os.path.exists(build_path):
os.makedirs(build_path)
download_builds(build_path, urls)
download_all_deb_packages(check_name, reports_path, build_path)
my_env = get_env_for_runner(build_path, repo_path, result_path, work_path)
json_path = os.path.join(work_path, 'params.json')
with open(json_path, 'w', encoding='utf-8') as json_params:
json_params.write(json.dumps(get_json_params_dict(check_name, pr_info.sha, pr_info.number, images_with_version)))
json_params.write(json.dumps(get_json_params_dict(check_name, pr_info.sha, pr_info.number, images_with_versions)))
output_path_log = os.path.join(result_path, "main_script_log.txt")
@ -324,7 +148,6 @@ if __name__ == "__main__":
state, description, test_results, additional_logs = process_results(result_path)
s3_helper = S3Helper('https://s3.amazonaws.com')
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, output_path_log, additional_logs, check_name)
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [output_path_log] + additional_logs, check_name, False)
print(f"::notice ::Report url: {report_url}")
commit = get_commit(gh, pr_info.sha)
commit.create_status(context=check_name, description=description, state=state, target_url=report_url)
post_commit_status(gh, pr_info.sha, check_name, description, state, report_url)

46
tests/ci/pvs_check.py
View File

@ -8,27 +8,17 @@ import json
import logging
import sys
from github import Github
from report import create_test_html_report
from s3_helper import S3Helper
from pr_info import PRInfo
from get_robot_token import get_best_robot_token, get_parameter_from_ssm
from upload_result_helper import upload_results
from commit_status_helper import get_commit
NAME = 'PVS Studio (actions)'
LICENCE_NAME = 'Free license: ClickHouse, Yandex'
HTML_REPORT_FOLDER = 'pvs-studio-html-report'
TXT_REPORT_NAME = 'pvs-studio-task-report.txt'
def process_logs(s3_client, additional_logs, s3_path_prefix):
additional_urls = []
for log_path in additional_logs:
if log_path:
additional_urls.append(
s3_client.upload_test_report_to_s3(
log_path,
s3_path_prefix + "/" + os.path.basename(log_path)))
return additional_urls
def _process_txt_report(path):
warnings = []
errors = []
@ -44,36 +34,6 @@ def _process_txt_report(path):
return warnings, errors
def get_commit(gh, commit_sha):
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
commit = repo.get_commit(commit_sha)
return commit
def upload_results(s3_client, pr_number, commit_sha, test_results, additional_files):
s3_path_prefix = str(pr_number) + "/" + commit_sha + "/" + NAME.lower().replace(' ', '_').replace('(', '_').replace(')', '_')
additional_urls = process_logs(s3_client, additional_files, s3_path_prefix)
branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
branch_name = "master"
if pr_number != 0:
branch_name = "PR #{}".format(pr_number)
branch_url = "https://github.com/ClickHouse/ClickHouse/pull/" + str(pr_number)
commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
raw_log_url = additional_urls[0]
additional_urls.pop(0)
html_report = create_test_html_report(NAME, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls, False)
with open('report.html', 'w') as f:
f.write(html_report)
url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
logging.info("Search result in url %s", url)
return url
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
repo_path = os.path.join(os.getenv("REPO_COPY", os.path.abspath("../../")))
@ -133,7 +93,7 @@ if __name__ == "__main__":
test_results = [(index_html, "Look at the report"), ("Errors count not checked", "OK")]
description = "Total errors {}".format(len(errors))
additional_logs = [txt_report, os.path.join(temp_path, 'pvs-studio.log')]
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, additional_logs)
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, additional_logs, NAME)
print("::notice ::Report url: {}".format(report_url))
commit = get_commit(gh, pr_info.sha)

6
tests/ci/run_check.py
View File

@ -6,6 +6,7 @@ import logging
from github import Github
from pr_info import PRInfo
from get_robot_token import get_best_robot_token
from commit_status_helper import get_commit
NAME = 'Run Check (actions)'
@ -101,11 +102,6 @@ def should_run_checks_for_pr(pr_info):
return True, "No special conditions apply"
def get_commit(gh, commit_sha):
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
commit = repo.get_commit(commit_sha)
return commit
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
with open(os.getenv('GITHUB_EVENT_PATH'), 'r') as event_file:

175
tests/ci/split_build_smoke_check.py
View File

@ -3,25 +3,23 @@
import os
import logging
import json
import sys
import subprocess
import time
from github import Github
import requests
from report import create_test_html_report
from s3_helper import S3Helper
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from build_download_helper import download_shared_build
from upload_result_helper import upload_results
from docker_pull_helper import get_image_with_version
from commit_status_helper import post_commit_status
DOCKER_IMAGE = "clickhouse/split-build-smoke-test"
DOWNLOAD_RETRIES_COUNT = 5
RESULT_LOG_NAME = "run.log"
CHECK_NAME = 'Split build smoke test (actions)'
def process_result(result_folder, server_log_folder):
status = "success"
description = 'Server started and responded'
@ -46,142 +44,12 @@ def process_result(result_folder, server_log_folder):
return status, description, summary, result_logs
def process_(log_path):
name = os.path.basename(log_path)
with open(log_path, 'r') as log:
line = log.read().split('\n')[0].strip()
if line != 'OK':
return (name, "FAIL")
else:
return (name, "OK")
def dowload_build_with_progress(url, path):
logging.info("Downloading from %s to temp path %s", url, path)
for i in range(DOWNLOAD_RETRIES_COUNT):
try:
with open(path, 'wb') as f:
response = requests.get(url, stream=True)
response.raise_for_status()
total_length = response.headers.get('content-length')
if total_length is None or int(total_length) == 0:
logging.info("No content-length, will download file without progress")
f.write(response.content)
else:
dl = 0
total_length = int(total_length)
logging.info("Content length is %ld bytes", total_length)
for data in response.iter_content(chunk_size=4096):
dl += len(data)
f.write(data)
if sys.stdout.isatty():
done = int(50 * dl / total_length)
percent = int(100 * float(dl) / total_length)
eq_str = '=' * done
space_str = ' ' * (50 - done)
sys.stdout.write(f"\r[{eq_str}{space_str}] {percent}%")
sys.stdout.flush()
break
except Exception as ex:
sys.stdout.write("\n")
time.sleep(3)
logging.info("Exception while downloading %s, retry %s", ex, i + 1)
if os.path.exists(path):
os.remove(path)
else:
raise Exception(f"Cannot download dataset from {url}, all retries exceeded")
sys.stdout.write("\n")
logging.info("Downloading finished")
def download_builds(result_path, build_urls):
for url in build_urls:
if url.endswith('.tgz'):
fname = os.path.basename(url)
logging.info("Will download %s to %s", fname, result_path)
dowload_build_with_progress(url, os.path.join(result_path, fname))
def get_build_config(build_number, repo_path):
ci_config_path = os.path.join(repo_path, "tests/ci/ci_config.json")
with open(ci_config_path, 'r', encoding='utf-8') as ci_config:
config_dict = json.load(ci_config)
return config_dict['special_build_config'][build_number]
def get_build_urls(build_config_str, reports_path):
for root, _, files in os.walk(reports_path):
for f in files:
if build_config_str in f :
logging.info("Found build report json %s", f)
with open(os.path.join(root, f), 'r', encoding='utf-8') as file_handler:
build_report = json.load(file_handler)
return build_report['build_urls']
return []
def build_config_to_string(build_config):
if build_config["package-type"] == "performance":
return "performance"
return "_".join([
build_config['compiler'],
build_config['build-type'] if build_config['build-type'] else "relwithdebuginfo",
build_config['sanitizer'] if build_config['sanitizer'] else "none",
build_config['bundled'],
build_config['splitted'],
"tidy" if build_config['tidy'] == "enable" else "notidy",
"with_coverage" if build_config['with_coverage'] else "without_coverage",
build_config['package-type'],
])
def get_run_command(build_path, result_folder, server_log_folder, docker_image):
return f"docker run --network=host --volume={build_path}:/package_folder" \
f" --volume={server_log_folder}:/var/log/clickhouse-server" \
f" --volume={result_folder}:/test_output" \
f" {docker_image} >{result_folder}/{RESULT_LOG_NAME}"
def process_logs(s3_client, additional_logs, s3_path_prefix):
additional_urls = []
for log_path in additional_logs:
if log_path:
additional_urls.append(
s3_client.upload_test_report_to_s3(
log_path,
s3_path_prefix + "/" + os.path.basename(log_path)))
return additional_urls
def upload_results(s3_client, pr_number, commit_sha, test_results, additional_files, check_name):
s3_path_prefix = f"{pr_number}/{commit_sha}/" + check_name.lower().replace(' ', '_').replace('(', '_').replace(')', '_').replace(',', '_')
additional_urls = process_logs(s3_client, additional_files, s3_path_prefix)
branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
branch_name = "master"
if pr_number != 0:
branch_name = f"PR #{pr_number}"
branch_url = f"https://github.com/ClickHouse/ClickHouse/pull/{pr_number}"
commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
if additional_urls:
raw_log_url = additional_urls[0]
additional_urls.pop(0)
else:
raw_log_url = task_url
html_report = create_test_html_report(check_name, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls, True)
with open('report.html', 'w', encoding='utf-8') as f:
f.write(html_report)
url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
logging.info("Search result in url %s", url)
return url
def get_commit(gh, commit_sha):
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
commit = repo.get_commit(commit_sha)
return commit
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
@ -189,8 +57,6 @@ if __name__ == "__main__":
repo_path = os.getenv("REPO_COPY", os.path.abspath("../../"))
reports_path = os.getenv("REPORTS_PATH", "./reports")
build_number = int(sys.argv[1])
with open(os.getenv('GITHUB_EVENT_PATH'), 'r', encoding='utf-8') as event_file:
event = json.load(event_file)
@ -204,39 +70,13 @@ if __name__ == "__main__":
images_path = os.path.join(root, 'changed_images.json')
break
docker_image = DOCKER_IMAGE
if images_path and os.path.exists(images_path):
logging.info("Images file exists")
with open(images_path, 'r', encoding='utf-8') as images_fd:
images = json.load(images_fd)
logging.info("Got images %s", images)
if docker_image in images:
docker_image += ':' + images[docker_image]
else:
logging.info("Images file not found")
for i in range(10):
try:
logging.info("Pulling image %s", docker_image)
subprocess.check_output(f"docker pull {docker_image}", stderr=subprocess.STDOUT, shell=True)
break
except Exception as ex:
time.sleep(i * 3)
logging.info("Got execption pulling docker %s", ex)
else:
raise Exception(f"Cannot pull dockerhub for image docker pull {docker_image}")
build_config = get_build_config(build_number, repo_path)
build_config_str = build_config_to_string(build_config)
urls = get_build_urls(build_config_str, reports_path)
if not urls:
raise Exception("No build URLs found")
docker_image = get_image_with_version(reports_path, DOCKER_IMAGE)
packages_path = os.path.join(temp_path, "packages")
if not os.path.exists(packages_path):
os.makedirs(packages_path)
download_builds(packages_path, urls)
download_shared_build(CHECK_NAME, reports_path, packages_path)
server_log_path = os.path.join(temp_path, "server_log")
if not os.path.exists(server_log_path):
@ -265,5 +105,4 @@ if __name__ == "__main__":
s3_helper = S3Helper('https://s3.amazonaws.com')
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, additional_logs, CHECK_NAME)
print(f"::notice ::Report url: {report_url}")
commit = get_commit(gh, pr_info.sha)
commit.create_status(context=CHECK_NAME, description=description, state=state, target_url=report_url)
post_commit_status(gh, pr_info.sha, CHECK_NAME, description, state, report_url)

177
tests/ci/stress_check.py
View File

@ -1,103 +1,21 @@
#!/usr/bin/env python3
#!/usr/bin/env python3
import csv
import logging
import subprocess
import os
import json
import time
import sys
from github import Github
import requests
from report import create_test_html_report
from s3_helper import S3Helper
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
DOWNLOAD_RETRIES_COUNT = 5
def dowload_build_with_progress(url, path):
logging.info("Downloading from %s to temp path %s", url, path)
for i in range(DOWNLOAD_RETRIES_COUNT):
try:
with open(path, 'wb') as f:
response = requests.get(url, stream=True)
response.raise_for_status()
total_length = response.headers.get('content-length')
if total_length is None or int(total_length) == 0:
logging.info("No content-length, will download file without progress")
f.write(response.content)
else:
dl = 0
total_length = int(total_length)
logging.info("Content length is %ld bytes", total_length)
for data in response.iter_content(chunk_size=4096):
dl += len(data)
f.write(data)
if sys.stdout.isatty():
done = int(50 * dl / total_length)
percent = int(100 * float(dl) / total_length)
eq_str = '=' * done
space_str = ' ' * (50 - done)
sys.stdout.write(f"\r[{eq_str}{space_str}] {percent}%")
sys.stdout.flush()
break
except Exception as ex:
sys.stdout.write("\n")
time.sleep(3)
logging.info("Exception while downloading %s, retry %s", ex, i + 1)
if os.path.exists(path):
os.remove(path)
else:
raise Exception(f"Cannot download dataset from {url}, all retries exceeded")
sys.stdout.write("\n")
logging.info("Downloading finished")
def download_builds(result_path, build_urls):
for url in build_urls:
if url.endswith('.deb'):
fname = os.path.basename(url)
logging.info("Will download %s to %s", fname, result_path)
dowload_build_with_progress(url, os.path.join(result_path, fname))
def get_build_config(build_number, repo_path):
ci_config_path = os.path.join(repo_path, "tests/ci/ci_config.json")
with open(ci_config_path, 'r', encoding='utf-8') as ci_config:
config_dict = json.load(ci_config)
return config_dict['build_config'][build_number]
def get_build_urls(build_config_str, reports_path):
for root, _, files in os.walk(reports_path):
for f in files:
if build_config_str in f :
logging.info("Found build report json %s", f)
with open(os.path.join(root, f), 'r', encoding='utf-8') as file_handler:
build_report = json.load(file_handler)
return build_report['build_urls']
return []
def build_config_to_string(build_config):
if build_config["package-type"] == "performance":
return "performance"
return "_".join([
build_config['compiler'],
build_config['build-type'] if build_config['build-type'] else "relwithdebuginfo",
build_config['sanitizer'] if build_config['sanitizer'] else "none",
build_config['bundled'],
build_config['splitted'],
"tidy" if build_config['tidy'] == "enable" else "notidy",
"with_coverage" if build_config['with_coverage'] else "without_coverage",
build_config['package-type'],
])
from build_download_helper import download_all_deb_packages
from upload_result_helper import upload_results
from docker_pull_helper import get_image_with_version
from commit_status_helper import post_commit_status
def get_run_command(build_path, result_folder, server_log_folder, image):
cmd = "docker run -e S3_URL='https://clickhouse-datasets.s3.amazonaws.com' " + \
@ -107,47 +25,6 @@ def get_run_command(build_path, result_folder, server_log_folder, image):
return cmd
def process_logs(s3_client, additional_logs, s3_path_prefix):
additional_urls = []
for log_path in additional_logs:
if log_path:
additional_urls.append(
s3_client.upload_test_report_to_s3(
log_path,
s3_path_prefix + "/" + os.path.basename(log_path)))
return additional_urls
def upload_results(s3_client, pr_number, commit_sha, test_results, raw_log, additional_files, check_name):
additional_files = [raw_log] + additional_files
s3_path_prefix = f"{pr_number}/{commit_sha}/" + check_name.lower().replace(' ', '_').replace('(', '_').replace(')', '_').replace(',', '_')
additional_urls = process_logs(s3_client, additional_files, s3_path_prefix)
branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
branch_name = "master"
if pr_number != 0:
branch_name = f"PR #{pr_number}"
branch_url = f"https://github.com/ClickHouse/ClickHouse/pull/{pr_number}"
commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
raw_log_url = additional_urls[0]
additional_urls.pop(0)
html_report = create_test_html_report(check_name, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls, True)
with open('report.html', 'w', encoding='utf-8') as f:
f.write(html_report)
url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
logging.info("Search result in url %s", url)
return url
def get_commit(gh, commit_sha):
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
commit = repo.get_commit(commit_sha)
return commit
def process_results(result_folder, server_log_path, run_log_path):
test_results = []
additional_files = []
@ -191,7 +68,6 @@ if __name__ == "__main__":
reports_path = os.getenv("REPORTS_PATH", "./reports")
check_name = sys.argv[1]
build_number = int(sys.argv[2])
if not os.path.exists(temp_path):
os.makedirs(temp_path)
@ -203,46 +79,14 @@ if __name__ == "__main__":
gh = Github(get_best_robot_token())
for root, _, files in os.walk(reports_path):
for f in files:
if f == 'changed_images.json':
images_path = os.path.join(root, 'changed_images.json')
break
image_name = "clickhouse/stress-test"
docker_image = image_name
if images_path and os.path.exists(images_path):
logging.info("Images file exists")
with open(images_path, 'r', encoding='utf-8') as images_fd:
images = json.load(images_fd)
logging.info("Got images %s", images)
if image_name in images:
docker_image += ':' + images[image_name]
else:
logging.info("Images file not found")
for i in range(10):
try:
logging.info("Pulling image %s", docker_image)
subprocess.check_output(f"docker pull {docker_image}", stderr=subprocess.STDOUT, shell=True)
break
except Exception as ex:
time.sleep(i * 3)
logging.info("Got execption pulling docker %s", ex)
else:
raise Exception(f"Cannot pull dockerhub for image docker pull {docker_image}")
build_config = get_build_config(build_number, repo_path)
build_config_str = build_config_to_string(build_config)
urls = get_build_urls(build_config_str, reports_path)
if not urls:
raise Exception("No build URLs found")
docker_image = get_image_with_version(reports_path, 'clickhouse/stress-test')
packages_path = os.path.join(temp_path, "packages")
if not os.path.exists(packages_path):
os.makedirs(packages_path)
download_all_deb_packages(check_name, reports_path, packages_path)
server_log_path = os.path.join(temp_path, "server_log")
if not os.path.exists(server_log_path):
os.makedirs(server_log_path)
@ -253,7 +97,6 @@ if __name__ == "__main__":
run_log_path = os.path.join(temp_path, "runlog.log")
download_builds(packages_path, urls)
run_command = get_run_command(packages_path, result_path, server_log_path, docker_image)
logging.info("Going to run func tests: %s", run_command)
@ -269,7 +112,7 @@ if __name__ == "__main__":
s3_helper = S3Helper('https://s3.amazonaws.com')
state, description, test_results, additional_logs = process_results(result_path, server_log_path, run_log_path)
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, run_log_path, additional_logs, check_name)
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [run_log_path] + additional_logs, check_name)
print(f"::notice ::Report url: {report_url}")
commit = get_commit(gh, pr_info.sha)
commit.create_status(context=check_name, description=description, state=state, target_url=report_url)
post_commit_status(gh, pr_info.sha, check_name, description, state, report_url)

74
tests/ci/style_check.py
View File

@ -3,29 +3,18 @@ import logging
import subprocess
import os
import csv
import time
import json
from github import Github
from report import create_test_html_report
from s3_helper import S3Helper
from pr_info import PRInfo
from get_robot_token import get_best_robot_token
from upload_result_helper import upload_results
from docker_pull_helper import get_image_with_version
from commit_status_helper import post_commit_status
NAME = "Style Check (actions)"
def process_logs(s3_client, additional_logs, s3_path_prefix):
additional_urls = []
for log_path in additional_logs:
if log_path:
additional_urls.append(
s3_client.upload_test_report_to_s3(
log_path,
s3_path_prefix + "/" + os.path.basename(log_path)))
return additional_urls
def process_result(result_folder):
test_results = []
additional_files = []
@ -54,36 +43,6 @@ def process_result(result_folder):
state, description = "error", "Failed to read test_results.tsv"
return state, description, test_results, additional_files
def upload_results(s3_client, pr_number, commit_sha, test_results, additional_files):
s3_path_prefix = f"{pr_number}/{commit_sha}/style_check"
additional_urls = process_logs(s3_client, additional_files, s3_path_prefix)
branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
branch_name = "master"
if pr_number != 0:
branch_name = "PR #{}".format(pr_number)
branch_url = "https://github.com/ClickHouse/ClickHouse/pull/" + str(pr_number)
commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
raw_log_url = additional_urls[0]
additional_urls.pop(0)
html_report = create_test_html_report(NAME, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls)
with open('report.html', 'w') as f:
f.write(html_report)
url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
logging.info("Search result in url %s", url)
return url
def get_commit(gh, commit_sha):
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
commit = repo.get_commit(commit_sha)
return commit
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
repo_path = os.path.join(os.getenv("GITHUB_WORKSPACE", os.path.abspath("../../")))
@ -98,32 +57,11 @@ if __name__ == "__main__":
gh = Github(get_best_robot_token())
images_path = os.path.join(temp_path, 'changed_images.json')
docker_image = 'clickhouse/style-test'
if os.path.exists(images_path):
logging.info("Images file exists")
with open(images_path, 'r') as images_fd:
images = json.load(images_fd)
logging.info("Got images %s", images)
if 'clickhouse/style-test' in images:
docker_image += ':' + images['clickhouse/style-test']
logging.info("Got docker image %s", docker_image)
for i in range(10):
try:
subprocess.check_output(f"docker pull {docker_image}", shell=True)
break
except Exception as ex:
time.sleep(i * 3)
logging.info("Got execption pulling docker %s", ex)
else:
raise Exception(f"Cannot pull dockerhub for image {docker_image}")
docker_image = get_image_with_version(temp_path, 'clickhouse/style-test')
s3_helper = S3Helper('https://s3.amazonaws.com')
subprocess.check_output(f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --cap-add=SYS_PTRACE --volume={repo_path}:/ClickHouse --volume={temp_path}:/test_output {docker_image}", shell=True)
state, description, test_results, additional_files = process_result(temp_path)
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, additional_files)
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, additional_files, NAME)
print("::notice ::Report url: {}".format(report_url))
commit = get_commit(gh, pr_info.sha)
commit.create_status(context=NAME, description=description, state=state, target_url=report_url)
post_commit_status(gh, pr_info.sha, NAME, description, state, report_url)

166
tests/ci/unit_tests_check.py
View File

@ -3,133 +3,21 @@
import logging
import os
import sys
import time
import subprocess
import json
from github import Github
import requests
from report import create_test_html_report
from s3_helper import S3Helper
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from build_download_helper import download_unit_tests
from upload_result_helper import upload_results
from docker_pull_helper import get_image_with_version
from commit_status_helper import post_commit_status
IMAGE_NAME = 'clickhouse/unit-test'
DOWNLOAD_RETRIES_COUNT = 5
def process_logs(s3_client, additional_logs, s3_path_prefix):
additional_urls = []
for log_path in additional_logs:
if log_path:
additional_urls.append(
s3_client.upload_test_report_to_s3(
log_path,
s3_path_prefix + "/" + os.path.basename(log_path)))
return additional_urls
def dowload_build_with_progress(url, path):
logging.info("Downloading from %s to temp path %s", url, path)
for i in range(DOWNLOAD_RETRIES_COUNT):
try:
with open(path, 'wb') as f:
response = requests.get(url, stream=True)
response.raise_for_status()
total_length = response.headers.get('content-length')
if total_length is None or int(total_length) == 0:
logging.info("No content-length, will download file without progress")
f.write(response.content)
else:
dl = 0
total_length = int(total_length)
logging.info("Content length is %ld bytes", total_length)
for data in response.iter_content(chunk_size=4096):
dl += len(data)
f.write(data)
if sys.stdout.isatty():
done = int(50 * dl / total_length)
percent = int(100 * float(dl) / total_length)
eq_str = '=' * done
space_str = ' ' * (50 - done)
sys.stdout.write(f"\r[{eq_str}{space_str}] {percent}%")
sys.stdout.flush()
break
except Exception as ex:
sys.stdout.write("\n")
time.sleep(3)
logging.info("Exception while downloading %s, retry %s", ex, i + 1)
if os.path.exists(path):
os.remove(path)
else:
raise Exception(f"Cannot download dataset from {url}, all retries exceeded")
sys.stdout.write("\n")
logging.info("Downloading finished")
def upload_results(s3_client, pr_number, commit_sha, test_results, raw_log, additional_files, check_name):
additional_files = [raw_log] + additional_files
s3_path_prefix = f"{pr_number}/{commit_sha}/" + check_name.lower().replace(' ', '_').replace('(', '_').replace(')', '_').replace(',', '_')
additional_urls = process_logs(s3_client, additional_files, s3_path_prefix)
branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
branch_name = "master"
if pr_number != 0:
branch_name = f"PR #{pr_number}"
branch_url = f"https://github.com/ClickHouse/ClickHouse/pull/{pr_number}"
commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
raw_log_url = additional_urls[0]
additional_urls.pop(0)
html_report = create_test_html_report(check_name, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls, True)
with open('report.html', 'w', encoding='utf-8') as f:
f.write(html_report)
url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
logging.info("Search result in url %s", url)
return url
def get_commit(gh, commit_sha):
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
commit = repo.get_commit(commit_sha)
return commit
def get_build_config(build_number, repo_path):
ci_config_path = os.path.join(repo_path, "tests/ci/ci_config.json")
with open(ci_config_path, 'r', encoding='utf-8') as ci_config:
config_dict = json.load(ci_config)
return config_dict['build_config'][build_number]
def get_build_urls(build_config_str, reports_path):
for root, _, files in os.walk(reports_path):
for f in files:
if build_config_str in f :
logging.info("Found build report json %s", f)
with open(os.path.join(root, f), 'r', encoding='utf-8') as file_handler:
build_report = json.load(file_handler)
return build_report['build_urls']
return []
def build_config_to_string(build_config):
if build_config["package-type"] == "performance":
return "performance"
return "_".join([
build_config['compiler'],
build_config['build-type'] if build_config['build-type'] else "relwithdebuginfo",
build_config['sanitizer'] if build_config['sanitizer'] else "none",
build_config['bundled'],
build_config['splitted'],
"tidy" if build_config['tidy'] == "enable" else "notidy",
"with_coverage" if build_config['with_coverage'] else "without_coverage",
build_config['package-type'],
])
def get_test_name(line):
elements = reversed(line.split(' '))
for element in elements:
@ -204,7 +92,6 @@ if __name__ == "__main__":
reports_path = os.getenv("REPORTS_PATH", "./reports")
check_name = sys.argv[1]
build_number = int(sys.argv[2])
if not os.path.exists(temp_path):
os.makedirs(temp_path)
@ -216,47 +103,11 @@ if __name__ == "__main__":
gh = Github(get_best_robot_token())
for root, _, files in os.walk(reports_path):
for f in files:
if f == 'changed_images.json':
images_path = os.path.join(root, 'changed_images.json')
break
docker_image = get_image_with_version(reports_path, IMAGE_NAME)
docker_image = IMAGE_NAME
if images_path and os.path.exists(images_path):
logging.info("Images file exists")
with open(images_path, 'r', encoding='utf-8') as images_fd:
images = json.load(images_fd)
logging.info("Got images %s", images)
if IMAGE_NAME in images:
docker_image += ':' + images[IMAGE_NAME]
else:
logging.info("Images file not found")
for i in range(10):
try:
logging.info("Pulling image %s", docker_image)
subprocess.check_output(f"docker pull {docker_image}", stderr=subprocess.STDOUT, shell=True)
break
except Exception as ex:
time.sleep(i * 3)
logging.info("Got execption pulling docker %s", ex)
else:
raise Exception(f"Cannot pull dockerhub for image docker pull {docker_image}")
build_config = get_build_config(build_number, repo_path)
build_config_str = build_config_to_string(build_config)
urls = get_build_urls(build_config_str, reports_path)
if not urls:
raise Exception("No build URLs found")
download_unit_tests(check_name, reports_path, temp_path)
tests_binary_path = os.path.join(temp_path, "unit_tests_dbms")
for url in urls:
if url.endswith('unit_tests_dbms'):
dowload_build_with_progress(url, tests_binary_path)
break
os.chmod(tests_binary_path, 0o777)
test_output = os.path.join(temp_path, "test_output")
@ -281,7 +132,6 @@ if __name__ == "__main__":
s3_helper = S3Helper('https://s3.amazonaws.com')
state, description, test_results, additional_logs = process_result(test_output)
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, run_log_path, additional_logs, check_name)
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [run_log_path] + additional_logs, check_name)
print(f"::notice ::Report url: {report_url}")
commit = get_commit(gh, pr_info.sha)
commit.create_status(context=check_name, description=description, state=state, target_url=report_url)
post_commit_status(gh, pr_info.sha, check_name, description, state, report_url)

64
tests/ci/upload_result_helper.py Normal file
View File

@ -0,0 +1,64 @@
import os
import logging
import ast
from report import create_test_html_report
def process_logs(s3_client, additional_logs, s3_path_prefix, test_results, with_raw_logs):
proccessed_logs = {}
# Firstly convert paths of logs from test_results to urls to s3.
for test_result in test_results:
if len(test_result) <= 3 or with_raw_logs:
continue
# Convert from string repr of list to list.
test_log_paths = ast.literal_eval(test_result[3])
test_log_urls = []
for log_path in test_log_paths:
if log_path in proccessed_logs:
test_log_urls.append(proccessed_logs[log_path])
elif log_path:
url = s3_client.upload_test_report_to_s3(
log_path,
s3_path_prefix + "/" + os.path.basename(log_path))
test_log_urls.append(url)
proccessed_logs[log_path] = url
test_result[3] = test_log_urls
additional_urls = []
for log_path in additional_logs:
if log_path:
additional_urls.append(
s3_client.upload_test_report_to_s3(
log_path,
s3_path_prefix + "/" + os.path.basename(log_path)))
return additional_urls
def upload_results(s3_client, pr_number, commit_sha, test_results, additional_files, check_name, with_raw_logs=True):
s3_path_prefix = f"{pr_number}/{commit_sha}/" + check_name.lower().replace(' ', '_').replace('(', '_').replace(')', '_').replace(',', '_')
additional_urls = process_logs(s3_client, additional_files, s3_path_prefix, test_results, with_raw_logs)
branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
branch_name = "master"
if pr_number != 0:
branch_name = f"PR #{pr_number}"
branch_url = f"https://github.com/ClickHouse/ClickHouse/pull/{pr_number}"
commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
if additional_urls:
raw_log_url = additional_urls[0]
additional_urls.pop(0)
else:
raw_log_url = task_url
html_report = create_test_html_report(check_name, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls, with_raw_logs)
with open('report.html', 'w', encoding='utf-8') as f:
f.write(html_report)
url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
logging.info("Search result in url %s", url)
return url