thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-28 18:42:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #58516 from ClickHouse/move_out_ci_specifics_to_ci_py

Browse Source 
CI: move ci-specifics from job scripts to ci.py
This commit is contained in:
Max K 2024-01-21 18:24:42 +01:00 committed by GitHub
commit 070a55e194
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
37 changed files with 1162 additions and 1494 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/jepsen.yml vendored
View File

@ -14,6 +14,7 @@ jobs:
with:
test_name: Jepsen keeper check
runner_type: style-checker
report_required: true
run_command: |
python3 jepsen_check.py keeper
# ServerJepsenRelease:

10
.github/workflows/master.yml vendored
View File

@ -15,6 +15,8 @@ jobs:
outputs:
data: ${{ steps.runconfig.outputs.CI_DATA }}
steps:
- name: DebugInfo
uses: hmarr/debug-action@a701ed95a46e6f2fb0df25e1a558c16356fae35a
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
@ -33,11 +35,9 @@ jobs:
- name: PrepareRunConfig
id: runconfig
run: |
echo "::group::configure CI run"
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --rebuild-all-binaries --outfile ${{ runner.temp }}/ci_run_data.json
echo "::endgroup::"
echo "::group::CI run configure results"
echo "::group::CI configuration"
python3 -m json.tool ${{ runner.temp }}/ci_run_data.json
echo "::endgroup::"
@ -255,9 +255,9 @@ jobs:
run_command: |
cd "$GITHUB_WORKSPACE/tests/ci"
python3 docker_server.py --release-type head \
--image-repo clickhouse/clickhouse-server --image-path docker/server
--image-repo clickhouse/clickhouse-server --image-path docker/server --allow-build-reuse
python3 docker_server.py --release-type head \
--image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
--image-repo clickhouse/clickhouse-keeper --image-path docker/keeper --allow-build-reuse
############################################################################################
##################################### BUILD REPORTER #######################################
############################################################################################

11
.github/workflows/pull_request.yml vendored
View File

@ -22,6 +22,8 @@ jobs:
outputs:
data: ${{ steps.runconfig.outputs.CI_DATA }}
steps:
- name: DebugInfo
uses: hmarr/debug-action@a701ed95a46e6f2fb0df25e1a558c16356fae35a
- name: Check out repository code
uses: ClickHouse/checkout@v1
with:
@ -44,11 +46,9 @@ jobs:
- name: PrepareRunConfig
id: runconfig
run: |
echo "::group::configure CI run"
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --outfile ${{ runner.temp }}/ci_run_data.json
echo "::endgroup::"
echo "::group::CI run configure results"
echo "::group::CI configuration"
python3 -m json.tool ${{ runner.temp }}/ci_run_data.json
echo "::endgroup::"
@ -67,6 +67,7 @@ jobs:
DOCKER_TAG=$(echo '${{ toJson(fromJson(steps.runconfig.outputs.CI_DATA).docker_data.images) }}' | tr -d '\n')
export DOCKER_TAG=$DOCKER_TAG
python3 ./tests/ci/style_check.py --no-push
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ runner.temp }}/ci_run_data.json --post --job-name 'Style check'
BuildDockers:
needs: [RunConfig]
if: ${{ !failure() && !cancelled() }}
@ -796,7 +797,7 @@ jobs:
test_name: Unit tests (asan)
runner_type: fuzzer-unit-tester
data: ${{ needs.RunConfig.outputs.data }}
UnitTestsReleaseClang:
UnitTestsRelease:
needs: [RunConfig, BuilderBinRelease]
if: ${{ !failure() && !cancelled() }}
uses: ./.github/workflows/reusable_test.yml
@ -923,7 +924,7 @@ jobs:
- UnitTestsTsan
- UnitTestsMsan
- UnitTestsUBsan
- UnitTestsReleaseClang
- UnitTestsRelease
- CompatibilityCheckX86
- CompatibilityCheckAarch64
- SQLancerTestRelease

9
.github/workflows/reusable_build.yml vendored
View File

@ -73,12 +73,15 @@ jobs:
- name: Pre
run: |
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --pre --job-name '${{inputs.build_name}}'
- name: Build
- name: Run
run: |
python3 "$GITHUB_WORKSPACE/tests/ci/build_check.py" "$BUILD_NAME"
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" \
--infile ${{ toJson(inputs.data) }} \
--job-name "$BUILD_NAME" \
--run
- name: Post
# it still be build report to upload for failed build job
if: always()
if: ${{ !cancelled() }}
run: |
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --post --job-name '${{inputs.build_name}}'
- name: Mark as done

14
.github/workflows/reusable_simple_job.yml vendored
View File

@ -34,12 +34,16 @@ name: Simple job
working-directory:
description: sets custom working directory
type: string
default: ""
default: "$GITHUB_WORKSPACE/tests/ci"
git_ref:
description: commit to use, merge commit for pr or head
required: false
type: string
default: ${{ github.event.after }} # no merge commit
report_required:
description: set to true if job report with the commit status required
type: boolean
default: false
secrets:
secret_envs:
description: if given, it's passed to the environments
@ -81,12 +85,12 @@ jobs:
job_type: test
- name: Run
run: |
if [ -n '${{ inputs.working-directory }}' ]; then
cd "${{ inputs.working-directory }}"
else
cd "$GITHUB_WORKSPACE/tests/ci"
fi
${{ inputs.run_command }}
- name: Post
if: ${{ inputs.report_required }}
run: |
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --post --job-name '${{inputs.test_name}}'
- name: Clean
if: always()
uses: ./.github/actions/clean

19
.github/workflows/reusable_test.yml vendored
View File

@ -38,7 +38,7 @@ name: Testing workflow
working-directory:
description: sets custom working directory
type: string
default: ""
default: "$GITHUB_WORKSPACE/tests/ci"
secrets:
secret_envs:
description: if given, it's passed to the environments
@ -96,19 +96,14 @@ jobs:
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --pre --job-name '${{inputs.test_name}}'
- name: Run
run: |
if [ -n "${{ inputs.working-directory }}" ]; then
cd "${{ inputs.working-directory }}"
else
cd "$GITHUB_WORKSPACE/tests/ci"
fi
if [ -n "$(echo '${{ inputs.run_command }}' | tr -d '\n')" ]; then
echo "Running command from workflow input"
${{ inputs.run_command }}
else
echo "Running command from job config"
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --run --job-name '${{inputs.test_name}}'
fi
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" \
--infile ${{ toJson(inputs.data) }} \
--job-name '${{inputs.test_name}}' \
--run \
--run-command '''${{inputs.run_command}}'''
- name: Post run
if: ${{ !cancelled() }}
run: |
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ toJson(inputs.data) }} --post --job-name '${{inputs.test_name}}'
- name: Mark as done

15
.gitmessage
View File

@ -1,9 +1,18 @@
## To avoid merge commit in CI run (add a leading space to apply):
#no-merge-commit
### CI modificators (add a leading space to apply):
## Running specified job (add a leading space to apply):
## To avoid a merge commit in CI:
#no_merge_commit
## To discard CI cache:
#no_ci_cache
## To run specified set of tests in CI:
#ci_set_<SET_NAME>
#ci_set_reduced
## To run specified job in CI:
#job_<JOB NAME>
#job_stateless_tests_release
#job_package_debug

85
tests/ci/ast_fuzzer_check.py
View File

@ -6,29 +6,16 @@ import subprocess
import sys
from pathlib import Path
from github import Github
from build_download_helper import get_build_name_for_check, read_build_urls
from clickhouse_helper import (
CiLogsCredentials,
ClickHouseHelper,
prepare_tests_results_for_clickhouse,
)
from commit_status_helper import (
RerunHelper,
format_description,
get_commit,
post_commit_status,
)
from docker_images_helper import DockerImage, get_docker_image, pull_image
from env_helper import REPORT_PATH, TEMP_PATH
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from report import TestResult
from s3_helper import S3Helper
from report import JobReport
from stopwatch import Stopwatch
from tee_popen import TeePopen
from upload_result_helper import upload_results
IMAGE_NAME = "clickhouse/fuzzer"
@ -77,14 +64,6 @@ def main():
pr_info = PRInfo()
gh = Github(get_best_robot_token(), per_page=100)
commit = get_commit(gh, pr_info.sha)
rerun_helper = RerunHelper(commit, check_name)
if rerun_helper.is_already_finished_by_status():
logging.info("Check is already finished according to github status, exiting")
sys.exit(0)
docker_image = pull_image(get_docker_image(IMAGE_NAME))
build_name = get_build_name_for_check(check_name)
@ -131,10 +110,6 @@ def main():
subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True)
ci_logs_credentials.clean_ci_logs_from_credentials(run_log_path)
check_name_lower = (
check_name.lower().replace("(", "").replace(")", "").replace(" ", "")
)
s3_prefix = f"{pr_info.number}/{pr_info.sha}/fuzzer_{check_name_lower}/"
paths = {
"run.log": run_log_path,
"main.log": main_log_path,
@ -154,17 +129,6 @@ def main():
if not_compressed_server_log_path.exists():
paths["server.log"] = not_compressed_server_log_path
s3_helper = S3Helper()
urls = []
report_url = ""
for file, path in paths.items():
try:
url = s3_helper.upload_test_report_to_s3(path, s3_prefix + file)
report_url = url if file == "report.html" else report_url
urls.append(url)
except Exception as ex:
logging.info("Exception uploading file %s text %s", file, ex)
# Try to get status message saved by the fuzzer
try:
with open(workspace_path / "status.txt", "r", encoding="utf-8") as status_f:
@ -176,42 +140,19 @@ def main():
status = "failure"
description = "Task failed: $?=" + str(retcode)
description = format_description(description)
JobReport(
description=description,
test_results=[],
status=status,
start_time=stopwatch.start_time_str,
duration=stopwatch.duration_seconds,
# test generates its own report.html
additional_files=[v for _, v in paths.items()],
).dump()
test_result = TestResult(description, "OK")
if "fail" in status:
test_result.status = "FAIL"
if not report_url:
report_url = upload_results(
s3_helper,
pr_info.number,
pr_info.sha,
[test_result],
[],
check_name,
urls,
)
ch_helper = ClickHouseHelper()
prepared_events = prepare_tests_results_for_clickhouse(
pr_info,
[test_result],
status,
stopwatch.duration_seconds,
stopwatch.start_time_str,
report_url,
check_name,
)
ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
logging.info("Result: '%s', '%s', '%s'", status, description, report_url)
print(f"::notice ::Report url: {report_url}")
post_commit_status(
commit, status, report_url, description, check_name, pr_info, dump_to_file=True
)
logging.info("Result: '%s', '%s'", status, description)
if status == "failure":
sys.exit(1)
if __name__ == "__main__":

289
tests/ci/build_check.py
View File

@ -12,15 +12,13 @@ from ci_config import CI_CONFIG, BuildConfig
from cache_utils import CargoCache
from env_helper import (
GITHUB_JOB_API_URL,
REPO_COPY,
S3_BUILDS_BUCKET,
S3_DOWNLOAD,
TEMP_PATH,
)
from git_helper import Git, git_runner
from git_helper import Git
from pr_info import PRInfo
from report import BuildResult, FAILURE, StatusType, SUCCESS
from report import FAILURE, JobReport, StatusType, SUCCESS
from s3_helper import S3Helper
from tee_popen import TeePopen
import docker_images_helper
@ -29,13 +27,6 @@ from version_helper import (
get_version_from_repo,
update_version_local,
)
from clickhouse_helper import (
ClickHouseHelper,
CiLogsCredentials,
prepare_tests_results_for_clickhouse,
get_instance_type,
get_instance_id,
)
from stopwatch import Stopwatch
IMAGE_NAME = "clickhouse/binary-builder"
@ -122,61 +113,6 @@ def build_clickhouse(
return build_log_path, SUCCESS if success else FAILURE
def check_for_success_run(
s3_helper: S3Helper,
s3_prefix: str,
build_name: str,
version: ClickHouseVersion,
) -> None:
# TODO: Remove after S3 artifacts
logging.info("Checking for artifacts %s in bucket %s", s3_prefix, S3_BUILDS_BUCKET)
try:
# Performance artifacts are now part of regular build, so we're safe
build_results = s3_helper.list_prefix(s3_prefix)
except Exception as ex:
logging.info("Got exception while listing %s: %s\nRerun", s3_prefix, ex)
return
if build_results is None or len(build_results) == 0:
logging.info("Nothing found in %s, rerun", s3_prefix)
return
logging.info("Some build results found:\n%s", build_results)
build_urls = []
log_url = ""
for url in build_results:
url_escaped = url.replace("+", "%2B").replace(" ", "%20")
if BUILD_LOG_NAME in url:
log_url = f"{S3_DOWNLOAD}/{S3_BUILDS_BUCKET}/{url_escaped}"
else:
build_urls.append(f"{S3_DOWNLOAD}/{S3_BUILDS_BUCKET}/{url_escaped}")
if not log_url:
# log is uploaded the last, so if there's no log we need to rerun the build
return
success = len(build_urls) > 0
build_result = BuildResult(
build_name,
log_url,
build_urls,
version.describe,
SUCCESS if success else FAILURE,
0,
GITHUB_JOB_API_URL(),
)
result_json_path = build_result.write_json(Path(TEMP_PATH))
logging.info(
"Build result file %s is written, content:\n %s",
result_json_path,
result_json_path.read_text(encoding="utf-8"),
)
# Fail build job if not successeded
if not success:
sys.exit(1)
else:
sys.exit(0)
def get_release_or_pr(pr_info: PRInfo, version: ClickHouseVersion) -> Tuple[str, str]:
"Return prefixes for S3 artifacts paths"
# FIXME performance
@ -196,34 +132,6 @@ def get_release_or_pr(pr_info: PRInfo, version: ClickHouseVersion) -> Tuple[str,
return pr_number, pr_number
def upload_master_static_binaries(
pr_info: PRInfo,
build_config: BuildConfig,
s3_helper: S3Helper,
build_output_path: Path,
) -> None:
"""Upload binary artifacts to a static S3 links"""
static_binary_name = build_config.static_binary_name
if pr_info.number != 0:
return
elif not static_binary_name:
return
elif pr_info.base_ref != "master":
return
# Full binary with debug info:
s3_path_full = "/".join((pr_info.base_ref, static_binary_name, "clickhouse-full"))
binary_full = build_output_path / "clickhouse"
url_full = s3_helper.upload_build_file_to_s3(binary_full, s3_path_full)
print(f"::notice ::Binary static URL (with debug info): {url_full}")
# Stripped binary without debug info:
s3_path_compact = "/".join((pr_info.base_ref, static_binary_name, "clickhouse"))
binary_compact = build_output_path / "clickhouse-stripped"
url_compact = s3_helper.upload_build_file_to_s3(binary_compact, s3_path_compact)
print(f"::notice ::Binary static URL (compact): {url_compact}")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser("Clickhouse builder script")
parser.add_argument(
@ -254,21 +162,6 @@ def main():
s3_helper = S3Helper()
version = get_version_from_repo(git=Git(True))
release_or_pr, performance_pr = get_release_or_pr(pr_info, version)
s3_path_prefix = "/".join((release_or_pr, pr_info.sha, build_name))
# FIXME performance
s3_performance_path = "/".join(
(performance_pr, pr_info.sha, build_name, "performance.tar.zst")
)
# FIXME: to be removed in favor of "skip by job digest"
# If this is rerun, then we try to find already created artifacts and just
# put them as github actions artifact (result)
# The s3_path_prefix has additional "/" in the end to prevent finding
# e.g. `binary_darwin_aarch64/clickhouse` for `binary_darwin`
check_for_success_run(s3_helper, f"{s3_path_prefix}/", build_name, version)
logging.info("Got version from repo %s", version.string)
official_flag = pr_info.number == 0
@ -331,174 +224,16 @@ def main():
)
sys.exit(1)
# FIXME performance
performance_urls = []
performance_path = build_output_path / "performance.tar.zst"
if performance_path.exists():
performance_urls.append(
s3_helper.upload_build_file_to_s3(performance_path, s3_performance_path)
)
logging.info(
"Uploaded performance.tar.zst to %s, now delete to avoid duplication",
performance_urls[0],
)
performance_path.unlink()
build_urls = (
s3_helper.upload_build_directory_to_s3(
build_output_path,
s3_path_prefix,
keep_dirs_in_s3_path=False,
upload_symlinks=False,
)
+ performance_urls
)
logging.info("Got build URLs %s", build_urls)
print("::notice ::Build URLs: {}".format("\n".join(build_urls)))
if log_path.exists():
log_url = s3_helper.upload_build_file_to_s3(
log_path, s3_path_prefix + "/" + log_path.name
)
logging.info("Log url %s", log_url)
else:
logging.info("Build log doesn't exist")
print(f"::notice ::Log URL: {log_url}")
build_result = BuildResult(
build_name,
log_url,
build_urls,
version.describe,
build_status,
elapsed,
GITHUB_JOB_API_URL(),
)
result_json_path = build_result.write_json(temp_path)
logging.info(
"Build result file %s is written, content:\n %s",
result_json_path,
result_json_path.read_text(encoding="utf-8"),
)
upload_master_static_binaries(pr_info, build_config, s3_helper, build_output_path)
# Upload profile data
ch_helper = ClickHouseHelper()
ci_logs_credentials = CiLogsCredentials(Path("/dev/null"))
if ci_logs_credentials.host:
instance_type = get_instance_type()
instance_id = get_instance_id()
query = f"""INSERT INTO build_time_trace
(
pull_request_number,
commit_sha,
check_start_time,
check_name,
instance_type,
instance_id,
file,
library,
time,
pid,
tid,
ph,
ts,
dur,
cat,
name,
detail,
count,
avgMs,
args_name
)
SELECT {pr_info.number}, '{pr_info.sha}', '{stopwatch.start_time_str}', '{build_name}', '{instance_type}', '{instance_id}', *
FROM input('
file String,
library String,
time DateTime64(6),
pid UInt32,
tid UInt32,
ph String,
ts UInt64,
dur UInt64,
cat String,
name String,
detail String,
count UInt64,
avgMs UInt64,
args_name String')
FORMAT JSONCompactEachRow"""
auth = {
"X-ClickHouse-User": "ci",
"X-ClickHouse-Key": ci_logs_credentials.password,
}
url = f"https://{ci_logs_credentials.host}/"
profiles_dir = temp_path / "profiles_source"
profiles_dir.mkdir(parents=True, exist_ok=True)
logging.info(
"Processing profile JSON files from %s", repo_path / "build_docker"
)
git_runner(
"./utils/prepare-time-trace/prepare-time-trace.sh "
f"build_docker {profiles_dir.absolute()}"
)
profile_data_file = temp_path / "profile.json"
with open(profile_data_file, "wb") as profile_fd:
for profile_source in profiles_dir.iterdir():
if profile_source.name != "binary_sizes.txt":
with open(profiles_dir / profile_source, "rb") as ps_fd:
profile_fd.write(ps_fd.read())
logging.info(
"::notice ::Log Uploading profile data, path: %s, size: %s, query: %s",
profile_data_file,
profile_data_file.stat().st_size,
query,
)
ch_helper.insert_file(url, auth, query, profile_data_file)
query = f"""INSERT INTO binary_sizes
(
pull_request_number,
commit_sha,
check_start_time,
check_name,
instance_type,
instance_id,
file,
size
)
SELECT {pr_info.number}, '{pr_info.sha}', '{stopwatch.start_time_str}', '{build_name}', '{instance_type}', '{instance_id}', file, size
FROM input('size UInt64, file String')
SETTINGS format_regexp = '^\\s*(\\d+) (.+)$'
FORMAT Regexp"""
binary_sizes_file = profiles_dir / "binary_sizes.txt"
logging.info(
"::notice ::Log Uploading binary sizes data, path: %s, size: %s, query: %s",
binary_sizes_file,
binary_sizes_file.stat().st_size,
query,
)
ch_helper.insert_file(url, auth, query, binary_sizes_file)
# Upload statistics to CI database
prepared_events = prepare_tests_results_for_clickhouse(
pr_info,
[],
build_status,
stopwatch.duration_seconds,
stopwatch.start_time_str,
log_url,
f"Build ({build_name})",
)
ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
JobReport(
description=version.describe,
test_results=[],
status=build_status,
start_time=stopwatch.start_time_str,
duration=elapsed,
additional_files=[log_path],
build_dir_for_upload=build_output_path,
version=version.describe,
).dump()
# Fail the build job if it didn't succeed
if build_status != SUCCESS:

69
tests/ci/build_report_check.py
View File

@ -4,12 +4,9 @@ import json
import logging
import os
import sys
import atexit
from pathlib import Path
from typing import List
from github import Github
from env_helper import (
GITHUB_JOB_URL,
GITHUB_REPOSITORY,
@ -22,20 +19,14 @@ from report import (
ERROR,
PENDING,
SUCCESS,
JobReport,
create_build_html_report,
get_worst_status,
)
from s3_helper import S3Helper
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from commit_status_helper import (
RerunHelper,
format_description,
get_commit,
post_commit_status,
update_mergeable_check,
)
from ci_config import CI_CONFIG
from stopwatch import Stopwatch
# Old way to read the neads_data
@ -46,6 +37,7 @@ NEEDS_DATA = os.getenv("NEEDS_DATA", "")
def main():
logging.basicConfig(level=logging.INFO)
stopwatch = Stopwatch()
temp_path = Path(TEMP_PATH)
reports_path = Path(REPORT_PATH)
temp_path.mkdir(parents=True, exist_ok=True)
@ -74,16 +66,7 @@ def main():
if needs_data:
logging.info("The next builds are required: %s", ", ".join(needs_data))
gh = Github(get_best_robot_token(), per_page=100)
pr_info = PRInfo()
commit = get_commit(gh, pr_info.sha)
atexit.register(update_mergeable_check, commit, pr_info, build_check_name)
rerun_helper = RerunHelper(commit, build_check_name)
if rerun_helper.is_already_finished_by_status():
logging.info("Check is already finished according to github status, exiting")
sys.exit(0)
builds_for_check = CI_CONFIG.get_builds_for_report(build_check_name)
required_builds = required_builds or len(builds_for_check)
@ -91,10 +74,15 @@ def main():
# Collect reports from json artifacts
build_results = []
for build_name in builds_for_check:
build_result = BuildResult.read_json(reports_path, build_name)
if build_result.is_missing:
build_result = BuildResult.load_any(
build_name, pr_info.number, pr_info.head_ref
)
if not build_result:
logging.warning("Build results for %s are missing", build_name)
continue
assert (
pr_info.head_ref == build_result.head_ref or pr_info.number > 0
), "BUG. if not a PR, report must be created on the same branch"
build_results.append(build_result)
# The code to collect missing reports for failed jobs
@ -125,8 +113,6 @@ def main():
logging.error("No success builds, failing check without creating a status")
sys.exit(1)
s3_helper = S3Helper()
branch_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/commits/master"
branch_name = "master"
if pr_info.number != 0:
@ -146,18 +132,6 @@ def main():
report_path = temp_path / "report.html"
report_path.write_text(report, encoding="utf-8")
logging.info("Going to upload prepared report")
context_name_for_path = build_check_name.lower().replace(" ", "_")
s3_path_prefix = (
str(pr_info.number) + "/" + pr_info.sha + "/" + context_name_for_path
)
url = s3_helper.upload_test_report_to_s3(
report_path, s3_path_prefix + "/report.html"
)
logging.info("Report url %s", url)
print(f"::notice ::Report url: {url}")
# Prepare a commit status
summary_status = get_worst_status(br.status for br in build_results)
@ -174,19 +148,16 @@ def main():
f" ({required_builds - missing_builds} of {required_builds} builds are OK)"
)
description = format_description(
f"{ok_groups}/{total_groups} artifact groups are OK{addition}"
)
description = f"{ok_groups}/{total_groups} artifact groups are OK{addition}"
post_commit_status(
commit,
summary_status,
url,
description,
build_check_name,
pr_info,
dump_to_file=True,
)
JobReport(
description=description,
test_results=[],
status=summary_status,
start_time=stopwatch.start_time_str,
duration=stopwatch.duration_seconds,
additional_files=[report_path],
).dump()
if summary_status == ERROR:
sys.exit(1)

629
tests/ci/ci.py
View File

@ -1,6 +1,7 @@
import argparse
import concurrent.futures
import json
import logging
import os
import re
import subprocess
@ -9,22 +10,41 @@ from pathlib import Path
from typing import Any, Dict, Iterable, List, Optional
import docker_images_helper
from ci_config import CI_CONFIG
from ci_config import CI_CONFIG, Labels
from commit_status_helper import (
CommitStatusData,
RerunHelper,
format_description,
get_commit,
post_commit_status,
set_status_comment,
update_mergeable_check,
)
from digest_helper import DockerDigester, JobDigester
from env_helper import CI, REPORT_PATH, ROOT_DIR, S3_BUILDS_BUCKET, TEMP_PATH
from env_helper import (
CI,
GITHUB_JOB_API_URL,
REPO_COPY,
REPORT_PATH,
S3_BUILDS_BUCKET,
TEMP_PATH,
)
from get_robot_token import get_best_robot_token
from git_helper import GIT_PREFIX, Git
from git_helper import Runner as GitRunner
from github import Github
from pr_info import PRInfo
from report import BuildResult
from report import SUCCESS, BuildResult, JobReport
from s3_helper import S3Helper
from clickhouse_helper import (
CiLogsCredentials,
ClickHouseHelper,
get_instance_id,
get_instance_type,
prepare_tests_results_for_clickhouse,
)
from build_check import get_release_or_pr
import upload_result_helper
from version_helper import get_version_from_repo
@ -94,6 +114,12 @@ def parse_args(parser: argparse.ArgumentParser) -> argparse.Namespace:
type=str,
help="Job name as in config",
)
parser.add_argument(
"--run-command",
default="",
type=str,
help="A run command to run in --run action. Will override run_command from a job config if any",
)
parser.add_argument(
"--batch",
default=-1,
@ -149,6 +175,11 @@ def parse_args(parser: argparse.ArgumentParser) -> argparse.Namespace:
default=False,
help="will create run config without skipping build jobs in any case, used in --configure action (for release branches)",
)
parser.add_argument(
"--commit-message",
default="",
help="debug option to test commit message processing",
)
return parser.parse_args()
@ -271,6 +302,7 @@ def _update_config_for_docs_only(run_config: dict) -> None:
def _configure_docker_jobs(
rebuild_all_dockers: bool, docker_digest_or_latest: bool = False
) -> Dict:
print("::group::Docker images check")
# generate docker jobs data
docker_digester = DockerDigester()
imagename_digest_dict = (
@ -283,7 +315,6 @@ def _configure_docker_jobs(
# FIXME: we need login as docker manifest inspect goes directly to one of the *.docker.com hosts instead of "registry-mirrors" : ["http://dockerhub-proxy.dockerhub-proxy-zone:5000"]
# find if it's possible to use the setting of /etc/docker/daemon.json
docker_images_helper.docker_login()
print("Start checking missing images in dockerhub")
missing_multi_dict = check_missing_images_on_dockerhub(imagename_digest_dict)
missing_multi = list(missing_multi_dict)
missing_amd64 = []
@ -313,7 +344,6 @@ def _configure_docker_jobs(
)
for image in missing_multi:
imagename_digest_dict[image] = "latest"
print("...checking missing images in dockerhub - done")
else:
# add all images to missing
missing_multi = list(imagename_digest_dict)
@ -324,6 +354,7 @@ def _configure_docker_jobs(
for name in imagename_digest_dict
if not images_info[name]["only_amd64"]
]
print("::endgroup::")
return {
"images": imagename_digest_dict,
@ -341,30 +372,36 @@ def _configure_jobs(
rebuild_all_binaries: bool,
pr_labels: Iterable[str],
commit_tokens: List[str],
ci_cache_enabled: bool,
) -> Dict:
# a. digest each item from the config
## a. digest each item from the config
job_digester = JobDigester()
jobs_params: Dict[str, Dict] = {}
jobs_to_do: List[str] = []
jobs_to_skip: List[str] = []
digests: Dict[str, str] = {}
print("Calculating job digests - start")
print("::group::Job Digests")
for job in CI_CONFIG.job_generator():
digest = job_digester.get_job_digest(CI_CONFIG.get_digest_config(job))
digests[job] = digest
print(f" job [{job.rjust(50)}] has digest [{digest}]")
print("Calculating job digests - done")
print("::endgroup::")
# b. check if we have something done
## b. check if we have something done
if ci_cache_enabled:
done_files = []
else:
path = get_s3_path(build_digest)
done_files = s3.list_prefix(path)
done_files = [file.split("/")[-1] for file in done_files]
print(f"S3 CI files for the build [{build_digest}]: {done_files}")
# print(f"S3 CI files for the build [{build_digest}]: {done_files}")
docs_path = get_s3_path_docs(docs_digest)
done_files_docs = s3.list_prefix(docs_path)
done_files_docs = [file.split("/")[-1] for file in done_files_docs]
print(f"S3 CI files for the docs [{docs_digest}]: {done_files_docs}")
# print(f"S3 CI files for the docs [{docs_digest}]: {done_files_docs}")
done_files += done_files_docs
for job in digests:
digest = digests[job]
job_config = CI_CONFIG.get_job_config(job)
@ -395,8 +432,9 @@ def _configure_jobs(
"num_batches": num_batches,
}
else:
jobs_to_skip += (job,)
jobs_to_skip.append(job)
## c. check CI controlling labels commit messages
if pr_labels:
jobs_requested_by_label = [] # type: List[str]
ci_controlling_labels = [] # type: List[str]
@ -410,41 +448,65 @@ def _configure_jobs(
print(
f" : following jobs will be executed: [{jobs_requested_by_label}]"
)
jobs_to_do = jobs_requested_by_label
jobs_to_do = [job for job in jobs_requested_by_label if job in jobs_to_do]
if commit_tokens:
jobs_to_do_requested = [] # type: List[str]
# handle ci set tokens
ci_controlling_tokens = [
token for token in commit_tokens if token in CI_CONFIG.label_configs
]
for token_ in ci_controlling_tokens:
label_config = CI_CONFIG.get_label_config(token_)
assert label_config, f"Unknonwn token [{token_}]"
print(
f"NOTE: CI controlling token: [{ci_controlling_tokens}], add jobs: [{label_config.run_jobs}]"
)
jobs_to_do_requested += label_config.run_jobs
# handle specific job requests
requested_jobs = [
token[len("#job_") :]
for token in commit_tokens
if token.startswith("#job_")
token[len("job_") :] for token in commit_tokens if token.startswith("job_")
]
if requested_jobs:
assert any(
len(x) > 1 for x in requested_jobs
), f"Invalid job names requested [{requested_jobs}]"
jobs_to_do_requested = []
for job in requested_jobs:
job_with_parents = CI_CONFIG.get_job_with_parents(job)
print(
f"NOTE: CI controlling token: [#job_{job}], add jobs: [{job_with_parents}]"
)
# always add requested job itself, even if it could be skipped
jobs_to_do_requested.append(job_with_parents[0])
for parent in job_with_parents[1:]:
if parent in jobs_to_do and parent not in jobs_to_do_requested:
jobs_to_do_requested.append(parent)
if jobs_to_do_requested:
print(
f"NOTE: Only specific job(s) were requested by commit message tokens: [{jobs_to_do_requested}]"
)
jobs_to_do = jobs_to_do_requested
jobs_to_do = list(
set(job for job in jobs_to_do_requested if job in jobs_to_do)
)
return {
"digests": digests,
"jobs_to_do": jobs_to_do,
"jobs_to_skip": jobs_to_skip,
"jobs_params": jobs_params,
"jobs_params": {
job: params for job, params in jobs_params.items() if job in jobs_to_do
},
}
def _update_gh_statuses(indata: Dict, s3: S3Helper) -> None:
# This action is required to re-create all GH statuses for skipped jobs, so that ci report can be generated afterwards
if indata["ci_flags"][Labels.NO_CI_CACHE]:
print("CI cache is disabled - skip restoring commit statuses from CI cache")
return
temp_path = Path(TEMP_PATH)
if not temp_path.exists():
temp_path.mkdir(parents=True, exist_ok=True)
@ -485,7 +547,7 @@ def _update_gh_statuses(indata: Dict, s3: S3Helper) -> None:
job_status = CommitStatusData.load_from_file(
f"{TEMP_PATH}/{success_flag_name}"
) # type: CommitStatusData
assert job_status.status == "success", "BUG!"
assert job_status.status == SUCCESS, "BUG!"
commit.create_status(
state=job_status.status,
target_url=job_status.report_url,
@ -528,18 +590,256 @@ def _update_gh_statuses(indata: Dict, s3: S3Helper) -> None:
def _fetch_commit_tokens(message: str) -> List[str]:
pattern = r"#[\w-]+"
matches = re.findall(pattern, message)
res = [
match
for match in matches
if match == "#no-merge-commit"
or match.startswith("#job_")
or match.startswith("#job-")
]
matches = [match[1:] for match in re.findall(pattern, message)]
res = [match for match in matches if match in Labels or match.startswith("job_")]
return res
def _upload_build_artifacts(
pr_info: PRInfo,
build_name: str,
build_digest: str,
job_report: JobReport,
s3: S3Helper,
s3_destination: str,
) -> str:
# There are ugly artifacts for the performance test. FIXME:
s3_performance_path = "/".join(
(
get_release_or_pr(pr_info, get_version_from_repo())[1],
pr_info.sha,
CI_CONFIG.normalize_string(build_name),
"performance.tar.zst",
)
)
performance_urls = []
assert job_report.build_dir_for_upload, "Must be set for build job"
performance_path = Path(job_report.build_dir_for_upload) / "performance.tar.zst"
if performance_path.exists():
performance_urls.append(
s3.upload_build_file_to_s3(performance_path, s3_performance_path)
)
print(
"Uploaded performance.tar.zst to %s, now delete to avoid duplication",
performance_urls[0],
)
performance_path.unlink()
build_urls = (
s3.upload_build_directory_to_s3(
Path(job_report.build_dir_for_upload),
s3_destination,
keep_dirs_in_s3_path=False,
upload_symlinks=False,
)
+ performance_urls
)
print("::notice ::Build URLs: {}".format("\n".join(build_urls)))
log_path = Path(job_report.additional_files[0])
log_url = ""
if log_path.exists():
log_url = s3.upload_build_file_to_s3(
log_path, s3_destination + "/" + log_path.name
)
print(f"::notice ::Log URL: {log_url}")
# generate and upload build report
build_result = BuildResult(
build_name,
log_url,
build_urls,
job_report.version,
job_report.status,
int(job_report.duration),
GITHUB_JOB_API_URL(),
head_ref=pr_info.head_ref,
pr_number=pr_info.number,
)
result_json_path = build_result.write_json()
s3_path = get_s3_path(build_digest) + result_json_path.name
build_report_url = s3.upload_file(
bucket=S3_BUILDS_BUCKET, file_path=result_json_path, s3_path=s3_path
)
print(f"Report file [{result_json_path}] has been uploaded to [{build_report_url}]")
# Upload head master binaries
static_bin_name = CI_CONFIG.build_config[build_name].static_binary_name
if pr_info.is_master() and static_bin_name:
# Full binary with debug info:
s3_path_full = "/".join((pr_info.base_ref, static_bin_name, "clickhouse-full"))
binary_full = Path(job_report.build_dir_for_upload) / "clickhouse"
url_full = s3.upload_build_file_to_s3(binary_full, s3_path_full)
print(f"::notice ::Binary static URL (with debug info): {url_full}")
# Stripped binary without debug info:
s3_path_compact = "/".join((pr_info.base_ref, static_bin_name, "clickhouse"))
binary_compact = Path(job_report.build_dir_for_upload) / "clickhouse-stripped"
url_compact = s3.upload_build_file_to_s3(binary_compact, s3_path_compact)
print(f"::notice ::Binary static URL (compact): {url_compact}")
return log_url
def _upload_build_profile_data(
pr_info: PRInfo,
build_name: str,
job_report: JobReport,
git_runner: GitRunner,
ch_helper: ClickHouseHelper,
) -> None:
ci_logs_credentials = CiLogsCredentials(Path("/dev/null"))
if ci_logs_credentials.host:
instance_type = get_instance_type()
instance_id = get_instance_id()
query = f"""INSERT INTO build_time_trace
(
pull_request_number,
commit_sha,
check_start_time,
check_name,
instance_type,
instance_id,
file,
library,
time,
pid,
tid,
ph,
ts,
dur,
cat,
name,
detail,
count,
avgMs,
args_name
)
SELECT {pr_info.number}, '{pr_info.sha}', '{job_report.start_time}', '{build_name}', '{instance_type}', '{instance_id}', *
FROM input('
file String,
library String,
time DateTime64(6),
pid UInt32,
tid UInt32,
ph String,
ts UInt64,
dur UInt64,
cat String,
name String,
detail String,
count UInt64,
avgMs UInt64,
args_name String')
FORMAT JSONCompactEachRow"""
auth = {
"X-ClickHouse-User": "ci",
"X-ClickHouse-Key": ci_logs_credentials.password,
}
url = f"https://{ci_logs_credentials.host}/"
profiles_dir = Path(TEMP_PATH) / "profiles_source"
profiles_dir.mkdir(parents=True, exist_ok=True)
print(
"Processing profile JSON files from %s",
Path(REPO_COPY) / "build_docker",
)
git_runner(
"./utils/prepare-time-trace/prepare-time-trace.sh "
f"build_docker {profiles_dir.absolute()}"
)
profile_data_file = Path(TEMP_PATH) / "profile.json"
with open(profile_data_file, "wb") as profile_fd:
for profile_source in profiles_dir.iterdir():
if profile_source.name != "binary_sizes.txt":
with open(profiles_dir / profile_source, "rb") as ps_fd:
profile_fd.write(ps_fd.read())
print(
"::notice ::Log Uploading profile data, path: %s, size: %s, query: %s",
profile_data_file,
profile_data_file.stat().st_size,
query,
)
ch_helper.insert_file(url, auth, query, profile_data_file)
query = f"""INSERT INTO binary_sizes
(
pull_request_number,
commit_sha,
check_start_time,
check_name,
instance_type,
instance_id,
file,
size
)
SELECT {pr_info.number}, '{pr_info.sha}', '{job_report.start_time}', '{build_name}', '{instance_type}', '{instance_id}', file, size
FROM input('size UInt64, file String')
SETTINGS format_regexp = '^\\s*(\\d+) (.+)$'
FORMAT Regexp"""
binary_sizes_file = profiles_dir / "binary_sizes.txt"
print(
"::notice ::Log Uploading binary sizes data, path: %s, size: %s, query: %s",
binary_sizes_file,
binary_sizes_file.stat().st_size,
query,
)
ch_helper.insert_file(url, auth, query, binary_sizes_file)
def _run_test(job_name: str, run_command: str) -> int:
assert (
run_command or CI_CONFIG.get_job_config(job_name).run_command
), "Run command must be provided as input argument or be configured in job config"
if not run_command:
if CI_CONFIG.get_job_config(job_name).timeout:
os.environ["KILL_TIMEOUT"] = str(CI_CONFIG.get_job_config(job_name).timeout)
run_command = "/".join(
(os.path.dirname(__file__), CI_CONFIG.get_job_config(job_name).run_command)
)
if ".py" in run_command and not run_command.startswith("python"):
run_command = "python3 " + run_command
print("Use run command from a job config")
else:
print("Use run command from the workflow")
os.environ["CHECK_NAME"] = job_name
print(f"Going to start run command [{run_command}]")
process = subprocess.run(
run_command,
stdout=sys.stdout,
stderr=sys.stderr,
text=True,
check=False,
shell=True,
)
if process.returncode == 0:
print(f"Run action done for: [{job_name}]")
exit_code = 0
else:
print(
f"Run action failed for: [{job_name}] with exit code [{process.returncode}]"
)
exit_code = process.returncode
return exit_code
def _get_ext_check_name(check_name: str) -> str:
run_by_hash_num = int(os.getenv("RUN_BY_HASH_NUM", "0"))
run_by_hash_total = int(os.getenv("RUN_BY_HASH_TOTAL", "0"))
if run_by_hash_total > 1:
check_name_with_group = (
check_name + f" [{run_by_hash_num + 1}/{run_by_hash_total}]"
)
else:
check_name_with_group = check_name
return check_name_with_group
def main() -> int:
logging.basicConfig(level=logging.INFO)
exit_code = 0
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
@ -561,26 +861,34 @@ def main() -> int:
result: Dict[str, Any] = {}
s3 = S3Helper()
if args.configure:
GR = GitRunner()
pr_info = PRInfo()
git_runner = GitRunner(set_cwd_to_git_root=True)
### CONFIGURE action: start
if args.configure:
docker_data = {}
git_ref = GR.run(f"{GIT_PREFIX} rev-parse HEAD")
git_ref = git_runner.run(f"{GIT_PREFIX} rev-parse HEAD")
# if '#no-merge-commit' is set in commit message - set git ref to PR branch head to avoid merge-commit
# if '#no_merge_commit' is set in commit message - set git ref to PR branch head to avoid merge-commit
tokens = []
if pr_info.number != 0 and not args.skip_jobs:
message = GR.run(f"{GIT_PREFIX} log {pr_info.sha} --format=%B -n 1")
tokens = _fetch_commit_tokens(message)
print(f"Found commit message tokens: [{tokens}]")
if "#no-merge-commit" in tokens and CI:
GR.run(f"{GIT_PREFIX} checkout {pr_info.sha}")
git_ref = GR.run(f"{GIT_PREFIX} rev-parse HEAD")
print(
"#no-merge-commit is set in commit message - Setting git ref to PR branch HEAD to not use merge commit"
ci_flags = {
Labels.NO_MERGE_COMMIT: False,
Labels.NO_CI_CACHE: False,
}
if (pr_info.number != 0 and not args.skip_jobs) or args.commit_message:
message = args.commit_message or git_runner.run(
f"{GIT_PREFIX} log {pr_info.sha} --format=%B -n 1"
)
tokens = _fetch_commit_tokens(message)
print(f"Commit message tokens: [{tokens}]")
if Labels.NO_MERGE_COMMIT in tokens and CI:
git_runner.run(f"{GIT_PREFIX} checkout {pr_info.sha}")
git_ref = git_runner.run(f"{GIT_PREFIX} rev-parse HEAD")
ci_flags[Labels.NO_MERGE_COMMIT] = True
print("NOTE: Disable Merge Commit")
if Labels.NO_CI_CACHE in tokens:
ci_flags[Labels.NO_CI_CACHE] = True
print("NOTE: Disable CI Cache")
# let's get CH version
version = get_version_from_repo(git=Git(True)).string
@ -607,9 +915,11 @@ def main() -> int:
docs_digest,
job_digester,
s3,
args.rebuild_all_binaries,
# FIXME: add suport for master wf w/o rebuilds
args.rebuild_all_binaries or pr_info.is_master(),
pr_info.labels,
tokens,
ci_flags[Labels.NO_CI_CACHE],
)
if not args.skip_jobs
else {}
@ -620,6 +930,7 @@ def main() -> int:
result["version"] = version
result["build"] = build_digest
result["docs"] = docs_digest
result["ci_flags"] = ci_flags
result["jobs_data"] = jobs_data
result["docker_data"] = docker_data
if pr_info.number != 0 and not args.docker_digest_or_latest:
@ -628,82 +939,191 @@ def main() -> int:
_check_and_update_for_early_style_check(result)
if pr_info.has_changes_in_documentation_only():
_update_config_for_docs_only(result)
### CONFIGURE action: end
elif args.update_gh_statuses:
assert indata, "Run config must be provided via --infile"
_update_gh_statuses(indata=indata, s3=s3)
### PRE action: start
elif args.pre:
# remove job status file if any
CommitStatusData.cleanup()
JobReport.cleanup()
BuildResult.cleanup()
if is_test_job(args.job_name):
assert indata, "Run config must be provided via --infile"
report_path = Path(REPORT_PATH)
report_path.mkdir(exist_ok=True, parents=True)
path = get_s3_path(indata["build"])
files = s3.download_files( # type: ignore
reports_files = s3.download_files( # type: ignore
bucket=S3_BUILDS_BUCKET,
s3_path=path,
file_suffix=".json",
local_directory=report_path,
)
print(
f"Pre action done. Report files [{files}] have been downloaded from [{path}] to [{report_path}]"
)
# for release/master branches reports must be created on the same branches
files = []
if pr_info.number == 0:
for file in reports_files:
if pr_info.head_ref not in file:
# keep reports from the same branch only, if not in a PR
(report_path / file).unlink()
print(f"drop report: [{report_path / file}]")
else:
print(f"Pre action done. Nothing to do for [{args.job_name}]")
files.append(file)
reports_files = files
print(
f"Pre action done. Report files [{reports_files}] have been downloaded from [{path}] to [{report_path}]"
)
### PRE action: end
### RUN action: start
elif args.run:
assert CI_CONFIG.get_job_config(
args.job_name
).run_command, f"Run command must be configured in CI_CONFIG for [{args.job_name}] or in GH workflow"
if CI_CONFIG.get_job_config(args.job_name).timeout:
os.environ["KILL_TIMEOUT"] = str(
CI_CONFIG.get_job_config(args.job_name).timeout
)
os.environ["CHECK_NAME"] = args.job_name
run_command = (
"./tests/ci/" + CI_CONFIG.get_job_config(args.job_name).run_command
)
if ".py" in run_command:
run_command = "python3 " + run_command
print(f"Going to start run command [{run_command}]")
process = subprocess.run(
run_command,
stdout=sys.stdout,
stderr=sys.stderr,
text=True,
check=False,
shell=True,
)
if process.returncode == 0:
print(f"Run action done for: [{args.job_name}]")
else:
print(
f"Run action failed for: [{args.job_name}] with exit code [{process.returncode}]"
)
exit_code = process.returncode
elif args.post:
if is_build_job(args.job_name):
report_path = Path(TEMP_PATH) # build-check.py stores report in TEMP_PATH
assert report_path.is_dir(), f"File [{report_path}] is not a dir"
files = list(report_path.glob(f"*{args.job_name}.json")) # type: ignore[arg-type]
assert len(files) == 1, f"Which is the report file: {files}?"
local_report = f"{files[0]}"
report_name = BuildResult.get_report_name(args.job_name)
assert indata
s3_path = Path(get_s3_path(indata["build"])) / report_name
report_url = s3.upload_file(
bucket=S3_BUILDS_BUCKET, file_path=local_report, s3_path=s3_path
)
check_name = args.job_name
check_name_with_group = _get_ext_check_name(check_name)
print(
f"Post action done. Report file [{local_report}] has been uploaded to [{report_url}]"
f"Check if rerun for name: [{check_name}], extended name [{check_name_with_group}]"
)
previous_status = None
if is_build_job(check_name):
# this is a build job - check if build report is present
build_result = (
BuildResult.load_any(check_name, pr_info.number, pr_info.head_ref)
if not indata["ci_flags"][Labels.NO_CI_CACHE]
else None
)
if build_result:
if build_result.status == SUCCESS:
previous_status = build_result.status
else:
# FIXME: Consider reusing failures for build jobs.
# Just remove this if/else - that makes build job starting and failing immediately
print(
"Build report found but status is unsuccessful - will try to rerun"
)
print("::group::Build Report")
print(build_result.as_json())
print("::endgroup::")
else:
# this is a test job - check if GH commit status is present
commit = get_commit(
Github(get_best_robot_token(), per_page=100), pr_info.sha
)
rerun_helper = RerunHelper(commit, check_name_with_group)
if rerun_helper.is_already_finished_by_status():
status = rerun_helper.get_finished_status()
assert status
previous_status = status.state
print("::group::Commit Status")
print(status)
print("::endgroup::")
if previous_status:
print(
f"Commit status or Build Report is already present - job will be skipped with status: [{previous_status}]"
)
if previous_status == SUCCESS:
exit_code = 0
else:
exit_code = 1
else:
exit_code = _run_test(check_name, args.run_command)
### RUN action: end
### POST action: start
elif args.post:
assert (
not is_build_job(args.job_name) or indata
), "--infile with config must be provided for POST action of a build type job [{args.job_name}]"
job_report = JobReport.load() if JobReport.exist() else None
if job_report:
ch_helper = ClickHouseHelper()
check_url = ""
if is_build_job(args.job_name):
build_name = args.job_name
s3_path_prefix = "/".join(
(
get_release_or_pr(pr_info, get_version_from_repo())[0],
pr_info.sha,
build_name,
)
)
log_url = _upload_build_artifacts(
pr_info,
build_name,
build_digest=indata["build"], # type: ignore
job_report=job_report,
s3=s3,
s3_destination=s3_path_prefix,
)
_upload_build_profile_data(
pr_info, build_name, job_report, git_runner, ch_helper
)
check_url = log_url
else:
# test job
additional_urls = []
s3_path_prefix = "/".join(
(
get_release_or_pr(pr_info, get_version_from_repo())[0],
pr_info.sha,
CI_CONFIG.normalize_string(
job_report.check_name or _get_ext_check_name(args.job_name)
),
)
)
if job_report.build_dir_for_upload:
additional_urls = s3.upload_build_directory_to_s3(
Path(job_report.build_dir_for_upload),
s3_path_prefix,
keep_dirs_in_s3_path=False,
upload_symlinks=False,
)
if job_report.test_results or job_report.additional_files:
check_url = upload_result_helper.upload_results(
s3,
pr_info.number,
pr_info.sha,
job_report.test_results,
job_report.additional_files,
job_report.check_name or args.job_name,
additional_urls=additional_urls or None,
)
commit = get_commit(
Github(get_best_robot_token(), per_page=100), pr_info.sha
)
post_commit_status(
commit,
job_report.status,
check_url,
format_description(job_report.description),
job_report.check_name or args.job_name,
pr_info,
dump_to_file=True,
)
update_mergeable_check(
commit,
pr_info,
job_report.check_name or _get_ext_check_name(args.job_name),
)
print(f"Job report url: [{check_url}]")
prepared_events = prepare_tests_results_for_clickhouse(
pr_info,
job_report.test_results,
job_report.status,
job_report.duration,
job_report.start_time,
check_url or "",
job_report.check_name or args.job_name,
)
ch_helper.insert_events_into(
db="default", table="checks", events=prepared_events
)
else:
print(f"Post action done. Nothing to do for [{args.job_name}]")
# no job report
print(f"No job report for {[args.job_name]} - do nothing")
### POST action: end
### MARK SUCCESS action: start
elif args.mark_success:
assert indata, "Run config must be provided via --infile"
job = args.job_name
@ -756,8 +1176,15 @@ def main() -> int:
)
else:
print(f"Job [{job}] is not ok, status [{job_status.status}]")
### MARK SUCCESS action: end
# print results
### UPDATE GH STATUSES action: start
elif args.update_gh_statuses:
assert indata, "Run config must be provided via --infile"
_update_gh_statuses(indata=indata, s3=s3)
### UPDATE GH STATUSES action: end
### print results
if args.outfile:
with open(args.outfile, "w") as f:
if isinstance(result, str):
@ -773,10 +1200,8 @@ def main() -> int:
print(json.dumps(result, indent=2 if args.pretty else None))
else:
raise AssertionError(f"Unexpected type for 'res': {type(result)}")
return exit_code
if __name__ == "__main__":
os.chdir(ROOT_DIR)
sys.exit(main())

111
tests/ci/ci_config.py
View File

@ -3,15 +3,46 @@
import logging
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path
from typing import Callable, Dict, Iterable, List, Literal, Optional, Union
from integration_test_images import IMAGES
from ci_utils import WithIter
class Labels(Enum):
DO_NOT_TEST_LABEL = "do not test"
class Labels(metaclass=WithIter):
DO_NOT_TEST_LABEL = "do_not_test"
NO_MERGE_COMMIT = "no_merge_commit"
NO_CI_CACHE = "no_ci_cache"
CI_SET_REDUCED = "ci_set_reduced"
class JobNames(metaclass=WithIter):
STYLE_CHECK = "Style check"
FAST_TEST = "Fast tests"
DOCKER_SERVER = "Docker server and keeper images"
PACKAGE_DEBUG = "package_debug"
PACKAGE_RELEASE = "package_release"
PACKAGE_RELEASE = "package_aarch64"
PACKAGE_BIN_RELEASE = "binary_release"
INSTALL_TEST_AMD = "Install packages (amd64)"
INSTALL_TEST_ARM = "Install packages (arm64)"
STATELESS_TEST_DEBUG = "Stateless tests (debug)"
STATEFUL_TEST_DEBUG = "Stateful tests (debug)"
INTEGRATION_TEST = "Integration tests (release)"
UPGRADE_TEST_DEBUG = "Upgrade check (debug)"
AST_FUZZER_TEST_DEBUG = "AST fuzzer (debug)"
COMPATIBILITY_TEST = "Compatibility check (amd64)"
UNIT_TEST = "Unit tests (release)"
PERFORMANCE_TEST_AMD64 = "Performance Comparison"
SQL_LANCER_TEST = "SQLancer (release)"
SQL_LOGIC_TEST = "Sqllogic test (release)"
CLCIKBENCH_TEST = "ClickBench (amd64)"
LIBFUZZER_TEST = "libFuzzer tests"
STRESS_TEST_DEBUG = "Stress test (debug)"
BUILD_CHECK = "ClickHouse build check"
DOCS_CHECK = "Docs check"
# FIXME: add all jobs
@dataclass
@ -94,6 +125,7 @@ class BuildConfig:
docker=["clickhouse/binary-builder"],
git_submodules=True,
),
run_command="build_check.py $BUILD_NAME",
)
)
@ -111,7 +143,16 @@ class BuildConfig:
@dataclass
class BuildReportConfig:
builds: List[str]
job_config: JobConfig = field(default_factory=JobConfig)
job_config: JobConfig = field(
default_factory=lambda: JobConfig(
digest=DigestConfig(
include_paths=[
"./tests/ci/build_report_check.py",
"./tests/ci/upload_result_helper.py",
],
),
)
)
@dataclass
@ -290,7 +331,7 @@ class CiConfig:
def get_label_config(self, label_name: str) -> Optional[LabelConfig]:
for label, config in self.label_configs.items():
if label_name == label:
if self.normalize_string(label_name) == self.normalize_string(label):
return config
return None
@ -310,8 +351,8 @@ class CiConfig:
), f"Invalid check_name or CI_CONFIG outdated, config not found for [{check_name}]"
return res # type: ignore
def get_job_with_parents(self, check_name: str) -> List[str]:
def _normalize_string(input_string: str) -> str:
@staticmethod
def normalize_string(input_string: str) -> str:
lowercase_string = input_string.lower()
normalized_string = (
lowercase_string.replace(" ", "_")
@ -322,8 +363,9 @@ class CiConfig:
)
return normalized_string
def get_job_with_parents(self, check_name: str) -> List[str]:
res = []
check_name = _normalize_string(check_name)
check_name = self.normalize_string(check_name)
for config in (
self.build_config,
@ -332,18 +374,18 @@ class CiConfig:
self.other_jobs_configs,
):
for job_name in config: # type: ignore
if check_name == _normalize_string(job_name):
if check_name == self.normalize_string(job_name):
res.append(job_name)
if isinstance(config[job_name], TestConfig): # type: ignore
assert config[
job_name
].required_build, f"Error: Experimantal feature... Not supported job [{job_name}]" # type: ignore
if config[job_name].required_build: # type: ignore
res.append(config[job_name].required_build) # type: ignore
res.append("Fast tests")
res.append("Style check")
elif isinstance(config[job_name], BuildConfig): # type: ignore
res.append("Fast tests")
res.append("Style check")
pass
elif isinstance(config[job_name], BuildReportConfig): # type: ignore
# add all build jobs as parents for build report check
res.extend(
[job for job in JobNames if job in self.build_config]
)
else:
assert (
False
@ -443,7 +485,24 @@ class CiConfig:
CI_CONFIG = CiConfig(
label_configs={
Labels.DO_NOT_TEST_LABEL.value: LabelConfig(run_jobs=["Style check"]),
Labels.DO_NOT_TEST_LABEL: LabelConfig(run_jobs=[JobNames.STYLE_CHECK]),
Labels.CI_SET_REDUCED: LabelConfig(
run_jobs=[
job
for job in JobNames
if not any(
[
nogo in job
for nogo in (
"asan",
"tsan",
"msan",
"ubsan",
)
]
)
]
),
},
build_config={
"package_release": BuildConfig(
@ -575,7 +634,7 @@ CI_CONFIG = CiConfig(
),
},
builds_report_config={
"ClickHouse build check": BuildReportConfig(
JobNames.BUILD_CHECK: BuildReportConfig(
builds=[
"package_release",
"package_aarch64",
@ -586,7 +645,7 @@ CI_CONFIG = CiConfig(
"package_debug",
"binary_release",
"fuzzers",
]
],
),
"ClickHouse special build check": BuildReportConfig(
builds=[
@ -601,11 +660,11 @@ CI_CONFIG = CiConfig(
"binary_s390x",
"binary_amd64_compat",
"binary_amd64_musl",
]
],
),
},
other_jobs_configs={
"Docker server and keeper images": TestConfig(
JobNames.DOCKER_SERVER: TestConfig(
"",
job_config=JobConfig(
digest=DigestConfig(
@ -626,7 +685,7 @@ CI_CONFIG = CiConfig(
),
),
),
"Fast tests": TestConfig(
JobNames.FAST_TEST: TestConfig(
"",
job_config=JobConfig(
digest=DigestConfig(
@ -636,7 +695,7 @@ CI_CONFIG = CiConfig(
)
),
),
"Style check": TestConfig(
JobNames.STYLE_CHECK: TestConfig(
"",
job_config=JobConfig(
run_always=True,
@ -649,10 +708,10 @@ CI_CONFIG = CiConfig(
),
},
test_configs={
"Install packages (amd64)": TestConfig(
JobNames.INSTALL_TEST_AMD: TestConfig(
"package_release", job_config=JobConfig(digest=install_check_digest)
),
"Install packages (arm64)": TestConfig(
JobNames.INSTALL_TEST_ARM: TestConfig(
"package_aarch64", job_config=JobConfig(digest=install_check_digest)
),
"Stateful tests (asan)": TestConfig(
@ -805,7 +864,7 @@ CI_CONFIG = CiConfig(
"Compatibility check (aarch64)": TestConfig(
"package_aarch64", job_config=JobConfig(digest=compatibility_check_digest)
),
"Unit tests (release)": TestConfig(
JobNames.UNIT_TEST: TestConfig(
"binary_release", job_config=JobConfig(**unit_test_common_params) # type: ignore
),
"Unit tests (asan)": TestConfig(

19
tests/ci/ci_utils.py Normal file
View File

@ -0,0 +1,19 @@
from contextlib import contextmanager
import os
from typing import Union, Iterator
from pathlib import Path
class WithIter(type):
def __iter__(cls):
return (v for k, v in cls.__dict__.items() if not k.startswith("_"))
@contextmanager
def cd(path: Union[Path, str]) -> Iterator[None]:
oldpwd = os.getcwd()
os.chdir(path)
try:
yield
finally:
os.chdir(oldpwd)

59
tests/ci/clickbench.py
View File

@ -6,34 +6,22 @@ import logging
import os
import subprocess
import sys
import atexit
from pathlib import Path
from typing import List, Tuple
from github import Github
from build_download_helper import download_all_deb_packages
from clickhouse_helper import (
CiLogsCredentials,
ClickHouseHelper,
prepare_tests_results_for_clickhouse,
)
from commit_status_helper import (
RerunHelper,
get_commit,
override_status,
post_commit_status,
update_mergeable_check,
)
from docker_images_helper import get_docker_image, pull_image, DockerImage
from env_helper import TEMP_PATH, REPORT_PATH
from get_robot_token import get_best_robot_token
from pr_info import FORCE_TESTS_LABEL, PRInfo
from s3_helper import S3Helper
from stopwatch import Stopwatch
from tee_popen import TeePopen
from upload_result_helper import upload_results
from report import TestResults
from report import JobReport, TestResults
def get_image_name() -> str:
@ -128,18 +116,8 @@ def main():
args = parse_args()
check_name = args.check_name
gh = Github(get_best_robot_token(), per_page=100)
pr_info = PRInfo()
commit = get_commit(gh, pr_info.sha)
atexit.register(update_mergeable_check, commit, pr_info, check_name)
rerun_helper = RerunHelper(commit, check_name)
if rerun_helper.is_already_finished_by_status():
logging.info("Check is already finished according to github status, exiting")
sys.exit(0)
image_name = get_image_name()
docker_image = pull_image(get_docker_image(image_name))
@ -186,39 +164,20 @@ def main():
logging.warning("Failed to change files owner in %s, ignoring it", temp_path)
ci_logs_credentials.clean_ci_logs_from_credentials(run_log_path)
s3_helper = S3Helper()
state, description, test_results, additional_logs = process_results(
result_path, server_log_path
)
state = override_status(state, check_name)
ch_helper = ClickHouseHelper()
report_url = upload_results(
s3_helper,
pr_info.number,
pr_info.sha,
test_results,
[run_log_path] + additional_logs,
check_name,
)
print(f"::notice:: {check_name} Report url: {report_url}")
post_commit_status(
commit, state, report_url, description, check_name, pr_info, dump_to_file=True
)
prepared_events = prepare_tests_results_for_clickhouse(
pr_info,
test_results,
state,
stopwatch.duration_seconds,
stopwatch.start_time_str,
report_url,
check_name,
)
ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
JobReport(
description=description,
test_results=test_results,
status=state,
start_time=stopwatch.start_time_str,
duration=stopwatch.duration_seconds,
additional_files=[run_log_path] + additional_logs,
).dump()
if state != "success":
if FORCE_TESTS_LABEL in pr_info.labels:

64
tests/ci/compatibility_check.py
View File

@ -8,22 +8,11 @@ import logging
import subprocess
import sys
from github import Github
from build_download_helper import download_builds_filter
from clickhouse_helper import (
ClickHouseHelper,
prepare_tests_results_for_clickhouse,
)
from commit_status_helper import RerunHelper, get_commit, post_commit_status
from docker_images_helper import DockerImage, get_docker_image, pull_image
from env_helper import TEMP_PATH, REPORT_PATH
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from report import TestResults, TestResult
from s3_helper import S3Helper
from report import JobReport, TestResults, TestResult
from stopwatch import Stopwatch
from upload_result_helper import upload_results
IMAGE_UBUNTU = "clickhouse/test-old-ubuntu"
IMAGE_CENTOS = "clickhouse/test-old-centos"
@ -149,16 +138,6 @@ def main():
temp_path.mkdir(parents=True, exist_ok=True)
reports_path.mkdir(parents=True, exist_ok=True)
pr_info = PRInfo()
gh = Github(get_best_robot_token(), per_page=100)
commit = get_commit(gh, pr_info.sha)
rerun_helper = RerunHelper(commit, args.check_name)
if rerun_helper.is_already_finished_by_status():
logging.info("Check is already finished according to github status, exiting")
sys.exit(0)
packages_path = temp_path / "packages"
packages_path.mkdir(parents=True, exist_ok=True)
@ -219,7 +198,6 @@ def main():
else:
raise Exception("Can't determine max glibc version")
s3_helper = S3Helper()
state, description, test_results, additional_logs = process_result(
result_path,
server_log_path,
@ -228,38 +206,14 @@ def main():
max_glibc_version,
)
ch_helper = ClickHouseHelper()
report_url = upload_results(
s3_helper,
pr_info.number,
pr_info.sha,
test_results,
additional_logs,
args.check_name,
)
print(f"::notice ::Report url: {report_url}")
post_commit_status(
commit,
state,
report_url,
description,
args.check_name,
pr_info,
dump_to_file=True,
)
prepared_events = prepare_tests_results_for_clickhouse(
pr_info,
test_results,
state,
stopwatch.duration_seconds,
stopwatch.start_time_str,
report_url,
args.check_name,
)
ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
JobReport(
description=description,
test_results=test_results,
status=state,
start_time=stopwatch.start_time_str,
duration=stopwatch.duration_seconds,
additional_files=additional_logs,
).dump()
if state == "failure":
sys.exit(1)

5
tests/ci/digest_helper.py
View File

@ -11,6 +11,8 @@ from sys import modules
from docker_images_helper import get_images_info
from ci_config import DigestConfig
from git_helper import Runner
from env_helper import ROOT_DIR
from ci_utils import cd
DOCKER_DIGEST_LEN = 12
JOB_DIGEST_LEN = 10
@ -67,6 +69,7 @@ def digest_paths(
The order is processed as given"""
hash_object = hash_object or md5()
paths_all: List[Path] = []
with cd(ROOT_DIR):
for p in paths:
if isinstance(p, str) and "*" in p:
for path in Path(".").glob(p):
@ -86,6 +89,7 @@ def digest_script(path_str: str) -> HASH:
path = Path(path_str)
parent = path.parent
md5_hash = md5()
with cd(ROOT_DIR):
try:
for script in modules.values():
script_path = getattr(script, "__file__", "")
@ -113,6 +117,7 @@ class DockerDigester:
def get_image_digest(self, name: str) -> str:
assert isinstance(name, str)
with cd(ROOT_DIR):
deps = [name]
digest = None
while deps:

4
tests/ci/docker_images_check.py
View File

@ -12,7 +12,7 @@ from github import Github
from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse
from commit_status_helper import format_description, get_commit, post_commit_status
from env_helper import ROOT_DIR, RUNNER_TEMP, GITHUB_RUN_URL
from env_helper import RUNNER_TEMP, GITHUB_RUN_URL
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from report import TestResults, TestResult
@ -170,8 +170,6 @@ def parse_args() -> argparse.Namespace:
def main():
# to be always aligned with docker paths from image.json
os.chdir(ROOT_DIR)
logging.basicConfig(level=logging.INFO)
stopwatch = Stopwatch()

3
tests/ci/docker_manifests_merge.py
View File

@ -21,7 +21,6 @@ from pr_info import PRInfo
from report import TestResult
from s3_helper import S3Helper
from stopwatch import Stopwatch
from env_helper import ROOT_DIR
from upload_result_helper import upload_results
from docker_images_helper import docker_login, get_images_oredered_list
@ -126,8 +125,6 @@ def create_manifest(
def main():
# to be aligned with docker paths from image.json
os.chdir(ROOT_DIR)
logging.basicConfig(level=logging.INFO)
stopwatch = Stopwatch()

47
tests/ci/docker_server.py
View File

@ -10,11 +10,7 @@ from pathlib import Path
from os import path as p, makedirs
from typing import Dict, List
from github import Github
from build_check import get_release_or_pr
from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse
from commit_status_helper import format_description, get_commit, post_commit_status
from docker_images_helper import DockerImageData, docker_login
from env_helper import (
GITHUB_RUN_URL,
@ -23,15 +19,12 @@ from env_helper import (
S3_BUILDS_BUCKET,
S3_DOWNLOAD,
)
from get_robot_token import get_best_robot_token
from git_helper import Git
from pr_info import PRInfo
from report import TestResults, TestResult
from s3_helper import S3Helper
from report import JobReport, TestResults, TestResult
from stopwatch import Stopwatch
from tee_popen import TeePopen
from build_download_helper import read_build_urls
from upload_result_helper import upload_results
from version_helper import (
ClickHouseVersion,
get_tagged_versions,
@ -346,7 +339,6 @@ def main():
image = DockerImageData(args.image_path, args.image_repo, False)
args.release_type = auto_release_type(args.version, args.release_type)
tags = gen_tags(args.version, args.release_type)
NAME = f"Docker image {image.repo} building check"
pr_info = None
repo_urls = dict()
direct_urls: Dict[str, List[str]] = dict()
@ -384,7 +376,6 @@ def main():
if args.push:
docker_login()
NAME = f"Docker image {image.repo} build and push"
logging.info("Following tags will be created: %s", ", ".join(tags))
status = "success"
@ -398,38 +389,18 @@ def main():
)
if test_results[-1].status != "OK":
status = "failure"
pr_info = pr_info or PRInfo()
s3_helper = S3Helper()
url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, [], NAME)
print(f"::notice ::Report url: {url}")
if not args.reports:
return
description = f"Processed tags: {', '.join(tags)}"
JobReport(
description=description,
test_results=test_results,
status=status,
start_time=stopwatch.start_time_str,
duration=stopwatch.duration_seconds,
additional_files=[],
).dump()
description = format_description(description)
gh = Github(get_best_robot_token(), per_page=100)
commit = get_commit(gh, pr_info.sha)
post_commit_status(
commit, status, url, description, NAME, pr_info, dump_to_file=True
)
prepared_events = prepare_tests_results_for_clickhouse(
pr_info,
test_results,
status,
stopwatch.duration_seconds,
stopwatch.start_time_str,
url,
NAME,
)
ch_helper = ClickHouseHelper()
ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
if status != "success":
sys.exit(1)

70
tests/ci/docs_check.py
View File

@ -1,29 +1,16 @@
#!/usr/bin/env python3
import argparse
import atexit
import logging
import subprocess
import sys
from pathlib import Path
from github import Github
from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse
from commit_status_helper import (
RerunHelper,
get_commit,
post_commit_status,
update_mergeable_check,
)
from docker_images_helper import get_docker_image, pull_image
from env_helper import TEMP_PATH, REPO_COPY
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from report import TestResults, TestResult
from s3_helper import S3Helper
from report import JobReport, TestResults, TestResult
from stopwatch import Stopwatch
from tee_popen import TeePopen
from upload_result_helper import upload_results
NAME = "Docs Check"
@ -60,26 +47,16 @@ def main():
pr_info = PRInfo(need_changed_files=True)
gh = Github(get_best_robot_token(), per_page=100)
commit = get_commit(gh, pr_info.sha)
rerun_helper = RerunHelper(commit, NAME)
if rerun_helper.is_already_finished_by_status():
logging.info("Check is already finished according to github status, exiting")
sys.exit(0)
atexit.register(update_mergeable_check, commit, pr_info, NAME)
if not pr_info.has_changes_in_documentation() and not args.force:
logging.info("No changes in documentation")
post_commit_status(
commit,
"success",
"",
"No changes in docs",
NAME,
pr_info,
dump_to_file=True,
)
JobReport(
description="No changes in docs",
test_results=[],
status="success",
start_time=stopwatch.start_time_str,
duration=stopwatch.duration_seconds,
additional_files=[],
).dump()
sys.exit(0)
if pr_info.has_changes_in_documentation():
@ -134,28 +111,15 @@ def main():
else:
test_results.append(TestResult("Non zero exit code", "FAIL"))
s3_helper = S3Helper()
ch_helper = ClickHouseHelper()
JobReport(
description=description,
test_results=test_results,
status=status,
start_time=stopwatch.start_time_str,
duration=stopwatch.duration_seconds,
additional_files=additional_files,
).dump()
report_url = upload_results(
s3_helper, pr_info.number, pr_info.sha, test_results, additional_files, NAME
)
print("::notice ::Report url: {report_url}")
post_commit_status(
commit, status, report_url, description, NAME, pr_info, dump_to_file=True
)
prepared_events = prepare_tests_results_for_clickhouse(
pr_info,
test_results,
status,
stopwatch.duration_seconds,
stopwatch.start_time_str,
report_url,
NAME,
)
ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
if status == "failure":
sys.exit(1)

86
tests/ci/fast_test_check.py
View File

@ -5,34 +5,15 @@ import subprocess
import os
import csv
import sys
import atexit
from pathlib import Path
from typing import Tuple
from github import Github
from build_check import get_release_or_pr
from clickhouse_helper import (
ClickHouseHelper,
prepare_tests_results_for_clickhouse,
)
from commit_status_helper import (
RerunHelper,
get_commit,
post_commit_status,
update_mergeable_check,
format_description,
)
from docker_images_helper import DockerImage, get_docker_image, pull_image
from env_helper import S3_BUILDS_BUCKET, TEMP_PATH, REPO_COPY
from get_robot_token import get_best_robot_token
from pr_info import FORCE_TESTS_LABEL, PRInfo
from report import TestResult, TestResults, read_test_results
from s3_helper import S3Helper
from report import JobReport, TestResult, TestResults, read_test_results
from stopwatch import Stopwatch
from tee_popen import TeePopen
from upload_result_helper import upload_results
from version_helper import get_version_from_repo
NAME = "Fast test"
@ -121,23 +102,8 @@ def main():
pr_info = PRInfo()
gh = Github(get_best_robot_token(), per_page=100)
commit = get_commit(gh, pr_info.sha)
atexit.register(update_mergeable_check, commit, pr_info, NAME)
rerun_helper = RerunHelper(commit, NAME)
if rerun_helper.is_already_finished_by_status():
logging.info("Check is already finished according to github status, exiting")
status = rerun_helper.get_finished_status()
if status is not None and status.state != "success":
sys.exit(1)
sys.exit(0)
docker_image = pull_image(get_docker_image("clickhouse/fasttest"))
s3_helper = S3Helper()
workspace = temp_path / "fasttest-workspace"
workspace.mkdir(parents=True, exist_ok=True)
@ -204,47 +170,17 @@ def main():
if timeout_expired:
test_results.append(TestResult.create_check_timeout_expired(args.timeout))
state = "failure"
description = format_description(test_results[-1].name)
description = test_results[-1].name
ch_helper = ClickHouseHelper()
s3_path_prefix = "/".join(
(
get_release_or_pr(pr_info, get_version_from_repo())[0],
pr_info.sha,
"fast_tests",
)
)
build_urls = s3_helper.upload_build_directory_to_s3(
output_path / "binaries",
s3_path_prefix,
keep_dirs_in_s3_path=False,
upload_symlinks=False,
)
report_url = upload_results(
s3_helper,
pr_info.number,
pr_info.sha,
test_results,
additional_logs,
NAME,
build_urls,
)
print(f"::notice ::Report url: {report_url}")
post_commit_status(
commit, state, report_url, description, NAME, pr_info, dump_to_file=True
)
prepared_events = prepare_tests_results_for_clickhouse(
pr_info,
test_results,
state,
stopwatch.duration_seconds,
stopwatch.start_time_str,
report_url,
NAME,
)
ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
JobReport(
description=description,
test_results=test_results,
status=state,
start_time=stopwatch.start_time_str,
duration=stopwatch.duration_seconds,
additional_files=additional_logs,
build_dir_for_upload=str(output_path / "binaries"),
).dump()
# Refuse other checks to run if fast test failed
if state != "success":

9
tests/ci/functional_test_check.py
View File

@ -20,7 +20,6 @@ from clickhouse_helper import (
prepare_tests_results_for_clickhouse,
)
from commit_status_helper import (
RerunHelper,
get_commit,
override_status,
post_commit_status,
@ -247,13 +246,14 @@ def main():
flaky_check = "flaky" in check_name.lower()
run_changed_tests = flaky_check or validate_bugfix_check
gh = Github(get_best_robot_token(), per_page=100)
# For validate_bugfix_check we need up to date information about labels, so pr_event_from_api is used
pr_info = PRInfo(
need_changed_files=run_changed_tests, pr_event_from_api=validate_bugfix_check
)
# FIXME: move to job report and remove
gh = Github(get_best_robot_token(), per_page=100)
commit = get_commit(gh, pr_info.sha)
atexit.register(update_mergeable_check, commit, pr_info, check_name)
@ -279,11 +279,6 @@ def main():
run_by_hash_total = 0
check_name_with_group = check_name
rerun_helper = RerunHelper(commit, check_name_with_group)
if rerun_helper.is_already_finished_by_status():
logging.info("Check is already finished according to github status, exiting")
sys.exit(0)
tests_to_run = []
if run_changed_tests:
tests_to_run = get_tests_to_run(pr_info)

85
tests/ci/install_check.py
View File

@ -2,7 +2,6 @@
import argparse
import atexit
import logging
import sys
import subprocess
@ -10,30 +9,15 @@ from pathlib import Path
from shutil import copy2
from typing import Dict
from github import Github
from build_download_helper import download_builds_filter
from clickhouse_helper import (
ClickHouseHelper,
prepare_tests_results_for_clickhouse,
)
from commit_status_helper import (
RerunHelper,
format_description,
get_commit,
post_commit_status,
update_mergeable_check,
)
from compress_files import compress_fast
from docker_images_helper import DockerImage, pull_image, get_docker_image
from env_helper import CI, REPORT_PATH, TEMP_PATH as TEMP
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from report import TestResults, TestResult, FAILURE, FAIL, OK, SUCCESS
from s3_helper import S3Helper
from env_helper import REPORT_PATH, TEMP_PATH as TEMP
from report import JobReport, TestResults, TestResult, FAILURE, FAIL, OK, SUCCESS
from stopwatch import Stopwatch
from tee_popen import TeePopen
from upload_result_helper import upload_results
RPM_IMAGE = "clickhouse/install-rpm-test"
@ -274,20 +258,6 @@ def main():
TEMP_PATH.mkdir(parents=True, exist_ok=True)
LOGS_PATH.mkdir(parents=True, exist_ok=True)
pr_info = PRInfo()
if CI:
gh = Github(get_best_robot_token(), per_page=100)
commit = get_commit(gh, pr_info.sha)
atexit.register(update_mergeable_check, commit, pr_info, args.check_name)
rerun_helper = RerunHelper(commit, args.check_name)
if rerun_helper.is_already_finished_by_status():
logging.info(
"Check is already finished according to github status, exiting"
)
sys.exit(0)
deb_image = pull_image(get_docker_image(DEB_IMAGE))
rpm_image = pull_image(get_docker_image(RPM_IMAGE))
@ -331,54 +301,21 @@ def main():
test_results.extend(test_install_tgz(rpm_image))
state = SUCCESS
test_status = OK
description = "Packages installed successfully"
if FAIL in (result.status for result in test_results):
state = FAILURE
test_status = FAIL
description = "Failed to install packages: " + ", ".join(
result.name for result in test_results
)
s3_helper = S3Helper()
report_url = upload_results(
s3_helper,
pr_info.number,
pr_info.sha,
test_results,
[],
args.check_name,
)
print(f"::notice ::Report url: {report_url}")
if not CI:
return
ch_helper = ClickHouseHelper()
description = format_description(description)
post_commit_status(
commit,
state,
report_url,
description,
args.check_name,
pr_info,
dump_to_file=True,
)
prepared_events = prepare_tests_results_for_clickhouse(
pr_info,
test_results,
test_status,
stopwatch.duration_seconds,
stopwatch.start_time_str,
report_url,
args.check_name,
)
ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
JobReport(
description=description,
test_results=test_results,
status=state,
start_time=stopwatch.start_time_str,
duration=stopwatch.duration_seconds,
additional_files=[],
).dump()
if state == FAILURE:
sys.exit(1)

7
tests/ci/integration_test_check.py
View File

@ -13,7 +13,6 @@ from typing import Dict, List, Tuple
from build_download_helper import download_all_deb_packages
from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse
from commit_status_helper import (
RerunHelper,
get_commit,
override_status,
post_commit_status,
@ -189,14 +188,10 @@ def main():
logging.info("Skipping '%s' (no pr-bugfix in '%s')", check_name, pr_info.labels)
sys.exit(0)
# FIXME: switch to JobReport and remove:
gh = GitHub(get_best_robot_token())
commit = get_commit(gh, pr_info.sha)
rerun_helper = RerunHelper(commit, check_name_with_group)
if rerun_helper.is_already_finished_by_status():
logging.info("Check is already finished according to github status, exiting")
sys.exit(0)
images = [pull_image(get_docker_image(i)) for i in IMAGES]
result_path = temp_path / "output_dir"
result_path.mkdir(parents=True, exist_ok=True)

57
tests/ci/jepsen_check.py
View File

@ -10,26 +10,23 @@ from pathlib import Path
from typing import Any, List
import boto3 # type: ignore
from github import Github
import requests # type: ignore
from git_helper import git_runner
from build_download_helper import (
download_build_with_progress,
get_build_name_for_check,
read_build_urls,
)
from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse
from commit_status_helper import RerunHelper, get_commit, post_commit_status
from compress_files import compress_fast
from env_helper import REPO_COPY, REPORT_PATH, S3_BUILDS_BUCKET, S3_URL, TEMP_PATH
from get_robot_token import get_best_robot_token, get_parameter_from_ssm
from env_helper import REPO_COPY, REPORT_PATH, S3_URL, TEMP_PATH, S3_BUILDS_BUCKET
from get_robot_token import get_parameter_from_ssm
from git_helper import git_runner
from pr_info import PRInfo
from report import TestResults, TestResult
from s3_helper import S3Helper
from report import JobReport, TestResults, TestResult
from ssh import SSHKey
from stopwatch import Stopwatch
from tee_popen import TeePopen
from upload_result_helper import upload_results
JEPSEN_GROUP_NAME = "jepsen_group"
@ -186,16 +183,8 @@ def main():
logging.info("Not jepsen test label in labels list, skipping")
sys.exit(0)
gh = Github(get_best_robot_token(), per_page=100)
commit = get_commit(gh, pr_info.sha)
check_name = KEEPER_CHECK_NAME if args.program == "keeper" else SERVER_CHECK_NAME
rerun_helper = RerunHelper(commit, check_name)
if rerun_helper.is_already_finished_by_status():
logging.info("Check is already finished according to github status, exiting")
sys.exit(0)
if not os.path.exists(TEMP_PATH):
os.makedirs(TEMP_PATH)
@ -292,32 +281,16 @@ def main():
description = "No Jepsen output log"
test_result = [TestResult("No Jepsen output log", "FAIL")]
s3_helper = S3Helper()
report_url = upload_results(
s3_helper,
pr_info.number,
pr_info.sha,
test_result,
[run_log_path] + additional_data,
check_name,
)
JobReport(
description=description,
test_results=test_result,
status=status,
start_time=stopwatch.start_time_str,
duration=stopwatch.duration_seconds,
additional_files=[run_log_path] + additional_data,
check_name=check_name,
).dump()
print(f"::notice ::Report url: {report_url}")
post_commit_status(
commit, status, report_url, description, check_name, pr_info, dump_to_file=True
)
ch_helper = ClickHouseHelper()
prepared_events = prepare_tests_results_for_clickhouse(
pr_info,
test_result,
status,
stopwatch.duration_seconds,
stopwatch.start_time_str,
report_url,
check_name,
)
ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
clear_autoscaling_group()

22
tests/ci/libfuzzer_test_check.py
View File

@ -4,28 +4,18 @@ import argparse
import logging
import os
import sys
import atexit
import zipfile
from pathlib import Path
from typing import List
from github import Github
from build_download_helper import download_fuzzers
from clickhouse_helper import (
CiLogsCredentials,
)
from commit_status_helper import (
RerunHelper,
get_commit,
update_mergeable_check,
)
from docker_images_helper import DockerImage, pull_image, get_docker_image
from env_helper import REPORT_PATH, TEMP_PATH, REPO_COPY
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from report import TestResults
from stopwatch import Stopwatch
@ -116,28 +106,16 @@ def main():
check_name = args.check_name
kill_timeout = args.kill_timeout
gh = Github(get_best_robot_token(), per_page=100)
pr_info = PRInfo()
commit = get_commit(gh, pr_info.sha)
atexit.register(update_mergeable_check, commit, pr_info, check_name)
temp_path.mkdir(parents=True, exist_ok=True)
if "RUN_BY_HASH_NUM" in os.environ:
run_by_hash_num = int(os.getenv("RUN_BY_HASH_NUM", "0"))
run_by_hash_total = int(os.getenv("RUN_BY_HASH_TOTAL", "0"))
check_name_with_group = (
check_name + f" [{run_by_hash_num + 1}/{run_by_hash_total}]"
)
else:
run_by_hash_num = 0
run_by_hash_total = 0
check_name_with_group = check_name
rerun_helper = RerunHelper(commit, check_name_with_group)
if rerun_helper.is_already_finished_by_status():
logging.info("Check is already finished according to github status, exiting")
sys.exit(0)
docker_image = pull_image(get_docker_image("clickhouse/libfuzzer"))

87
tests/ci/performance_comparison_check.py
View File

@ -8,11 +8,10 @@ import subprocess
import traceback
import re
from pathlib import Path
from typing import Dict
from github import Github
from commit_status_helper import RerunHelper, get_commit, post_commit_status
from commit_status_helper import get_commit
from ci_config import CI_CONFIG
from docker_images_helper import pull_image, get_docker_image
from env_helper import (
@ -26,11 +25,11 @@ from env_helper import (
)
from get_robot_token import get_best_robot_token, get_parameter_from_ssm
from pr_info import PRInfo
from s3_helper import S3Helper
from tee_popen import TeePopen
from clickhouse_helper import get_instance_type, get_instance_id
from stopwatch import Stopwatch
from build_download_helper import download_builds_filter
from report import JobReport
IMAGE_NAME = "clickhouse/performance-comparison"
@ -123,23 +122,7 @@ def main():
is_aarch64 = "aarch64" in os.getenv("CHECK_NAME", "Performance Comparison").lower()
if pr_info.number != 0 and is_aarch64 and "pr-performance" not in pr_info.labels:
status = "success"
message = "Skipped, not labeled with 'pr-performance'"
report_url = GITHUB_RUN_URL
post_commit_status(
commit,
status,
report_url,
message,
check_name_with_group,
pr_info,
dump_to_file=True,
)
sys.exit(0)
rerun_helper = RerunHelper(commit, check_name_with_group)
if rerun_helper.is_already_finished_by_status():
logging.info("Check is already finished according to github status, exiting")
print("Skipped, not labeled with 'pr-performance'")
sys.exit(0)
check_name_prefix = (
@ -202,6 +185,13 @@ def main():
subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True)
def too_many_slow(msg):
match = re.search(r"(|.* )(\d+) slower.*", msg)
# This threshold should be synchronized with the value in
# https://github.com/ClickHouse/ClickHouse/blob/master/docker/test/performance-comparison/report.py#L629
threshold = 5
return int(match.group(2).strip()) > threshold if match else False
paths = {
"compare.log": compare_log_path,
"output.7z": result_path / "output.7z",
@ -212,32 +202,12 @@ def main():
"run.log": run_log_path,
}
s3_prefix = f"{pr_info.number}/{pr_info.sha}/{check_name_prefix}/"
s3_helper = S3Helper()
uploaded = {} # type: Dict[str, str]
for name, path in paths.items():
try:
uploaded[name] = s3_helper.upload_test_report_to_s3(
Path(path), s3_prefix + name
# FIXME: where images come from? dir does not exist atm.
image_files = (
list((Path(result_path) / "images").iterdir())
if (Path(result_path) / "images").exists()
else []
)
except Exception:
uploaded[name] = ""
traceback.print_exc()
# Upload all images and flamegraphs to S3
try:
s3_helper.upload_test_directory_to_s3(
Path(result_path) / "images", s3_prefix + "images"
)
except Exception:
traceback.print_exc()
def too_many_slow(msg):
match = re.search(r"(|.* )(\d+) slower.*", msg)
# This threshold should be synchronized with the value in
# https://github.com/ClickHouse/ClickHouse/blob/master/docker/test/performance-comparison/report.py#L629
threshold = 5
return int(match.group(2).strip()) > threshold if match else False
# Try to fetch status from the report.
status = ""
@ -269,24 +239,15 @@ def main():
status = "failure"
message = "No message in report."
report_url = GITHUB_RUN_URL
report_url = (
uploaded["report.html"]
or uploaded["output.7z"]
or uploaded["compare.log"]
or uploaded["run.log"]
)
post_commit_status(
commit,
status,
report_url,
message,
check_name_with_group,
pr_info,
dump_to_file=True,
)
JobReport(
description=message,
test_results=[],
status=status,
start_time=stopwatch.start_time_str,
duration=stopwatch.duration_seconds,
additional_files=[v for _, v in paths.items()] + image_files,
check_name=check_name_with_group,
).dump()
if status == "error":
sys.exit(1)

9
tests/ci/pr_info.py
View File

@ -69,11 +69,13 @@ def get_pr_for_commit(sha, ref):
if pr["head"]["ref"] in ref:
return pr
our_prs.append(pr)
print("Cannot find PR with required ref", ref, "returning first one")
print(
f"Cannot find PR with required ref {ref}, sha {sha} - returning first one"
)
first_pr = our_prs[0]
return first_pr
except Exception as ex:
print("Cannot fetch PR info from commit", ex)
print(f"Cannot fetch PR info from commit {ref}, {sha}", ex)
return None
@ -279,6 +281,9 @@ class PRInfo:
if need_changed_files:
self.fetch_changed_files()
def is_master(self) -> bool:
return self.number == 0 and self.base_ref == "master"
def is_scheduled(self):
return self.event_type == EventType.SCHEDULE

182
tests/ci/report.py
View File

@ -1,8 +1,18 @@
# -*- coding: utf-8 -*-
from ast import literal_eval
from dataclasses import dataclass
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Dict, Final, Iterable, List, Literal, Optional, Tuple
from typing import (
Dict,
Final,
Iterable,
List,
Literal,
Optional,
Sequence,
Tuple,
Union,
)
from html import escape
import csv
import datetime
@ -12,6 +22,7 @@ import os
from build_download_helper import get_gh_api
from ci_config import BuildConfig, CI_CONFIG
from env_helper import REPORT_PATH, TEMP_PATH
logger = logging.getLogger(__name__)
@ -221,6 +232,7 @@ HTML_TEST_PART = """
"""
BASE_HEADERS = ["Test name", "Test status"]
JOB_REPORT_FILE = Path(TEMP_PATH) / "job_report.json"
@dataclass
@ -229,10 +241,10 @@ class TestResult:
status: str
# the following fields are optional
time: Optional[float] = None
log_files: Optional[List[Path]] = None
log_files: Optional[Union[Sequence[str], Sequence[Path]]] = None
raw_logs: Optional[str] = None
# the field for uploaded logs URLs
log_urls: Optional[List[str]] = None
log_urls: Optional[Sequence[str]] = None
def set_raw_logs(self, raw_logs: str) -> None:
self.raw_logs = raw_logs
@ -245,9 +257,8 @@ class TestResult:
f"Malformed input: must be a list literal: {log_files_literal}"
)
for log_path in log_paths:
file = Path(log_path)
assert file.exists(), file
self.log_files.append(file)
assert Path(log_path).exists(), log_path
self.log_files.append(log_path)
@staticmethod
def create_check_timeout_expired(timeout: float) -> "TestResult":
@ -257,6 +268,53 @@ class TestResult:
TestResults = List[TestResult]
@dataclass
class JobReport:
status: str
description: str
test_results: TestResults
start_time: str
duration: float
additional_files: Union[Sequence[str], Sequence[Path]]
# clcikhouse version, build job only
version: str = ""
# checkname to set in commit status, set if differs from jjob name
check_name: str = ""
# directory with artifacts to upload on s3
build_dir_for_upload: Union[Path, str] = ""
# if False no GH commit status will be created by CI
need_commit_status: bool = True
@classmethod
def exist(cls) -> bool:
return JOB_REPORT_FILE.is_file()
@classmethod
def load(cls): # type: ignore
res = {}
with open(JOB_REPORT_FILE, "r") as json_file:
res = json.load(json_file)
# Deserialize the nested lists of TestResult
test_results_data = res.get("test_results", [])
test_results = [TestResult(**result) for result in test_results_data]
del res["test_results"]
return JobReport(test_results=test_results, **res)
@classmethod
def cleanup(cls):
if JOB_REPORT_FILE.exists():
JOB_REPORT_FILE.unlink()
def dump(self):
def path_converter(obj):
if isinstance(obj, Path):
return str(obj)
raise TypeError("Type not serializable")
with open(JOB_REPORT_FILE, "w") as json_file:
json.dump(asdict(self), json_file, default=path_converter, indent=2)
def read_test_results(results_path: Path, with_raw_logs: bool = True) -> TestResults:
results = [] # type: TestResults
with open(results_path, "r", encoding="utf-8") as descriptor:
@ -296,14 +354,72 @@ class BuildResult:
log_url: str
build_urls: List[str]
version: str
status: StatusType
status: str
elapsed_seconds: int
job_api_url: str
pr_number: int = 0
head_ref: str = "dummy_branch_name"
_job_name: Optional[str] = None
_job_html_url: Optional[str] = None
_job_html_link: Optional[str] = None
_grouped_urls: Optional[List[List[str]]] = None
@classmethod
def cleanup(cls):
if Path(REPORT_PATH).exists():
for file in Path(REPORT_PATH).iterdir():
if "build_report" in file.name and file.name.endswith(".json"):
file.unlink()
@classmethod
def load(cls, build_name: str, pr_number: int, head_ref: str): # type: ignore
"""
loads report from a report file matched with given @pr_number and/or a @head_ref
"""
report_path = Path(REPORT_PATH) / BuildResult.get_report_name(
build_name, pr_number or head_ref
)
return cls.load_from_file(report_path)
@classmethod
def load_any(cls, build_name: str, pr_number: int, head_ref: str): # type: ignore
"""
loads report from suitable report file with the following priority:
1. report from PR with the same @pr_number
2. report from branch with the same @head_ref
3. report from the master
4. any other report
"""
reports = []
for file in Path(REPORT_PATH).iterdir():
if f"{build_name}.json" in file.name:
reports.append(file)
if not reports:
return None
file_path = None
for file in reports:
if pr_number and f"_{pr_number}_" in file.name:
file_path = file
break
if f"_{head_ref}_" in file.name:
file_path = file
break
if "_master_" in file.name:
file_path = file
break
return cls.load_from_file(file_path or reports[-1])
@classmethod
def load_from_file(cls, file: Union[Path, str]): # type: ignore
if not Path(file).exists():
return None
with open(file, "r") as json_file:
res = json.load(json_file)
return BuildResult(**res)
def as_json(self) -> str:
return json.dumps(asdict(self), indent=2)
@property
def build_config(self) -> Optional[BuildConfig]:
return CI_CONFIG.build_config.get(self.build_name, None)
@ -373,10 +489,6 @@ class BuildResult:
def _wrong_config_message(self) -> str:
return "missing"
@property
def file_name(self) -> Path:
return self.get_report_name(self.build_name)
@property
def is_missing(self) -> bool:
"The report is created for missing json file"
@ -427,37 +539,18 @@ class BuildResult:
self._job_html_url = job_data.get("html_url", "")
@staticmethod
def get_report_name(name: str) -> Path:
return Path(f"build_report_{name}.json")
@staticmethod
def read_json(directory: Path, build_name: str) -> "BuildResult":
path = directory / BuildResult.get_report_name(build_name)
try:
with open(path, "r", encoding="utf-8") as pf:
data = json.load(pf) # type: dict
except FileNotFoundError:
logger.warning(
"File %s for build named '%s' is not found", path, build_name
)
return BuildResult.missing_result(build_name)
return BuildResult(
data.get("build_name", build_name),
data.get("log_url", ""),
data.get("build_urls", []),
data.get("version", ""),
data.get("status", ERROR),
data.get("elapsed_seconds", 0),
data.get("job_api_url", ""),
)
def get_report_name(name: str, suffix: Union[str, int]) -> Path:
assert "/" not in str(suffix)
return Path(f"build_report_{suffix}_{name}.json")
@staticmethod
def missing_result(build_name: str) -> "BuildResult":
return BuildResult(build_name, "", [], "missing", ERROR, 0, "missing")
def write_json(self, directory: Path) -> Path:
path = directory / self.file_name
def write_json(self, directory: Union[Path, str] = REPORT_PATH) -> Path:
path = Path(directory) / self.get_report_name(
self.build_name, self.pr_number or self.head_ref
)
path.write_text(
json.dumps(
{
@ -468,6 +561,8 @@ class BuildResult:
"status": self.status,
"elapsed_seconds": self.elapsed_seconds,
"job_api_url": self.job_api_url,
"pr_number": self.pr_number,
"head_ref": self.head_ref,
}
),
encoding="utf-8",
@ -532,10 +627,17 @@ def _get_status_style(status: str, colortheme: Optional[ColorTheme] = None) -> s
def _get_html_url_name(url):
base_name = ""
if isinstance(url, str):
return os.path.basename(url).replace("%2B", "+").replace("%20", " ")
base_name = os.path.basename(url)
if isinstance(url, tuple):
return url[1].replace("%2B", "+").replace("%20", " ")
base_name = url[1]
if "?" in base_name:
base_name = base_name.split("?")[0]
if base_name is not None:
return base_name.replace("%2B", "+").replace("%20", " ")
return None

65
tests/ci/sqlancer_check.py
View File

@ -6,29 +6,15 @@ import subprocess
import sys
from pathlib import Path
from github import Github
from build_download_helper import get_build_name_for_check, read_build_urls
from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse
from commit_status_helper import (
RerunHelper,
format_description,
get_commit,
post_commit_status,
)
from docker_images_helper import DockerImage, pull_image, get_docker_image
from env_helper import (
GITHUB_RUN_URL,
REPORT_PATH,
TEMP_PATH,
)
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from report import TestResults, TestResult
from s3_helper import S3Helper
from report import JobReport, TestResults, TestResult
from stopwatch import Stopwatch
from tee_popen import TeePopen
from upload_result_helper import upload_results
IMAGE_NAME = "clickhouse/sqlancer-test"
@ -58,16 +44,6 @@ def main():
check_name
), "Check name must be provided as an input arg or in CHECK_NAME env"
pr_info = PRInfo()
gh = Github(get_best_robot_token(), per_page=100)
commit = get_commit(gh, pr_info.sha)
rerun_helper = RerunHelper(commit, check_name)
if rerun_helper.is_already_finished_by_status():
logging.info("Check is already finished according to github status, exiting")
sys.exit(0)
docker_image = pull_image(get_docker_image(IMAGE_NAME))
build_name = get_build_name_for_check(check_name)
@ -118,9 +94,6 @@ def main():
paths += [workspace_path / f"{t}.err" for t in tests]
paths += [workspace_path / f"{t}.out" for t in tests]
s3_helper = S3Helper()
report_url = GITHUB_RUN_URL
status = "success"
test_results = [] # type: TestResults
# Try to get status message saved by the SQLancer
@ -139,33 +112,17 @@ def main():
status = "failure"
description = "Task failed: $?=" + str(retcode)
description = format_description(description)
if not test_results:
test_results = [TestResult(name=__file__, status=status)]
report_url = upload_results(
s3_helper,
pr_info.number,
pr_info.sha,
test_results,
paths,
check_name,
)
post_commit_status(
commit, status, report_url, description, check_name, pr_info, dump_to_file=True
)
print(f"::notice:: {check_name} Report url: {report_url}")
ch_helper = ClickHouseHelper()
prepared_events = prepare_tests_results_for_clickhouse(
pr_info,
test_results,
status,
stopwatch.duration_seconds,
stopwatch.start_time_str,
report_url,
check_name,
)
ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
JobReport(
description=description,
test_results=test_results,
status=status,
start_time=stopwatch.start_time_str,
duration=stopwatch.duration_seconds,
additional_files=paths,
).dump()
if __name__ == "__main__":

66
tests/ci/sqllogic_test.py
View File

@ -5,28 +5,25 @@ import csv
import logging
import os
import subprocess
import sys
from pathlib import Path
from typing import List, Tuple
from github import Github
from typing import Tuple
from build_download_helper import download_all_deb_packages
from commit_status_helper import (
RerunHelper,
get_commit,
override_status,
post_commit_status,
)
from commit_status_helper import override_status
from docker_images_helper import DockerImage, pull_image, get_docker_image
from env_helper import REPORT_PATH, TEMP_PATH, REPO_COPY
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from report import OK, FAIL, ERROR, SUCCESS, TestResults, TestResult, read_test_results
from s3_helper import S3Helper
from report import (
OK,
FAIL,
ERROR,
SUCCESS,
JobReport,
TestResults,
TestResult,
read_test_results,
)
from stopwatch import Stopwatch
from tee_popen import TeePopen
from upload_result_helper import upload_results
NO_CHANGES_MSG = "Nothing to run"
@ -104,15 +101,6 @@ def main():
kill_timeout > 0
), "kill timeout must be provided as an input arg or in KILL_TIMEOUT env"
pr_info = PRInfo()
gh = Github(get_best_robot_token(), per_page=100)
commit = get_commit(gh, pr_info.sha)
rerun_helper = RerunHelper(commit, check_name)
if rerun_helper.is_already_finished_by_status():
logging.info("Check is already finished according to github status, exiting")
sys.exit(0)
docker_image = pull_image(get_docker_image(IMAGE_NAME))
repo_tests_path = repo_path / "tests"
@ -150,8 +138,6 @@ def main():
logging.info("Files in result folder %s", os.listdir(result_path))
s3_helper = S3Helper()
status = None
description = None
@ -186,29 +172,19 @@ def main():
)
)
report_url = upload_results(
s3_helper,
pr_info.number,
pr_info.sha,
test_results,
additional_logs,
check_name,
)
print(
f"::notice:: {check_name}"
f", Result: '{status}'"
f", Description: '{description}'"
f", Report url: '{report_url}'"
)
# Until it pass all tests, do not block CI, report "success"
assert description is not None
# FIXME: force SUCCESS until all cases are fixed
status = SUCCESS
post_commit_status(
commit, status, report_url, description, check_name, pr_info, dump_to_file=True
)
JobReport(
description=description,
test_results=test_results,
status=status,
start_time=stopwatch.start_time_str,
duration=stopwatch.duration_seconds,
additional_files=additional_logs,
).dump()
if __name__ == "__main__":

67
tests/ci/sqltest.py
View File

@ -7,25 +7,15 @@ import sys
from pathlib import Path
from typing import Dict
from github import Github
from build_download_helper import get_build_name_for_check, read_build_urls
from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse
from commit_status_helper import (
RerunHelper,
get_commit,
post_commit_status,
)
from docker_images_helper import pull_image, get_docker_image
from env_helper import (
GITHUB_RUN_URL,
REPORT_PATH,
TEMP_PATH,
)
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from report import TestResult
from s3_helper import S3Helper
from report import JobReport, TestResult
from stopwatch import Stopwatch
IMAGE_NAME = "clickhouse/sqltest"
@ -62,14 +52,6 @@ def main():
pr_info = PRInfo()
gh = Github(get_best_robot_token(), per_page=100)
commit = get_commit(gh, pr_info.sha)
rerun_helper = RerunHelper(commit, check_name)
if rerun_helper.is_already_finished_by_status():
logging.info("Check is already finished according to github status, exiting")
sys.exit(0)
docker_image = pull_image(get_docker_image(IMAGE_NAME))
build_name = get_build_name_for_check(check_name)
@ -109,10 +91,6 @@ def main():
subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True)
check_name_lower = (
check_name.lower().replace("(", "").replace(")", "").replace(" ", "")
)
s3_prefix = f"{pr_info.number}/{pr_info.sha}/sqltest_{check_name_lower}/"
paths = {
"run.log": run_log_path,
"server.log.zst": workspace_path / "server.log.zst",
@ -120,43 +98,18 @@ def main():
"report.html": workspace_path / "report.html",
"test.log": workspace_path / "test.log",
}
path_urls = {} # type: Dict[str, str]
s3_helper = S3Helper()
for f in paths:
try:
path_urls[f] = s3_helper.upload_test_report_to_s3(paths[f], s3_prefix + f)
except Exception as ex:
logging.info("Exception uploading file %s text %s", f, ex)
path_urls[f] = ""
report_url = GITHUB_RUN_URL
if path_urls["report.html"]:
report_url = path_urls["report.html"]
status = "success"
description = "See the report"
test_result = TestResult(description, "OK")
test_results = [TestResult(description, "OK")]
ch_helper = ClickHouseHelper()
prepared_events = prepare_tests_results_for_clickhouse(
pr_info,
[test_result],
status,
stopwatch.duration_seconds,
stopwatch.start_time_str,
report_url,
check_name,
)
ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
logging.info("Result: '%s', '%s', '%s'", status, description, report_url)
print(f"::notice ::Report url: {report_url}")
post_commit_status(
commit, status, report_url, description, check_name, pr_info, dump_to_file=True
)
JobReport(
description=description,
test_results=test_results,
status=status,
start_time=stopwatch.start_time_str,
duration=stopwatch.duration_seconds,
additional_files=[v for _, v in paths.items()],
).dump()
if __name__ == "__main__":

65
tests/ci/stress_check.py
View File

@ -8,29 +8,15 @@ import sys
from pathlib import Path
from typing import List, Tuple
from github import Github
from build_download_helper import download_all_deb_packages
from clickhouse_helper import (
CiLogsCredentials,
ClickHouseHelper,
prepare_tests_results_for_clickhouse,
)
from commit_status_helper import (
RerunHelper,
get_commit,
post_commit_status,
format_description,
)
from clickhouse_helper import CiLogsCredentials
from docker_images_helper import DockerImage, pull_image, get_docker_image
from env_helper import REPORT_PATH, TEMP_PATH, REPO_COPY
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from report import TestResult, TestResults, read_test_results
from s3_helper import S3Helper
from report import JobReport, TestResult, TestResults, read_test_results
from stopwatch import Stopwatch
from tee_popen import TeePopen
from upload_result_helper import upload_results
def get_additional_envs() -> List[str]:
@ -139,14 +125,6 @@ def run_stress_test(docker_image_name: str) -> None:
pr_info = PRInfo()
gh = Github(get_best_robot_token(), per_page=100)
commit = get_commit(gh, pr_info.sha)
rerun_helper = RerunHelper(commit, check_name)
if rerun_helper.is_already_finished_by_status():
logging.info("Check is already finished according to github status, exiting")
sys.exit(0)
docker_image = pull_image(get_docker_image(docker_image_name))
packages_path = temp_path / "packages"
@ -194,7 +172,6 @@ def run_stress_test(docker_image_name: str) -> None:
subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True)
ci_logs_credentials.clean_ci_logs_from_credentials(run_log_path)
s3_helper = S3Helper()
state, description, test_results, additional_logs = process_results(
result_path, server_log_path, run_log_path
)
@ -202,34 +179,16 @@ def run_stress_test(docker_image_name: str) -> None:
if timeout_expired:
test_results.append(TestResult.create_check_timeout_expired(timeout))
state = "failure"
description = format_description(test_results[-1].name)
description = test_results[-1].name
ch_helper = ClickHouseHelper()
report_url = upload_results(
s3_helper,
pr_info.number,
pr_info.sha,
test_results,
additional_logs,
check_name,
)
print(f"::notice ::Report url: {report_url}")
post_commit_status(
commit, state, report_url, description, check_name, pr_info, dump_to_file=True
)
prepared_events = prepare_tests_results_for_clickhouse(
pr_info,
test_results,
state,
stopwatch.duration_seconds,
stopwatch.start_time_str,
report_url,
check_name,
)
ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
JobReport(
description=description,
test_results=test_results,
status=state,
start_time=stopwatch.start_time_str,
duration=stopwatch.duration_seconds,
additional_files=additional_logs,
).dump()
if state == "failure":
sys.exit(1)

58
tests/ci/style_check.py
View File

@ -1,6 +1,5 @@
#!/usr/bin/env python3
import argparse
import atexit
import csv
import logging
import os
@ -9,24 +8,14 @@ import sys
from pathlib import Path
from typing import List, Tuple
from clickhouse_helper import ClickHouseHelper, prepare_tests_results_for_clickhouse
from commit_status_helper import (
RerunHelper,
get_commit,
post_commit_status,
update_mergeable_check,
)
from docker_images_helper import get_docker_image, pull_image
from env_helper import REPO_COPY, TEMP_PATH
from get_robot_token import get_best_robot_token
from git_helper import GIT_PREFIX, git_runner
from github_helper import GitHub
from pr_info import PRInfo
from report import TestResults, read_test_results
from s3_helper import S3Helper
from report import JobReport, TestResults, read_test_results
from ssh import SSHKey
from stopwatch import Stopwatch
from upload_result_helper import upload_results
NAME = "Style Check"
@ -142,21 +131,6 @@ def main():
temp_path.mkdir(parents=True, exist_ok=True)
pr_info = PRInfo()
gh = GitHub(get_best_robot_token(), create_cache_dir=False)
commit = get_commit(gh, pr_info.sha)
atexit.register(update_mergeable_check, commit, pr_info, NAME)
rerun_helper = RerunHelper(commit, NAME)
if rerun_helper.is_already_finished_by_status():
logging.info("Check is already finished according to github status, exiting")
# Finish with the same code as previous
state = rerun_helper.get_finished_status().state # type: ignore
# state == "success" -> code = 0
code = int(state != "success")
sys.exit(code)
s3_helper = S3Helper()
IMAGE_NAME = "clickhouse/style-test"
image = pull_image(get_docker_image(IMAGE_NAME))
@ -180,28 +154,18 @@ def main():
checkout_last_ref(pr_info)
state, description, test_results, additional_files = process_result(temp_path)
ch_helper = ClickHouseHelper()
report_url = upload_results(
s3_helper, pr_info.number, pr_info.sha, test_results, additional_files, NAME
)
print(f"::notice ::Report url: {report_url}")
post_commit_status(
commit, state, report_url, description, NAME, pr_info, dump_to_file=True
)
prepared_events = prepare_tests_results_for_clickhouse(
pr_info,
test_results,
state,
stopwatch.duration_seconds,
stopwatch.start_time_str,
report_url,
NAME,
)
ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
JobReport(
description=description,
test_results=test_results,
status=state,
start_time=stopwatch.start_time_str,
duration=stopwatch.duration_seconds,
additional_files=additional_files,
).dump()
if state in ["error", "failure"]:
print(f"Style check failed: [{description}]")
sys.exit(1)

3
tests/ci/test_digest.py
View File

@ -5,6 +5,7 @@ from hashlib import md5
from pathlib import Path
import digest_helper as dh
from env_helper import ROOT_DIR
_12 = b"12\n"
_13 = b"13\n"
@ -13,7 +14,7 @@ _14 = b"14\n"
# pylint:disable=protected-access
class TestDigests(unittest.TestCase):
tests_dir = Path("tests/digests")
tests_dir = Path(ROOT_DIR) / "tests/ci/tests/digests"
broken_link = tests_dir / "broken-symlink"
empty_digest = "d41d8cd98f00b204e9800998ecf8427e"

73
tests/ci/unit_tests_check.py
View File

@ -5,33 +5,15 @@ import logging
import os
import sys
import subprocess
import atexit
from pathlib import Path
from typing import List, Tuple
from github import Github
from typing import Tuple
from build_download_helper import download_unit_tests
from clickhouse_helper import (
ClickHouseHelper,
prepare_tests_results_for_clickhouse,
)
from commit_status_helper import (
RerunHelper,
get_commit,
post_commit_status,
update_mergeable_check,
)
from docker_images_helper import pull_image, get_docker_image
from env_helper import REPORT_PATH, TEMP_PATH
from get_robot_token import get_best_robot_token
from pr_info import PRInfo
from report import ERROR, FAILURE, FAIL, OK, SUCCESS, TestResults, TestResult
from s3_helper import S3Helper
from report import ERROR, FAILURE, FAIL, OK, SUCCESS, JobReport, TestResults, TestResult
from stopwatch import Stopwatch
from tee_popen import TeePopen
from upload_result_helper import upload_results
IMAGE_NAME = "clickhouse/unit-test"
@ -182,18 +164,6 @@ def main():
temp_path = Path(TEMP_PATH)
temp_path.mkdir(parents=True, exist_ok=True)
pr_info = PRInfo()
gh = Github(get_best_robot_token(), per_page=100)
commit = get_commit(gh, pr_info.sha)
atexit.register(update_mergeable_check, commit, pr_info, check_name)
rerun_helper = RerunHelper(commit, check_name)
if rerun_helper.is_already_finished_by_status():
logging.info("Check is already finished according to github status, exiting")
sys.exit(0)
docker_image = pull_image(get_docker_image(IMAGE_NAME))
download_unit_tests(check_name, REPORT_PATH, TEMP_PATH)
@ -222,35 +192,18 @@ def main():
subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {TEMP_PATH}", shell=True)
s3_helper = S3Helper()
state, description, test_results = process_results(test_output)
ch_helper = ClickHouseHelper()
report_url = upload_results(
s3_helper,
pr_info.number,
pr_info.sha,
test_results,
[run_log_path] + [p for p in test_output.iterdir() if not p.is_dir()],
check_name,
)
print(f"::notice ::Report url: {report_url}")
post_commit_status(
commit, state, report_url, description, check_name, pr_info, dump_to_file=True
)
prepared_events = prepare_tests_results_for_clickhouse(
pr_info,
test_results,
state,
stopwatch.duration_seconds,
stopwatch.start_time_str,
report_url,
check_name,
)
ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)
additional_files = [run_log_path] + [
p for p in test_output.iterdir() if not p.is_dir()
]
JobReport(
description=description,
test_results=test_results,
status=state,
start_time=stopwatch.start_time_str,
duration=stopwatch.duration_seconds,
additional_files=additional_files,
).dump()
if state == "failure":
sys.exit(1)

40
tests/ci/upload_result_helper.py
View File

@ -1,5 +1,5 @@
from pathlib import Path
from typing import Dict, List, Optional
from typing import Dict, List, Optional, Sequence, Union
import os
import logging
@ -15,13 +15,13 @@ from s3_helper import S3Helper
def process_logs(
s3_client: S3Helper,
additional_logs: List[Path],
additional_logs: Union[Sequence[str], Sequence[Path]],
s3_path_prefix: str,
test_results: TestResults,
) -> List[str]:
logging.info("Upload files to s3 %s", additional_logs)
processed_logs = {} # type: Dict[Path, str]
processed_logs = {} # type: Dict[str, str]
# Firstly convert paths of logs from test_results to urls to s3.
for test_result in test_results:
if test_result.log_files is None:
@ -31,22 +31,24 @@ def process_logs(
test_result.log_urls = []
for path in test_result.log_files:
if path in processed_logs:
test_result.log_urls.append(processed_logs[path])
test_result.log_urls.append(processed_logs[str(path)])
elif path:
url = s3_client.upload_test_report_to_s3(
path, s3_path_prefix + "/" + path.name
Path(path), s3_path_prefix + "/" + str(path)
)
test_result.log_urls.append(url)
processed_logs[path] = url
processed_logs[str(path)] = url
additional_urls = []
for log_path in additional_logs:
if log_path.is_file():
if Path(log_path).is_file():
additional_urls.append(
s3_client.upload_test_report_to_s3(
log_path, s3_path_prefix + "/" + os.path.basename(log_path)
Path(log_path), s3_path_prefix + "/" + os.path.basename(log_path)
)
)
else:
logging.error("File %s is missing - skip", log_path)
return additional_urls
@ -56,7 +58,7 @@ def upload_results(
pr_number: int,
commit_sha: str,
test_results: TestResults,
additional_files: List[Path],
additional_files: Union[Sequence[Path], Sequence[str]],
check_name: str,
additional_urls: Optional[List[str]] = None,
) -> str:
@ -65,12 +67,11 @@ def upload_results(
normalized_check_name = normalized_check_name.replace(*r)
# Preserve additional_urls to not modify the original one
original_additional_urls = additional_urls or []
additional_urls = additional_urls or []
s3_path_prefix = f"{pr_number}/{commit_sha}/{normalized_check_name}"
additional_urls = process_logs(
s3_client, additional_files, s3_path_prefix, test_results
additional_urls.extend(
process_logs(s3_client, additional_files, s3_path_prefix, test_results)
)
additional_urls.extend(original_additional_urls)
branch_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/commits/master"
branch_name = "master"
@ -79,6 +80,13 @@ def upload_results(
branch_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/pull/{pr_number}"
commit_url = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/commit/{commit_sha}"
ready_report_url = None
for url in additional_urls:
if "report.html" in url:
ready_report_url = url
additional_urls.remove(ready_report_url)
break
if additional_urls:
raw_log_url = additional_urls.pop(0)
else:
@ -88,6 +96,7 @@ def upload_results(
ReportColorTheme.bugfixcheck if "bugfix validate check" in check_name else None
)
if test_results or not ready_report_url:
html_report = create_test_html_report(
check_name,
test_results,
@ -102,7 +111,10 @@ def upload_results(
)
report_path = Path("report.html")
report_path.write_text(html_report, encoding="utf-8")
url = s3_client.upload_test_report_to_s3(report_path, s3_path_prefix + ".html")
else:
logging.info("report.html was prepared by test job itself")
url = ready_report_url
logging.info("Search result in url %s", url)
return url