ClickHouse/.github/workflows/master.yml

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

176 lines
6.8 KiB
YAML
Raw Normal View History

# yamllint disable rule:comments-indentation
2021-11-22 11:54:02 +00:00
name: MasterCI
env:
# Force the stdout and stderr streams to be unbuffered
PYTHONUNBUFFERED: 1
2021-11-22 11:54:02 +00:00
on: # yamllint disable-line rule:truthy
push:
branches:
- 'master'
jobs:
RunConfig:
runs-on: [self-hosted, style-checker-aarch64]
outputs:
data: ${{ steps.runconfig.outputs.CI_DATA }}
2022-01-18 21:33:53 +00:00
steps:
- name: DebugInfo
2024-04-29 10:27:40 +00:00
uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
2022-01-18 21:33:53 +00:00
- name: Check out repository code
2022-12-23 16:54:19 +00:00
uses: ClickHouse/checkout@v1
with:
clear-repository: true # to ensure correct digests
fetch-depth: 0 # to get version
filter: tree:0
2024-04-11 11:58:44 +00:00
- name: Merge sync PR
2024-04-10 13:17:39 +00:00
run: |
cd "$GITHUB_WORKSPACE/tests/ci"
2024-04-11 11:58:44 +00:00
python3 sync_pr.py --merge || :
# Runs in MQ:
# - name: Python unit tests
# run: |
# cd "$GITHUB_WORKSPACE/tests/ci"
# echo "Testing the main ci directory"
# python3 -m unittest discover -s . -p 'test_*.py'
# for dir in *_lambda/; do
# echo "Testing $dir"
# python3 -m unittest discover -s "$dir" -p 'test_*.py'
# done
- name: PrepareRunConfig
id: runconfig
run: |
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --outfile ${{ runner.temp }}/ci_run_data.json
echo "::group::CI configuration"
python3 -m json.tool ${{ runner.temp }}/ci_run_data.json
echo "::endgroup::"
{
echo 'CI_DATA<<EOF'
cat ${{ runner.temp }}/ci_run_data.json
echo 'EOF'
} >> "$GITHUB_OUTPUT"
- name: Re-create GH statuses for skipped jobs if any
run: |
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ runner.temp }}/ci_run_data.json --update-gh-statuses
2024-05-24 08:53:19 +00:00
# Runs in MQ:
# BuildDockers:
# needs: [RunConfig]
# if: ${{ !failure() && !cancelled() }}
# uses: ./.github/workflows/reusable_docker.yml
# with:
# data: ${{ needs.RunConfig.outputs.data }}
# StyleCheck:
# needs: [RunConfig, BuildDockers]
# if: ${{ !failure() && !cancelled() }}
# uses: ./.github/workflows/reusable_test.yml
# with:
# test_name: Style check
# runner_type: style-checker
# data: ${{ needs.RunConfig.outputs.data }}
# run_command: |
# python3 style_check.py --no-push
2024-05-24 08:53:19 +00:00
################################# Main stages #################################
# for main CI chain
#
Builds_1:
needs: [RunConfig]
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_1') }}
# using callable wf (reusable_stage.yml) allows grouping all nested jobs under a tab
uses: ./.github/workflows/reusable_build_stage.yml
with:
2024-05-24 08:53:19 +00:00
stage: Builds_1
data: ${{ needs.RunConfig.outputs.data }}
2024-05-24 08:53:19 +00:00
Tests_1:
needs: [RunConfig, Builds_1]
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_1') }}
uses: ./.github/workflows/reusable_test_stage.yml
2024-05-14 08:03:32 +00:00
with:
2024-05-24 08:53:19 +00:00
stage: Tests_1
2024-05-14 08:03:32 +00:00
data: ${{ needs.RunConfig.outputs.data }}
2024-05-24 08:53:19 +00:00
Builds_2:
needs: [RunConfig, Builds_1]
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_2') }}
uses: ./.github/workflows/reusable_build_stage.yml
with:
2024-05-24 08:53:19 +00:00
stage: Builds_2
data: ${{ needs.RunConfig.outputs.data }}
2024-05-24 08:53:19 +00:00
Tests_2:
needs: [RunConfig, Builds_2]
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_2') }}
uses: ./.github/workflows/reusable_test_stage.yml
with:
2024-05-24 08:53:19 +00:00
stage: Tests_2
data: ${{ needs.RunConfig.outputs.data }}
2024-05-24 08:53:19 +00:00
# stage for jobs that do not prohibit merge
Tests_3:
needs: [RunConfig, Builds_1]
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_3') }}
uses: ./.github/workflows/reusable_test_stage.yml
with:
2024-05-24 08:53:19 +00:00
stage: Tests_3
data: ${{ needs.RunConfig.outputs.data }}
2024-05-24 08:53:19 +00:00
################################# Reports #################################
# Reports should be run even if Builds_1/2 failed - put them separately in wf (not in Tests_1/2)
Builds_1_Report:
# run report check for failed builds to indicate the CI error
2024-05-24 08:53:19 +00:00
if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse build check') }}
needs: [RunConfig, Builds_1]
uses: ./.github/workflows/reusable_test.yml
with:
test_name: ClickHouse build check
runner_type: style-checker-aarch64
data: ${{ needs.RunConfig.outputs.data }}
2024-05-24 08:53:19 +00:00
Builds_2_Report:
# run report check for failed builds to indicate the CI error
2024-05-24 08:53:19 +00:00
if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse special build check') }}
needs: [RunConfig, Builds_2]
uses: ./.github/workflows/reusable_test.yml
with:
test_name: ClickHouse special build check
runner_type: style-checker-aarch64
data: ${{ needs.RunConfig.outputs.data }}
2024-05-24 08:53:19 +00:00
MarkReleaseReady:
if: ${{ !failure() && !cancelled() }}
2024-05-24 14:59:47 +00:00
needs: [RunConfig, Builds_1, Builds_2]
2024-05-24 08:53:19 +00:00
runs-on: [self-hosted, style-checker-aarch64]
steps:
- name: Debug
run: |
echo need with different filters
cat << 'EOF'
${{ toJSON(needs) }}
${{ toJSON(needs.*.result) }}
no failures ${{ !contains(needs.*.result, 'failure') }}
no skips ${{ !contains(needs.*.result, 'skipped') }}
no both ${{ !(contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
EOF
- name: Not ready
2024-05-24 08:53:19 +00:00
# fail the job to be able to restart it
if: ${{ contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure') }}
run: exit 1
- name: Check out repository code
if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
uses: ClickHouse/checkout@v1
- name: Mark Commit Release Ready
if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
run: |
cd "$GITHUB_WORKSPACE/tests/ci"
python3 mark_release_ready.py
2021-11-22 11:54:02 +00:00
FinishCheck:
if: ${{ !cancelled() }}
2024-05-24 08:53:19 +00:00
needs: [RunConfig, Builds_1, Builds_2, Builds_1_Report, Builds_2_Report, Tests_1, Tests_2, Tests_3]
runs-on: [self-hosted, style-checker-aarch64]
2021-11-22 11:54:02 +00:00
steps:
- name: Check out repository code
2022-12-23 16:54:19 +00:00
uses: ClickHouse/checkout@v1
2021-11-22 11:54:02 +00:00
- name: Finish label
run: |
cd "$GITHUB_WORKSPACE/tests/ci"
2021-11-22 11:54:02 +00:00
python3 finish_check.py