ClickHouse/.github/workflows/master.yml

# yamllint disable rule:comments-indentation
name: MasterCI

env:
  # Force the stdout and stderr streams to be unbuffered
  PYTHONUNBUFFERED: 1

on: # yamllint disable-line rule:truthy
  push:
    branches:
      - 'master'
jobs:
  RunConfig:
    runs-on: [self-hosted, style-checker-aarch64]
    outputs:
      data: ${{ steps.runconfig.outputs.CI_DATA }}
    steps:
      - name: DebugInfo
        uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
      - name: Check out repository code
        uses: ClickHouse/checkout@v1
        with:
          clear-repository: true # to ensure correct digests
          fetch-depth: 0 # to get version
          filter: tree:0
      - name: Merge sync PR
        run: |
          cd "$GITHUB_WORKSPACE/tests/ci"
          python3 sync_pr.py --merge || :          
# Runs in MQ:
#      - name: Python unit tests
#        run: |
#          cd "$GITHUB_WORKSPACE/tests/ci"
#          echo "Testing the main ci directory"
#          python3 -m unittest discover -s . -p 'test_*.py'
#          for dir in *_lambda/; do
#            echo "Testing $dir"
#            python3 -m unittest discover -s "$dir" -p 'test_*.py'
#          done
      - name: PrepareRunConfig
        id: runconfig
        run: |
            python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --outfile ${{ runner.temp }}/ci_run_data.json

            echo "::group::CI configuration"
            python3 -m json.tool ${{ runner.temp }}/ci_run_data.json
            echo "::endgroup::"

            {
              echo 'CI_DATA<<EOF'
              cat  ${{ runner.temp }}/ci_run_data.json
              echo 'EOF'
            } >> "$GITHUB_OUTPUT"            
      - name: Re-create GH statuses for skipped jobs if any
        run: |
            python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ runner.temp }}/ci_run_data.json --update-gh-statuses            
# Runs in MQ:
#  BuildDockers:
#    needs: [RunConfig]
#    if: ${{ !failure() && !cancelled() }}
#    uses: ./.github/workflows/docker_test_images.yml
#    with:
#      data: ${{ needs.RunConfig.outputs.data }}
  # StyleCheck:
  #   needs: [RunConfig, BuildDockers]
  #   if: ${{ !failure() && !cancelled() }}
  #   uses: ./.github/workflows/reusable_test.yml
  #   with:
  #     test_name: Style check
  #     runner_type: style-checker
  #     data: ${{ needs.RunConfig.outputs.data }}
  #     run_command: |
  #         python3 style_check.py --no-push

  ################################# Main stages #################################
  # for main CI chain
  #
  Builds_1:
    needs: [RunConfig]
    if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_1') }}
    # using callable wf (reusable_stage.yml) allows grouping all nested jobs under a tab
    uses: ./.github/workflows/reusable_build_stage.yml
    with:
      stage: Builds_1
      data: ${{ needs.RunConfig.outputs.data }}
  Tests_1:
    needs: [RunConfig, Builds_1]
    if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_1') }}
    uses: ./.github/workflows/reusable_test_stage.yml
    with:
      stage: Tests_1
      data: ${{ needs.RunConfig.outputs.data }}
  Builds_2:
    needs: [RunConfig, Builds_1]
    if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_2') }}
    uses: ./.github/workflows/reusable_build_stage.yml
    with:
      stage: Builds_2
      data: ${{ needs.RunConfig.outputs.data }}
  Tests_2:
    needs: [RunConfig, Builds_2]
    if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_2') }}
    uses: ./.github/workflows/reusable_test_stage.yml
    with:
      stage: Tests_2
      data: ${{ needs.RunConfig.outputs.data }}
  Tests_3:
    # Test_3 should not wait for Test_1/Test_2 and should not be blocked by them on master branch since all jobs need to run there.
    needs: [RunConfig, Builds_1]
    if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_3') }}
    uses: ./.github/workflows/reusable_test_stage.yml
    with:
      stage: Tests_3
      data: ${{ needs.RunConfig.outputs.data }}

  ################################# Reports #################################
  # Reports should run even if Builds_1/2 fail - run them separately, not in Tests_1/2/3
  Builds_Report:
    # run report check for failed builds to indicate the CI error
    if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Builds') }}
    needs: [RunConfig, Builds_1, Builds_2]
    uses: ./.github/workflows/reusable_test.yml
    with:
      test_name: Builds
      runner_type: style-checker-aarch64
      data: ${{ needs.RunConfig.outputs.data }}

  MarkReleaseReady:
    if: ${{ !failure() && !cancelled() }}
    needs: [RunConfig, Builds_1, Builds_2]
    runs-on: [self-hosted, style-checker-aarch64]
    steps:
      - name: Debug
        run: |
          echo need with different filters
          cat << 'EOF'
          ${{ toJSON(needs) }}
          ${{ toJSON(needs.*.result) }}
          no failures ${{ !contains(needs.*.result, 'failure') }}
          no skips ${{ !contains(needs.*.result, 'skipped') }}
          no both ${{ !(contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
          EOF          
      - name: Not ready
        # fail the job to be able to restart it
        if: ${{ contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure') }}
        run: exit 1
      - name: Check out repository code
        if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
        uses: ClickHouse/checkout@v1
      - name: Mark Commit Release Ready
        if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
        run: |
          cd "$GITHUB_WORKSPACE/tests/ci"
          python3 mark_release_ready.py          

  FinishCheck:
    if: ${{ !cancelled() }}
    needs: [RunConfig, Builds_1, Builds_2, Builds_Report, Tests_1, Tests_2, Tests_3]
    runs-on: [self-hosted, style-checker-aarch64]
    steps:
      - name: Check out repository code
        uses: ClickHouse/checkout@v1
      - name: Finish label
        run: |
          cd "$GITHUB_WORKSPACE/tests/ci"
          python3 finish_check.py --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}