Backport #60421 to 24.2: CI: make workflow yml abstract

.github/workflows/pull_request.yml

@@ -13,9 +13,7 @@ on:  # yamllint disable-line rule:truthy
       - opened
     branches:
       - master
-##########################################################################################
+
-##################################### SMALL CHECKS #######################################
-##########################################################################################
 jobs:
   RunConfig:
     runs-on: [self-hosted, style-checker-aarch64]
@@ -70,13 +68,13 @@ jobs:
           python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ runner.temp }}/ci_run_data.json --post --job-name 'Style check'
   BuildDockers:
     needs: [RunConfig]
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !failure() && !cancelled() && toJson(fromJson(needs.RunConfig.outputs.data).docker_data.missing_multi) != '[]' }}
     uses: ./.github/workflows/reusable_docker.yml
     with:
       data: ${{ needs.RunConfig.outputs.data }}
   StyleCheck:
     needs: [RunConfig, BuildDockers]
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Style check')}}
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: Style check
@@ -89,19 +87,9 @@ jobs:
         ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
         ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
         RCSK
-  DocsCheck:
-    needs: [RunConfig, StyleCheck]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Docs check
-      runner_type: func-tester-aarch64
-      data: ${{ needs.RunConfig.outputs.data }}
-      run_command: |
-        python3 docs_check.py
   FastTest:
     needs: [RunConfig, StyleCheck]
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Fast test') }}
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: Fast test
@@ -109,818 +97,83 @@ jobs:
       data: ${{ needs.RunConfig.outputs.data }}
       run_command: |
           python3 fast_test_check.py
-  CompatibilityCheckX86:
+
-    needs: [RunConfig, BuilderDebRelease]
+  ################################# Main statges #################################
-    if: ${{ !failure() && !cancelled() }}
+  # for main CI chain
-    uses: ./.github/workflows/reusable_test.yml
+  #
-    with:
+  Builds_1:
-      test_name: Compatibility check (amd64)
-      runner_type: style-checker
-      data: ${{ needs.RunConfig.outputs.data }}
-  CompatibilityCheckAarch64:
-    needs: [RunConfig, BuilderDebAarch64]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Compatibility check (aarch64)
-      runner_type: style-checker
-      data: ${{ needs.RunConfig.outputs.data }}
-#########################################################################################
-#################################### ORDINARY BUILDS ####################################
-#########################################################################################
-  BuilderDebDebug:
     needs: [RunConfig, FastTest]
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_1') }}
-    uses: ./.github/workflows/reusable_build.yml
+    # using callable wf (reusable_stage.yml) allows to group all nested jobs under a tab
+    uses: ./.github/workflows/reusable_build_stage.yml
     with:
-      build_name: package_debug
+      stage: Builds_1
       data: ${{ needs.RunConfig.outputs.data }}
-  BuilderDebRelease:
+  Tests_1:
-    needs: [RunConfig, FastTest]
+    needs: [RunConfig, Builds_1]
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_1') }}
-    uses: ./.github/workflows/reusable_build.yml
+    # using callable wf (reusable_stage.yml) allows to group all nested jobs under a tab
+    uses: ./.github/workflows/reusable_test_stage.yml
     with:
-      build_name: package_release
+      stage: Tests_1
-      checkout_depth: 0
       data: ${{ needs.RunConfig.outputs.data }}
-  BuilderDebReleaseCoverage:
+  Builds_2:
-    needs: [RunConfig, FastTest]
+    needs: [RunConfig, Builds_1]
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_2') }}
-    uses: ./.github/workflows/reusable_build.yml
+    # using callable wf (reusable_stage.yml) allows to group all nested jobs under a tab
+    uses: ./.github/workflows/reusable_build_stage.yml
     with:
-      build_name: package_release_coverage
+      stage: Builds_2
-      checkout_depth: 0
       data: ${{ needs.RunConfig.outputs.data }}
-  BuilderDebAarch64:
+  Tests_2:
-    needs: [RunConfig, FastTest]
+    needs: [RunConfig, Builds_2]
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_2') }}
-    uses: ./.github/workflows/reusable_build.yml
+    # using callable wf (reusable_stage.yml) allows to group all nested jobs under a tab
+    uses: ./.github/workflows/reusable_test_stage.yml
     with:
-      build_name: package_aarch64
+      stage: Tests_2
-      checkout_depth: 0
       data: ${{ needs.RunConfig.outputs.data }}
-  BuilderBinRelease:
+
-    needs: [RunConfig, FastTest]
+  ################################# Reports #################################
-    if: ${{ !failure() && !cancelled() }}
+  # Reports should by run even if Builds_1/2 fail, so put them separatly in wf (not in Tests_1/2)
-    uses: ./.github/workflows/reusable_build.yml
+  Builds_1_Report:
-    with:
-      build_name: binary_release
-      data: ${{ needs.RunConfig.outputs.data }}
-  BuilderDebAsan:
-    needs: [RunConfig, FastTest]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_build.yml
-    with:
-      build_name: package_asan
-      data: ${{ needs.RunConfig.outputs.data }}
-  BuilderDebUBsan:
-    needs: [RunConfig, FastTest]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_build.yml
-    with:
-      build_name: package_ubsan
-      data: ${{ needs.RunConfig.outputs.data }}
-  BuilderDebTsan:
-    needs: [RunConfig, FastTest]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_build.yml
-    with:
-      build_name: package_tsan
-      data: ${{ needs.RunConfig.outputs.data }}
-  BuilderDebMsan:
-    needs: [RunConfig, FastTest]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_build.yml
-    with:
-      build_name: package_msan
-      data: ${{ needs.RunConfig.outputs.data }}
-##########################################################################################
-##################################### SPECIAL BUILDS #####################################
-##########################################################################################
-  BuilderBinClangTidy:
-    needs: [RunConfig, FastTest]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_build.yml
-    with:
-      build_name: binary_tidy
-      data: ${{ needs.RunConfig.outputs.data }}
-  BuilderBinDarwin:
-    needs: [RunConfig, FastTest]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_build.yml
-    with:
-      build_name: binary_darwin
-      data: ${{ needs.RunConfig.outputs.data }}
-  BuilderBinAarch64:
-    needs: [RunConfig, FastTest]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_build.yml
-    with:
-      build_name: binary_aarch64
-      data: ${{ needs.RunConfig.outputs.data }}
-  BuilderBinFreeBSD:
-    needs: [RunConfig, FastTest]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_build.yml
-    with:
-      build_name: binary_freebsd
-      data: ${{ needs.RunConfig.outputs.data }}
-  BuilderBinDarwinAarch64:
-    needs: [RunConfig, FastTest]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_build.yml
-    with:
-      build_name: binary_darwin_aarch64
-      data: ${{ needs.RunConfig.outputs.data }}
-  BuilderBinPPC64:
-    needs: [RunConfig, FastTest]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_build.yml
-    with:
-      build_name: binary_ppc64le
-      data: ${{ needs.RunConfig.outputs.data }}
-  BuilderBinAmd64Compat:
-    needs: [RunConfig, FastTest]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_build.yml
-    with:
-      build_name: binary_amd64_compat
-      data: ${{ needs.RunConfig.outputs.data }}
-  BuilderBinAmd64Musl:
-    needs: [RunConfig, FastTest]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_build.yml
-    with:
-      build_name: binary_amd64_musl
-      data: ${{ needs.RunConfig.outputs.data }}
-  BuilderBinAarch64V80Compat:
-    needs: [RunConfig, FastTest]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_build.yml
-    with:
-      build_name: binary_aarch64_v80compat
-      data: ${{ needs.RunConfig.outputs.data }}
-  BuilderBinRISCV64:
-    needs: [RunConfig, FastTest]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_build.yml
-    with:
-      build_name: binary_riscv64
-      data: ${{ needs.RunConfig.outputs.data }}
-  BuilderBinS390X:
-    needs: [RunConfig, FastTest]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_build.yml
-    with:
-      build_name: binary_s390x
-      data: ${{ needs.RunConfig.outputs.data }}
-############################################################################################
-##################################### Docker images  #######################################
-############################################################################################
-  DockerServerImage:
-    needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Docker server image
-      runner_type: style-checker
-      data: ${{ needs.RunConfig.outputs.data }}
-  DockerKeeperImage:
-    needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Docker keeper image
-      runner_type: style-checker
-      data: ${{ needs.RunConfig.outputs.data }}
-############################################################################################
-##################################### BUILD REPORTER #######################################
-############################################################################################
-  BuilderReport:
     # run report check for failed builds to indicate the CI error
-    if: ${{ !cancelled() }}
+    if: ${{ !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse build check') }}
     needs:
       - RunConfig
-      - BuilderDebAarch64
+      - Builds_1
-      - BuilderDebAsan
-      - BuilderDebDebug
-      - BuilderDebMsan
-      - BuilderDebRelease
-      - BuilderDebTsan
-      - BuilderDebUBsan
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: ClickHouse build check
-      runner_type: style-checker
+      runner_type: style-checker-aarch64
       data: ${{ needs.RunConfig.outputs.data }}
-  BuilderSpecialReport:
+  Builds_2_Report:
     # run report check for failed builds to indicate the CI error
-    if: ${{ !cancelled() }}
+    if: ${{ !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse special build check') }}
     needs:
       - RunConfig
-      - BuilderBinAarch64
+      - Builds_2
-      - BuilderBinDarwin
-      - BuilderBinDarwinAarch64
-      - BuilderBinFreeBSD
-      - BuilderBinPPC64
-      - BuilderBinRISCV64
-      - BuilderBinS390X
-      - BuilderBinAmd64Compat
-      - BuilderBinAarch64V80Compat
-      - BuilderBinClangTidy
-      - BuilderDebReleaseCoverage
-      - BuilderBinRelease
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: ClickHouse special build check
-      runner_type: style-checker
-      data: ${{ needs.RunConfig.outputs.data }}
-############################################################################################
-#################################### INSTALL PACKAGES ######################################
-############################################################################################
-  InstallPackagesTestRelease:
-    needs: [RunConfig, BuilderDebRelease]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Install packages (amd64)
-      runner_type: style-checker
-      data: ${{ needs.RunConfig.outputs.data }}
-      run_command: |
-        python3 install_check.py "$CHECK_NAME"
-  InstallPackagesTestAarch64:
-    needs: [RunConfig, BuilderDebAarch64]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Install packages (arm64)
       runner_type: style-checker-aarch64
       data: ${{ needs.RunConfig.outputs.data }}
-      run_command: |
+
-        python3 install_check.py "$CHECK_NAME"
+  ################################# Stage Final #################################
-##############################################################################################
+  #
-########################### FUNCTIONAl STATELESS TESTS #######################################
-##############################################################################################
-  FunctionalStatelessTestRelease:
-    needs: [RunConfig, BuilderDebRelease]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateless tests (release)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatelessTestReleaseAnalyzerS3Replicated:
-    needs: [RunConfig, BuilderDebRelease]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateless tests (release, analyzer, s3, DatabaseReplicated)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatelessTestS3Debug:
-    needs: [RunConfig, BuilderDebDebug]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateless tests (debug, s3 storage)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatelessTestS3Tsan:
-    needs: [RunConfig, BuilderDebTsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateless tests (tsan, s3 storage)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatelessTestAarch64:
-    needs: [RunConfig, BuilderDebAarch64]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateless tests (aarch64)
-      runner_type: func-tester-aarch64
-      data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatelessTestAsan:
-    needs: [RunConfig, BuilderDebAsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateless tests (asan)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatelessTestTsan:
-    needs: [RunConfig, BuilderDebTsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateless tests (tsan)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatelessTestMsan:
-    needs: [RunConfig, BuilderDebMsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateless tests (msan)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatelessTestUBsan:
-    needs: [RunConfig, BuilderDebUBsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateless tests (ubsan)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatelessTestDebug:
-    needs: [RunConfig, BuilderDebDebug]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateless tests (debug)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatelessTestFlakyCheck:
-    needs: [RunConfig, BuilderDebAsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateless tests flaky check (asan)
-      data: ${{ needs.RunConfig.outputs.data }}
-      runner_type: func-tester
-  TestsBugfixCheck:
-    needs: [RunConfig, StyleCheck]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Bugfix validation
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-##############################################################################################
-############################ FUNCTIONAl STATEFUL TESTS #######################################
-##############################################################################################
-  FunctionalStatefulTestRelease:
-    needs: [RunConfig, BuilderDebRelease]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateful tests (release)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatefulTestAarch64:
-    needs: [RunConfig, BuilderDebAarch64]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateful tests (aarch64)
-      runner_type: func-tester-aarch64
-      data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatefulTestAsan:
-    needs: [RunConfig, BuilderDebAsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateful tests (asan)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatefulTestTsan:
-    needs: [RunConfig, BuilderDebTsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateful tests (tsan)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatefulTestMsan:
-    needs: [RunConfig, BuilderDebMsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateful tests (msan)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatefulTestUBsan:
-    needs: [RunConfig, BuilderDebUBsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateful tests (ubsan)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatefulTestDebug:
-    needs: [RunConfig, BuilderDebDebug]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateful tests (debug)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  # Parallel replicas
-  FunctionalStatefulTestDebugParallelReplicas:
-    needs: [RunConfig, BuilderDebDebug]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateful tests (debug, ParallelReplicas)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatefulTestUBsanParallelReplicas:
-    needs: [RunConfig, BuilderDebUBsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateful tests (ubsan, ParallelReplicas)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatefulTestMsanParallelReplicas:
-    needs: [RunConfig, BuilderDebMsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateful tests (msan, ParallelReplicas)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatefulTestTsanParallelReplicas:
-    needs: [RunConfig, BuilderDebTsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateful tests (tsan, ParallelReplicas)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatefulTestAsanParallelReplicas:
-    needs: [RunConfig, BuilderDebAsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateful tests (asan, ParallelReplicas)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  FunctionalStatefulTestReleaseParallelReplicas:
-    needs: [RunConfig, BuilderDebRelease]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stateful tests (release, ParallelReplicas)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-##############################################################################################
-########################### ClickBench #######################################################
-##############################################################################################
-  ClickBenchAMD64:
-    needs: [RunConfig, BuilderDebRelease]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: ClickBench (amd64)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-      run_command: |
-        python3 clickbench.py "$CHECK_NAME"
-  ClickBenchAarch64:
-    needs: [RunConfig, BuilderDebAarch64]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: ClickBench (aarch64)
-      runner_type: func-tester-aarch64
-      data: ${{ needs.RunConfig.outputs.data }}
-      run_command: |
-        python3 clickbench.py "$CHECK_NAME"
-##############################################################################################
-######################################### STRESS TESTS #######################################
-##############################################################################################
-  StressTestAsan:
-    needs: [RunConfig, BuilderDebAsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stress test (asan)
-      runner_type: stress-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  StressTestTsan:
-    needs: [RunConfig, BuilderDebTsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stress test (tsan)
-      runner_type: stress-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  StressTestMsan:
-    needs: [RunConfig, BuilderDebMsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stress test (msan)
-      runner_type: stress-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  StressTestUBsan:
-    needs: [RunConfig, BuilderDebUBsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stress test (ubsan)
-      runner_type: stress-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  StressTestDebug:
-    needs: [RunConfig, BuilderDebDebug]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Stress test (debug)
-      runner_type: stress-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-##############################################################################################
-######################################### UPGRADE CHECK ######################################
-##############################################################################################
-  UpgradeCheckAsan:
-    needs: [RunConfig, BuilderDebAsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Upgrade check (asan)
-      runner_type: stress-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  UpgradeCheckTsan:
-    needs: [RunConfig, BuilderDebTsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Upgrade check (tsan)
-      runner_type: stress-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  UpgradeCheckMsan:
-    needs: [RunConfig, BuilderDebMsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Upgrade check (msan)
-      runner_type: stress-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  UpgradeCheckDebug:
-    needs: [RunConfig, BuilderDebDebug]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Upgrade check (debug)
-      runner_type: stress-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-##############################################################################################
-##################################### AST FUZZERS ############################################
-##############################################################################################
-  ASTFuzzerTestAsan:
-    needs: [RunConfig, BuilderDebAsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: AST fuzzer (asan)
-      runner_type: fuzzer-unit-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  ASTFuzzerTestTsan:
-    needs: [RunConfig, BuilderDebTsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: AST fuzzer (tsan)
-      runner_type: fuzzer-unit-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  ASTFuzzerTestUBSan:
-    needs: [RunConfig, BuilderDebUBsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: AST fuzzer (ubsan)
-      runner_type: fuzzer-unit-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  ASTFuzzerTestMSan:
-    needs: [RunConfig, BuilderDebMsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: AST fuzzer (msan)
-      runner_type: fuzzer-unit-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  ASTFuzzerTestDebug:
-    needs: [RunConfig, BuilderDebDebug]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: AST fuzzer (debug)
-      runner_type: fuzzer-unit-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-#############################################################################################
-############################# INTEGRATION TESTS #############################################
-#############################################################################################
-  IntegrationTestsAnalyzerAsan:
-    needs: [RunConfig, BuilderDebAsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Integration tests (asan, analyzer)
-      runner_type: stress-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  IntegrationTestsTsan:
-    needs: [RunConfig, BuilderDebTsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Integration tests (tsan)
-      runner_type: stress-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  IntegrationTestsAarch64:
-    needs: [RunConfig, BuilderDebAarch64]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Integration tests (aarch64)
-      # FIXME: there is no stress-tester for aarch64. func-tester-aarch64 is ok?
-      runner_type: func-tester-aarch64
-      data: ${{ needs.RunConfig.outputs.data }}
-  IntegrationTestsFlakyCheck:
-    needs: [RunConfig, BuilderDebAsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Integration tests flaky check (asan)
-      runner_type: stress-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-#############################################################################################
-#################################### UNIT TESTS #############################################
-#############################################################################################
-  UnitTestsAsan:
-    needs: [RunConfig, BuilderDebAsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Unit tests (asan)
-      runner_type: fuzzer-unit-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  UnitTestsRelease:
-    needs: [RunConfig, BuilderBinRelease]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Unit tests (release)
-      runner_type: fuzzer-unit-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  UnitTestsTsan:
-    needs: [RunConfig, BuilderDebTsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Unit tests (tsan)
-      runner_type: fuzzer-unit-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  UnitTestsMsan:
-    needs: [RunConfig, BuilderDebMsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Unit tests (msan)
-      runner_type: fuzzer-unit-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  UnitTestsUBsan:
-    needs: [RunConfig, BuilderDebUBsan]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Unit tests (ubsan)
-      runner_type: fuzzer-unit-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-#############################################################################################
-#################################### PERFORMANCE TESTS ######################################
-#############################################################################################
-  PerformanceComparisonX86:
-    needs: [RunConfig, BuilderDebRelease]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Performance Comparison
-      runner_type: stress-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  PerformanceComparisonAarch:
-    needs: [RunConfig, BuilderDebAarch64]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Performance Comparison Aarch64
-      runner_type: func-tester-aarch64
-      data: ${{ needs.RunConfig.outputs.data }}
-##############################################################################################
-###################################### SQLANCER FUZZERS ######################################
-##############################################################################################
-  SQLancerTestRelease:
-    needs: [RunConfig, BuilderDebRelease]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: SQLancer (release)
-      runner_type: fuzzer-unit-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-  SQLancerTestDebug:
-    needs: [RunConfig, BuilderDebDebug]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: SQLancer (debug)
-      runner_type: fuzzer-unit-tester
-      data: ${{ needs.RunConfig.outputs.data }}
   FinishCheck:
     if: ${{ !failure() && !cancelled() }}
-    needs:
+    needs: [Tests_1, Tests_2]
-      - BuilderReport
-      - BuilderSpecialReport
-      - DocsCheck
-      - FastTest
-      - TestsBugfixCheck
-      - FunctionalStatelessTestDebug
-      - FunctionalStatelessTestRelease
-      - FunctionalStatelessTestAarch64
-      - FunctionalStatelessTestAsan
-      - FunctionalStatelessTestTsan
-      - FunctionalStatelessTestMsan
-      - FunctionalStatelessTestUBsan
-      - FunctionalStatefulTestDebug
-      - FunctionalStatefulTestRelease
-      - FunctionalStatefulTestAarch64
-      - FunctionalStatefulTestAsan
-      - FunctionalStatefulTestTsan
-      - FunctionalStatefulTestMsan
-      - FunctionalStatefulTestUBsan
-      - FunctionalStatelessTestS3Debug
-      - FunctionalStatelessTestS3Tsan
-      - FunctionalStatelessTestReleaseAnalyzerS3Replicated
-      - FunctionalStatefulTestReleaseParallelReplicas
-      - FunctionalStatefulTestAsanParallelReplicas
-      - FunctionalStatefulTestTsanParallelReplicas
-      - FunctionalStatefulTestMsanParallelReplicas
-      - FunctionalStatefulTestUBsanParallelReplicas
-      - FunctionalStatefulTestDebugParallelReplicas
-      - StressTestDebug
-      - StressTestAsan
-      - StressTestTsan
-      - StressTestMsan
-      - StressTestUBsan
-      - UpgradeCheckAsan
-      - UpgradeCheckTsan
-      - UpgradeCheckMsan
-      - UpgradeCheckDebug
-      - ASTFuzzerTestDebug
-      - ASTFuzzerTestAsan
-      - ASTFuzzerTestTsan
-      - ASTFuzzerTestMSan
-      - ASTFuzzerTestUBSan
-      - IntegrationTestsAnalyzerAsan
-      - IntegrationTestsTsan
-      - IntegrationTestsAarch64
-      - IntegrationTestsFlakyCheck
-      - PerformanceComparisonX86
-      - PerformanceComparisonAarch
-      - UnitTestsAsan
-      - UnitTestsTsan
-      - UnitTestsMsan
-      - UnitTestsUBsan
-      - UnitTestsRelease
-      - CompatibilityCheckX86
-      - CompatibilityCheckAarch64
-      - SQLancerTestRelease
-      - SQLancerTestDebug
     runs-on: [self-hosted, style-checker]
     steps:
       - name: Check out repository code
         uses: ClickHouse/checkout@v1
-        with:
-          clear-repository: true
       - name: Finish label
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 finish_check.py
-          python3 merge_pr.py --check-approved
+
-##############################################################################################
+
-############################ SQLLOGIC TEST ###################################################
-##############################################################################################
-  SQLLogicTestRelease:
-    needs: [RunConfig, BuilderDebRelease]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: Sqllogic test (release)
-      runner_type: func-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-##############################################################################################
-##################################### SQL TEST ###############################################
-##############################################################################################
-  SQLTest:
-    needs: [RunConfig, BuilderDebRelease]
-    if: ${{ !failure() && !cancelled() }}
-    uses: ./.github/workflows/reusable_test.yml
-    with:
-      test_name: SQLTest
-      runner_type: fuzzer-unit-tester
-      data: ${{ needs.RunConfig.outputs.data }}
-#############################################################################################
-###################################### NOT IN FINISH ########################################
 #############################################################################################
 ###################################### JEPSEN TESTS #########################################
 #############################################################################################
@@ -931,19 +184,11 @@ jobs:
     # we need concurrency as the job uses dedicated instances in the cloud
     concurrency:
       group: jepsen
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse Keeper Jepsen') }}
-    needs: [RunConfig, BuilderBinRelease]
+    # jepsen needs binary_release build which is in Builds_2
+    needs: [RunConfig, Builds_2]
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: ClickHouse Keeper Jepsen
       runner_type: style-checker
       data: ${{ needs.RunConfig.outputs.data }}
-#############################################################################################
-####################################### libFuzzer ###########################################
-#############################################################################################
-  libFuzzer:
-    if: ${{ !failure() && !cancelled() }}
-    needs: [RunConfig, StyleCheck]
-    uses: ./.github/workflows/libfuzzer.yml
-    with:
-      data: ${{ needs.RunConfig.outputs.data }}

.github/workflows/reusable_build_stage.yml

@@ -0,0 +1,32 @@
+### FIXME: merge reusable_test.yml and reusable_build.yml as they are almost identical
+#          and then merge reusable_build_stage.yml and reusable_test_stage.yml
+
+name: BuildStageWF
+'on':
+  workflow_call:
+    inputs:
+      stage:
+        description: stage name
+        type: string
+        required: true
+      data:
+        description: ci data
+        type: string
+        required: true
+
+jobs:
+  s:
+    if: ${{ !failure() && !cancelled() }}
+    strategy:
+      fail-fast: false
+      matrix:
+        job_name_and_runner_type: ${{ fromJson(inputs.data).stages_data[inputs.stage] }}
+    uses: ./.github/workflows/reusable_build.yml
+    with:
+      build_name: ${{ matrix.job_name_and_runner_type.job_name }}
+      runner_type: ${{ matrix.job_name_and_runner_type.runner_type }}
+      # don't forget to pass force flag (no ci cache/no reuse) - once it's needed
+      force: false
+      # for now let's do I deep checkout for builds
+      checkout_depth: 0
+      data: ${{ inputs.data }}

.github/workflows/reusable_test_stage.yml

@@ -0,0 +1,25 @@
+name: StageWF
+'on':
+  workflow_call:
+    inputs:
+      stage:
+        description: stage name
+        type: string
+        required: true
+      data:
+        description: ci data
+        type: string
+        required: true
+
+jobs:
+  s:
+    if: ${{ !failure() && !cancelled() }}
+    strategy:
+      fail-fast: false
+      matrix:
+        job_name_and_runner_type: ${{ fromJson(inputs.data).stages_data[inputs.stage] }}
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: ${{ matrix.job_name_and_runner_type.job_name }}
+      runner_type: ${{ matrix.job_name_and_runner_type.runner_type }}
+      data: ${{ inputs.data }}

tests/ci/ci.py

@@ -17,7 +17,7 @@ from typing import Any, Dict, List, Optional, Sequence, Set, Union
 import docker_images_helper
 import upload_result_helper
 from build_check import get_release_or_pr
-from ci_config import CI_CONFIG, Build, JobNames, Labels
+from ci_config import CI_CONFIG, Build, CIStages, Labels, JobNames
 from ci_utils import GHActions, is_hex, normalize_string
 from clickhouse_helper import (
     CiLogsCredentials,
@@ -1119,7 +1119,7 @@ def _configure_jobs(
 
     ## b. check what we need to run
     ci_cache = None
-    if not ci_cache_disabled:
+    if not ci_cache_disabled and CI:
         ci_cache = CiCache(s3, digests).update()
         ci_cache.print_status()
 
@@ -1283,6 +1283,29 @@ def _configure_jobs(
     }
 
 
+def _generate_ci_stage_config(jobs_data: Dict[str, Any]) -> Dict[str, Dict[str, Any]]:
+    """
+    populates GH Actions' workflow with real jobs
+    "Builds_1": [{"job_name": NAME, "runner_type": RUNER_TYPE}]
+    "Tests_1": [{"job_name": NAME, "runner_type": RUNER_TYPE}]
+    ...
+    """
+    result = {}  # type: Dict[str, Any]
+    stages_to_do = []
+    for job in jobs_data["jobs_to_do"]:
+        stage_type = CI_CONFIG.get_job_ci_stage(job)
+        if stage_type == CIStages.NA:
+            continue
+        if stage_type not in result:
+            result[stage_type] = []
+            stages_to_do.append(stage_type)
+        result[stage_type].append(
+            {"job_name": job, "runner_type": CI_CONFIG.get_runner_type(job)}
+        )
+    result["stages_to_do"] = stages_to_do
+    return result
+
+
 def _create_gh_status(
     commit: Any, job: str, batch: int, num_batches: int, job_status: CommitStatusData
 ) -> None:
@@ -1734,6 +1757,7 @@ def main() -> int:
         result["build"] = build_digest
         result["docs"] = docs_digest
         result["ci_flags"] = ci_flags
+        result["stages_data"] = _generate_ci_stage_config(jobs_data)
         result["jobs_data"] = jobs_data
         result["docker_data"] = docker_data
     ### CONFIGURE action: end

tests/ci/ci_config.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+from copy import deepcopy
 import logging
 from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
 from dataclasses import dataclass, field
@@ -10,6 +11,24 @@ from ci_utils import WithIter
 from integration_test_images import IMAGES
 
 
+class CIStages:
+    NA = "UNKNOWN"
+    BUILDS_1 = "Builds_1"
+    BUILDS_2 = "Builds_2"
+    TESTS_1 = "Tests_1"
+    TESTS_2 = "Tests_2"
+
+
+class Runners(metaclass=WithIter):
+    BUILDER = "builder"
+    STYLE_CHECKER = "style-checker"
+    STYLE_CHECKER_ARM = "style-checker-aarch64"
+    FUNC_TESTER = "func-tester"
+    FUNC_TESTER_ARM = "func-tester-aarch64"
+    STRESS_TESTER = "stress-tester"
+    FUZZER_UNIT_TESTER = "fuzzer-unit-tester"
+
+
 class Labels(metaclass=WithIter):
     """
     Label names or commit tokens in normalized form
@@ -207,6 +226,45 @@ class JobConfig:
     random_bucket: str = ""
 
 
+builds_job_config = JobConfig(
+    required_on_release_branch=True,
+    digest=DigestConfig(
+        include_paths=[
+            "./src",
+            "./contrib/*-cmake",
+            "./contrib/consistent-hashing",
+            "./contrib/murmurhash",
+            "./contrib/libfarmhash",
+            "./contrib/pdqsort",
+            "./contrib/cityhash102",
+            "./contrib/sparse-checkout",
+            "./contrib/libmetrohash",
+            "./contrib/update-submodules.sh",
+            "./contrib/CMakeLists.txt",
+            "./CMakeLists.txt",
+            "./PreLoad.cmake",
+            "./cmake",
+            "./base",
+            "./programs",
+            "./packages",
+            "./docker/packager/packager",
+            "./rust",
+            # FIXME: This is a WA to rebuild the CH and recreate the Performance.tar.zst artifact
+            # when there are changes in performance test scripts.
+            # Due to the current design of the perf test we need to rebuild CH when the performance test changes,
+            # otherwise the changes will not be visible in the PerformanceTest job in CI
+            "./tests/performance",
+        ],
+        exclude_files=[".md"],
+        docker=["clickhouse/binary-builder"],
+        git_submodules=True,
+    ),
+    run_command="build_check.py $BUILD_NAME",
+)
+fuzzer_build_job_config = deepcopy(builds_job_config)
+fuzzer_build_job_config.run_by_label = Labels.libFuzzer
+
+
 @dataclass
 class BuildConfig:
     name: str
@@ -222,43 +280,7 @@ class BuildConfig:
     sparse_checkout: bool = False
     comment: str = ""
     static_binary_name: str = ""
-    job_config: JobConfig = field(
+    job_config: JobConfig = field(default_factory=lambda: deepcopy(builds_job_config))
-        default_factory=lambda: JobConfig(
-            required_on_release_branch=True,
-            digest=DigestConfig(
-                include_paths=[
-                    "./src",
-                    "./contrib/*-cmake",
-                    "./contrib/consistent-hashing",
-                    "./contrib/murmurhash",
-                    "./contrib/libfarmhash",
-                    "./contrib/pdqsort",
-                    "./contrib/cityhash102",
-                    "./contrib/sparse-checkout",
-                    "./contrib/libmetrohash",
-                    "./contrib/update-submodules.sh",
-                    "./contrib/CMakeLists.txt",
-                    "./CMakeLists.txt",
-                    "./PreLoad.cmake",
-                    "./cmake",
-                    "./base",
-                    "./programs",
-                    "./packages",
-                    "./docker/packager/packager",
-                    "./rust",
-                    # FIXME: This is a WA to rebuild the CH and recreate the Performance.tar.zst artifact
-                    # when there are changes in performance test scripts.
-                    # Due to the current design of the perf test we need to rebuild CH when the performance test changes,
-                    # otherwise the changes will not be visible in the PerformanceTest job in CI
-                    "./tests/performance",
-                ],
-                exclude_files=[".md"],
-                docker=["clickhouse/binary-builder"],
-                git_submodules=True,
-            ),
-            run_command="build_check.py $BUILD_NAME",
-        )
-    )
 
     def export_env(self, export: bool = False) -> str:
         def process(field_name: str, field: Union[bool, str]) -> str:
@@ -464,6 +486,19 @@ sql_test_params = {
     "run_command": "sqltest.py",
     "timeout": 10800,
 }
+clickbench_test_params = {
+    "digest": DigestConfig(
+        include_paths=[
+            "tests/ci/clickbench.py",
+        ],
+        docker=["clickhouse/clickbench"],
+    ),
+    "run_command": 'clickbench.py "$CHECK_NAME"',
+}
+install_test_params = {
+    "digest": install_check_digest,
+    "run_command": 'install_check.py "$CHECK_NAME"',
+}
 
 
 @dataclass
@@ -485,6 +520,37 @@ class CIConfig:
                 return config
         return None
 
+    def get_job_ci_stage(self, job_name: str) -> str:
+        if job_name in [
+            JobNames.STYLE_CHECK,
+            JobNames.FAST_TEST,
+            JobNames.JEPSEN_KEEPER,
+            JobNames.BUILD_CHECK,
+            JobNames.BUILD_CHECK_SPECIAL,
+        ]:
+            # FIXME: we can't currently handle Jepsen in the Stage as it's job has concurrency directive
+            # BUILD_CHECK and BUILD_CHECK_SPECIAL runs not in stage because we need them even if Builds stage failed
+            return CIStages.NA
+        stage_type = None
+        if self.is_build_job(job_name):
+            stage_type = CIStages.BUILDS_1
+            if job_name in CI_CONFIG.get_builds_for_report(
+                JobNames.BUILD_CHECK_SPECIAL
+            ):
+                # special builds go to Build_2 stage to not delay Builds_1/Test_1
+                stage_type = CIStages.BUILDS_2
+        elif self.is_docs_job(job_name):
+            stage_type = CIStages.TESTS_1
+        elif job_name == JobNames.BUILD_CHECK_SPECIAL:
+            stage_type = CIStages.TESTS_2
+        elif self.is_test_job(job_name):
+            stage_type = CIStages.TESTS_1
+            if job_name == JobNames.LIBFUZZER_TEST:
+                # since fuzzers build in Builds_2, test must be in Tests_2
+                stage_type = CIStages.TESTS_2
+        assert stage_type, f"BUG [{job_name}]"
+        return stage_type
+
     def get_job_config(self, check_name: str) -> JobConfig:
         res = None
         for config in (
@@ -498,6 +564,63 @@ class CIConfig:
                 break
         return res  # type: ignore
 
+    def get_runner_type(self, check_name: str) -> str:
+        result = None
+        if self.is_build_job(check_name) or check_name == JobNames.FAST_TEST:
+            result = Runners.BUILDER
+        elif any(
+            words in check_name.lower()
+            for words in [
+                "install packages",
+                "compatibility check",
+                "docker",
+                "build check",
+                "jepsen",
+                "style check",
+            ]
+        ):
+            result = Runners.STYLE_CHECKER
+        elif check_name == JobNames.DOCS_CHECK:
+            # docs job is demanding
+            result = Runners.FUNC_TESTER_ARM
+        elif any(
+            words in check_name.lower()
+            for words in [
+                "stateless",
+                "stateful",
+                "clickbench",
+                "sqllogic test",
+                "libfuzzer",
+                "bugfix validation",
+            ]
+        ):
+            result = Runners.FUNC_TESTER
+        elif any(
+            words in check_name.lower()
+            for words in ["stress", "upgrade", "integration", "performance comparison"]
+        ):
+            result = Runners.STRESS_TESTER
+        elif any(
+            words in check_name.lower()
+            for words in ["ast fuzzer", "unit tests", "sqlancer", "sqltest"]
+        ):
+            result = Runners.FUZZER_UNIT_TESTER
+
+        assert result, f"BUG, no runner for [{check_name}]"
+
+        if ("aarch" in check_name or "arm" in check_name) and "aarch" not in result:
+            if result == Runners.STRESS_TESTER:
+                # FIXME: no arm stress tester group atm
+                result = Runners.FUNC_TESTER_ARM
+            elif result == Runners.BUILDER:
+                # crosscompile - no arm required
+                pass
+            else:
+                # switch to aarch64 runnner
+                result += "-aarch64"
+
+        return result
+
     @staticmethod
     def normalize_string(input_string: str) -> str:
         lowercase_string = input_string.lower()
@@ -598,11 +721,7 @@ class CIConfig:
 
     @classmethod
     def is_test_job(cls, job: str) -> bool:
-        return (
+        return not cls.is_build_job(job) and job != JobNames.STYLE_CHECK
-            not cls.is_build_job(job)
-            and not cls.is_build_job(job)
-            and job != JobNames.STYLE_CHECK
-        )
 
     @classmethod
     def is_docs_job(cls, job: str) -> bool:
@@ -843,7 +962,7 @@ CI_CONFIG = CIConfig(
             name=Build.FUZZERS,
             compiler="clang-17",
             package_type="fuzzers",
-            job_config=JobConfig(run_by_label=Labels.libFuzzer),
+            job_config=fuzzer_build_job_config,
         ),
     },
     builds_report_config={
@@ -887,6 +1006,7 @@ CI_CONFIG = CIConfig(
                     include_paths=["**/*.md", "./docs", "tests/ci/docs_check.py"],
                     docker=["clickhouse/docs-builder"],
                 ),
+                run_command="docs_check.py",
             ),
         ),
         JobNames.FAST_TEST: TestConfig(
@@ -916,10 +1036,10 @@ CI_CONFIG = CIConfig(
     },
     test_configs={
         JobNames.INSTALL_TEST_AMD: TestConfig(
-            Build.PACKAGE_RELEASE, job_config=JobConfig(digest=install_check_digest)
+            Build.PACKAGE_RELEASE, job_config=JobConfig(**install_test_params)  # type: ignore
         ),
         JobNames.INSTALL_TEST_ARM: TestConfig(
-            Build.PACKAGE_AARCH64, job_config=JobConfig(digest=install_check_digest)
+            Build.PACKAGE_AARCH64, job_config=JobConfig(**install_test_params)  # type: ignore
         ),
         JobNames.STATEFUL_TEST_ASAN: TestConfig(
             Build.PACKAGE_ASAN, job_config=JobConfig(**stateful_test_common_params)  # type: ignore
@@ -1137,9 +1257,20 @@ CI_CONFIG = CIConfig(
         JobNames.SQLTEST: TestConfig(
             Build.PACKAGE_RELEASE, job_config=JobConfig(**sql_test_params)  # type: ignore
         ),
-        JobNames.CLCIKBENCH_TEST: TestConfig(Build.PACKAGE_RELEASE),
+        JobNames.CLCIKBENCH_TEST: TestConfig(
-        JobNames.CLCIKBENCH_TEST_ARM: TestConfig(Build.PACKAGE_AARCH64),
+            Build.PACKAGE_RELEASE, job_config=JobConfig(**clickbench_test_params)  # type: ignore
-        JobNames.LIBFUZZER_TEST: TestConfig(Build.FUZZERS, job_config=JobConfig(run_by_label=Labels.libFuzzer)),  # type: ignore
+        ),
+        JobNames.CLCIKBENCH_TEST_ARM: TestConfig(
+            Build.PACKAGE_AARCH64, job_config=JobConfig(**clickbench_test_params)  # type: ignore
+        ),
+        JobNames.LIBFUZZER_TEST: TestConfig(
+            Build.FUZZERS,
+            job_config=JobConfig(
+                run_by_label=Labels.libFuzzer,
+                timeout=10800,
+                run_command='libfuzzer_test_check.py "$CHECK_NAME" 10800',
+            ),
+        ),  # type: ignore
     },
 )
 CI_CONFIG.validate()

tests/ci/test_ci_config.py

@@ -1,15 +1,12 @@
 #!/usr/bin/env python3
 
 import unittest
+from ci_config import JobNames, CI_CONFIG, Runners
 
 
 class TestCIConfig(unittest.TestCase):
-    def test_no_errors_in_ci_config(self):
+    def test_runner_config(self):
-        raised = None
+        """check runner is provided w/o exception"""
-        try:
+        for job in JobNames:
-            from ci_config import (  # pylint: disable=import-outside-toplevel
+            runner = CI_CONFIG.get_runner_type(job)
-                CI_CONFIG as _,
+            self.assertIn(runner, Runners)
-            )
-        except Exception as exc:
-            raised = exc
-        self.assertIsNone(raised, f"CI_CONFIG import raised error {raised}")