mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 08:40:50 +00:00
Backport #60421 to 24.2: CI: make workflow yml abstract
This commit is contained in:
parent
49b8c37bba
commit
43d2a54d78
855
.github/workflows/pull_request.yml
vendored
855
.github/workflows/pull_request.yml
vendored
@ -13,9 +13,7 @@ on: # yamllint disable-line rule:truthy
|
||||
- opened
|
||||
branches:
|
||||
- master
|
||||
##########################################################################################
|
||||
##################################### SMALL CHECKS #######################################
|
||||
##########################################################################################
|
||||
|
||||
jobs:
|
||||
RunConfig:
|
||||
runs-on: [self-hosted, style-checker-aarch64]
|
||||
@ -70,13 +68,13 @@ jobs:
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --infile ${{ runner.temp }}/ci_run_data.json --post --job-name 'Style check'
|
||||
BuildDockers:
|
||||
needs: [RunConfig]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
if: ${{ !failure() && !cancelled() && toJson(fromJson(needs.RunConfig.outputs.data).docker_data.missing_multi) != '[]' }}
|
||||
uses: ./.github/workflows/reusable_docker.yml
|
||||
with:
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
StyleCheck:
|
||||
needs: [RunConfig, BuildDockers]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Style check')}}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Style check
|
||||
@ -89,19 +87,9 @@ jobs:
|
||||
ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
|
||||
${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
|
||||
RCSK
|
||||
DocsCheck:
|
||||
needs: [RunConfig, StyleCheck]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Docs check
|
||||
runner_type: func-tester-aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
run_command: |
|
||||
python3 docs_check.py
|
||||
FastTest:
|
||||
needs: [RunConfig, StyleCheck]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Fast test') }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Fast test
|
||||
@ -109,818 +97,83 @@ jobs:
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
run_command: |
|
||||
python3 fast_test_check.py
|
||||
CompatibilityCheckX86:
|
||||
needs: [RunConfig, BuilderDebRelease]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Compatibility check (amd64)
|
||||
runner_type: style-checker
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
CompatibilityCheckAarch64:
|
||||
needs: [RunConfig, BuilderDebAarch64]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Compatibility check (aarch64)
|
||||
runner_type: style-checker
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
#########################################################################################
|
||||
#################################### ORDINARY BUILDS ####################################
|
||||
#########################################################################################
|
||||
BuilderDebDebug:
|
||||
|
||||
################################# Main statges #################################
|
||||
# for main CI chain
|
||||
#
|
||||
Builds_1:
|
||||
needs: [RunConfig, FastTest]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_build.yml
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_1') }}
|
||||
# using callable wf (reusable_stage.yml) allows to group all nested jobs under a tab
|
||||
uses: ./.github/workflows/reusable_build_stage.yml
|
||||
with:
|
||||
build_name: package_debug
|
||||
stage: Builds_1
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
BuilderDebRelease:
|
||||
needs: [RunConfig, FastTest]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_build.yml
|
||||
Tests_1:
|
||||
needs: [RunConfig, Builds_1]
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_1') }}
|
||||
# using callable wf (reusable_stage.yml) allows to group all nested jobs under a tab
|
||||
uses: ./.github/workflows/reusable_test_stage.yml
|
||||
with:
|
||||
build_name: package_release
|
||||
checkout_depth: 0
|
||||
stage: Tests_1
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
BuilderDebReleaseCoverage:
|
||||
needs: [RunConfig, FastTest]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_build.yml
|
||||
Builds_2:
|
||||
needs: [RunConfig, Builds_1]
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_2') }}
|
||||
# using callable wf (reusable_stage.yml) allows to group all nested jobs under a tab
|
||||
uses: ./.github/workflows/reusable_build_stage.yml
|
||||
with:
|
||||
build_name: package_release_coverage
|
||||
checkout_depth: 0
|
||||
stage: Builds_2
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
BuilderDebAarch64:
|
||||
needs: [RunConfig, FastTest]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_build.yml
|
||||
Tests_2:
|
||||
needs: [RunConfig, Builds_2]
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_2') }}
|
||||
# using callable wf (reusable_stage.yml) allows to group all nested jobs under a tab
|
||||
uses: ./.github/workflows/reusable_test_stage.yml
|
||||
with:
|
||||
build_name: package_aarch64
|
||||
checkout_depth: 0
|
||||
stage: Tests_2
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
BuilderBinRelease:
|
||||
needs: [RunConfig, FastTest]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_build.yml
|
||||
with:
|
||||
build_name: binary_release
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
BuilderDebAsan:
|
||||
needs: [RunConfig, FastTest]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_build.yml
|
||||
with:
|
||||
build_name: package_asan
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
BuilderDebUBsan:
|
||||
needs: [RunConfig, FastTest]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_build.yml
|
||||
with:
|
||||
build_name: package_ubsan
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
BuilderDebTsan:
|
||||
needs: [RunConfig, FastTest]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_build.yml
|
||||
with:
|
||||
build_name: package_tsan
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
BuilderDebMsan:
|
||||
needs: [RunConfig, FastTest]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_build.yml
|
||||
with:
|
||||
build_name: package_msan
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
##########################################################################################
|
||||
##################################### SPECIAL BUILDS #####################################
|
||||
##########################################################################################
|
||||
BuilderBinClangTidy:
|
||||
needs: [RunConfig, FastTest]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_build.yml
|
||||
with:
|
||||
build_name: binary_tidy
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
BuilderBinDarwin:
|
||||
needs: [RunConfig, FastTest]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_build.yml
|
||||
with:
|
||||
build_name: binary_darwin
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
BuilderBinAarch64:
|
||||
needs: [RunConfig, FastTest]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_build.yml
|
||||
with:
|
||||
build_name: binary_aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
BuilderBinFreeBSD:
|
||||
needs: [RunConfig, FastTest]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_build.yml
|
||||
with:
|
||||
build_name: binary_freebsd
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
BuilderBinDarwinAarch64:
|
||||
needs: [RunConfig, FastTest]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_build.yml
|
||||
with:
|
||||
build_name: binary_darwin_aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
BuilderBinPPC64:
|
||||
needs: [RunConfig, FastTest]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_build.yml
|
||||
with:
|
||||
build_name: binary_ppc64le
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
BuilderBinAmd64Compat:
|
||||
needs: [RunConfig, FastTest]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_build.yml
|
||||
with:
|
||||
build_name: binary_amd64_compat
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
BuilderBinAmd64Musl:
|
||||
needs: [RunConfig, FastTest]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_build.yml
|
||||
with:
|
||||
build_name: binary_amd64_musl
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
BuilderBinAarch64V80Compat:
|
||||
needs: [RunConfig, FastTest]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_build.yml
|
||||
with:
|
||||
build_name: binary_aarch64_v80compat
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
BuilderBinRISCV64:
|
||||
needs: [RunConfig, FastTest]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_build.yml
|
||||
with:
|
||||
build_name: binary_riscv64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
BuilderBinS390X:
|
||||
needs: [RunConfig, FastTest]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_build.yml
|
||||
with:
|
||||
build_name: binary_s390x
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
############################################################################################
|
||||
##################################### Docker images #######################################
|
||||
############################################################################################
|
||||
DockerServerImage:
|
||||
needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Docker server image
|
||||
runner_type: style-checker
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
DockerKeeperImage:
|
||||
needs: [RunConfig, BuilderDebRelease, BuilderDebAarch64]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Docker keeper image
|
||||
runner_type: style-checker
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
############################################################################################
|
||||
##################################### BUILD REPORTER #######################################
|
||||
############################################################################################
|
||||
BuilderReport:
|
||||
|
||||
################################# Reports #################################
|
||||
# Reports should by run even if Builds_1/2 fail, so put them separatly in wf (not in Tests_1/2)
|
||||
Builds_1_Report:
|
||||
# run report check for failed builds to indicate the CI error
|
||||
if: ${{ !cancelled() }}
|
||||
if: ${{ !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse build check') }}
|
||||
needs:
|
||||
- RunConfig
|
||||
- BuilderDebAarch64
|
||||
- BuilderDebAsan
|
||||
- BuilderDebDebug
|
||||
- BuilderDebMsan
|
||||
- BuilderDebRelease
|
||||
- BuilderDebTsan
|
||||
- BuilderDebUBsan
|
||||
- Builds_1
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: ClickHouse build check
|
||||
runner_type: style-checker
|
||||
runner_type: style-checker-aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
BuilderSpecialReport:
|
||||
Builds_2_Report:
|
||||
# run report check for failed builds to indicate the CI error
|
||||
if: ${{ !cancelled() }}
|
||||
if: ${{ !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse special build check') }}
|
||||
needs:
|
||||
- RunConfig
|
||||
- BuilderBinAarch64
|
||||
- BuilderBinDarwin
|
||||
- BuilderBinDarwinAarch64
|
||||
- BuilderBinFreeBSD
|
||||
- BuilderBinPPC64
|
||||
- BuilderBinRISCV64
|
||||
- BuilderBinS390X
|
||||
- BuilderBinAmd64Compat
|
||||
- BuilderBinAarch64V80Compat
|
||||
- BuilderBinClangTidy
|
||||
- BuilderDebReleaseCoverage
|
||||
- BuilderBinRelease
|
||||
- Builds_2
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: ClickHouse special build check
|
||||
runner_type: style-checker
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
############################################################################################
|
||||
#################################### INSTALL PACKAGES ######################################
|
||||
############################################################################################
|
||||
InstallPackagesTestRelease:
|
||||
needs: [RunConfig, BuilderDebRelease]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Install packages (amd64)
|
||||
runner_type: style-checker
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
run_command: |
|
||||
python3 install_check.py "$CHECK_NAME"
|
||||
InstallPackagesTestAarch64:
|
||||
needs: [RunConfig, BuilderDebAarch64]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Install packages (arm64)
|
||||
runner_type: style-checker-aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
run_command: |
|
||||
python3 install_check.py "$CHECK_NAME"
|
||||
##############################################################################################
|
||||
########################### FUNCTIONAl STATELESS TESTS #######################################
|
||||
##############################################################################################
|
||||
FunctionalStatelessTestRelease:
|
||||
needs: [RunConfig, BuilderDebRelease]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateless tests (release)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatelessTestReleaseAnalyzerS3Replicated:
|
||||
needs: [RunConfig, BuilderDebRelease]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateless tests (release, analyzer, s3, DatabaseReplicated)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatelessTestS3Debug:
|
||||
needs: [RunConfig, BuilderDebDebug]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateless tests (debug, s3 storage)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatelessTestS3Tsan:
|
||||
needs: [RunConfig, BuilderDebTsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateless tests (tsan, s3 storage)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatelessTestAarch64:
|
||||
needs: [RunConfig, BuilderDebAarch64]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateless tests (aarch64)
|
||||
runner_type: func-tester-aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatelessTestAsan:
|
||||
needs: [RunConfig, BuilderDebAsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateless tests (asan)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatelessTestTsan:
|
||||
needs: [RunConfig, BuilderDebTsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateless tests (tsan)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatelessTestMsan:
|
||||
needs: [RunConfig, BuilderDebMsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateless tests (msan)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatelessTestUBsan:
|
||||
needs: [RunConfig, BuilderDebUBsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateless tests (ubsan)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatelessTestDebug:
|
||||
needs: [RunConfig, BuilderDebDebug]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateless tests (debug)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatelessTestFlakyCheck:
|
||||
needs: [RunConfig, BuilderDebAsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateless tests flaky check (asan)
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
runner_type: func-tester
|
||||
TestsBugfixCheck:
|
||||
needs: [RunConfig, StyleCheck]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Bugfix validation
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
##############################################################################################
|
||||
############################ FUNCTIONAl STATEFUL TESTS #######################################
|
||||
##############################################################################################
|
||||
FunctionalStatefulTestRelease:
|
||||
needs: [RunConfig, BuilderDebRelease]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateful tests (release)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatefulTestAarch64:
|
||||
needs: [RunConfig, BuilderDebAarch64]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateful tests (aarch64)
|
||||
runner_type: func-tester-aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatefulTestAsan:
|
||||
needs: [RunConfig, BuilderDebAsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateful tests (asan)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatefulTestTsan:
|
||||
needs: [RunConfig, BuilderDebTsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateful tests (tsan)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatefulTestMsan:
|
||||
needs: [RunConfig, BuilderDebMsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateful tests (msan)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatefulTestUBsan:
|
||||
needs: [RunConfig, BuilderDebUBsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateful tests (ubsan)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatefulTestDebug:
|
||||
needs: [RunConfig, BuilderDebDebug]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateful tests (debug)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
# Parallel replicas
|
||||
FunctionalStatefulTestDebugParallelReplicas:
|
||||
needs: [RunConfig, BuilderDebDebug]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateful tests (debug, ParallelReplicas)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatefulTestUBsanParallelReplicas:
|
||||
needs: [RunConfig, BuilderDebUBsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateful tests (ubsan, ParallelReplicas)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatefulTestMsanParallelReplicas:
|
||||
needs: [RunConfig, BuilderDebMsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateful tests (msan, ParallelReplicas)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatefulTestTsanParallelReplicas:
|
||||
needs: [RunConfig, BuilderDebTsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateful tests (tsan, ParallelReplicas)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatefulTestAsanParallelReplicas:
|
||||
needs: [RunConfig, BuilderDebAsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateful tests (asan, ParallelReplicas)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
FunctionalStatefulTestReleaseParallelReplicas:
|
||||
needs: [RunConfig, BuilderDebRelease]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stateful tests (release, ParallelReplicas)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
##############################################################################################
|
||||
########################### ClickBench #######################################################
|
||||
##############################################################################################
|
||||
ClickBenchAMD64:
|
||||
needs: [RunConfig, BuilderDebRelease]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: ClickBench (amd64)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
run_command: |
|
||||
python3 clickbench.py "$CHECK_NAME"
|
||||
ClickBenchAarch64:
|
||||
needs: [RunConfig, BuilderDebAarch64]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: ClickBench (aarch64)
|
||||
runner_type: func-tester-aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
run_command: |
|
||||
python3 clickbench.py "$CHECK_NAME"
|
||||
##############################################################################################
|
||||
######################################### STRESS TESTS #######################################
|
||||
##############################################################################################
|
||||
StressTestAsan:
|
||||
needs: [RunConfig, BuilderDebAsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stress test (asan)
|
||||
runner_type: stress-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
StressTestTsan:
|
||||
needs: [RunConfig, BuilderDebTsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stress test (tsan)
|
||||
runner_type: stress-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
StressTestMsan:
|
||||
needs: [RunConfig, BuilderDebMsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stress test (msan)
|
||||
runner_type: stress-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
StressTestUBsan:
|
||||
needs: [RunConfig, BuilderDebUBsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stress test (ubsan)
|
||||
runner_type: stress-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
StressTestDebug:
|
||||
needs: [RunConfig, BuilderDebDebug]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Stress test (debug)
|
||||
runner_type: stress-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
##############################################################################################
|
||||
######################################### UPGRADE CHECK ######################################
|
||||
##############################################################################################
|
||||
UpgradeCheckAsan:
|
||||
needs: [RunConfig, BuilderDebAsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Upgrade check (asan)
|
||||
runner_type: stress-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
UpgradeCheckTsan:
|
||||
needs: [RunConfig, BuilderDebTsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Upgrade check (tsan)
|
||||
runner_type: stress-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
UpgradeCheckMsan:
|
||||
needs: [RunConfig, BuilderDebMsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Upgrade check (msan)
|
||||
runner_type: stress-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
UpgradeCheckDebug:
|
||||
needs: [RunConfig, BuilderDebDebug]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Upgrade check (debug)
|
||||
runner_type: stress-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
##############################################################################################
|
||||
##################################### AST FUZZERS ############################################
|
||||
##############################################################################################
|
||||
ASTFuzzerTestAsan:
|
||||
needs: [RunConfig, BuilderDebAsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: AST fuzzer (asan)
|
||||
runner_type: fuzzer-unit-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
ASTFuzzerTestTsan:
|
||||
needs: [RunConfig, BuilderDebTsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: AST fuzzer (tsan)
|
||||
runner_type: fuzzer-unit-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
ASTFuzzerTestUBSan:
|
||||
needs: [RunConfig, BuilderDebUBsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: AST fuzzer (ubsan)
|
||||
runner_type: fuzzer-unit-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
ASTFuzzerTestMSan:
|
||||
needs: [RunConfig, BuilderDebMsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: AST fuzzer (msan)
|
||||
runner_type: fuzzer-unit-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
ASTFuzzerTestDebug:
|
||||
needs: [RunConfig, BuilderDebDebug]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: AST fuzzer (debug)
|
||||
runner_type: fuzzer-unit-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
#############################################################################################
|
||||
############################# INTEGRATION TESTS #############################################
|
||||
#############################################################################################
|
||||
IntegrationTestsAnalyzerAsan:
|
||||
needs: [RunConfig, BuilderDebAsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Integration tests (asan, analyzer)
|
||||
runner_type: stress-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
IntegrationTestsTsan:
|
||||
needs: [RunConfig, BuilderDebTsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Integration tests (tsan)
|
||||
runner_type: stress-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
IntegrationTestsAarch64:
|
||||
needs: [RunConfig, BuilderDebAarch64]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Integration tests (aarch64)
|
||||
# FIXME: there is no stress-tester for aarch64. func-tester-aarch64 is ok?
|
||||
runner_type: func-tester-aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
IntegrationTestsFlakyCheck:
|
||||
needs: [RunConfig, BuilderDebAsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Integration tests flaky check (asan)
|
||||
runner_type: stress-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
#############################################################################################
|
||||
#################################### UNIT TESTS #############################################
|
||||
#############################################################################################
|
||||
UnitTestsAsan:
|
||||
needs: [RunConfig, BuilderDebAsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Unit tests (asan)
|
||||
runner_type: fuzzer-unit-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
UnitTestsRelease:
|
||||
needs: [RunConfig, BuilderBinRelease]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Unit tests (release)
|
||||
runner_type: fuzzer-unit-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
UnitTestsTsan:
|
||||
needs: [RunConfig, BuilderDebTsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Unit tests (tsan)
|
||||
runner_type: fuzzer-unit-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
UnitTestsMsan:
|
||||
needs: [RunConfig, BuilderDebMsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Unit tests (msan)
|
||||
runner_type: fuzzer-unit-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
UnitTestsUBsan:
|
||||
needs: [RunConfig, BuilderDebUBsan]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Unit tests (ubsan)
|
||||
runner_type: fuzzer-unit-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
#############################################################################################
|
||||
#################################### PERFORMANCE TESTS ######################################
|
||||
#############################################################################################
|
||||
PerformanceComparisonX86:
|
||||
needs: [RunConfig, BuilderDebRelease]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Performance Comparison
|
||||
runner_type: stress-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
PerformanceComparisonAarch:
|
||||
needs: [RunConfig, BuilderDebAarch64]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Performance Comparison Aarch64
|
||||
runner_type: func-tester-aarch64
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
##############################################################################################
|
||||
###################################### SQLANCER FUZZERS ######################################
|
||||
##############################################################################################
|
||||
SQLancerTestRelease:
|
||||
needs: [RunConfig, BuilderDebRelease]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: SQLancer (release)
|
||||
runner_type: fuzzer-unit-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
SQLancerTestDebug:
|
||||
needs: [RunConfig, BuilderDebDebug]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: SQLancer (debug)
|
||||
runner_type: fuzzer-unit-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
|
||||
################################# Stage Final #################################
|
||||
#
|
||||
FinishCheck:
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
needs:
|
||||
- BuilderReport
|
||||
- BuilderSpecialReport
|
||||
- DocsCheck
|
||||
- FastTest
|
||||
- TestsBugfixCheck
|
||||
- FunctionalStatelessTestDebug
|
||||
- FunctionalStatelessTestRelease
|
||||
- FunctionalStatelessTestAarch64
|
||||
- FunctionalStatelessTestAsan
|
||||
- FunctionalStatelessTestTsan
|
||||
- FunctionalStatelessTestMsan
|
||||
- FunctionalStatelessTestUBsan
|
||||
- FunctionalStatefulTestDebug
|
||||
- FunctionalStatefulTestRelease
|
||||
- FunctionalStatefulTestAarch64
|
||||
- FunctionalStatefulTestAsan
|
||||
- FunctionalStatefulTestTsan
|
||||
- FunctionalStatefulTestMsan
|
||||
- FunctionalStatefulTestUBsan
|
||||
- FunctionalStatelessTestS3Debug
|
||||
- FunctionalStatelessTestS3Tsan
|
||||
- FunctionalStatelessTestReleaseAnalyzerS3Replicated
|
||||
- FunctionalStatefulTestReleaseParallelReplicas
|
||||
- FunctionalStatefulTestAsanParallelReplicas
|
||||
- FunctionalStatefulTestTsanParallelReplicas
|
||||
- FunctionalStatefulTestMsanParallelReplicas
|
||||
- FunctionalStatefulTestUBsanParallelReplicas
|
||||
- FunctionalStatefulTestDebugParallelReplicas
|
||||
- StressTestDebug
|
||||
- StressTestAsan
|
||||
- StressTestTsan
|
||||
- StressTestMsan
|
||||
- StressTestUBsan
|
||||
- UpgradeCheckAsan
|
||||
- UpgradeCheckTsan
|
||||
- UpgradeCheckMsan
|
||||
- UpgradeCheckDebug
|
||||
- ASTFuzzerTestDebug
|
||||
- ASTFuzzerTestAsan
|
||||
- ASTFuzzerTestTsan
|
||||
- ASTFuzzerTestMSan
|
||||
- ASTFuzzerTestUBSan
|
||||
- IntegrationTestsAnalyzerAsan
|
||||
- IntegrationTestsTsan
|
||||
- IntegrationTestsAarch64
|
||||
- IntegrationTestsFlakyCheck
|
||||
- PerformanceComparisonX86
|
||||
- PerformanceComparisonAarch
|
||||
- UnitTestsAsan
|
||||
- UnitTestsTsan
|
||||
- UnitTestsMsan
|
||||
- UnitTestsUBsan
|
||||
- UnitTestsRelease
|
||||
- CompatibilityCheckX86
|
||||
- CompatibilityCheckAarch64
|
||||
- SQLancerTestRelease
|
||||
- SQLancerTestDebug
|
||||
needs: [Tests_1, Tests_2]
|
||||
runs-on: [self-hosted, style-checker]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: ClickHouse/checkout@v1
|
||||
with:
|
||||
clear-repository: true
|
||||
- name: Finish label
|
||||
run: |
|
||||
cd "$GITHUB_WORKSPACE/tests/ci"
|
||||
python3 finish_check.py
|
||||
python3 merge_pr.py --check-approved
|
||||
##############################################################################################
|
||||
############################ SQLLOGIC TEST ###################################################
|
||||
##############################################################################################
|
||||
SQLLogicTestRelease:
|
||||
needs: [RunConfig, BuilderDebRelease]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: Sqllogic test (release)
|
||||
runner_type: func-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
##############################################################################################
|
||||
##################################### SQL TEST ###############################################
|
||||
##############################################################################################
|
||||
SQLTest:
|
||||
needs: [RunConfig, BuilderDebRelease]
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: SQLTest
|
||||
runner_type: fuzzer-unit-tester
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
#############################################################################################
|
||||
###################################### NOT IN FINISH ########################################
|
||||
|
||||
|
||||
#############################################################################################
|
||||
###################################### JEPSEN TESTS #########################################
|
||||
#############################################################################################
|
||||
@ -931,19 +184,11 @@ jobs:
|
||||
# we need concurrency as the job uses dedicated instances in the cloud
|
||||
concurrency:
|
||||
group: jepsen
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
needs: [RunConfig, BuilderBinRelease]
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'ClickHouse Keeper Jepsen') }}
|
||||
# jepsen needs binary_release build which is in Builds_2
|
||||
needs: [RunConfig, Builds_2]
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: ClickHouse Keeper Jepsen
|
||||
runner_type: style-checker
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
#############################################################################################
|
||||
####################################### libFuzzer ###########################################
|
||||
#############################################################################################
|
||||
libFuzzer:
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
needs: [RunConfig, StyleCheck]
|
||||
uses: ./.github/workflows/libfuzzer.yml
|
||||
with:
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
|
32
.github/workflows/reusable_build_stage.yml
vendored
Normal file
32
.github/workflows/reusable_build_stage.yml
vendored
Normal file
@ -0,0 +1,32 @@
|
||||
### FIXME: merge reusable_test.yml and reusable_build.yml as they are almost identical
|
||||
# and then merge reusable_build_stage.yml and reusable_test_stage.yml
|
||||
|
||||
name: BuildStageWF
|
||||
'on':
|
||||
workflow_call:
|
||||
inputs:
|
||||
stage:
|
||||
description: stage name
|
||||
type: string
|
||||
required: true
|
||||
data:
|
||||
description: ci data
|
||||
type: string
|
||||
required: true
|
||||
|
||||
jobs:
|
||||
s:
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
job_name_and_runner_type: ${{ fromJson(inputs.data).stages_data[inputs.stage] }}
|
||||
uses: ./.github/workflows/reusable_build.yml
|
||||
with:
|
||||
build_name: ${{ matrix.job_name_and_runner_type.job_name }}
|
||||
runner_type: ${{ matrix.job_name_and_runner_type.runner_type }}
|
||||
# don't forget to pass force flag (no ci cache/no reuse) - once it's needed
|
||||
force: false
|
||||
# for now let's do I deep checkout for builds
|
||||
checkout_depth: 0
|
||||
data: ${{ inputs.data }}
|
25
.github/workflows/reusable_test_stage.yml
vendored
Normal file
25
.github/workflows/reusable_test_stage.yml
vendored
Normal file
@ -0,0 +1,25 @@
|
||||
name: StageWF
|
||||
'on':
|
||||
workflow_call:
|
||||
inputs:
|
||||
stage:
|
||||
description: stage name
|
||||
type: string
|
||||
required: true
|
||||
data:
|
||||
description: ci data
|
||||
type: string
|
||||
required: true
|
||||
|
||||
jobs:
|
||||
s:
|
||||
if: ${{ !failure() && !cancelled() }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
job_name_and_runner_type: ${{ fromJson(inputs.data).stages_data[inputs.stage] }}
|
||||
uses: ./.github/workflows/reusable_test.yml
|
||||
with:
|
||||
test_name: ${{ matrix.job_name_and_runner_type.job_name }}
|
||||
runner_type: ${{ matrix.job_name_and_runner_type.runner_type }}
|
||||
data: ${{ inputs.data }}
|
@ -17,7 +17,7 @@ from typing import Any, Dict, List, Optional, Sequence, Set, Union
|
||||
import docker_images_helper
|
||||
import upload_result_helper
|
||||
from build_check import get_release_or_pr
|
||||
from ci_config import CI_CONFIG, Build, JobNames, Labels
|
||||
from ci_config import CI_CONFIG, Build, CIStages, Labels, JobNames
|
||||
from ci_utils import GHActions, is_hex, normalize_string
|
||||
from clickhouse_helper import (
|
||||
CiLogsCredentials,
|
||||
@ -1119,7 +1119,7 @@ def _configure_jobs(
|
||||
|
||||
## b. check what we need to run
|
||||
ci_cache = None
|
||||
if not ci_cache_disabled:
|
||||
if not ci_cache_disabled and CI:
|
||||
ci_cache = CiCache(s3, digests).update()
|
||||
ci_cache.print_status()
|
||||
|
||||
@ -1283,6 +1283,29 @@ def _configure_jobs(
|
||||
}
|
||||
|
||||
|
||||
def _generate_ci_stage_config(jobs_data: Dict[str, Any]) -> Dict[str, Dict[str, Any]]:
|
||||
"""
|
||||
populates GH Actions' workflow with real jobs
|
||||
"Builds_1": [{"job_name": NAME, "runner_type": RUNER_TYPE}]
|
||||
"Tests_1": [{"job_name": NAME, "runner_type": RUNER_TYPE}]
|
||||
...
|
||||
"""
|
||||
result = {} # type: Dict[str, Any]
|
||||
stages_to_do = []
|
||||
for job in jobs_data["jobs_to_do"]:
|
||||
stage_type = CI_CONFIG.get_job_ci_stage(job)
|
||||
if stage_type == CIStages.NA:
|
||||
continue
|
||||
if stage_type not in result:
|
||||
result[stage_type] = []
|
||||
stages_to_do.append(stage_type)
|
||||
result[stage_type].append(
|
||||
{"job_name": job, "runner_type": CI_CONFIG.get_runner_type(job)}
|
||||
)
|
||||
result["stages_to_do"] = stages_to_do
|
||||
return result
|
||||
|
||||
|
||||
def _create_gh_status(
|
||||
commit: Any, job: str, batch: int, num_batches: int, job_status: CommitStatusData
|
||||
) -> None:
|
||||
@ -1734,6 +1757,7 @@ def main() -> int:
|
||||
result["build"] = build_digest
|
||||
result["docs"] = docs_digest
|
||||
result["ci_flags"] = ci_flags
|
||||
result["stages_data"] = _generate_ci_stage_config(jobs_data)
|
||||
result["jobs_data"] = jobs_data
|
||||
result["docker_data"] = docker_data
|
||||
### CONFIGURE action: end
|
||||
|
@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from copy import deepcopy
|
||||
import logging
|
||||
from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
|
||||
from dataclasses import dataclass, field
|
||||
@ -10,6 +11,24 @@ from ci_utils import WithIter
|
||||
from integration_test_images import IMAGES
|
||||
|
||||
|
||||
class CIStages:
|
||||
NA = "UNKNOWN"
|
||||
BUILDS_1 = "Builds_1"
|
||||
BUILDS_2 = "Builds_2"
|
||||
TESTS_1 = "Tests_1"
|
||||
TESTS_2 = "Tests_2"
|
||||
|
||||
|
||||
class Runners(metaclass=WithIter):
|
||||
BUILDER = "builder"
|
||||
STYLE_CHECKER = "style-checker"
|
||||
STYLE_CHECKER_ARM = "style-checker-aarch64"
|
||||
FUNC_TESTER = "func-tester"
|
||||
FUNC_TESTER_ARM = "func-tester-aarch64"
|
||||
STRESS_TESTER = "stress-tester"
|
||||
FUZZER_UNIT_TESTER = "fuzzer-unit-tester"
|
||||
|
||||
|
||||
class Labels(metaclass=WithIter):
|
||||
"""
|
||||
Label names or commit tokens in normalized form
|
||||
@ -207,23 +226,7 @@ class JobConfig:
|
||||
random_bucket: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class BuildConfig:
|
||||
name: str
|
||||
compiler: str
|
||||
package_type: Literal["deb", "binary", "fuzzers"]
|
||||
additional_pkgs: bool = False
|
||||
debug_build: bool = False
|
||||
coverage: bool = False
|
||||
sanitizer: str = ""
|
||||
tidy: bool = False
|
||||
# sparse_checkout is needed only to test the option itself.
|
||||
# No particular sense to use it in every build, since it slows down the job.
|
||||
sparse_checkout: bool = False
|
||||
comment: str = ""
|
||||
static_binary_name: str = ""
|
||||
job_config: JobConfig = field(
|
||||
default_factory=lambda: JobConfig(
|
||||
builds_job_config = JobConfig(
|
||||
required_on_release_branch=True,
|
||||
digest=DigestConfig(
|
||||
include_paths=[
|
||||
@ -257,8 +260,27 @@ class BuildConfig:
|
||||
git_submodules=True,
|
||||
),
|
||||
run_command="build_check.py $BUILD_NAME",
|
||||
)
|
||||
)
|
||||
)
|
||||
fuzzer_build_job_config = deepcopy(builds_job_config)
|
||||
fuzzer_build_job_config.run_by_label = Labels.libFuzzer
|
||||
|
||||
|
||||
@dataclass
|
||||
class BuildConfig:
|
||||
name: str
|
||||
compiler: str
|
||||
package_type: Literal["deb", "binary", "fuzzers"]
|
||||
additional_pkgs: bool = False
|
||||
debug_build: bool = False
|
||||
coverage: bool = False
|
||||
sanitizer: str = ""
|
||||
tidy: bool = False
|
||||
# sparse_checkout is needed only to test the option itself.
|
||||
# No particular sense to use it in every build, since it slows down the job.
|
||||
sparse_checkout: bool = False
|
||||
comment: str = ""
|
||||
static_binary_name: str = ""
|
||||
job_config: JobConfig = field(default_factory=lambda: deepcopy(builds_job_config))
|
||||
|
||||
def export_env(self, export: bool = False) -> str:
|
||||
def process(field_name: str, field: Union[bool, str]) -> str:
|
||||
@ -464,6 +486,19 @@ sql_test_params = {
|
||||
"run_command": "sqltest.py",
|
||||
"timeout": 10800,
|
||||
}
|
||||
clickbench_test_params = {
|
||||
"digest": DigestConfig(
|
||||
include_paths=[
|
||||
"tests/ci/clickbench.py",
|
||||
],
|
||||
docker=["clickhouse/clickbench"],
|
||||
),
|
||||
"run_command": 'clickbench.py "$CHECK_NAME"',
|
||||
}
|
||||
install_test_params = {
|
||||
"digest": install_check_digest,
|
||||
"run_command": 'install_check.py "$CHECK_NAME"',
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
@ -485,6 +520,37 @@ class CIConfig:
|
||||
return config
|
||||
return None
|
||||
|
||||
def get_job_ci_stage(self, job_name: str) -> str:
|
||||
if job_name in [
|
||||
JobNames.STYLE_CHECK,
|
||||
JobNames.FAST_TEST,
|
||||
JobNames.JEPSEN_KEEPER,
|
||||
JobNames.BUILD_CHECK,
|
||||
JobNames.BUILD_CHECK_SPECIAL,
|
||||
]:
|
||||
# FIXME: we can't currently handle Jepsen in the Stage as it's job has concurrency directive
|
||||
# BUILD_CHECK and BUILD_CHECK_SPECIAL runs not in stage because we need them even if Builds stage failed
|
||||
return CIStages.NA
|
||||
stage_type = None
|
||||
if self.is_build_job(job_name):
|
||||
stage_type = CIStages.BUILDS_1
|
||||
if job_name in CI_CONFIG.get_builds_for_report(
|
||||
JobNames.BUILD_CHECK_SPECIAL
|
||||
):
|
||||
# special builds go to Build_2 stage to not delay Builds_1/Test_1
|
||||
stage_type = CIStages.BUILDS_2
|
||||
elif self.is_docs_job(job_name):
|
||||
stage_type = CIStages.TESTS_1
|
||||
elif job_name == JobNames.BUILD_CHECK_SPECIAL:
|
||||
stage_type = CIStages.TESTS_2
|
||||
elif self.is_test_job(job_name):
|
||||
stage_type = CIStages.TESTS_1
|
||||
if job_name == JobNames.LIBFUZZER_TEST:
|
||||
# since fuzzers build in Builds_2, test must be in Tests_2
|
||||
stage_type = CIStages.TESTS_2
|
||||
assert stage_type, f"BUG [{job_name}]"
|
||||
return stage_type
|
||||
|
||||
def get_job_config(self, check_name: str) -> JobConfig:
|
||||
res = None
|
||||
for config in (
|
||||
@ -498,6 +564,63 @@ class CIConfig:
|
||||
break
|
||||
return res # type: ignore
|
||||
|
||||
def get_runner_type(self, check_name: str) -> str:
|
||||
result = None
|
||||
if self.is_build_job(check_name) or check_name == JobNames.FAST_TEST:
|
||||
result = Runners.BUILDER
|
||||
elif any(
|
||||
words in check_name.lower()
|
||||
for words in [
|
||||
"install packages",
|
||||
"compatibility check",
|
||||
"docker",
|
||||
"build check",
|
||||
"jepsen",
|
||||
"style check",
|
||||
]
|
||||
):
|
||||
result = Runners.STYLE_CHECKER
|
||||
elif check_name == JobNames.DOCS_CHECK:
|
||||
# docs job is demanding
|
||||
result = Runners.FUNC_TESTER_ARM
|
||||
elif any(
|
||||
words in check_name.lower()
|
||||
for words in [
|
||||
"stateless",
|
||||
"stateful",
|
||||
"clickbench",
|
||||
"sqllogic test",
|
||||
"libfuzzer",
|
||||
"bugfix validation",
|
||||
]
|
||||
):
|
||||
result = Runners.FUNC_TESTER
|
||||
elif any(
|
||||
words in check_name.lower()
|
||||
for words in ["stress", "upgrade", "integration", "performance comparison"]
|
||||
):
|
||||
result = Runners.STRESS_TESTER
|
||||
elif any(
|
||||
words in check_name.lower()
|
||||
for words in ["ast fuzzer", "unit tests", "sqlancer", "sqltest"]
|
||||
):
|
||||
result = Runners.FUZZER_UNIT_TESTER
|
||||
|
||||
assert result, f"BUG, no runner for [{check_name}]"
|
||||
|
||||
if ("aarch" in check_name or "arm" in check_name) and "aarch" not in result:
|
||||
if result == Runners.STRESS_TESTER:
|
||||
# FIXME: no arm stress tester group atm
|
||||
result = Runners.FUNC_TESTER_ARM
|
||||
elif result == Runners.BUILDER:
|
||||
# crosscompile - no arm required
|
||||
pass
|
||||
else:
|
||||
# switch to aarch64 runnner
|
||||
result += "-aarch64"
|
||||
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def normalize_string(input_string: str) -> str:
|
||||
lowercase_string = input_string.lower()
|
||||
@ -598,11 +721,7 @@ class CIConfig:
|
||||
|
||||
@classmethod
|
||||
def is_test_job(cls, job: str) -> bool:
|
||||
return (
|
||||
not cls.is_build_job(job)
|
||||
and not cls.is_build_job(job)
|
||||
and job != JobNames.STYLE_CHECK
|
||||
)
|
||||
return not cls.is_build_job(job) and job != JobNames.STYLE_CHECK
|
||||
|
||||
@classmethod
|
||||
def is_docs_job(cls, job: str) -> bool:
|
||||
@ -843,7 +962,7 @@ CI_CONFIG = CIConfig(
|
||||
name=Build.FUZZERS,
|
||||
compiler="clang-17",
|
||||
package_type="fuzzers",
|
||||
job_config=JobConfig(run_by_label=Labels.libFuzzer),
|
||||
job_config=fuzzer_build_job_config,
|
||||
),
|
||||
},
|
||||
builds_report_config={
|
||||
@ -887,6 +1006,7 @@ CI_CONFIG = CIConfig(
|
||||
include_paths=["**/*.md", "./docs", "tests/ci/docs_check.py"],
|
||||
docker=["clickhouse/docs-builder"],
|
||||
),
|
||||
run_command="docs_check.py",
|
||||
),
|
||||
),
|
||||
JobNames.FAST_TEST: TestConfig(
|
||||
@ -916,10 +1036,10 @@ CI_CONFIG = CIConfig(
|
||||
},
|
||||
test_configs={
|
||||
JobNames.INSTALL_TEST_AMD: TestConfig(
|
||||
Build.PACKAGE_RELEASE, job_config=JobConfig(digest=install_check_digest)
|
||||
Build.PACKAGE_RELEASE, job_config=JobConfig(**install_test_params) # type: ignore
|
||||
),
|
||||
JobNames.INSTALL_TEST_ARM: TestConfig(
|
||||
Build.PACKAGE_AARCH64, job_config=JobConfig(digest=install_check_digest)
|
||||
Build.PACKAGE_AARCH64, job_config=JobConfig(**install_test_params) # type: ignore
|
||||
),
|
||||
JobNames.STATEFUL_TEST_ASAN: TestConfig(
|
||||
Build.PACKAGE_ASAN, job_config=JobConfig(**stateful_test_common_params) # type: ignore
|
||||
@ -1137,9 +1257,20 @@ CI_CONFIG = CIConfig(
|
||||
JobNames.SQLTEST: TestConfig(
|
||||
Build.PACKAGE_RELEASE, job_config=JobConfig(**sql_test_params) # type: ignore
|
||||
),
|
||||
JobNames.CLCIKBENCH_TEST: TestConfig(Build.PACKAGE_RELEASE),
|
||||
JobNames.CLCIKBENCH_TEST_ARM: TestConfig(Build.PACKAGE_AARCH64),
|
||||
JobNames.LIBFUZZER_TEST: TestConfig(Build.FUZZERS, job_config=JobConfig(run_by_label=Labels.libFuzzer)), # type: ignore
|
||||
JobNames.CLCIKBENCH_TEST: TestConfig(
|
||||
Build.PACKAGE_RELEASE, job_config=JobConfig(**clickbench_test_params) # type: ignore
|
||||
),
|
||||
JobNames.CLCIKBENCH_TEST_ARM: TestConfig(
|
||||
Build.PACKAGE_AARCH64, job_config=JobConfig(**clickbench_test_params) # type: ignore
|
||||
),
|
||||
JobNames.LIBFUZZER_TEST: TestConfig(
|
||||
Build.FUZZERS,
|
||||
job_config=JobConfig(
|
||||
run_by_label=Labels.libFuzzer,
|
||||
timeout=10800,
|
||||
run_command='libfuzzer_test_check.py "$CHECK_NAME" 10800',
|
||||
),
|
||||
), # type: ignore
|
||||
},
|
||||
)
|
||||
CI_CONFIG.validate()
|
||||
|
@ -1,15 +1,12 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import unittest
|
||||
from ci_config import JobNames, CI_CONFIG, Runners
|
||||
|
||||
|
||||
class TestCIConfig(unittest.TestCase):
|
||||
def test_no_errors_in_ci_config(self):
|
||||
raised = None
|
||||
try:
|
||||
from ci_config import ( # pylint: disable=import-outside-toplevel
|
||||
CI_CONFIG as _,
|
||||
)
|
||||
except Exception as exc:
|
||||
raised = exc
|
||||
self.assertIsNone(raised, f"CI_CONFIG import raised error {raised}")
|
||||
def test_runner_config(self):
|
||||
"""check runner is provided w/o exception"""
|
||||
for job in JobNames:
|
||||
runner = CI_CONFIG.get_runner_type(job)
|
||||
self.assertIn(runner, Runners)
|
||||
|
Loading…
Reference in New Issue
Block a user