# yamllint disable rule:comments-indentation name: MasterCI env: # Force the stdout and stderr streams to be unbuffered PYTHONUNBUFFERED: 1 on: # yamllint disable-line rule:truthy push: branches: - 'master' jobs: PythonUnitTests: runs-on: [self-hosted, style-checker] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 with: clear-repository: true - name: Python unit tests run: | cd "$GITHUB_WORKSPACE/tests/ci" echo "Testing the main ci directory" python3 -m unittest discover -s . -p 'test_*.py' for dir in *_lambda/; do echo "Testing $dir" python3 -m unittest discover -s "$dir" -p 'test_*.py' done DockerHubPushAarch64: runs-on: [self-hosted, style-checker-aarch64] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 with: clear-repository: true - name: Images check run: | cd "$GITHUB_WORKSPACE/tests/ci" python3 docker_images_check.py --suffix aarch64 - name: Upload images files to artifacts uses: actions/upload-artifact@v3 with: name: changed_images_aarch64 path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json DockerHubPushAmd64: runs-on: [self-hosted, style-checker] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 with: clear-repository: true - name: Images check run: | cd "$GITHUB_WORKSPACE/tests/ci" python3 docker_images_check.py --suffix amd64 - name: Upload images files to artifacts uses: actions/upload-artifact@v3 with: name: changed_images_amd64 path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json DockerHubPush: needs: [DockerHubPushAmd64, DockerHubPushAarch64, PythonUnitTests] runs-on: [self-hosted, style-checker] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 with: clear-repository: true fetch-depth: 0 # to find ancestor merge commits necessary for finding proper docker tags filter: tree:0 - name: Download changed aarch64 images uses: actions/download-artifact@v3 with: name: changed_images_aarch64 path: ${{ runner.temp }} - name: Download changed amd64 images uses: actions/download-artifact@v3 with: name: changed_images_amd64 path: ${{ runner.temp }} - name: Images check run: | cd "$GITHUB_WORKSPACE/tests/ci" python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 - name: Upload images files to artifacts uses: actions/upload-artifact@v3 with: name: changed_images path: ${{ runner.temp }}/changed_images.json StyleCheck: needs: DockerHubPush if: ${{ success() || failure() }} uses: ./.github/workflows/reusable_test.yml with: test_name: Style check runner_type: style-checker run_command: | cd "$REPO_COPY/tests/ci" python3 style_check.py CompatibilityCheckX86: needs: [BuilderDebRelease] uses: ./.github/workflows/reusable_test.yml with: test_name: Compatibility check X86 runner_type: style-checker run_command: | cd "$REPO_COPY/tests/ci" python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions CompatibilityCheckAarch64: needs: [BuilderDebAarch64] uses: ./.github/workflows/reusable_test.yml with: test_name: Compatibility check X86 runner_type: style-checker run_command: | cd "$REPO_COPY/tests/ci" python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc ######################################################################################### #################################### ORDINARY BUILDS #################################### ######################################################################################### BuilderDebRelease: needs: [DockerHubPush] uses: ./.github/workflows/reusable_build.yml with: checkout_depth: 0 build_name: package_release BuilderDebAarch64: needs: [DockerHubPush] uses: ./.github/workflows/reusable_build.yml with: checkout_depth: 0 build_name: package_aarch64 BuilderBinRelease: needs: [DockerHubPush] uses: ./.github/workflows/reusable_build.yml with: checkout_depth: 0 build_name: binary_release BuilderDebAsan: needs: [DockerHubPush] uses: ./.github/workflows/reusable_build.yml with: build_name: package_asan BuilderDebUBsan: needs: [DockerHubPush] uses: ./.github/workflows/reusable_build.yml with: build_name: package_ubsan BuilderDebTsan: needs: [DockerHubPush] uses: ./.github/workflows/reusable_build.yml with: build_name: package_tsan BuilderDebMsan: needs: [DockerHubPush] uses: ./.github/workflows/reusable_build.yml with: build_name: package_msan BuilderDebDebug: needs: [DockerHubPush] uses: ./.github/workflows/reusable_build.yml with: build_name: package_debug ########################################################################################## ##################################### SPECIAL BUILDS ##################################### ########################################################################################## BuilderBinClangTidy: needs: [DockerHubPush] uses: ./.github/workflows/reusable_build.yml with: build_name: binary_tidy BuilderBinDarwin: needs: [DockerHubPush] uses: ./.github/workflows/reusable_build.yml with: build_name: binary_darwin checkout_depth: 0 BuilderBinAarch64: needs: [DockerHubPush] uses: ./.github/workflows/reusable_build.yml with: build_name: binary_aarch64 checkout_depth: 0 BuilderBinFreeBSD: needs: [DockerHubPush] uses: ./.github/workflows/reusable_build.yml with: build_name: binary_freebsd checkout_depth: 0 BuilderBinDarwinAarch64: needs: [DockerHubPush] uses: ./.github/workflows/reusable_build.yml with: build_name: binary_darwin_aarch64 checkout_depth: 0 BuilderBinPPC64: needs: [DockerHubPush] uses: ./.github/workflows/reusable_build.yml with: build_name: binary_ppc64le checkout_depth: 0 BuilderBinAmd64Compat: needs: [DockerHubPush] uses: ./.github/workflows/reusable_build.yml with: build_name: binary_amd64_compat checkout_depth: 0 BuilderBinAarch64V80Compat: needs: [DockerHubPush] uses: ./.github/workflows/reusable_build.yml with: build_name: binary_aarch64_v80compat checkout_depth: 0 BuilderBinRISCV64: needs: [DockerHubPush] uses: ./.github/workflows/reusable_build.yml with: build_name: binary_riscv64 checkout_depth: 0 BuilderBinS390X: needs: [DockerHubPush] uses: ./.github/workflows/reusable_build.yml with: build_name: binary_s390x checkout_depth: 0 ############################################################################################ ##################################### Docker images ####################################### ############################################################################################ DockerServerImages: needs: - BuilderDebRelease - BuilderDebAarch64 runs-on: [self-hosted, style-checker] steps: - name: Check out repository code uses: ClickHouse/checkout@v1 with: clear-repository: true fetch-depth: 0 # It MUST BE THE SAME for all dependencies and the job itself filter: tree:0 - name: Check docker clickhouse/clickhouse-server building run: | cd "$GITHUB_WORKSPACE/tests/ci" python3 docker_server.py --release-type head \ --image-repo clickhouse/clickhouse-server --image-path docker/server python3 docker_server.py --release-type head \ --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper - name: Cleanup if: always() run: | docker ps --quiet | xargs --no-run-if-empty docker kill ||: docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||: sudo rm -fr "$TEMP_PATH" ############################################################################################ ##################################### BUILD REPORTER ####################################### ############################################################################################ BuilderReport: if: ${{ success() || failure() }} needs: - BuilderBinRelease - BuilderDebAarch64 - BuilderDebAsan - BuilderDebDebug - BuilderDebMsan - BuilderDebRelease - BuilderDebTsan - BuilderDebUBsan uses: ./.github/workflows/reusable_test.yml with: test_name: ClickHouse build check runner_type: style-checker additional_envs: | NEEDS_DATA<