ClickHouse/.github/workflows/nightly.yml

name: NightlyBuilds

env:
  # Force the stdout and stderr streams to be unbuffered
  PYTHONUNBUFFERED: 1

"on":
  schedule:
    - cron: '13 3 * * *'
  workflow_dispatch:

jobs:
  Debug:
    # The task for having a preserved ENV and event.json for later investigation
    uses: ./.github/workflows/debug.yml
  DockerHubPushAarch64:
    runs-on: [self-hosted, style-checker-aarch64]
    steps:
      - name: Check out repository code
        uses: ClickHouse/checkout@v1
        with:
          clear-repository: true
      - name: Images check
        run: |
          cd "$GITHUB_WORKSPACE/tests/ci"
          python3 docker_images_check.py --suffix aarch64 --all          
      - name: Upload images files to artifacts
        uses: actions/upload-artifact@v3
        with:
          name: changed_images_aarch64
          path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json
  DockerHubPushAmd64:
    runs-on: [self-hosted, style-checker]
    steps:
      - name: Check out repository code
        uses: ClickHouse/checkout@v1
        with:
          clear-repository: true
      - name: Images check
        run: |
          cd "$GITHUB_WORKSPACE/tests/ci"
          python3 docker_images_check.py --suffix amd64 --all          
      - name: Upload images files to artifacts
        uses: actions/upload-artifact@v3
        with:
          name: changed_images_amd64
          path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json
  DockerHubPush:
    needs: [DockerHubPushAmd64, DockerHubPushAarch64]
    runs-on: [self-hosted, style-checker]
    steps:
      - name: Check out repository code
        uses: ClickHouse/checkout@v1
        with:
          clear-repository: true
      - name: Download changed aarch64 images
        uses: actions/download-artifact@v3
        with:
          name: changed_images_aarch64
          path: ${{ runner.temp }}
      - name: Download changed amd64 images
        uses: actions/download-artifact@v3
        with:
          name: changed_images_amd64
          path: ${{ runner.temp }}
      - name: Images check
        run: |
          cd "$GITHUB_WORKSPACE/tests/ci"
          python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64          
      - name: Upload images files to artifacts
        uses: actions/upload-artifact@v3
        with:
          name: changed_images
          path: ${{ runner.temp }}/changed_images.json
  Codebrowser:
    needs: [DockerHubPush]
    uses: ./.github/workflows/woboq.yml
  BuilderCoverity:
    needs: DockerHubPush
    runs-on: [self-hosted, builder]
    steps:
      - name: Set envs
        run: |
          cat >> "$GITHUB_ENV" << 'EOF'
          BUILD_NAME=coverity
          CACHES_PATH=${{runner.temp}}/../ccaches
          IMAGES_PATH=${{runner.temp}}/images_path
          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
          TEMP_PATH=${{runner.temp}}/build_check
          EOF
          echo "COVERITY_TOKEN=${{ secrets.COVERITY_TOKEN }}" >> "$GITHUB_ENV"          
      - name: Download changed images
        uses: actions/download-artifact@v3
        with:
          name: changed_images
          path: ${{ env.IMAGES_PATH }}
      - name: Check out repository code
        uses: ClickHouse/checkout@v1
        with:
          clear-repository: true
          submodules: true
      - name: Build
        run: |
          sudo rm -fr "$TEMP_PATH"
          mkdir -p "$TEMP_PATH"
          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"          
      - name: Upload Coverity Analysis
        if: ${{ success() || failure() }}
        run: |
          curl --form token="${COVERITY_TOKEN}" \
            --form email='security+coverity@clickhouse.com' \
            --form file="@$TEMP_PATH/$BUILD_NAME/coverity-scan.tar.gz" \
            --form version="${GITHUB_REF#refs/heads/}-${GITHUB_SHA::6}" \
            --form description="Nighly Scan: $(date +'%Y-%m-%dT%H:%M:%S')" \
            https://scan.coverity.com/builds?project=ClickHouse%2FClickHouse          
      - name: Cleanup
        if: always()
        run: |
          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"          
  SonarCloud:
    # TODO: Remove if: whenever SonarCloud supports c++23
    if: ${{ false }}
    runs-on: [self-hosted, builder]
    env:
      SONAR_SCANNER_VERSION: 4.8.0.2856
      SONAR_SERVER_URL: "https://sonarcloud.io"
      BUILD_WRAPPER_OUT_DIR: build_wrapper_output_directory # Directory where build-wrapper output will be placed
      CC: clang-16
      CXX: clang++-16
    steps:
      - name: Check out repository code
        uses: ClickHouse/checkout@v1
        with:
          clear-repository: true
          fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
          submodules: true
      - name: Set up JDK 11
        uses: actions/setup-java@v1
        with:
          java-version: 11
      - name: Download and set up sonar-scanner
        env:
          SONAR_SCANNER_DOWNLOAD_URL: https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-${{ env.SONAR_SCANNER_VERSION }}-linux.zip
        run: |
          mkdir -p "$HOME/.sonar"
          curl -sSLo "$HOME/.sonar/sonar-scanner.zip" "${{ env.SONAR_SCANNER_DOWNLOAD_URL }}"
          unzip -o "$HOME/.sonar/sonar-scanner.zip" -d "$HOME/.sonar/"
          echo "$HOME/.sonar/sonar-scanner-${{ env.SONAR_SCANNER_VERSION }}-linux/bin" >> "$GITHUB_PATH"          
      - name: Download and set up build-wrapper
        env:
          BUILD_WRAPPER_DOWNLOAD_URL: ${{ env.SONAR_SERVER_URL }}/static/cpp/build-wrapper-linux-x86.zip
        run: |
          curl -sSLo "$HOME/.sonar/build-wrapper-linux-x86.zip" "${{ env.BUILD_WRAPPER_DOWNLOAD_URL }}"
          unzip -o "$HOME/.sonar/build-wrapper-linux-x86.zip" -d "$HOME/.sonar/"
          echo "$HOME/.sonar/build-wrapper-linux-x86" >> "$GITHUB_PATH"          
      - name: Set Up Build Tools
        run: |
          sudo apt-get update
          sudo apt-get install -yq git cmake ccache ninja-build python3 yasm
          sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"          
      - name: Run build-wrapper
        run: |
          mkdir build
          cd build
          cmake ..
          cd ..
          build-wrapper-linux-x86-64 --out-dir ${{ env.BUILD_WRAPPER_OUT_DIR }} cmake --build build/          
      - name: Run sonar-scanner
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
        run: |
          sonar-scanner \
            --define sonar.host.url="${{ env.SONAR_SERVER_URL }}" \
            --define sonar.cfamily.build-wrapper-output="${{ env.BUILD_WRAPPER_OUT_DIR }}" \
            --define sonar.projectKey="ClickHouse_ClickHouse" \
            --define sonar.organization="clickhouse-java" \
            --define sonar.exclusions="**/*.java,**/*.ts,**/*.js,**/*.css,**/*.sql" \