Merge pull request #34895 from DevTeamBK/coverity_scan

Attempt at Nightly Coverity Scan and upload

.github/workflows/nightly.yml

@@ -72,3 +72,52 @@ jobs:
         with:
           name: changed_images
           path: ${{ runner.temp }}/changed_images.json
+  BuilderCoverity:
+    needs: DockerHubPush
+    runs-on: [self-hosted, builder]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/build_check
+          IMAGES_PATH=${{runner.temp}}/images_path
+          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
+          CACHES_PATH=${{runner.temp}}/../ccaches
+          CHECK_NAME=ClickHouse build check (actions)
+          BUILD_NAME=coverity
+          EOF
+      - name: Download changed images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ env.IMAGES_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        id: coverity-checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: 'true'
+          fetch-depth: 0 # otherwise we will have no info about contributors
+      - name: Build
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci" && python3 build_check.py "$CHECK_NAME" "$BUILD_NAME" "${{ secrets.COV_TOKEN }}"
+      - name: Upload Coverity Analysis
+        if: ${{ success() || failure() }}
+        run: |
+          curl --form token='${{ secrets.COV_TOKEN }}' \
+            --form email='${{ secrets.ROBOT_CLICKHOUSE_EMAIL }}' \
+            --form file="@$TEMP_PATH/$BUILD_NAME/clickhouse-scan.tgz" \
+            --form version="${GITHUB_REF#refs/heads/}-${GITHUB_SHA::6}" \
+            --form description="Nighly Scan: $(date +'%Y-%m-%dT%H:%M:%S')" \
+            https://scan.coverity.com/builds?project=ClickHouse%2FClickHouse
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"

docker/packager/binary/build.sh

@@ -25,13 +25,21 @@ read -ra CMAKE_FLAGS <<< "${CMAKE_FLAGS:-}"
 env
 cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
 
+if [ "coverity" == "$COMBINED_OUTPUT" ]
+then
+    wget --post-data "token=$COV_TOKEN&project=ClickHouse%2FClickHouse" -qO- https://scan.coverity.com/download/linux64 | tar xz -C /opt/cov-analysis --strip-components 1
+    export PATH=$PATH:/opt/cov-analysis/bin
+    cov-configure --config ./coverity.config --template --comptype clangcc --compiler "$CC"
+    SCAN_WRAPPER="cov-build --config ./coverity.config --dir cov-int"
+fi
+
 cache_status
 # clear cache stats
 ccache --zero-stats ||:
 
 # No quotes because I want it to expand to nothing if empty.
-# shellcheck disable=SC2086
-ninja $NINJA_FLAGS clickhouse-bundle
+# shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty.
+$SCAN_WRAPPER ninja $NINJA_FLAGS clickhouse-bundle
 
 cache_status
 
@@ -91,6 +99,12 @@ then
     mv "$COMBINED_OUTPUT.tgz" /output
 fi
 
+if [ "coverity" == "$COMBINED_OUTPUT" ]
+then
+    tar -cv -I pigz -f "coverity-scan.tgz" cov-int
+    mv "coverity-scan.tgz" /output
+fi
+
 # Also build fuzzers if any sanitizer specified
 # if [ -n "$SANITIZER" ]
 # then

docker/packager/packager

@@ -86,6 +86,7 @@ def parse_env_variables(
     additional_pkgs,
     with_coverage,
     with_binaries,
+    coverity_scan,
 ):
     DARWIN_SUFFIX = "-darwin"
     DARWIN_ARM_SUFFIX = "-darwin-aarch64"
@@ -176,6 +177,9 @@ def parse_env_variables(
     if package_type == "performance":
         result.append("COMBINED_OUTPUT=performance")
         cmake_flags.append("-DENABLE_TESTS=0")
+    elif package_type == "coverity":
+        result.append("COMBINED_OUTPUT=coverity")
+        result.append("COV_TOKEN={}".format(cov_token))
     elif split_binary:
         result.append("COMBINED_OUTPUT=shared_build")
 
@@ -262,9 +266,8 @@ if __name__ == "__main__":
     # and configs to be used for performance test.
     parser.add_argument(
         "--package-type",
-        choices=("deb", "binary", "performance"),
+        choices=["deb", "binary", "performance", "coverity"],
         required=True,
-        help="a build type",
     )
     parser.add_argument(
         "--clickhouse-repo-path",
@@ -325,12 +328,13 @@ if __name__ == "__main__":
     parser.add_argument(
         "--docker-image-version", default="latest", help="docker image tag to use"
     )
+    parser.add_argument("--cov_token", default="")
 
     args = parser.parse_args()
     if not os.path.isabs(args.output_dir):
         args.output_dir = os.path.abspath(os.path.join(os.getcwd(), args.output_dir))
 
-    image_type = "binary" if args.package_type == "performance" else args.package_type
+    image_type = "binary" if args.package_type in ("performance", "coverity") else args.package_type
     image_name = "clickhouse/binary-builder"
 
     if not os.path.isabs(args.clickhouse_repo_path):
@@ -372,6 +376,7 @@ if __name__ == "__main__":
         args.additional_pkgs,
         args.with_coverage,
         args.with_binaries,
+        args.cov_token,
     )
 
     run_docker_image_with_env(

tests/ci/build_check.py

@@ -55,6 +55,7 @@ def get_packager_cmd(
     image_version: str,
     ccache_path: str,
     official: bool,
+    cov_token: str,
 ) -> str:
     package_type = build_config["package_type"]
     comp = build_config["compiler"]
@@ -87,6 +88,8 @@ def get_packager_cmd(
     if official:
         cmd += " --official"
 
+    if cov_token:
+        cmd += " --cov-token={}".format(cov_token)
     return cmd
 
 
@@ -203,6 +206,9 @@ def main():
 
     build_check_name = sys.argv[1]
     build_name = sys.argv[2]
+    cov_token = ""
+    if len(sys.argv) > 3:
+        cov_token = sys.argv[3]
 
     build_config = get_build_config(build_check_name, build_name)
 
@@ -296,7 +302,8 @@ def main():
         version.string,
         image_version,
         ccache_path,
-        official=official_flag,
+        official_flag,
+        cov_token,
     )
 
     logging.info("Going to run packager with %s", packager_cmd)

tests/ci/ci_config.py

@@ -28,6 +28,16 @@ CI_CONFIG = {
             "tidy": "disable",
             "with_coverage": False,
         },
+        "coverity": {
+            "compiler": "clang-13",
+            "build_type": "",
+            "sanitizer": "",
+            "package_type": "coverity",
+            "bundled": "bundled",
+            "splitted": "unsplitted",
+            "tidy": "disable",
+            "with_coverage": False,
+        },
         "binary_gcc": {
             "compiler": "gcc-11",
             "build_type": "",
@@ -190,6 +200,7 @@ CI_CONFIG = {
         "ClickHouse build check (actions)": [
             "package_release",
             "performance",
+            "coverity",
             "package_aarch64",
             "package_asan",
             "package_ubsan",