From dd4bd4abba54830e4bbd159aec5b1df1c2b1617c Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 7 Apr 2022 18:21:13 +0200 Subject: [PATCH 1/2] Simplify and fix coverity --- .github/workflows/nightly.yml | 15 ++++++++------- docker/packager/binary/build.sh | 4 +++- docker/packager/packager | 11 ++++++----- tests/ci/build_check.py | 7 ------- 4 files changed, 17 insertions(+), 20 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1e70213adf5..836421f34dd 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -79,13 +79,14 @@ jobs: - name: Set envs run: | cat >> "$GITHUB_ENV" << 'EOF' - TEMP_PATH=${{runner.temp}}/build_check - IMAGES_PATH=${{runner.temp}}/images_path - REPO_COPY=${{runner.temp}}/build_check/ClickHouse + BUILD_NAME=coverity CACHES_PATH=${{runner.temp}}/../ccaches CHECK_NAME=ClickHouse build check (actions) - BUILD_NAME=coverity + IMAGES_PATH=${{runner.temp}}/images_path + REPO_COPY=${{runner.temp}}/build_check/ClickHouse + TEMP_PATH=${{runner.temp}}/build_check EOF + echo "COVERITY_TOKEN=${{ secrets.COVERITY_TOKEN }}" >> "$GITHUB_ENV" - name: Download changed images uses: actions/download-artifact@v2 with: @@ -105,12 +106,12 @@ jobs: sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" - cd "$REPO_COPY/tests/ci" && python3 build_check.py "$CHECK_NAME" "$BUILD_NAME" "${{ secrets.COV_TOKEN }}" + cd "$REPO_COPY/tests/ci" && python3 build_check.py "$CHECK_NAME" "$BUILD_NAME" - name: Upload Coverity Analysis if: ${{ success() || failure() }} run: | - curl --form token='${{ secrets.COV_TOKEN }}' \ - --form email='${{ secrets.ROBOT_CLICKHOUSE_EMAIL }}' \ + curl --form token="${COVERITY_TOKEN}" \ + --form email='security+coverity@clickhouse.com' \ --form file="@$TEMP_PATH/$BUILD_NAME/clickhouse-scan.tgz" \ --form version="${GITHUB_REF#refs/heads/}-${GITHUB_SHA::6}" \ --form description="Nighly Scan: $(date +'%Y-%m-%dT%H:%M:%S')" \ diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 269d3eb52c6..2bedb50dd40 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -27,7 +27,9 @@ cmake --debug-trycompile --verbose=1 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUI if [ "coverity" == "$COMBINED_OUTPUT" ] then - wget --post-data "token=$COV_TOKEN&project=ClickHouse%2FClickHouse" -qO- https://scan.coverity.com/download/linux64 | tar xz -C /opt/cov-analysis --strip-components 1 + mkdir -p /opt/cov-analysis + + wget --post-data "token=$COVERITY_TOKEN&project=ClickHouse%2FClickHouse" -qO- https://scan.coverity.com/download/linux64 | tar xz -C /opt/cov-analysis --strip-components 1 export PATH=$PATH:/opt/cov-analysis/bin cov-configure --config ./coverity.config --template --comptype clangcc --compiler "$CC" SCAN_WRAPPER="cov-build --config ./coverity.config --dir cov-int" diff --git a/docker/packager/packager b/docker/packager/packager index 1a79b497fa2..9a72a16bd70 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -86,7 +86,6 @@ def parse_env_variables( additional_pkgs, with_coverage, with_binaries, - coverity_scan, ): DARWIN_SUFFIX = "-darwin" DARWIN_ARM_SUFFIX = "-darwin-aarch64" @@ -179,7 +178,7 @@ def parse_env_variables( cmake_flags.append("-DENABLE_TESTS=0") elif package_type == "coverity": result.append("COMBINED_OUTPUT=coverity") - result.append("COV_TOKEN={}".format(cov_token)) + result.append('COVERITY_TOKEN="$COVERITY_TOKEN"') elif split_binary: result.append("COMBINED_OUTPUT=shared_build") @@ -328,13 +327,16 @@ if __name__ == "__main__": parser.add_argument( "--docker-image-version", default="latest", help="docker image tag to use" ) - parser.add_argument("--cov_token", default="") args = parser.parse_args() if not os.path.isabs(args.output_dir): args.output_dir = os.path.abspath(os.path.join(os.getcwd(), args.output_dir)) - image_type = "binary" if args.package_type in ("performance", "coverity") else args.package_type + image_type = ( + "binary" + if args.package_type in ("performance", "coverity") + else args.package_type + ) image_name = "clickhouse/binary-builder" if not os.path.isabs(args.clickhouse_repo_path): @@ -376,7 +378,6 @@ if __name__ == "__main__": args.additional_pkgs, args.with_coverage, args.with_binaries, - args.cov_token, ) run_docker_image_with_env( diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 74ebebf1e6f..e9d660156e1 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -55,7 +55,6 @@ def get_packager_cmd( image_version: str, ccache_path: str, official: bool, - cov_token: str, ) -> str: package_type = build_config["package_type"] comp = build_config["compiler"] @@ -88,8 +87,6 @@ def get_packager_cmd( if official: cmd += " --official" - if cov_token: - cmd += " --cov-token={}".format(cov_token) return cmd @@ -206,9 +203,6 @@ def main(): build_check_name = sys.argv[1] build_name = sys.argv[2] - cov_token = "" - if len(sys.argv) > 3: - cov_token = sys.argv[3] build_config = get_build_config(build_check_name, build_name) @@ -303,7 +297,6 @@ def main(): image_version, ccache_path, official_flag, - cov_token, ) logging.info("Going to run packager with %s", packager_cmd) From a5b1a0620c4433d1af58409472351174741f4aa0 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 7 Apr 2022 18:22:29 +0200 Subject: [PATCH 2/2] Add default IMAGES_PATH value --- tests/ci/env_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index 242d6bf5e40..6462baad729 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -15,7 +15,7 @@ GITHUB_RUN_ID = os.getenv("GITHUB_RUN_ID", "0") GITHUB_SERVER_URL = os.getenv("GITHUB_SERVER_URL", "https://github.com") GITHUB_WORKSPACE = os.getenv("GITHUB_WORKSPACE", git_root) GITHUB_RUN_URL = f"{GITHUB_SERVER_URL}/{GITHUB_REPOSITORY}/actions/runs/{GITHUB_RUN_ID}" -IMAGES_PATH = os.getenv("IMAGES_PATH") +IMAGES_PATH = os.getenv("IMAGES_PATH", TEMP_PATH) REPORTS_PATH = os.getenv("REPORTS_PATH", p.abspath(p.join(module_dir, "./reports"))) REPO_COPY = os.getenv("REPO_COPY", git_root) RUNNER_TEMP = os.getenv("RUNNER_TEMP", p.abspath(p.join(module_dir, "./tmp")))