Merge pull request #37448 from ClickHouse/pr_info-dispatch

Don't fail docker images build on non-comparible events, fix docs-release

docker/docs/release/Dockerfile

@@ -1,4 +1,3 @@
-# rebuild in #33610
 # docker build -t clickhouse/docs-release .
 FROM ubuntu:20.04
 

docs/tools/deploy-to-test.sh

@@ -12,12 +12,11 @@
 #
 set -ex
 
-BASE_DIR=$(dirname $(readlink -f $0))
+BASE_DIR=$(dirname "$(readlink -f "$0")")
 GIT_USER=${GIT_USER:-$USER}
 
-GIT_TEST_URI=git@github.com:${GIT_USER}/clickhouse.github.io.git \
+GIT_PROD_URI=git@github.com:${GIT_USER}/clickhouse.github.io.git \
  BASE_DOMAIN=${GIT_USER}-test.clickhouse.com \
- EXTRA_BUILD_ARGS="${@}" \
+ EXTRA_BUILD_ARGS="${*}" \
  CLOUDFLARE_TOKEN="" \
- HISTORY_SIZE=3 \
- ${BASE_DIR}/release.sh
+ "${BASE_DIR}/release.sh"

docs/tools/release.sh

@@ -1,24 +1,24 @@
 #!/usr/bin/env bash
 set -ex
 
-BASE_DIR=$(dirname $(readlink -f $0))
+BASE_DIR=$(dirname "$(readlink -f "$0")")
 BUILD_DIR="${BASE_DIR}/../build"
 PUBLISH_DIR="${BASE_DIR}/../publish"
 BASE_DOMAIN="${BASE_DOMAIN:-content.clickhouse.com}"
-GIT_TEST_URI="${GIT_TEST_URI:-git@github.com:ClickHouse/clickhouse-com-content.git}"
-GIT_PROD_URI="git@github.com:ClickHouse/clickhouse-website-content.git"
+GIT_PROD_URI="${GIT_PROD_URI:-git@github.com:ClickHouse/clickhouse-com-content.git}"
 EXTRA_BUILD_ARGS="${EXTRA_BUILD_ARGS:---verbose}"
 
 if [[ -z "$1" ]]
 then
     source "${BASE_DIR}/venv/bin/activate"
+    # shellcheck disable=2086
     python3 "${BASE_DIR}/build.py" ${EXTRA_BUILD_ARGS}
     rm -rf "${PUBLISH_DIR}"
     mkdir "${PUBLISH_DIR}" && cd "${PUBLISH_DIR}"
 
     # Will make a repository with website content as the only commit.
     git init
-    git remote add origin "${GIT_TEST_URI}"
+    git remote add origin "${GIT_PROD_URI}"
     git config user.email "robot-clickhouse@clickhouse.com"
     git config user.name "robot-clickhouse"
 
@@ -28,7 +28,7 @@ then
     echo -n "" > README.md
     echo -n "" > ".nojekyll"
     cp "${BASE_DIR}/../../LICENSE" .
-    git add *
+    git add ./*
     git add ".nojekyll"
 
     git commit --quiet -m "Add new release at $(date)"
@@ -40,7 +40,7 @@ then
     # Turn off logging.
     set +x
 
-    if [[ ! -z "${CLOUDFLARE_TOKEN}" ]]
+    if [[ -n "${CLOUDFLARE_TOKEN}" ]]
     then
         sleep 1m
         # https://api.cloudflare.com/#zone-purge-files-by-cache-tags,-host-or-prefix

tests/ci/docker_images_check.py

@@ -404,7 +404,11 @@ def main():
     elif args.image_path:
         pr_info.changed_files = set(i for i in args.image_path)
     else:
-        pr_info.fetch_changed_files()
+        try:
+            pr_info.fetch_changed_files()
+        except TypeError:
+            # If the event does not contain diff, nothing will be built
+            pass
 
     changed_images = get_changed_docker_images(pr_info, images_dict)
     if changed_images:

tests/ci/docs_release.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+import argparse
 import logging
 import subprocess
 import os
@@ -15,17 +16,31 @@ from upload_result_helper import upload_results
 from docker_pull_helper import get_image_with_version
 from commit_status_helper import get_commit
 from rerun_helper import RerunHelper
+from tee_popen import TeePopen
 
 NAME = "Docs Release (actions)"
 
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        description="ClickHouse building script using prebuilt Docker image",
+    )
+    parser.add_argument(
+        "--as-root", action="store_true", help="if the container should run as root"
+    )
+    return parser.parse_args()
+
+
 if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO)
+    args = parse_args()
 
     temp_path = TEMP_PATH
     repo_path = REPO_COPY
 
     gh = Github(get_best_robot_token())
-    pr_info = PRInfo(need_changed_files=True)
+    pr_info = PRInfo()
     rerun_helper = RerunHelper(gh, pr_info, NAME)
     if rerun_helper.is_already_finished_by_status():
         logging.info("Check is already finished according to github status, exiting")
@@ -40,20 +55,23 @@ if __name__ == "__main__":
     if not os.path.exists(test_output):
         os.makedirs(test_output)
 
-    token = CLOUDFLARE_TOKEN
-    cmd = (
-        "docker run --cap-add=SYS_PTRACE --volume=$SSH_AUTH_SOCK:/ssh-agent "
-        f"-e SSH_AUTH_SOCK=/ssh-agent -e CLOUDFLARE_TOKEN={token} "
-        f"-e EXTRA_BUILD_ARGS='--verbose' --volume={repo_path}:/repo_path"
-        f" --volume={test_output}:/output_path {docker_image}"
-    )
+    if args.as_root:
+        user = "0:0"
+    else:
+        user = f"{os.geteuid()}:{os.getegid()}"
 
     run_log_path = os.path.join(test_output, "runlog.log")
 
-    with open(run_log_path, "w", encoding="utf-8") as log, SSHKey(
-        "ROBOT_CLICKHOUSE_SSH_KEY"
-    ):
-        with subprocess.Popen(cmd, shell=True, stderr=log, stdout=log) as process:
+    with SSHKey("ROBOT_CLICKHOUSE_SSH_KEY"):
+        cmd = (
+            f"docker run --cap-add=SYS_PTRACE --user={user} "
+            f"--volume='{os.getenv('SSH_AUTH_SOCK', '')}:/ssh-agent' "
+            f"--volume={repo_path}:/repo_path --volume={test_output}:/output_path "
+            f"-e SSH_AUTH_SOCK=/ssh-agent -e EXTRA_BUILD_ARGS='--verbose' "
+            f"-e CLOUDFLARE_TOKEN={CLOUDFLARE_TOKEN} {docker_image}"
+        )
+        logging.info("Running command: %s", cmd)
+        with TeePopen(cmd, run_log_path) as process:
             retcode = process.wait()
             if retcode == 0:
                 logging.info("Run successfully")
@@ -98,3 +116,6 @@ if __name__ == "__main__":
     commit.create_status(
         context=NAME, description=description, state=status, target_url=report_url
     )
+
+    if status == "failure":
+        sys.exit(1)

tests/ci/pr_info.py

@@ -186,6 +186,7 @@ class PRInfo:
                 else:
                     self.diff_url = pull_request["diff_url"]
         else:
+            print("event.json does not match pull_request or push:")
             print(json.dumps(github_event, sort_keys=True, indent=4))
             self.sha = os.getenv("GITHUB_SHA")
             self.number = 0
@@ -204,8 +205,8 @@ class PRInfo:
             self.fetch_changed_files()
 
     def fetch_changed_files(self):
-        if not self.diff_url:
-            raise Exception("Diff URL cannot be find for event")
+        if not getattr(self, "diff_url", False):
+            raise TypeError("The event does not have diff URL")
 
         response = get_with_retries(
             self.diff_url,