Merge pull request #61142 from ClickHouse/revert-61133-ci_fast_style

Revert "CI: make style check faster"

tests/ci/style_check.py

@@ -10,7 +10,7 @@ from pathlib import Path
 from typing import List, Tuple
 
 from docker_images_helper import get_docker_image, pull_image
-from env_helper import CI, REPO_COPY, TEMP_PATH
+from env_helper import REPO_COPY, TEMP_PATH
 from git_helper import GIT_PREFIX, git_runner
 from pr_info import PRInfo
 from report import ERROR, FAILURE, SUCCESS, JobReport, TestResults, read_test_results
@@ -128,39 +128,31 @@ def main():
     temp_path = Path(TEMP_PATH)
     temp_path.mkdir(parents=True, exist_ok=True)
 
-    pr_info = PRInfo()
+    # pr_info = PRInfo()
 
     IMAGE_NAME = "clickhouse/style-test"
     image = pull_image(get_docker_image(IMAGE_NAME))
-    cmd_cpp = (
+    cmd_1 = (
         f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --cap-add=SYS_PTRACE "
         f"--volume={repo_path}:/ClickHouse --volume={temp_path}:/test_output "
         f"--entrypoint= -w/ClickHouse/utils/check-style "
-        f"{image} ./check_cpp.sh"
+        f"{image} ./check_cpp_docs.sh"
     )
-    cmd_py = (
+    cmd_2 = (
         f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --cap-add=SYS_PTRACE "
         f"--volume={repo_path}:/ClickHouse --volume={temp_path}:/test_output "
         f"--entrypoint= -w/ClickHouse/utils/check-style "
         f"{image} ./check_py.sh"
     )
+    logging.info("Is going to run the command: %s", cmd_1)
+    logging.info("Is going to run the command: %s", cmd_2)
 
     with ProcessPoolExecutor(max_workers=2) as executor:
-        logging.info("Is going to run the command: %s", cmd_cpp)
+        # Submit commands for execution in parallel
-        future1 = executor.submit(subprocess.run, cmd_cpp, shell=True)
+        future1 = executor.submit(subprocess.run, cmd_1, shell=True)
-        # Parallelization  does not make it faster - run subsequently
+        future2 = executor.submit(subprocess.run, cmd_2, shell=True)
+        # Wait for both commands to complete
         _ = future1.result()
-
-        run_pycheck = True
-        if CI and pr_info.number > 0:
-            # skip py check if PR and no changed py files
-            pr_info.fetch_changed_files()
-            if not any(file.endswith(".py") for file in pr_info.changed_files):
-                run_pycheck = False
-
-        if run_pycheck:
-            logging.info("Is going to run the command: %s", cmd_py)
-            future2 = executor.submit(subprocess.run, cmd_py, shell=True)
         _ = future2.result()
 
     # if args.push:

utils/check-style/check_cpp_docs.sh

@@ -4,48 +4,31 @@
 
 cd /ClickHouse/utils/check-style || echo -e "failure\tRepo not found" > /test_output/check_status.tsv
 
-start_total=`date +%s`
+# FIXME: 30 sec to wait
-
+# echo "Check duplicates" | ts
-# 40 sec - too much
-# start=`date +%s`
 # ./check-duplicate-includes.sh |& tee /test_output/duplicate_includes_output.txt
-# runtime=$((`date +%s`-start))
-# echo "Duplicates check. Done. $runtime seconds."
 
-start=`date +%s`
+echo "Check style" | ts
 ./check-style -n              |& tee /test_output/style_output.txt
-runtime=$((`date +%s`-start))
+echo "Check typos" | ts
-echo "Check style. Done. $runtime seconds."
-
-start=`date +%s`
-./check-whitespaces -n        |& tee /test_output/whitespaces_output.txt
-runtime=$((`date +%s`-start))
-echo "Check whitespaces. Done. $runtime seconds."
-
-start=`date +%s`
-./check-workflows             |& tee /test_output/workflows_output.txt
-runtime=$((`date +%s`-start))
-echo "Check workflows. Done. $runtime seconds."
-
-start=`date +%s`
-./check-submodules            |& tee /test_output/submodules_output.txt
-runtime=$((`date +%s`-start))
-echo "Check submodules. Done. $runtime seconds."
-
-start=`date +%s`
 ./check-typos                 |& tee /test_output/typos_output.txt
-runtime=$((`date +%s`-start))
+echo "Check docs spelling" | ts
-echo "Check typos. Done. $runtime seconds."
-
-start=`date +%s`
 ./check-doc-aspell            |& tee /test_output/docs_spelling_output.txt
-runtime=$((`date +%s`-start))
+echo "Check whitespaces" | ts
-echo "Check docs spelling. Done. $runtime seconds."
+./check-whitespaces -n        |& tee /test_output/whitespaces_output.txt
-
+echo "Check workflows" | ts
-runtime=$((`date +%s`-start_total))
+./check-workflows             |& tee /test_output/workflows_output.txt
-echo "Check style, total. Done. $runtime seconds."
+echo "Check submodules" | ts
-
+./check-submodules            |& tee /test_output/submodules_output.txt
+echo "Check style. Done" | ts
 
 # FIXME: 6 min to wait
 # echo "Check shell scripts with shellcheck" | ts
 # ./shellcheck-run.sh           |& tee /test_output/shellcheck_output.txt
+
+
+# FIXME: move out
+# /process_style_check_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv
+# echo "Check help for changelog generator works" | ts
+# cd ../changelog || exit 1
+# ./changelog.py -h 2>/dev/null 1>&2

utils/check-style/check_py.sh

@@ -1,22 +1,17 @@
 #!/bin/bash
 
+# yaml check is not the best one
+
 cd /ClickHouse/utils/check-style || echo -e "failure\tRepo not found" > /test_output/check_status.tsv
 
 # FIXME: 1 min to wait + head checkout
 # echo "Check python formatting with black" | ts
 # ./check-black -n              |& tee /test_output/black_output.txt
 
-start_total=`date +%s`
+echo "Check pylint" | ts
-
-start=`date +%s`
 ./check-pylint -n               |& tee /test_output/pylint_output.txt
-runtime=$((`date +%s`-start))
+echo "Check pylint. Done" | ts
-echo "Check pylint. Done. $runtime seconds."
 
-start=`date +%s`
+echo "Check python type hinting with mypy" | ts
 ./check-mypy -n               |& tee /test_output/mypy_output.txt
-runtime=$((`date +%s`-start))
+echo "Check python type hinting with mypy. Done" | ts
-echo "Check python type hinting with mypy. Done. $runtime seconds."
-
-runtime=$((`date +%s`-start_total))
-echo "Check python total. Done. $runtime seconds."