bugfix validate integration test

.github/workflows/pull_request.yml

@@ -1734,7 +1734,6 @@ jobs:
           docker rm -f "$(docker ps -a -q)" ||:
           sudo rm -fr "$TEMP_PATH"
   FunctionalStatelessTestBugfixCheck:
-    needs: []
     runs-on: [self-hosted, func-tester]
     steps:
       - name: Set envs
@@ -2727,6 +2726,39 @@ jobs:
           docker kill "$(docker ps -q)" ||:
           docker rm -f "$(docker ps -a -q)" ||:
           sudo rm -fr "$TEMP_PATH"
+  IntegrationTestsBugfixCheck:
+    runs-on: [self-hosted, stress-tester]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/integration_tests_asan_bugfix_check
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          CHECK_NAME=Integration tests bugfix validate check (asan, actions)
+          REPO_COPY=${{runner.temp}}/integration_tests_asan_bugfix_check/ClickHouse
+          EOF
+      - name: Download json reports
+        uses: actions/download-artifact@v2
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Integration test
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci"
+          python3 integration_test_check.py "$CHECK_NAME" --validate-bugfix
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH"
 #############################################################################################
 #################################### UNIT TESTS #############################################
 #############################################################################################

tests/ci/commit_status_helper.py

@@ -7,9 +7,15 @@ from ci_config import CI_CONFIG
 RETRY = 5
 
 
-def override_status(status, check_name):
+def override_status(status, check_name, invert=False):
-    if CI_CONFIG["tests_config"][check_name].get("force_tests", False):
+    if CI_CONFIG["tests_config"].get(check_name, {}).get("force_tests", False):
         return "success"
+
+    if invert:
+        if status == 'success':
+            return 'error'
+        return 'success'
+
     return status
 
 

tests/ci/download_previous_release.py

@@ -7,11 +7,12 @@
 #                                                                         #
 ###########################################################################
 
-import requests
 import re
 import os
 import logging
 
+import requests
+
 CLICKHOUSE_TAGS_URL = "https://api.github.com/repos/ClickHouse/ClickHouse/tags"
 
 CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL = "https://github.com/ClickHouse/ClickHouse/releases/download/v{version}-{type}/clickhouse-common-static_{version}_amd64.deb"
@@ -86,7 +87,7 @@ def download_packet(url, out_path):
     """
 
     response = requests.get(url)
-    logging.info(f'Downloading {url}')
+    logging.info('Downloading %s', url)
     if response.ok:
         open(out_path, 'wb').write(response.content)
 
@@ -94,7 +95,7 @@ def download_packets(release, dest_path=PACKETS_DIR):
     if not os.path.exists(dest_path):
         os.makedirs(dest_path)
 
-    logging.info(f'Will download {release}')
+    logging.info('Will download %s', release)
 
     download_packet(
         CLICKHOUSE_COMMON_STATIC_DOWNLOAD_URL.format(version=release.version, type=release.type),
@@ -123,6 +124,7 @@ def download_previous_release(dest_path):
 
 
 if __name__ == '__main__':
+    logging.basicConfig(level=logging.INFO)
     server_version = Version(input())
     previous_release = get_previous_release(server_version)
     download_packets(previous_release)

tests/ci/functional_test_check.py

@@ -50,12 +50,6 @@ def get_image_name(check_name):
         raise Exception(f"Cannot deduce image name based on check name {check_name}")
 
 
-def invert_status(status):
-    if status == 'success':
-        return 'error'
-    return 'success'
-
-
 def get_run_command(builds_path, repo_tests_path, result_path, server_log_path, kill_timeout, additional_envs, image, flaky_check, tests_to_run):
     additional_options = ['--hung-check']
     additional_options.append('--print-time')
@@ -155,6 +149,10 @@ if __name__ == "__main__":
 
     pr_info = PRInfo(need_changed_files=run_changed_tests)
 
+    if validate_bugix_check and 'pr-bugfix' not in pr_info.labels:
+        logging.info("Skipping %s (no pr-bugfix)", check_name)
+        sys.exit(0)
+
     if 'RUN_BY_HASH_NUM' in os.environ:
         run_by_hash_num = int(os.getenv('RUN_BY_HASH_NUM'))
         run_by_hash_total = int(os.getenv('RUN_BY_HASH_TOTAL'))
@@ -223,9 +221,7 @@ if __name__ == "__main__":
     s3_helper = S3Helper('https://s3.amazonaws.com')
 
     state, description, test_results, additional_logs = process_results(result_path, server_log_path)
-    state = override_status(state, check_name)
+    state = override_status(state, check_name, validate_bugix_check)
-    if validate_bugix_check:
-        state = invert_status(state)
 
     ch_helper = ClickHouseHelper()
     mark_flaky_tests(ch_helper, check_name, test_results)

tests/ci/integration_test_check.py

@@ -14,9 +14,10 @@ from s3_helper import S3Helper
 from get_robot_token import get_best_robot_token
 from pr_info import PRInfo
 from build_download_helper import download_all_deb_packages
+from download_previous_release import download_previous_release
 from upload_result_helper import upload_results
 from docker_pull_helper import get_images_with_versions
-from commit_status_helper import post_commit_status
+from commit_status_helper import post_commit_status, override_status
 from clickhouse_helper import ClickHouseHelper, mark_flaky_tests, prepare_tests_results_for_clickhouse
 from stopwatch import Stopwatch
 from rerun_helper import RerunHelper
@@ -112,6 +113,7 @@ if __name__ == "__main__":
     reports_path = REPORTS_PATH
 
     check_name = sys.argv[1]
+    validate_bugix_check = len(sys.argv) >= 3 and sys.argv[2] == "--validate-bugfix"
 
     if 'RUN_BY_HASH_NUM' in os.environ:
         run_by_hash_num = int(os.getenv('RUN_BY_HASH_NUM'))
@@ -126,7 +128,11 @@ if __name__ == "__main__":
         os.makedirs(temp_path)
 
     is_flaky_check = 'flaky' in check_name
-    pr_info = PRInfo(need_changed_files=is_flaky_check)
+    pr_info = PRInfo(need_changed_files=is_flaky_check or validate_bugix_check)
+
+    if validate_bugix_check and 'pr-bugfix' not in pr_info.labels:
+        logging.info("Skipping %s (no pr-bugfix)", check_name)
+        sys.exit(0)
 
     gh = Github(get_best_robot_token())
 
@@ -149,7 +155,10 @@ if __name__ == "__main__":
     if not os.path.exists(build_path):
         os.makedirs(build_path)
 
+    if not validate_bugix_check:
         download_all_deb_packages(check_name, reports_path, build_path)
+    else:
+        download_previous_release(build_path)
 
     my_env = get_env_for_runner(build_path, repo_path, result_path, work_path)
 
@@ -172,6 +181,7 @@ if __name__ == "__main__":
     subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True)
 
     state, description, test_results, additional_logs = process_results(result_path)
+    state = override_status(state, check_name, validate_bugix_check)
 
     ch_helper = ClickHouseHelper()
     mark_flaky_tests(ch_helper, check_name, test_results)

tests/ci/pr_info.py

@@ -78,7 +78,7 @@ class PRInfo:
             else:
                 github_event = PRInfo.default_event.copy()
         self.event = github_event
-        self.changed_files = set([])
+        self.changed_files = set()
         self.body = ""
         ref = github_event.get("ref", "refs/head/master")
         if ref and ref.startswith("refs/heads/"):
@@ -209,6 +209,7 @@ class PRInfo:
         else:
             diff_object = PatchSet(response.text)
             self.changed_files = {f.path for f in diff_object}
+        print("Fetched info about %d changed files", len(self.changed_files))
 
     def get_dict(self):
         return {