Add binary compatibility check for Aarch64

.github/workflows/backport_branches.yml

@@ -79,7 +79,7 @@ jobs:
         with:
           name: changed_images
           path: ${{ runner.temp }}/changed_images.json
-  CompatibilityCheck:
+  CompatibilityCheckX86:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, style-checker]
     steps:
@@ -98,12 +98,43 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           path: ${{ env.REPORTS_PATH }}
-      - name: CompatibilityCheck
+      - name: CompatibilityCheckX86
         run: |
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py
+          cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (x86)" --check-glibc --check-distributions
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"
+  CompatibilityCheckAarch64:
+    needs: [BuilderDebAarch64]
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/compatibility_check
+          REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          EOF
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+      - name: Download json reports
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: CompatibilityCheckAarch64
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc
       - name: Cleanup
         if: always()
         run: |
@@ -741,7 +772,8 @@ jobs:
       - FunctionalStatefulTestDebug
       - StressTestTsan
       - IntegrationTestsRelease
-      - CompatibilityCheck
+      - CompatibilityCheckX86
+      - CompatibilityCheckAarch64
     runs-on: [self-hosted, style-checker]
     steps:
       - name: Check out repository code

.github/workflows/master.yml

@@ -110,7 +110,7 @@ jobs:
           docker ps --quiet | xargs --no-run-if-empty docker kill ||:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
-  CompatibilityCheck:
+  CompatibilityCheckX86:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, style-checker]
     steps:
@@ -129,12 +129,43 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           path: ${{ env.REPORTS_PATH }}
-      - name: CompatibilityCheck
+      - name: CompatibilityCheckX86
         run: |
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py
+          cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (x86)" --check-glibc --check-distributions
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"
+  CompatibilityCheckAarch64:
+    needs: [BuilderDebAarch64]
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/compatibility_check
+          REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          EOF
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+      - name: Download json reports
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: CompatibilityCheckAarch64
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc
       - name: Cleanup
         if: always()
         run: |
@@ -3124,7 +3155,8 @@ jobs:
       - PerformanceComparisonX86-1
       - PerformanceComparisonX86-2
       - PerformanceComparisonX86-3
-      - CompatibilityCheck
+      - CompatibilityCheckX86
+      - CompatibilityCheckAarch64
       - ASTFuzzerTestDebug
       - ASTFuzzerTestAsan
       - ASTFuzzerTestTsan

.github/workflows/pull_request.yml

@@ -174,7 +174,7 @@ jobs:
           docker ps --quiet | xargs --no-run-if-empty docker kill ||:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH" "$CACHES_PATH"
-  CompatibilityCheck:
+  CompatibilityCheckX86:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, style-checker]
     steps:
@@ -193,12 +193,43 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           path: ${{ env.REPORTS_PATH }}
-      - name: CompatibilityCheck
+      - name: CompatibilityCheckX86
         run: |
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py
+          cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (x86)" --check-glibc --check-distributions
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"
+  CompatibilityCheckAarch64:
+    needs: [BuilderDebAarch64]
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/compatibility_check
+          REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          EOF
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+      - name: Download json reports
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: CompatibilityCheckAarch64
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc
       - name: Cleanup
         if: always()
         run: |
@@ -4792,7 +4823,8 @@ jobs:
       - UnitTestsMsan
       - UnitTestsUBsan
       - UnitTestsReleaseClang
-      - CompatibilityCheck
+      - CompatibilityCheckX86
+      - CompatibilityCheckAarch64
       - IntegrationTestsFlakyCheck
       - SQLancerTestRelease
       - SQLancerTestDebug

.github/workflows/release_branches.yml

@@ -71,7 +71,7 @@ jobs:
         with:
           name: changed_images
           path: ${{ runner.temp }}/changed_images.json
-  CompatibilityCheck:
+  CompatibilityCheckX86:
     needs: [BuilderDebRelease]
     runs-on: [self-hosted, style-checker]
     steps:
@@ -90,12 +90,43 @@ jobs:
         uses: actions/download-artifact@v3
         with:
           path: ${{ env.REPORTS_PATH }}
-      - name: CompatibilityCheck
+      - name: CompatibilityCheckX86
         run: |
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
           cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
-          cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py
+          cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (x86)" --check-glibc --check-distributions
+      - name: Cleanup
+        if: always()
+        run: |
+          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
+          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
+          sudo rm -fr "$TEMP_PATH"
+  CompatibilityCheckAarch64:
+    needs: [BuilderDebAarch64]
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/compatibility_check
+          REPO_COPY=${{runner.temp}}/compatibility_check/ClickHouse
+          REPORTS_PATH=${{runner.temp}}/reports_dir
+          EOF
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true
+      - name: Download json reports
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{ env.REPORTS_PATH }}
+      - name: CompatibilityCheckAarch64
+        run: |
+          sudo rm -fr "$TEMP_PATH"
+          mkdir -p "$TEMP_PATH"
+          cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
+          cd "$REPO_COPY/tests/ci" && python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc
       - name: Cleanup
         if: always()
         run: |
@@ -1947,7 +1978,8 @@ jobs:
       - IntegrationTestsTsan1
       - IntegrationTestsTsan2
       - IntegrationTestsTsan3
-      - CompatibilityCheck
+      - CompatibilityCheckX86
+      - CompatibilityCheckAarch64
     runs-on: [self-hosted, style-checker]
     steps:
       - name: Check out repository code

tests/ci/ci_config.py

@@ -316,9 +316,12 @@ CI_CONFIG = {
         "Integration tests flaky check (asan)": {
             "required_build": "package_asan",
         },
-        "Compatibility check": {
+        "Compatibility check (x86)": {
             "required_build": "package_release",
         },
+        "Compatibility check (aarch64)": {
+            "required_build": "package_aarch64",
+        },
         "Unit tests (release-clang)": {
             "required_build": "binary_release",
         },

tests/ci/compatibility_check.py

@@ -2,6 +2,7 @@
 
 from distutils.version import StrictVersion
 from typing import List, Tuple
+import argparse
 import logging
 import os
 import subprocess
@@ -28,9 +29,7 @@ from upload_result_helper import upload_results
 
 IMAGE_UBUNTU = "clickhouse/test-old-ubuntu"
 IMAGE_CENTOS = "clickhouse/test-old-centos"
-MAX_GLIBC_VERSION = "2.4"
 DOWNLOAD_RETRIES_COUNT = 5
-CHECK_NAME = "Compatibility check"
 
 
 def process_os_check(log_path: str) -> TestResult:
@@ -43,7 +42,7 @@ def process_os_check(log_path: str) -> TestResult:
             return TestResult(name, "OK")
 
 
-def process_glibc_check(log_path: str) -> TestResults:
+def process_glibc_check(log_path: str, max_glibc_version: str) -> TestResults:
     test_results = []  # type: TestResults
     with open(log_path, "r") as log:
         for line in log:
@@ -53,7 +52,7 @@ def process_glibc_check(log_path: str) -> TestResults:
                 _, version = symbol_with_glibc.split("@GLIBC_")
                 if version == "PRIVATE":
                     test_results.append(TestResult(symbol_with_glibc, "FAIL"))
-                elif StrictVersion(version) > MAX_GLIBC_VERSION:
+                elif StrictVersion(version) > max_glibc_version:
                     test_results.append(TestResult(symbol_with_glibc, "FAIL"))
     if not test_results:
         test_results.append(TestResult("glibc check", "OK"))
@@ -61,10 +60,10 @@ def process_glibc_check(log_path: str) -> TestResults:
 
 
 def process_result(
-    result_folder: str, server_log_folder: str
+    result_folder: str, server_log_folder: str, max_glibc_version: str
 ) -> Tuple[str, str, TestResults, List[str]]:
     glibc_log_path = os.path.join(result_folder, "glibc.log")
-    test_results = process_glibc_check(glibc_log_path)
+    test_results = process_glibc_check(glibc_log_path, max_glibc_version)
 
     status = "success"
     description = "Compatibility check passed"
@@ -101,13 +100,18 @@ def process_result(
     return status, description, test_results, result_logs
 
 
-def get_run_commands(
+def get_run_commands_glibc(build_path, result_folder):
-    build_path, result_folder, server_log_folder, image_centos, image_ubuntu
-):
     return [
         f"readelf -s --wide {build_path}/usr/bin/clickhouse | grep '@GLIBC_' > {result_folder}/glibc.log",
         f"readelf -s --wide {build_path}/usr/bin/clickhouse-odbc-bridge | grep '@GLIBC_' >> {result_folder}/glibc.log",
         f"readelf -s --wide {build_path}/usr/bin/clickhouse-library-bridge | grep '@GLIBC_' >> {result_folder}/glibc.log",
+    ]
+
+
+def get_run_commands_distributions(
+    build_path, result_folder, server_log_folder, image_centos, image_ubuntu
+):
+    return [
         f"docker run --network=host --volume={build_path}/usr/bin/clickhouse:/clickhouse "
         f"--volume={build_path}/etc/clickhouse-server:/config "
         f"--volume={server_log_folder}:/var/log/clickhouse-server {image_ubuntu} > {result_folder}/ubuntu:12.04",
@@ -117,9 +121,21 @@ def get_run_commands(
     ]
 
 
+def parse_args():
+    parser = argparse.ArgumentParser("Check compatibility with old distributions")
+    parser.add_argument("--check-name", required=True)
+    parser.add_argument("--check-glibc", action="store_true")
+    parser.add_argument(
+        "--check-distributions", action="store_true"
+    )  # currently hardcoded to x86, don't enable for ARM
+    return parser.parse_args()
+
+
 def main():
     logging.basicConfig(level=logging.INFO)
 
+    args = parse_args()
+
     stopwatch = Stopwatch()
 
     temp_path = TEMP_PATH
@@ -129,13 +145,11 @@ def main():
 
     gh = Github(get_best_robot_token(), per_page=100)
 
-    rerun_helper = RerunHelper(gh, pr_info, CHECK_NAME)
+    rerun_helper = RerunHelper(gh, pr_info, args.check_name)
     if rerun_helper.is_already_finished_by_status():
         logging.info("Check is already finished according to github status, exiting")
         sys.exit(0)
 
-    docker_images = get_images_with_versions(reports_path, [IMAGE_CENTOS, IMAGE_UBUNTU])
-
     packages_path = os.path.join(temp_path, "packages")
     if not os.path.exists(packages_path):
         os.makedirs(packages_path)
@@ -145,7 +159,7 @@ def main():
             "clickhouse-common-static_" in url or "clickhouse-server_" in url
         )
 
-    download_builds_filter(CHECK_NAME, reports_path, packages_path, url_filter)
+    download_builds_filter(args.check_name, reports_path, packages_path, url_filter)
 
     for f in os.listdir(packages_path):
         if ".deb" in f:
@@ -162,9 +176,24 @@ def main():
     if not os.path.exists(result_path):
         os.makedirs(result_path)
 
-    run_commands = get_run_commands(
+    run_commands = []
-        packages_path, result_path, server_log_path, docker_images[0], docker_images[1]
+
-    )
+    if args.check_glibc:
+        check_glibc_commands = get_run_commands_glibc(packages_path, result_path)
+        run_commands.extend(check_glibc_commands)
+
+    if args.check_distributions:
+        docker_images = get_images_with_versions(
+            reports_path, [IMAGE_CENTOS, IMAGE_UBUNTU]
+        )
+        check_distributions_commands = get_run_commands_distributions(
+            packages_path,
+            result_path,
+            server_log_path,
+            docker_images[0],
+            docker_images[1],
+        )
+        run_commands.extend(check_distributions_commands)
 
     state = "success"
     for run_command in run_commands:
@@ -177,13 +206,22 @@ def main():
 
     subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {temp_path}", shell=True)
 
+    # See https://sourceware.org/glibc/wiki/Glibc%20Timeline
+    max_glibc_version = ""
+    if "x86" in args.check_name:
+        max_glibc_version = "2.4"
+    elif "aarch64" in args.check_nam:
+        max_glibc_version = "2.18" # because of build with newer sysroot?
+    else:
+        raise Exception("Can't determine max glibc version")
+
     s3_helper = S3Helper()
     state, description, test_results, additional_logs = process_result(
-        result_path, server_log_path
+        result_path, server_log_path, max_glibc_version
     )
 
     ch_helper = ClickHouseHelper()
-    mark_flaky_tests(ch_helper, CHECK_NAME, test_results)
+    mark_flaky_tests(ch_helper, args.check_name, test_results)
 
     report_url = upload_results(
         s3_helper,
@@ -191,10 +229,10 @@ def main():
         pr_info.sha,
         test_results,
         additional_logs,
-        CHECK_NAME,
+        args.check_name,
     )
     print(f"::notice ::Report url: {report_url}")
-    post_commit_status(gh, pr_info.sha, CHECK_NAME, description, state, report_url)
+    post_commit_status(gh, pr_info.sha, args.check_name, description, state, report_url)
 
     prepared_events = prepare_tests_results_for_clickhouse(
         pr_info,
@@ -203,7 +241,7 @@ def main():
         stopwatch.duration_seconds,
         stopwatch.start_time_str,
         report_url,
-        CHECK_NAME,
+        args.check_name,
     )
 
     ch_helper.insert_events_into(db="default", table="checks", events=prepared_events)