arm build

2024-11-21 15:12:02 +00:00 · 2024-11-11 21:55:12 +01:00 · 2024-11-11 21:55:12 +01:00 · 73fa0f93a5
commit 73fa0f93a5
parent 06debdc479
10 changed files with 231 additions and 44 deletions
--- a/.github/workflows/pr.yaml
+++ b/.github/workflows/pr.yaml
@ -182,7 +182,7 @@ jobs:

  build_amd_debug:
    runs-on: [builder]
-    needs: [config_workflow, docker_builds]
+    needs: [config_workflow, docker_builds, fast_test]
    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9kZWJ1Zyk=') }}
    name: "Build (amd_debug)"
    outputs:
@ -222,7 +222,7 @@ jobs:

  build_amd_release:
    runs-on: [builder]
-    needs: [config_workflow, docker_builds]
+    needs: [config_workflow, docker_builds, fast_test]
    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }}
    name: "Build (amd_release)"
    outputs:
@ -260,6 +260,86 @@ jobs:
            python3 -m praktika run --job '''Build (amd_release)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
          fi

+  build_arm_release:
+    runs-on: [builder-aarch64]
+    needs: [config_workflow, docker_builds, fast_test]
+    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }}
+    name: "Build (arm_release)"
+    outputs:
+      data: ${{ steps.run.outputs.DATA }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+            ref: ${{ github.head_ref }}
+
+      - name: Prepare env script
+        run: |
+          cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
+          export PYTHONPATH=./ci:.
+
+          cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
+          ${{ needs.config_workflow.outputs.data }}
+          EOF
+          cat > /tmp/praktika/workflow_status.json << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          ENV_SETUP_SCRIPT_EOF
+
+          rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
+          mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
+
+      - name: Run
+        id: run
+        run: |
+          . /tmp/praktika_setup_env.sh
+          set -o pipefail
+          if command -v ts &> /dev/null; then
+            python3 -m praktika run --job '''Build (arm_release)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
+          else
+            python3 -m praktika run --job '''Build (arm_release)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
+          fi
+
+  build_arm_asan:
+    runs-on: [builder-aarch64]
+    needs: [config_workflow, docker_builds, fast_test]
+    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFybV9hc2FuKQ==') }}
+    name: "Build (arm_asan)"
+    outputs:
+      data: ${{ steps.run.outputs.DATA }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+            ref: ${{ github.head_ref }}
+
+      - name: Prepare env script
+        run: |
+          cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
+          export PYTHONPATH=./ci:.
+
+          cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
+          ${{ needs.config_workflow.outputs.data }}
+          EOF
+          cat > /tmp/praktika/workflow_status.json << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          ENV_SETUP_SCRIPT_EOF
+
+          rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
+          mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
+
+      - name: Run
+        id: run
+        run: |
+          . /tmp/praktika_setup_env.sh
+          set -o pipefail
+          if command -v ts &> /dev/null; then
+            python3 -m praktika run --job '''Build (arm_asan)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
+          else
+            python3 -m praktika run --job '''Build (arm_asan)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
+          fi
+
  stateless_tests_amd_debugparallel:
    runs-on: [builder]
    needs: [config_workflow, docker_builds, build_amd_debug]
@ -420,9 +500,89 @@ jobs:
            python3 -m praktika run --job '''Stateless tests (amd_release,non-parallel)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
          fi

+  stateless_tests_arm_asanparallel:
+    runs-on: [builder-aarch64]
+    needs: [config_workflow, docker_builds, build_arm_asan]
+    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYXNhbixwYXJhbGxlbCk=') }}
+    name: "Stateless tests (arm_asan,parallel)"
+    outputs:
+      data: ${{ steps.run.outputs.DATA }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+            ref: ${{ github.head_ref }}
+
+      - name: Prepare env script
+        run: |
+          cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
+          export PYTHONPATH=./ci:.
+
+          cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
+          ${{ needs.config_workflow.outputs.data }}
+          EOF
+          cat > /tmp/praktika/workflow_status.json << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          ENV_SETUP_SCRIPT_EOF
+
+          rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
+          mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
+
+      - name: Run
+        id: run
+        run: |
+          . /tmp/praktika_setup_env.sh
+          set -o pipefail
+          if command -v ts &> /dev/null; then
+            python3 -m praktika run --job '''Stateless tests (arm_asan,parallel)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
+          else
+            python3 -m praktika run --job '''Stateless tests (arm_asan,parallel)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
+          fi
+
+  stateless_tests_arm_asannon_parallel:
+    runs-on: [func-tester-aarch64]
+    needs: [config_workflow, docker_builds, build_arm_asan]
+    if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYXNhbixub24tcGFyYWxsZWwp') }}
+    name: "Stateless tests (arm_asan,non-parallel)"
+    outputs:
+      data: ${{ steps.run.outputs.DATA }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+            ref: ${{ github.head_ref }}
+
+      - name: Prepare env script
+        run: |
+          cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
+          export PYTHONPATH=./ci:.
+
+          cat > /tmp/praktika/workflow_config_pr.json << 'EOF'
+          ${{ needs.config_workflow.outputs.data }}
+          EOF
+          cat > /tmp/praktika/workflow_status.json << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          ENV_SETUP_SCRIPT_EOF
+
+          rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika
+          mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output
+
+      - name: Run
+        id: run
+        run: |
+          . /tmp/praktika_setup_env.sh
+          set -o pipefail
+          if command -v ts &> /dev/null; then
+            python3 -m praktika run --job '''Stateless tests (arm_asan,non-parallel)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
+          else
+            python3 -m praktika run --job '''Stateless tests (arm_asan,non-parallel)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log
+          fi
+
  finish_workflow:
    runs-on: [ci_services]
-    needs: [config_workflow, docker_builds, style_check, fast_test, build_amd_debug, build_amd_release, stateless_tests_amd_debugparallel, stateless_tests_amd_debugnon_parallel, stateless_tests_amd_releaseparallel, stateless_tests_amd_releasenon_parallel]
+    needs: [config_workflow, docker_builds, style_check, fast_test, build_amd_debug, build_amd_release, build_arm_release, build_arm_asan, stateless_tests_amd_debugparallel, stateless_tests_amd_debugnon_parallel, stateless_tests_amd_releaseparallel, stateless_tests_amd_releasenon_parallel, stateless_tests_arm_asanparallel, stateless_tests_arm_asannon_parallel]
    if: ${{ !cancelled() }}
    name: "Finish Workflow"
    outputs:
--- a/ci/docker/stateless-test/Dockerfile
+++ b/ci/docker/stateless-test/Dockerfile
@ -7,6 +7,12 @@ RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list

 ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.6.20200320/clickhouse-odbc-1.1.6-Linux.tar.gz"

+
+RUN mkdir /etc/clickhouse-server /etc/clickhouse-keeper /etc/clickhouse-server && chmod 777 /etc/clickhouse-*
+
+RUN addgroup --gid 1001 clickhouse && adduser --uid 1001 --gid 1001 --disabled-password clickhouse
+USER clickhouse
+
 # moreutils - provides ts fo FT
 # expect, bzip2 - requried by FT
 # bsdmainutils - provides hexdump for FT
@ -106,6 +112,3 @@ RUN curl -L --no-verbose -O 'https://archive.apache.org/dist/hadoop/common/hadoo

 RUN npm install -g azurite@3.30.0 \
    && npm install -g tslib && npm install -g node
-
-RUN addgroup --gid 1001 clickhouse && adduser --uid 1001 --gid 1001 --disabled-password clickhouse
-USER clickhouse
--- a/ci/docker/stateless-test/requirements.txt
+++ b/ci/docker/stateless-test/requirements.txt
@ -3,3 +3,4 @@ numpy==1.26.4
 requests==2.32.3
 pandas==1.5.3
 scipy==1.12.0
+pyarrow==18.0.0
--- a/ci/jobs/build_clickhouse.py
+++ b/ci/jobs/build_clickhouse.py
@ -60,24 +60,24 @@ def main():

    CACHE_TYPE = "sccache"

+    BUILD_TYPE = "RelWithDebInfo"
+    SANITIZER = ""
+    AUX_DEFS = " -DENABLE_TESTS=0 "
+
    if "debug" in build_type:
        print("Build type set: debug")
        BUILD_TYPE = "Debug"
-        AUX_DEFS = (
-            " -DENABLE_TESTS=1 -DSPLIT_DEBUG_SYMBOLS=ON -DBUILD_STANDALONE_KEEPER=1 "
-        )
+        AUX_DEFS = " -DENABLE_TESTS=1 "
    elif "release" in build_type:
        print("Build type set: release")
-        BUILD_TYPE = "RelWithDebInfo"
-        AUX_DEFS = " -DENABLE_TESTS=0 "
-    else:
-        assert False
-
-    if "asan" in build_type:
+        AUX_DEFS = (
+            " -DENABLE_TESTS=0 -DSPLIT_DEBUG_SYMBOLS=ON -DBUILD_STANDALONE_KEEPER=1 "
+        )
+    elif "asan" in build_type:
        print("Sanitizer set: address")
        SANITIZER = "address"
    else:
-        SANITIZER = ""
+        assert False

    cmake_cmd = CMAKE_CMD.format(
        BUILD_TYPE=BUILD_TYPE,
--- a/ci/jobs/functional_stateless_tests.py
+++ b/ci/jobs/functional_stateless_tests.py
@ -1,4 +1,5 @@
 import argparse
+import os
 import time
 from pathlib import Path

@ -94,12 +95,15 @@ def main():

    if res and JobStages.INSTALL_CLICKHOUSE in stages:
        commands = [
+            f"rm -rf /tmp/praktika/var/log/clickhouse-server/clickhouse-server.*",
            f"chmod +x {ch_path}/clickhouse",
            f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-server",
            f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-client",
            f"rm -rf {Settings.TEMP_DIR}/etc/ && mkdir -p {Settings.TEMP_DIR}/etc/clickhouse-client {Settings.TEMP_DIR}/etc/clickhouse-server",
            f"cp programs/server/config.xml programs/server/users.xml {Settings.TEMP_DIR}/etc/clickhouse-server/",
            f"./tests/config/install.sh {Settings.TEMP_DIR}/etc/clickhouse-server {Settings.TEMP_DIR}/etc/clickhouse-client --s3-storage",
+            # clickhouse benchmark segfaults with --config-path, so provide client config by its default location
+            f"cp {Settings.TEMP_DIR}/etc/clickhouse-client/* /etc/clickhouse-client/",
            # update_path_ch_config,
            # f"sed -i 's|>/var/|>{Settings.TEMP_DIR}/var/|g; s|>/etc/|>{Settings.TEMP_DIR}/etc/|g' {Settings.TEMP_DIR}/etc/clickhouse-server/config.xml",
            # f"sed -i 's|>/etc/|>{Settings.TEMP_DIR}/etc/|g' {Settings.TEMP_DIR}/etc/clickhouse-server/config.d/ssl_certs.xml",
--- a/ci/praktika/gh.py
+++ b/ci/praktika/gh.py
@ -18,7 +18,7 @@ class GH:
            ret_code, out, err = Shell.get_res_stdout_stderr(command, verbose=True)
            res = ret_code == 0
            if not res and "Validation Failed" in err:
-                print("ERROR: GH command validation error")
+                print(f"ERROR: GH command validation error.")
                break
            if not res and "Bad credentials" in err:
                print("ERROR: GH credentials/auth failure")
--- a/ci/praktika/native_jobs.py
+++ b/ci/praktika/native_jobs.py
@ -344,7 +344,9 @@ def _finish_workflow(workflow, job_name):
            failed_results.append(result.name)

    if failed_results:
-        ready_for_merge_description = f"Failed: {', '.join(failed_results)}"
+        ready_for_merge_description = (
+            f'Failed {len(failed_results)} "Required for Merge" jobs'
+        )

    if not GH.post_commit_status(
        name=Settings.READY_FOR_MERGE_STATUS_NAME + f" [{workflow.name}]",
--- a/ci/settings/definitions.py
+++ b/ci/settings/definitions.py
@ -7,8 +7,10 @@ S3_BUCKET_HTTP_ENDPOINT = "clickhouse-builds.s3.amazonaws.com"
 class RunnerLabels:
    CI_SERVICES = "ci_services"
    CI_SERVICES_EBS = "ci_services_ebs"
-    BUILDER = "builder"
+    BUILDER_AMD = "builder"
+    BUILDER_ARM = "builder-aarch64"
    FUNC_TESTER_AMD = "func-tester"
+    FUNC_TESTER_ARM = "func-tester-aarch64"


 BASE_BRANCH = "master"
--- a/ci/workflows/pull_request.py
+++ b/ci/workflows/pull_request.py
@ -15,6 +15,8 @@ from ci.settings.definitions import (
 class ArtifactNames:
    CH_AMD_DEBUG = "CH_AMD_DEBUG"
    CH_AMD_RELEASE = "CH_AMD_RELEASE"
+    CH_ARM_RELEASE = "CH_ARM_RELEASE"
+    CH_ARM_ASAN = "CH_ARM_ASAN"


 style_check_job = Job.Config(
@ -26,7 +28,7 @@ style_check_job = Job.Config(

 fast_test_job = Job.Config(
    name=JobNames.FAST_TEST,
-    runs_on=[RunnerLabels.BUILDER],
+    runs_on=[RunnerLabels.BUILDER_AMD],
    command="python3 ./ci/jobs/fast_test.py",
    run_in_docker="clickhouse/fasttest",
    digest_config=Job.CacheDigestConfig(
@ -38,9 +40,10 @@ fast_test_job = Job.Config(
    ),
 )

-amd_build_jobs = Job.Config(
+build_jobs = Job.Config(
    name=JobNames.BUILD,
-    runs_on=[RunnerLabels.BUILDER],
+    runs_on=["...from params..."],
+    requires=[JobNames.FAST_TEST],
    command="python3 ./ci/jobs/build_clickhouse.py --build-type {PARAMETER}",
    run_in_docker="clickhouse/fasttest",
    timeout=3600 * 2,
@ -60,13 +63,24 @@ amd_build_jobs = Job.Config(
        ],
    ),
 ).parametrize(
-    parameter=["amd_debug", "amd_release"],
-    provides=[[ArtifactNames.CH_AMD_DEBUG], [ArtifactNames.CH_AMD_RELEASE]],
+    parameter=["amd_debug", "amd_release", "arm_release", "arm_asan"],
+    provides=[
+        [ArtifactNames.CH_AMD_DEBUG],
+        [ArtifactNames.CH_AMD_RELEASE],
+        [ArtifactNames.CH_ARM_RELEASE],
+        [ArtifactNames.CH_ARM_ASAN],
+    ],
+    runs_on=[
+        [RunnerLabels.BUILDER_AMD],
+        [RunnerLabels.BUILDER_AMD],
+        [RunnerLabels.BUILDER_ARM],
+        [RunnerLabels.BUILDER_ARM],
+    ],
 )

 stateless_tests_jobs = Job.Config(
    name=JobNames.STATELESS,
-    runs_on=[RunnerLabels.BUILDER],
+    runs_on=[RunnerLabels.BUILDER_AMD],
    command="python3 ./ci/jobs/functional_stateless_tests.py --test-options {PARAMETER}",
    # many tests expect to see "/var/lib/clickhouse" in various output lines - add mount for now, consider creating this dir in docker file
    run_in_docker="clickhouse/stateless-test+--security-opt seccomp=unconfined+--volume=/tmp/praktika:/var/lib/clickhouse",
@ -81,37 +95,27 @@ stateless_tests_jobs = Job.Config(
        "amd_debug,non-parallel",
        "amd_release,parallel",
        "amd_release,non-parallel",
+        "arm_asan,parallel",
+        "arm_asan,non-parallel",
    ],
    runs_on=[
-        [RunnerLabels.BUILDER],
+        [RunnerLabels.BUILDER_AMD],
        [RunnerLabels.FUNC_TESTER_AMD],
-        [RunnerLabels.BUILDER],
+        [RunnerLabels.BUILDER_AMD],
        [RunnerLabels.FUNC_TESTER_AMD],
+        [RunnerLabels.BUILDER_ARM],
+        [RunnerLabels.FUNC_TESTER_ARM],
    ],
    requires=[
        [ArtifactNames.CH_AMD_DEBUG],
        [ArtifactNames.CH_AMD_DEBUG],
        [ArtifactNames.CH_AMD_RELEASE],
        [ArtifactNames.CH_AMD_RELEASE],
+        [ArtifactNames.CH_ARM_ASAN],
+        [ArtifactNames.CH_ARM_ASAN],
    ],
 )

-# stateless_tests_amd_release_jobs = Job.Config(
-#     name=JobNames.STATELESS_AMD_RELEASE,
-#     runs_on=[RunnerLabels.BUILDER],
-#     command="python3 ./ci/jobs/functional_stateless_tests.py --test-options {PARAMETER}",
-#     run_in_docker="clickhouse/stateless-test+--security-opt seccomp=unconfined+--volume=/tmp/praktika:/var/lib/clickhouse",
-#     digest_config=Job.CacheDigestConfig(
-#         include_paths=[
-#             "./ci/jobs/functional_stateless_tests.py",
-#         ],
-#     ),
-#     requires=[ArtifactNames.CH_AMD_RELEASE],
-# ).parametrize(
-#     parameter=["parallel", "non-parallel"],
-#     runs_on=[[RunnerLabels.BUILDER], [RunnerLabels.FUNC_TESTER_AMD]],
-# )
-
 workflow = Workflow.Config(
    name="PR",
    event=Workflow.Event.PULL_REQUEST,
@ -119,7 +123,7 @@ workflow = Workflow.Config(
    jobs=[
        style_check_job,
        fast_test_job,
-        *amd_build_jobs,
+        *build_jobs,
        *stateless_tests_jobs,
    ],
    artifacts=[
@ -133,6 +137,16 @@ workflow = Workflow.Config(
            type=Artifact.Type.S3,
            path=f"{Settings.TEMP_DIR}/build/programs/clickhouse",
        ),
+        Artifact.Config(
+            name=ArtifactNames.CH_ARM_RELEASE,
+            type=Artifact.Type.S3,
+            path=f"{Settings.TEMP_DIR}/build/programs/clickhouse",
+        ),
+        Artifact.Config(
+            name=ArtifactNames.CH_ARM_ASAN,
+            type=Artifact.Type.S3,
+            path=f"{Settings.TEMP_DIR}/build/programs/clickhouse",
+        ),
    ],
    dockers=DOCKERS,
    secrets=SECRETS,
--- a/tests/config/client_config.xml
+++ b/tests/config/client_config.xml
@ -6,6 +6,7 @@
            <cacheSessions>true</cacheSessions>
            <disableProtocols>sslv2,sslv3</disableProtocols>
            <preferServerCiphers>true</preferServerCiphers>
+            <verificationMode>none</verificationMode>
            <invalidCertificateHandler>
                <name>AcceptCertificateHandler</name>  <!-- For tests only-->
            </invalidCertificateHandler>