From 73fa0f93a51137367a6b883fe31cdd28d466f618 Mon Sep 17 00:00:00 2001 From: Max Kainov Date: Mon, 11 Nov 2024 21:55:12 +0100 Subject: [PATCH] arm build --- .github/workflows/pr.yaml | 166 +++++++++++++++++++++- ci/docker/stateless-test/Dockerfile | 9 +- ci/docker/stateless-test/requirements.txt | 1 + ci/jobs/build_clickhouse.py | 20 +-- ci/jobs/functional_stateless_tests.py | 4 + ci/praktika/gh.py | 2 +- ci/praktika/native_jobs.py | 4 +- ci/settings/definitions.py | 4 +- ci/workflows/pull_request.py | 64 +++++---- tests/config/client_config.xml | 1 + 10 files changed, 231 insertions(+), 44 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 51bb9b52d10..a70ff0cfe23 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -182,7 +182,7 @@ jobs: build_amd_debug: runs-on: [builder] - needs: [config_workflow, docker_builds] + needs: [config_workflow, docker_builds, fast_test] if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9kZWJ1Zyk=') }} name: "Build (amd_debug)" outputs: @@ -222,7 +222,7 @@ jobs: build_amd_release: runs-on: [builder] - needs: [config_workflow, docker_builds] + needs: [config_workflow, docker_builds, fast_test] if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFtZF9yZWxlYXNlKQ==') }} name: "Build (amd_release)" outputs: @@ -260,6 +260,86 @@ jobs: python3 -m praktika run --job '''Build (amd_release)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log fi + build_arm_release: + runs-on: [builder-aarch64] + needs: [config_workflow, docker_builds, fast_test] + if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFybV9yZWxlYXNlKQ==') }} + name: "Build (arm_release)" + outputs: + data: ${{ steps.run.outputs.DATA }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + + - name: Prepare env script + run: | + cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:. + + cat > /tmp/praktika/workflow_config_pr.json << 'EOF' + ${{ needs.config_workflow.outputs.data }} + EOF + cat > /tmp/praktika/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika + mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output + + - name: Run + id: run + run: | + . /tmp/praktika_setup_env.sh + set -o pipefail + if command -v ts &> /dev/null; then + python3 -m praktika run --job '''Build (arm_release)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log + else + python3 -m praktika run --job '''Build (arm_release)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log + fi + + build_arm_asan: + runs-on: [builder-aarch64] + needs: [config_workflow, docker_builds, fast_test] + if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'QnVpbGQgKGFybV9hc2FuKQ==') }} + name: "Build (arm_asan)" + outputs: + data: ${{ steps.run.outputs.DATA }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + + - name: Prepare env script + run: | + cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:. + + cat > /tmp/praktika/workflow_config_pr.json << 'EOF' + ${{ needs.config_workflow.outputs.data }} + EOF + cat > /tmp/praktika/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika + mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output + + - name: Run + id: run + run: | + . /tmp/praktika_setup_env.sh + set -o pipefail + if command -v ts &> /dev/null; then + python3 -m praktika run --job '''Build (arm_asan)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log + else + python3 -m praktika run --job '''Build (arm_asan)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log + fi + stateless_tests_amd_debugparallel: runs-on: [builder] needs: [config_workflow, docker_builds, build_amd_debug] @@ -420,9 +500,89 @@ jobs: python3 -m praktika run --job '''Stateless tests (amd_release,non-parallel)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log fi + stateless_tests_arm_asanparallel: + runs-on: [builder-aarch64] + needs: [config_workflow, docker_builds, build_arm_asan] + if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYXNhbixwYXJhbGxlbCk=') }} + name: "Stateless tests (arm_asan,parallel)" + outputs: + data: ${{ steps.run.outputs.DATA }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + + - name: Prepare env script + run: | + cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:. + + cat > /tmp/praktika/workflow_config_pr.json << 'EOF' + ${{ needs.config_workflow.outputs.data }} + EOF + cat > /tmp/praktika/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika + mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output + + - name: Run + id: run + run: | + . /tmp/praktika_setup_env.sh + set -o pipefail + if command -v ts &> /dev/null; then + python3 -m praktika run --job '''Stateless tests (arm_asan,parallel)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log + else + python3 -m praktika run --job '''Stateless tests (arm_asan,parallel)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log + fi + + stateless_tests_arm_asannon_parallel: + runs-on: [func-tester-aarch64] + needs: [config_workflow, docker_builds, build_arm_asan] + if: ${{ !failure() && !cancelled() && !contains(fromJson(needs.config_workflow.outputs.data).cache_success_base64, 'U3RhdGVsZXNzIHRlc3RzIChhcm1fYXNhbixub24tcGFyYWxsZWwp') }} + name: "Stateless tests (arm_asan,non-parallel)" + outputs: + data: ${{ steps.run.outputs.DATA }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + ref: ${{ github.head_ref }} + + - name: Prepare env script + run: | + cat > /tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:. + + cat > /tmp/praktika/workflow_config_pr.json << 'EOF' + ${{ needs.config_workflow.outputs.data }} + EOF + cat > /tmp/praktika/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + rm -rf /tmp/praktika/input /tmp/praktika/output /tmp/praktika + mkdir -p /tmp/praktika /tmp/praktika/input /tmp/praktika/output + + - name: Run + id: run + run: | + . /tmp/praktika_setup_env.sh + set -o pipefail + if command -v ts &> /dev/null; then + python3 -m praktika run --job '''Stateless tests (arm_asan,non-parallel)''' --workflow "PR" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log + else + python3 -m praktika run --job '''Stateless tests (arm_asan,non-parallel)''' --workflow "PR" --ci |& tee /tmp/praktika/praktika_run.log + fi + finish_workflow: runs-on: [ci_services] - needs: [config_workflow, docker_builds, style_check, fast_test, build_amd_debug, build_amd_release, stateless_tests_amd_debugparallel, stateless_tests_amd_debugnon_parallel, stateless_tests_amd_releaseparallel, stateless_tests_amd_releasenon_parallel] + needs: [config_workflow, docker_builds, style_check, fast_test, build_amd_debug, build_amd_release, build_arm_release, build_arm_asan, stateless_tests_amd_debugparallel, stateless_tests_amd_debugnon_parallel, stateless_tests_amd_releaseparallel, stateless_tests_amd_releasenon_parallel, stateless_tests_arm_asanparallel, stateless_tests_arm_asannon_parallel] if: ${{ !cancelled() }} name: "Finish Workflow" outputs: diff --git a/ci/docker/stateless-test/Dockerfile b/ci/docker/stateless-test/Dockerfile index 760fceeebbf..cd02fc27f8e 100644 --- a/ci/docker/stateless-test/Dockerfile +++ b/ci/docker/stateless-test/Dockerfile @@ -7,6 +7,12 @@ RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.6.20200320/clickhouse-odbc-1.1.6-Linux.tar.gz" + +RUN mkdir /etc/clickhouse-server /etc/clickhouse-keeper /etc/clickhouse-server && chmod 777 /etc/clickhouse-* + +RUN addgroup --gid 1001 clickhouse && adduser --uid 1001 --gid 1001 --disabled-password clickhouse +USER clickhouse + # moreutils - provides ts fo FT # expect, bzip2 - requried by FT # bsdmainutils - provides hexdump for FT @@ -106,6 +112,3 @@ RUN curl -L --no-verbose -O 'https://archive.apache.org/dist/hadoop/common/hadoo RUN npm install -g azurite@3.30.0 \ && npm install -g tslib && npm install -g node - -RUN addgroup --gid 1001 clickhouse && adduser --uid 1001 --gid 1001 --disabled-password clickhouse -USER clickhouse \ No newline at end of file diff --git a/ci/docker/stateless-test/requirements.txt b/ci/docker/stateless-test/requirements.txt index d556d23485f..6f64cc08951 100644 --- a/ci/docker/stateless-test/requirements.txt +++ b/ci/docker/stateless-test/requirements.txt @@ -3,3 +3,4 @@ numpy==1.26.4 requests==2.32.3 pandas==1.5.3 scipy==1.12.0 +pyarrow==18.0.0 diff --git a/ci/jobs/build_clickhouse.py b/ci/jobs/build_clickhouse.py index 3bdc23d383c..ed9fd491fcf 100644 --- a/ci/jobs/build_clickhouse.py +++ b/ci/jobs/build_clickhouse.py @@ -60,24 +60,24 @@ def main(): CACHE_TYPE = "sccache" + BUILD_TYPE = "RelWithDebInfo" + SANITIZER = "" + AUX_DEFS = " -DENABLE_TESTS=0 " + if "debug" in build_type: print("Build type set: debug") BUILD_TYPE = "Debug" - AUX_DEFS = ( - " -DENABLE_TESTS=1 -DSPLIT_DEBUG_SYMBOLS=ON -DBUILD_STANDALONE_KEEPER=1 " - ) + AUX_DEFS = " -DENABLE_TESTS=1 " elif "release" in build_type: print("Build type set: release") - BUILD_TYPE = "RelWithDebInfo" - AUX_DEFS = " -DENABLE_TESTS=0 " - else: - assert False - - if "asan" in build_type: + AUX_DEFS = ( + " -DENABLE_TESTS=0 -DSPLIT_DEBUG_SYMBOLS=ON -DBUILD_STANDALONE_KEEPER=1 " + ) + elif "asan" in build_type: print("Sanitizer set: address") SANITIZER = "address" else: - SANITIZER = "" + assert False cmake_cmd = CMAKE_CMD.format( BUILD_TYPE=BUILD_TYPE, diff --git a/ci/jobs/functional_stateless_tests.py b/ci/jobs/functional_stateless_tests.py index 390a6336b45..c2d374aa51b 100644 --- a/ci/jobs/functional_stateless_tests.py +++ b/ci/jobs/functional_stateless_tests.py @@ -1,4 +1,5 @@ import argparse +import os import time from pathlib import Path @@ -94,12 +95,15 @@ def main(): if res and JobStages.INSTALL_CLICKHOUSE in stages: commands = [ + f"rm -rf /tmp/praktika/var/log/clickhouse-server/clickhouse-server.*", f"chmod +x {ch_path}/clickhouse", f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-server", f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-client", f"rm -rf {Settings.TEMP_DIR}/etc/ && mkdir -p {Settings.TEMP_DIR}/etc/clickhouse-client {Settings.TEMP_DIR}/etc/clickhouse-server", f"cp programs/server/config.xml programs/server/users.xml {Settings.TEMP_DIR}/etc/clickhouse-server/", f"./tests/config/install.sh {Settings.TEMP_DIR}/etc/clickhouse-server {Settings.TEMP_DIR}/etc/clickhouse-client --s3-storage", + # clickhouse benchmark segfaults with --config-path, so provide client config by its default location + f"cp {Settings.TEMP_DIR}/etc/clickhouse-client/* /etc/clickhouse-client/", # update_path_ch_config, # f"sed -i 's|>/var/|>{Settings.TEMP_DIR}/var/|g; s|>/etc/|>{Settings.TEMP_DIR}/etc/|g' {Settings.TEMP_DIR}/etc/clickhouse-server/config.xml", # f"sed -i 's|>/etc/|>{Settings.TEMP_DIR}/etc/|g' {Settings.TEMP_DIR}/etc/clickhouse-server/config.d/ssl_certs.xml", diff --git a/ci/praktika/gh.py b/ci/praktika/gh.py index 77c360a0052..b7e49628ac8 100644 --- a/ci/praktika/gh.py +++ b/ci/praktika/gh.py @@ -18,7 +18,7 @@ class GH: ret_code, out, err = Shell.get_res_stdout_stderr(command, verbose=True) res = ret_code == 0 if not res and "Validation Failed" in err: - print("ERROR: GH command validation error") + print(f"ERROR: GH command validation error.") break if not res and "Bad credentials" in err: print("ERROR: GH credentials/auth failure") diff --git a/ci/praktika/native_jobs.py b/ci/praktika/native_jobs.py index 52bf6c6e204..cff6c851d0e 100644 --- a/ci/praktika/native_jobs.py +++ b/ci/praktika/native_jobs.py @@ -344,7 +344,9 @@ def _finish_workflow(workflow, job_name): failed_results.append(result.name) if failed_results: - ready_for_merge_description = f"Failed: {', '.join(failed_results)}" + ready_for_merge_description = ( + f'Failed {len(failed_results)} "Required for Merge" jobs' + ) if not GH.post_commit_status( name=Settings.READY_FOR_MERGE_STATUS_NAME + f" [{workflow.name}]", diff --git a/ci/settings/definitions.py b/ci/settings/definitions.py index 99fec8b5402..9f529798830 100644 --- a/ci/settings/definitions.py +++ b/ci/settings/definitions.py @@ -7,8 +7,10 @@ S3_BUCKET_HTTP_ENDPOINT = "clickhouse-builds.s3.amazonaws.com" class RunnerLabels: CI_SERVICES = "ci_services" CI_SERVICES_EBS = "ci_services_ebs" - BUILDER = "builder" + BUILDER_AMD = "builder" + BUILDER_ARM = "builder-aarch64" FUNC_TESTER_AMD = "func-tester" + FUNC_TESTER_ARM = "func-tester-aarch64" BASE_BRANCH = "master" diff --git a/ci/workflows/pull_request.py b/ci/workflows/pull_request.py index 707babb1250..94dcc2ab722 100644 --- a/ci/workflows/pull_request.py +++ b/ci/workflows/pull_request.py @@ -15,6 +15,8 @@ from ci.settings.definitions import ( class ArtifactNames: CH_AMD_DEBUG = "CH_AMD_DEBUG" CH_AMD_RELEASE = "CH_AMD_RELEASE" + CH_ARM_RELEASE = "CH_ARM_RELEASE" + CH_ARM_ASAN = "CH_ARM_ASAN" style_check_job = Job.Config( @@ -26,7 +28,7 @@ style_check_job = Job.Config( fast_test_job = Job.Config( name=JobNames.FAST_TEST, - runs_on=[RunnerLabels.BUILDER], + runs_on=[RunnerLabels.BUILDER_AMD], command="python3 ./ci/jobs/fast_test.py", run_in_docker="clickhouse/fasttest", digest_config=Job.CacheDigestConfig( @@ -38,9 +40,10 @@ fast_test_job = Job.Config( ), ) -amd_build_jobs = Job.Config( +build_jobs = Job.Config( name=JobNames.BUILD, - runs_on=[RunnerLabels.BUILDER], + runs_on=["...from params..."], + requires=[JobNames.FAST_TEST], command="python3 ./ci/jobs/build_clickhouse.py --build-type {PARAMETER}", run_in_docker="clickhouse/fasttest", timeout=3600 * 2, @@ -60,13 +63,24 @@ amd_build_jobs = Job.Config( ], ), ).parametrize( - parameter=["amd_debug", "amd_release"], - provides=[[ArtifactNames.CH_AMD_DEBUG], [ArtifactNames.CH_AMD_RELEASE]], + parameter=["amd_debug", "amd_release", "arm_release", "arm_asan"], + provides=[ + [ArtifactNames.CH_AMD_DEBUG], + [ArtifactNames.CH_AMD_RELEASE], + [ArtifactNames.CH_ARM_RELEASE], + [ArtifactNames.CH_ARM_ASAN], + ], + runs_on=[ + [RunnerLabels.BUILDER_AMD], + [RunnerLabels.BUILDER_AMD], + [RunnerLabels.BUILDER_ARM], + [RunnerLabels.BUILDER_ARM], + ], ) stateless_tests_jobs = Job.Config( name=JobNames.STATELESS, - runs_on=[RunnerLabels.BUILDER], + runs_on=[RunnerLabels.BUILDER_AMD], command="python3 ./ci/jobs/functional_stateless_tests.py --test-options {PARAMETER}", # many tests expect to see "/var/lib/clickhouse" in various output lines - add mount for now, consider creating this dir in docker file run_in_docker="clickhouse/stateless-test+--security-opt seccomp=unconfined+--volume=/tmp/praktika:/var/lib/clickhouse", @@ -81,37 +95,27 @@ stateless_tests_jobs = Job.Config( "amd_debug,non-parallel", "amd_release,parallel", "amd_release,non-parallel", + "arm_asan,parallel", + "arm_asan,non-parallel", ], runs_on=[ - [RunnerLabels.BUILDER], + [RunnerLabels.BUILDER_AMD], [RunnerLabels.FUNC_TESTER_AMD], - [RunnerLabels.BUILDER], + [RunnerLabels.BUILDER_AMD], [RunnerLabels.FUNC_TESTER_AMD], + [RunnerLabels.BUILDER_ARM], + [RunnerLabels.FUNC_TESTER_ARM], ], requires=[ [ArtifactNames.CH_AMD_DEBUG], [ArtifactNames.CH_AMD_DEBUG], [ArtifactNames.CH_AMD_RELEASE], [ArtifactNames.CH_AMD_RELEASE], + [ArtifactNames.CH_ARM_ASAN], + [ArtifactNames.CH_ARM_ASAN], ], ) -# stateless_tests_amd_release_jobs = Job.Config( -# name=JobNames.STATELESS_AMD_RELEASE, -# runs_on=[RunnerLabels.BUILDER], -# command="python3 ./ci/jobs/functional_stateless_tests.py --test-options {PARAMETER}", -# run_in_docker="clickhouse/stateless-test+--security-opt seccomp=unconfined+--volume=/tmp/praktika:/var/lib/clickhouse", -# digest_config=Job.CacheDigestConfig( -# include_paths=[ -# "./ci/jobs/functional_stateless_tests.py", -# ], -# ), -# requires=[ArtifactNames.CH_AMD_RELEASE], -# ).parametrize( -# parameter=["parallel", "non-parallel"], -# runs_on=[[RunnerLabels.BUILDER], [RunnerLabels.FUNC_TESTER_AMD]], -# ) - workflow = Workflow.Config( name="PR", event=Workflow.Event.PULL_REQUEST, @@ -119,7 +123,7 @@ workflow = Workflow.Config( jobs=[ style_check_job, fast_test_job, - *amd_build_jobs, + *build_jobs, *stateless_tests_jobs, ], artifacts=[ @@ -133,6 +137,16 @@ workflow = Workflow.Config( type=Artifact.Type.S3, path=f"{Settings.TEMP_DIR}/build/programs/clickhouse", ), + Artifact.Config( + name=ArtifactNames.CH_ARM_RELEASE, + type=Artifact.Type.S3, + path=f"{Settings.TEMP_DIR}/build/programs/clickhouse", + ), + Artifact.Config( + name=ArtifactNames.CH_ARM_ASAN, + type=Artifact.Type.S3, + path=f"{Settings.TEMP_DIR}/build/programs/clickhouse", + ), ], dockers=DOCKERS, secrets=SECRETS, diff --git a/tests/config/client_config.xml b/tests/config/client_config.xml index b91952b8773..0eec3b09d24 100644 --- a/tests/config/client_config.xml +++ b/tests/config/client_config.xml @@ -6,6 +6,7 @@ true sslv2,sslv3 true + none AcceptCertificateHandler