Merge pull request #32415 from ClickHouse/arm64_workers

Add arm64/aarch64 tests

.github/workflows/anchore-analysis.yml

@@ -8,6 +8,10 @@
 
 name: Docker Container Scan (clickhouse-server)
 
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
 "on":
   pull_request:
     paths:

.github/workflows/backport.yml

@@ -1,4 +1,9 @@
 name: CherryPick
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
 concurrency:
   group: cherry-pick
 on: # yamllint disable-line rule:truthy

.github/workflows/backport_branches.yml

@@ -1,4 +1,9 @@
 name: BackportPR
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
 on: # yamllint disable-line rule:truthy
   push:
     branches:

.github/workflows/cancel.yml

@@ -1,4 +1,9 @@
 name: Cancel
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
 on: # yamllint disable-line rule:truthy
   workflow_run:
     workflows: ["CIGithubActions", "ReleaseCI", "DocsCheck", "BackportPR"]

.github/workflows/docs_check.yml

@@ -1,4 +1,9 @@
 name: DocsCheck
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
 on: # yamllint disable-line rule:truthy
   pull_request:
     types:

.github/workflows/main.yml

@@ -1,4 +1,9 @@
 name: CIGithubActions
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
 on: # yamllint disable-line rule:truthy
   pull_request:
     types:
@@ -329,6 +334,47 @@ jobs:
           docker kill $(docker ps -q) ||:
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr $TEMP_PATH $CACHES_PATH
+  BuilderDebAarch64:
+    needs: [DockerHubPush, FastTest]
+    runs-on: [self-hosted, builder]
+    steps:
+      - name: Set envs
+        run: |
+          cat >> "$GITHUB_ENV" << 'EOF'
+          TEMP_PATH=${{runner.temp}}/build_check
+          IMAGES_PATH=${{runner.temp}}/images_path
+          REPO_COPY=${{runner.temp}}/build_check/ClickHouse
+          CACHES_PATH=${{runner.temp}}/../ccaches
+          CHECK_NAME=ClickHouse build check (actions)
+          BUILD_NAME=package_aarch64
+          EOF
+      - name: Download changed images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ runner.temp }}/images_path
+      - name: Check out repository code
+        uses: actions/checkout@v2
+        with:
+          submodules: 'true'
+          fetch-depth: 0 # otherwise we will have no info about contributors
+      - name: Build
+        run: |
+          sudo rm -fr $TEMP_PATH
+          mkdir -p $TEMP_PATH
+          cp -r $GITHUB_WORKSPACE $TEMP_PATH
+          cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
+      - name: Upload build URLs to artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ env.BUILD_NAME }}
+          path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill $(docker ps -q) ||:
+          docker rm -f $(docker ps -a -q) ||:
+          sudo rm -fr $TEMP_PATH $CACHES_PATH
   BuilderDebAsan:
     needs: [DockerHubPush, FastTest]
     runs-on: [self-hosted, builder]
@@ -867,6 +913,7 @@ jobs:
     needs:
       - BuilderDebRelease
       - BuilderBinRelease
+      - BuilderDebAarch64
       - BuilderDebAsan
       - BuilderDebTsan
       - BuilderDebUBsan

.github/workflows/master.yml

@@ -1,4 +1,9 @@
 name: MasterCI
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
 on: # yamllint disable-line rule:truthy
   push:
     branches:

.github/workflows/release.yml

@@ -1,4 +1,9 @@
 name: DocsReleaseChecks
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
 concurrency:
   group: master-release
   cancel-in-progress: true

.github/workflows/release_branches.yml

@@ -1,4 +1,9 @@
 name: ReleaseCI
+
+env:
+  # Force the stdout and stderr streams to be unbuffered
+  PYTHONUNBUFFERED: 1
+
 on: # yamllint disable-line rule:truthy
   push:
     branches:

debian/rules

@@ -45,6 +45,10 @@ ifdef DEB_CXX
 ifeq ($(DEB_BUILD_GNU_TYPE),$(DEB_HOST_GNU_TYPE))
         CC := $(DEB_CC)
         CXX := $(DEB_CXX)
+else ifeq (clang,$(findstring clang,$(DEB_CXX)))
+# If we crosscompile with clang, it knows what to do
+        CC := $(DEB_CC)
+        CXX := $(DEB_CXX)
 else
         CC := $(DEB_HOST_GNU_TYPE)-$(DEB_CC)
         CXX := $(DEB_HOST_GNU_TYPE)-$(DEB_CXX)

docker/packager/binary/Dockerfile

@@ -24,40 +24,34 @@ RUN apt-get update \
     && apt-key add /tmp/llvm-snapshot.gpg.key \
     && export CODENAME="$(lsb_release --codename --short | tr 'A-Z' 'a-z')" \
     && echo "deb [trusted=yes] https://apt.llvm.org/${CODENAME}/ llvm-toolchain-${CODENAME}-${LLVM_VERSION} main" >> \
-        /etc/apt/sources.list
+        /etc/apt/sources.list \
+    && apt-get clean
 
 # initial packages
-RUN apt-get update \
-    && apt-get install \
-        bash \
-        fakeroot \
-        ccache \
-        curl \
-        software-properties-common \
-        --yes --no-install-recommends
-
 RUN apt-get update \
     && apt-get install \
         bash \
         build-essential \
         ccache \
+        clang-${LLVM_VERSION} \
+        clang-tidy-${LLVM_VERSION} \
         cmake \
         curl \
+        fakeroot \
         gdb \
         git \
         gperf \
-        clang-${LLVM_VERSION} \
-        clang-tidy-${LLVM_VERSION} \
         lld-${LLVM_VERSION} \
         llvm-${LLVM_VERSION} \
         llvm-${LLVM_VERSION}-dev \
-        libicu-dev \
         moreutils \
         ninja-build \
         pigz \
         rename \
+        software-properties-common \
         tzdata \
-        --yes --no-install-recommends
+        --yes --no-install-recommends \
+    && apt-get clean
 
 # This symlink required by gcc to find lld compiler
 RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
@@ -66,7 +60,7 @@ ENV CC=clang-${LLVM_VERSION}
 ENV CXX=clang++-${LLVM_VERSION}
 
 # libtapi is required to support .tbh format from recent MacOS SDKs
-RUN git clone https://github.com/tpoechtrager/apple-libtapi.git \
+RUN git clone --depth 1 https://github.com/tpoechtrager/apple-libtapi.git \
     && cd apple-libtapi \
     && INSTALLPREFIX=/cctools ./build.sh \
     && ./install.sh \
@@ -74,7 +68,7 @@ RUN git clone https://github.com/tpoechtrager/apple-libtapi.git \
     && rm -rf apple-libtapi
 
 # Build and install tools for cross-linking to Darwin (x86-64)
-RUN git clone https://github.com/tpoechtrager/cctools-port.git \
+RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \
     && cd cctools-port/cctools \
     && ./configure --prefix=/cctools --with-libtapi=/cctools \
         --target=x86_64-apple-darwin \
@@ -83,7 +77,7 @@ RUN git clone https://github.com/tpoechtrager/cctools-port.git \
     && rm -rf cctools-port
 
 # Build and install tools for cross-linking to Darwin (aarch64)
-RUN git clone https://github.com/tpoechtrager/cctools-port.git \
+RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \
     && cd cctools-port/cctools \
     && ./configure --prefix=/cctools --with-libtapi=/cctools \
         --target=aarch64-apple-darwin \
@@ -97,7 +91,8 @@ RUN wget -nv https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacO
 # NOTE: Seems like gcc-11 is too new for ubuntu20 repository
 RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
     && apt-get update \
-    && apt-get install gcc-11 g++-11 --yes
+    && apt-get install gcc-11 g++-11 --yes \
+    && apt-get clean
 
 
 COPY build.sh /

docker/packager/deb/Dockerfile

@@ -64,8 +64,14 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
     && apt-get install gcc-11 g++-11 --yes
 
 
-# This symlink required by gcc to find lld compiler
-RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld
+# These symlinks are required:
+# /usr/bin/ld.lld: by gcc to find lld compiler
+# /usr/bin/aarch64-linux-gnu-obj*: for debug symbols stripping
+RUN ln -s /usr/bin/lld-${LLVM_VERSION} /usr/bin/ld.lld \
+    && ln -sf /usr/lib/llvm-${LLVM_VERSION}/bin/llvm-objcopy /usr/bin/aarch64-linux-gnu-strip \
+    && ln -sf /usr/lib/llvm-${LLVM_VERSION}/bin/llvm-objcopy /usr/bin/aarch64-linux-gnu-objcopy \
+    && ln -sf /usr/lib/llvm-${LLVM_VERSION}/bin/llvm-objdump /usr/bin/aarch64-linux-gnu-objdump
+
 
 COPY build.sh /
 

docker/packager/packager

@@ -29,7 +29,13 @@ def pull_image(image_name):
         return False
 
 def build_image(image_name, filepath):
-    subprocess.check_call("docker build --network=host -t {} -f {} .".format(image_name, filepath), shell=True)
+    context = os.path.dirname(filepath)
+    subprocess.check_call(
+        "docker build --network=host -t {} -f {} {}".format(
+            image_name, filepath, context
+        ),
+        shell=True,
+    )
 
 def run_docker_image_with_env(image_name, output, env_variables, ch_root, ccache_dir, docker_image_version):
     env_part = " -e ".join(env_variables)
@@ -90,6 +96,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
     elif is_cross_arm:
         cc = compiler[:-len(ARM_SUFFIX)]
         cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-aarch64.cmake")
+        result.append("DEB_ARCH_FLAG=-aarm64")
     elif is_cross_freebsd:
         cc = compiler[:-len(FREEBSD_SUFFIX)]
         cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/freebsd/toolchain-x86_64.cmake")
@@ -98,6 +105,7 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
         cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-ppc64le.cmake")
     else:
         cc = compiler
+        result.append("DEB_ARCH_FLAG=-aamd64")
 
     cxx = cc.replace('gcc', 'g++').replace('clang', 'clang++')
 

release

@@ -87,7 +87,7 @@ if [ -z "$NO_BUILD" ] ; then
     # Build (only binary packages).
     debuild --preserve-env -e PATH \
     -e DEB_CC=$DEB_CC -e DEB_CXX=$DEB_CXX -e CMAKE_FLAGS="$CMAKE_FLAGS" \
-    -b ${DEBUILD_NOSIGN_OPTIONS} ${DEBUILD_NODEPS_OPTIONS}
+    -b ${DEBUILD_NOSIGN_OPTIONS} ${DEBUILD_NODEPS_OPTIONS} ${DEB_ARCH_FLAG}
 fi
 
 if [ -n "$MAKE_RPM" ]; then

tests/ci/ci_config.py

@@ -11,7 +11,7 @@ CI_CONFIG = {
             "splitted": "unsplitted",
             "alien_pkgs": True,
             "tidy": "disable",
-            "with_coverage": False
+            "with_coverage": False,
         },
         "performance": {
             "compiler": "clang-13",
@@ -21,7 +21,7 @@ CI_CONFIG = {
             "bundled": "bundled",
             "splitted": "unsplitted",
             "tidy": "disable",
-            "with_coverage": False
+            "with_coverage": False,
         },
         "binary_gcc": {
             "compiler": "gcc-11",
@@ -31,7 +31,18 @@ CI_CONFIG = {
             "bundled": "bundled",
             "splitted": "unsplitted",
             "tidy": "disable",
-            "with_coverage": False
+            "with_coverage": False,
+        },
+        "package_aarch64": {
+            "compiler": "clang-13-aarch64",
+            "build_type": "",
+            "sanitizer": "",
+            "package_type": "deb",
+            "bundled": "bundled",
+            "splitted": "unsplitted",
+            "alien_pkgs": True,
+            "tidy": "disable",
+            "with_coverage": False,
         },
         "package_asan": {
             "compiler": "clang-13",
@@ -41,7 +52,7 @@ CI_CONFIG = {
             "bundled": "bundled",
             "splitted": "unsplitted",
             "tidy": "disable",
-            "with_coverage": False
+            "with_coverage": False,
         },
         "package_ubsan": {
             "compiler": "clang-13",
@@ -51,7 +62,7 @@ CI_CONFIG = {
             "bundled": "bundled",
             "splitted": "unsplitted",
             "tidy": "disable",
-            "with_coverage": False
+            "with_coverage": False,
         },
         "package_tsan": {
             "compiler": "clang-13",
@@ -61,7 +72,7 @@ CI_CONFIG = {
             "bundled": "bundled",
             "splitted": "unsplitted",
             "tidy": "disable",
-            "with_coverage": False
+            "with_coverage": False,
         },
         "package_msan": {
             "compiler": "clang-13",
@@ -71,7 +82,7 @@ CI_CONFIG = {
             "bundled": "bundled",
             "splitted": "unsplitted",
             "tidy": "disable",
-            "with_coverage": False
+            "with_coverage": False,
         },
         "package_debug": {
             "compiler": "clang-13",
@@ -81,7 +92,7 @@ CI_CONFIG = {
             "bundled": "bundled",
             "splitted": "unsplitted",
             "tidy": "disable",
-            "with_coverage": False
+            "with_coverage": False,
         },
         "binary_release": {
             "compiler": "clang-13",
@@ -91,7 +102,7 @@ CI_CONFIG = {
             "bundled": "bundled",
             "splitted": "unsplitted",
             "tidy": "disable",
-            "with_coverage": False
+            "with_coverage": False,
         },
         "binary_tidy": {
             "compiler": "clang-13",
@@ -101,7 +112,7 @@ CI_CONFIG = {
             "bundled": "bundled",
             "splitted": "unsplitted",
             "tidy": "enable",
-            "with_coverage": False
+            "with_coverage": False,
         },
         "binary_splitted": {
             "compiler": "clang-13",
@@ -111,7 +122,7 @@ CI_CONFIG = {
             "bundled": "bundled",
             "splitted": "splitted",
             "tidy": "disable",
-            "with_coverage": False
+            "with_coverage": False,
         },
         "binary_darwin": {
             "compiler": "clang-13-darwin",
@@ -121,7 +132,7 @@ CI_CONFIG = {
             "bundled": "bundled",
             "splitted": "unsplitted",
             "tidy": "disable",
-            "with_coverage": False
+            "with_coverage": False,
         },
         "binary_aarch64": {
             "compiler": "clang-13-aarch64",
@@ -131,7 +142,7 @@ CI_CONFIG = {
             "bundled": "bundled",
             "splitted": "unsplitted",
             "tidy": "disable",
-            "with_coverage": False
+            "with_coverage": False,
         },
         "binary_freebsd": {
             "compiler": "clang-13-freebsd",
@@ -141,7 +152,7 @@ CI_CONFIG = {
             "bundled": "bundled",
             "splitted": "unsplitted",
             "tidy": "disable",
-            "with_coverage": False
+            "with_coverage": False,
         },
         "binary_darwin_aarch64": {
             "compiler": "clang-13-darwin-aarch64",
@@ -151,7 +162,7 @@ CI_CONFIG = {
             "bundled": "bundled",
             "splitted": "unsplitted",
             "tidy": "disable",
-            "with_coverage": False
+            "with_coverage": False,
         },
         "binary_ppc64le": {
             "compiler": "clang-13-ppc64le",
@@ -161,19 +172,20 @@ CI_CONFIG = {
             "bundled": "bundled",
             "splitted": "unsplitted",
             "tidy": "disable",
-            "with_coverage": False
-        }
+            "with_coverage": False,
+        },
     },
     "builds_report_config": {
         "ClickHouse build check (actions)": [
             "package_release",
             "performance",
+            "package_aarch64",
             "package_asan",
             "package_ubsan",
             "package_tsan",
             "package_msan",
             "package_debug",
-            "binary_release"
+            "binary_release",
         ],
         "ClickHouse special build check (actions)": [
             "binary_tidy",
@@ -320,6 +332,6 @@ CI_CONFIG = {
         },
         "Performance Comparison (actions)": {
             "required_build": "performance",
-        }
-    }
+        },
+    },
 }