Add CI check for darwin-aarch64 (#25560)

* Add support for darwin-aarch64 * Fix PVS warnings * Add build configuration * Fix packager args
2024-12-18 04:12:19 +00:00 · 2021-06-25 18:21:56 +03:00 · 2021-06-25 18:21:56 +03:00 · 14e43cfafa
commit 14e43cfafa
parent 20d20a5389
15 changed files with 58 additions and 22 deletions
--- a/base/common/DateLUT.h
+++ b/base/common/DateLUT.h
@ -17,7 +17,7 @@ class DateLUT : private boost::noncopyable
 {
 public:
    /// Return singleton DateLUTImpl instance for the default time zone.
-    static ALWAYS_INLINE const DateLUTImpl & instance()
+    static ALWAYS_INLINE const DateLUTImpl & instance()  // -V1071
    {
        const auto & date_lut = getInstance();
        return *date_lut.default_impl.load(std::memory_order_acquire);
--- a/cmake/embed_binary.cmake
+++ b/cmake/embed_binary.cmake
@ -33,10 +33,16 @@ macro(clickhouse_embed_binaries)
        message(FATAL_ERROR "The list of binary resources to embed may not be empty")
    endif()

-    # If cross-compiling, ensure we use the toolchain file and target the
-    # actual target architecture
+    # If cross-compiling, ensure we use the toolchain file and target the actual target architecture
    if (CMAKE_CROSSCOMPILING)
-        set(CROSS_COMPILE_FLAGS "--target=${CMAKE_C_COMPILER_TARGET} --gcc-toolchain=${TOOLCHAIN_FILE}")
+        set(CROSS_COMPILE_FLAGS --target=${CMAKE_C_COMPILER_TARGET})
+
+        # FIXME: find a way to properly pass all cross-compile flags to custom command in CMake
+        if (CMAKE_SYSTEM_NAME STREQUAL "Darwin")
+            list(APPEND CROSS_COMPILE_FLAGS -isysroot ${CMAKE_OSX_SYSROOT} -mmacosx-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET})
+        else ()
+            list(APPEND CROSS_COMPILE_FLAGS -isysroot ${CMAKE_SYSROOT})
+        endif ()
    else()
        set(CROSS_COMPILE_FLAGS "")
    endif()
@ -67,6 +73,7 @@ macro(clickhouse_embed_binaries)
                 ${CMAKE_C_COMPILER} "${CROSS_COMPILE_FLAGS}" -c -o
                    "${CMAKE_CURRENT_BINARY_DIR}/${RESOURCE_OBJ}"
                    "${CMAKE_CURRENT_BINARY_DIR}/${ASSEMBLY_FILE_NAME}"
+            COMMAND_EXPAND_LISTS
        )
        set_source_files_properties("${RESOURCE_OBJ}" PROPERTIES EXTERNAL_OBJECT true GENERATED true)
    endforeach()
--- a/cmake/linux/toolchain-aarch64.cmake
+++ b/cmake/linux/toolchain-aarch64.cmake
@ -4,7 +4,6 @@ set (CMAKE_C_COMPILER_TARGET "aarch64-linux-gnu")
 set (CMAKE_CXX_COMPILER_TARGET "aarch64-linux-gnu")
 set (CMAKE_ASM_COMPILER_TARGET "aarch64-linux-gnu")
 set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64/aarch64-linux-gnu/libc")
-get_filename_component (TOOLCHAIN_FILE "${CMAKE_TOOLCHAIN_FILE}" REALPATH)

 # We don't use compiler from toolchain because it's gcc-8, and we provide support only for gcc-9.
 set (CMAKE_AR "${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64/bin/aarch64-linux-gnu-ar" CACHE FILEPATH "" FORCE)
--- a/contrib/cctz-cmake/CMakeLists.txt
+++ b/contrib/cctz-cmake/CMakeLists.txt
@ -26,7 +26,7 @@ if (NOT USE_INTERNAL_CCTZ_LIBRARY)
            set_property (TARGET cctz PROPERTY IMPORTED_LOCATION ${LIBRARY_CCTZ})
            set_property (TARGET cctz PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${INCLUDE_CCTZ})
        endif()
-        
+
        set(SYSTEM_STORAGE_TZ_FILE "${CMAKE_BINARY_DIR}/src/Storages/System/StorageSystemTimeZones.generated.cpp")
        file(REMOVE ${SYSTEM_STORAGE_TZ_FILE})
        file(APPEND ${SYSTEM_STORAGE_TZ_FILE} "// autogenerated by ClickHouse/contrib/cctz-cmake/CMakeLists.txt\n")
--- a/docker/packager/binary/Dockerfile
+++ b/docker/packager/binary/Dockerfile
@ -72,7 +72,7 @@ RUN git clone https://github.com/tpoechtrager/apple-libtapi.git \
    && cd .. \
    && rm -rf apple-libtapi

-# Build and install tools for cross-linking to Darwin
+# Build and install tools for cross-linking to Darwin (x86-64)
 RUN git clone https://github.com/tpoechtrager/cctools-port.git \
    && cd cctools-port/cctools \
    && ./configure --prefix=/cctools --with-libtapi=/cctools \
@ -81,8 +81,17 @@ RUN git clone https://github.com/tpoechtrager/cctools-port.git \
    && cd ../.. \
    && rm -rf cctools-port

-# Download toolchain for Darwin
-RUN wget -nv https://github.com/phracker/MacOSX-SDKs/releases/download/10.15/MacOSX10.15.sdk.tar.xz
+# Build and install tools for cross-linking to Darwin (aarch64)
+RUN git clone https://github.com/tpoechtrager/cctools-port.git \
+    && cd cctools-port/cctools \
+    && ./configure --prefix=/cctools --with-libtapi=/cctools \
+        --target=aarch64-apple-darwin \
+    && make install \
+    && cd ../.. \
+    && rm -rf cctools-port
+
+# Download toolchain and SDK for Darwin
+RUN wget -nv https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.0.sdk.tar.xz

 # Download toolchain for ARM
 # It contains all required headers and libraries. Note that it's named as "gcc" but actually we are using clang for cross compiling.
--- a/docker/packager/binary/build.sh
+++ b/docker/packager/binary/build.sh
@ -3,7 +3,9 @@
 set -x -e

 mkdir -p build/cmake/toolchain/darwin-x86_64
-tar xJf MacOSX10.15.sdk.tar.xz -C build/cmake/toolchain/darwin-x86_64 --strip-components=1
+tar xJf MacOSX11.0.sdk.tar.xz -C build/cmake/toolchain/darwin-x86_64 --strip-components=1
+
+ln -sf darwin-x86_64 build/cmake/toolchain/darwin-aarch64

 mkdir -p build/cmake/toolchain/linux-aarch64
 tar xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz -C build/cmake/toolchain/linux-aarch64 --strip-components=1
--- a/docker/packager/packager
+++ b/docker/packager/packager
@ -58,6 +58,7 @@ def run_docker_image_with_env(image_name, output, env_variables, ch_root, ccache
 def parse_env_variables(build_type, compiler, sanitizer, package_type, image_type, cache, distcc_hosts, unbundled, split_binary, clang_tidy, version, author, official, alien_pkgs, with_coverage, with_binaries):
    CLANG_PREFIX = "clang"
    DARWIN_SUFFIX = "-darwin"
+    DARWIN_ARM_SUFFIX = "-darwin-aarch64"
    ARM_SUFFIX = "-aarch64"
    FREEBSD_SUFFIX = "-freebsd"

@ -66,9 +67,10 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ

    is_clang = compiler.startswith(CLANG_PREFIX)
    is_cross_darwin = compiler.endswith(DARWIN_SUFFIX)
+    is_cross_darwin_arm = compiler.endswith(DARWIN_ARM_SUFFIX)
    is_cross_arm = compiler.endswith(ARM_SUFFIX)
    is_cross_freebsd = compiler.endswith(FREEBSD_SUFFIX)
-    is_cross_compile = is_cross_darwin or is_cross_arm or is_cross_freebsd
+    is_cross_compile = is_cross_darwin or is_cross_darwin_arm or is_cross_arm or is_cross_freebsd

    # Explicitly use LLD with Clang by default.
    # Don't force linker for cross-compilation.
@ -82,6 +84,13 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, image_typ
        cmake_flags.append("-DCMAKE_RANLIB:FILEPATH=/cctools/bin/x86_64-apple-darwin-ranlib")
        cmake_flags.append("-DLINKER_NAME=/cctools/bin/x86_64-apple-darwin-ld")
        cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/darwin/toolchain-x86_64.cmake")
+    elif is_cross_darwin_arm:
+        cc = compiler[:-len(DARWIN_ARM_SUFFIX)]
+        cmake_flags.append("-DCMAKE_AR:FILEPATH=/cctools/bin/aarch64-apple-darwin-ar")
+        cmake_flags.append("-DCMAKE_INSTALL_NAME_TOOL=/cctools/bin/aarch64-apple-darwin-install_name_tool")
+        cmake_flags.append("-DCMAKE_RANLIB:FILEPATH=/cctools/bin/aarch64-apple-darwin-ranlib")
+        cmake_flags.append("-DLINKER_NAME=/cctools/bin/aarch64-apple-darwin-ld")
+        cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/darwin/toolchain-aarch64.cmake")
    elif is_cross_arm:
        cc = compiler[:-len(ARM_SUFFIX)]
        cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-aarch64.cmake")
@ -185,8 +194,8 @@ if __name__ == "__main__":
    parser.add_argument("--clickhouse-repo-path", default=os.path.join(os.path.dirname(os.path.abspath(__file__)), os.pardir, os.pardir))
    parser.add_argument("--output-dir", required=True)
    parser.add_argument("--build-type", choices=("debug", ""), default="")
-    parser.add_argument("--compiler", choices=("clang-11", "clang-11-darwin", "clang-11-aarch64", "clang-11-freebsd",
-                                               "gcc-10"), default="clang-11")
+    parser.add_argument("--compiler", choices=("clang-11", "clang-11-darwin", "clang-11-darwin-aarch64", "clang-11-aarch64",
+                                               "clang-11-freebsd", "gcc-10"), default="clang-11")
    parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="")
    parser.add_argument("--unbundled", action="store_true")
    parser.add_argument("--split-binary", action="store_true")
--- a/src/Functions/FunctionsCoding.h
+++ b/src/Functions/FunctionsCoding.h
@ -992,7 +992,7 @@ public:
            UInt8 byte = x >> offset;

            /// Leading zeros.
-            if (byte == 0 && !was_nonzero && offset)
+            if (byte == 0 && !was_nonzero && offset)  // -V560
                continue;

            was_nonzero = true;
--- a/src/IO/ReadHelpers.h
+++ b/src/IO/ReadHelpers.h
@ -184,7 +184,7 @@ inline bool checkString(const String & s, ReadBuffer & buf)
    return checkString(s.c_str(), buf);
 }

-inline bool checkChar(char c, ReadBuffer & buf)
+inline bool checkChar(char c, ReadBuffer & buf)  // -V1071
 {
    if (buf.eof() || *buf.position() != c)
        return false;
@ -393,7 +393,7 @@ void readIntText(T & x, ReadBuffer & buf)
 }

 template <ReadIntTextCheckOverflow check_overflow = ReadIntTextCheckOverflow::CHECK_OVERFLOW, typename T>
-bool tryReadIntText(T & x, ReadBuffer & buf)
+bool tryReadIntText(T & x, ReadBuffer & buf)  // -V1071
 {
    return readIntTextImpl<T, bool, check_overflow>(x, buf);
 }
--- a/src/Interpreters/ExpressionActions.h
+++ b/src/Interpreters/ExpressionActions.h
@ -253,7 +253,7 @@ struct ExpressionActionsChain : WithContext
        steps.clear();
    }

-    ActionsDAGPtr getLastActions(bool allow_empty = false)
+    ActionsDAGPtr getLastActions(bool allow_empty = false)  // -V1071
    {
        if (steps.empty())
        {
--- a/src/Parsers/IParser.h
+++ b/src/Parsers/IParser.h
@ -88,13 +88,13 @@ public:
      */
    virtual bool parse(Pos & pos, ASTPtr & node, Expected & expected) = 0;

-    bool ignore(Pos & pos, Expected & expected)
+    bool ignore(Pos & pos, Expected & expected)  // -V1071
    {
        ASTPtr ignore_node;
        return parse(pos, ignore_node, expected);
    }

-    bool ignore(Pos & pos)
+    bool ignore(Pos & pos)  // -V1071
    {
        Expected expected;
        return ignore(pos, expected);
--- a/src/Parsers/IParserBase.h
+++ b/src/Parsers/IParserBase.h
@ -35,7 +35,7 @@ public:
        return res;
    }

-    bool parse(Pos & pos, ASTPtr & node, Expected & expected) override;
+    bool parse(Pos & pos, ASTPtr & node, Expected & expected) override;  // -V1071

 protected:
    virtual bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) = 0;
--- a/src/Parsers/MySQL/tests/gtest_alter_command_parser.cpp
+++ b/src/Parsers/MySQL/tests/gtest_alter_command_parser.cpp
@ -9,7 +9,7 @@
 using namespace DB;
 using namespace DB::MySQLParser;

-static inline ASTPtr tryParserQuery(IParser & parser, const String & query)
+static inline ASTPtr tryParserQuery(IParser & parser, const String & query)  // -V1071
 {
    return parseQuery(parser, query.data(), query.data() + query.size(), "", 0, 0);
 }
--- a/src/Storages/MergeTree/MergeTreePartInfo.h
+++ b/src/Storages/MergeTree/MergeTreePartInfo.h
@ -85,7 +85,7 @@ struct MergeTreePartInfo
        return static_cast<UInt64>(max_block - min_block + 1);
    }

-    static MergeTreePartInfo fromPartName(const String & part_name, MergeTreeDataFormatVersion format_version);
+    static MergeTreePartInfo fromPartName(const String & part_name, MergeTreeDataFormatVersion format_version);  // -V1071

    static bool tryParsePartName(const String & part_name, MergeTreePartInfo * part_info, MergeTreeDataFormatVersion format_version);

--- a/tests/ci/ci_config.json
+++ b/tests/ci/ci_config.json
@ -152,6 +152,16 @@
            "splitted": "unsplitted",
            "tidy": "disable",
            "with_coverage": false
+        },
+        {
+            "compiler": "clang-11-darwin-aarch64",
+            "build-type": "",
+            "sanitizer": "",
+            "package-type": "binary",
+            "bundled": "bundled",
+            "splitted": "unsplitted",
+            "tidy": "disable",
+            "with_coverage": false
        }
    ],
    "tests_config": {