From 15d12db16a2ac65af695aadcecc2c55b9a9c668a Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 28 Oct 2019 13:42:51 +0800 Subject: [PATCH 1/6] Fix DUMP debug. --- dbms/src/Core/iostream_debug_helpers.cpp | 18 ++++++------------ dbms/src/Core/iostream_debug_helpers.h | 16 +++++----------- dbms/src/Parsers/iostream_debug_helpers.cpp | 9 +++++++++ dbms/src/Parsers/iostream_debug_helpers.h | 3 +++ 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/dbms/src/Core/iostream_debug_helpers.cpp b/dbms/src/Core/iostream_debug_helpers.cpp index 6a1f7f3006a..8e673d1c547 100644 --- a/dbms/src/Core/iostream_debug_helpers.cpp +++ b/dbms/src/Core/iostream_debug_helpers.cpp @@ -18,15 +18,17 @@ namespace DB { -std::ostream & operator<<(std::ostream & stream, const IBlockInputStream & what) + +template <> +std::ostream & operator<< (std::ostream & stream, const Field & what) { - stream << "IBlockInputStream(name = " << what.getName() << ")"; + stream << applyVisitor(FieldVisitorDump(), what); return stream; } -std::ostream & operator<<(std::ostream & stream, const Field & what) +std::ostream & operator<<(std::ostream & stream, const IBlockInputStream & what) { - stream << applyVisitor(FieldVisitorDump(), what); + stream << "IBlockInputStream(name = " << what.getName() << ")"; return stream; } @@ -102,14 +104,6 @@ std::ostream & operator<<(std::ostream & stream, const Connection::Packet & what return stream; } -std::ostream & operator<<(std::ostream & stream, const IAST & what) -{ - stream << "IAST{"; - what.dumpTree(stream); - stream << "}"; - return stream; -} - std::ostream & operator<<(std::ostream & stream, const ExpressionAction & what) { stream << "ExpressionAction(" << what.toString() << ")"; diff --git a/dbms/src/Core/iostream_debug_helpers.h b/dbms/src/Core/iostream_debug_helpers.h index 7d109b4604b..35fc05faf1d 100644 --- a/dbms/src/Core/iostream_debug_helpers.h +++ b/dbms/src/Core/iostream_debug_helpers.h @@ -7,18 +7,15 @@ namespace DB { -// Used to disable implicit casting for certain overloaded types such as Field, which leads to -// overload resolution ambiguity. -template struct Dumpable; -template -std::ostream & operator<<(std::ostream & stream, const typename Dumpable::Type & what); +// Use template to disable implicit casting for certain overloaded types such as Field, which leads +// to overload resolution ambiguity. +class Field; +template >> +std::ostream & operator<<(std::ostream & stream, const T & what); class IBlockInputStream; std::ostream & operator<<(std::ostream & stream, const IBlockInputStream & what); -class Field; -template <> struct Dumpable { using Type = Field; }; - struct NameAndTypePair; std::ostream & operator<<(std::ostream & stream, const NameAndTypePair & what); @@ -43,9 +40,6 @@ std::ostream & operator<<(std::ostream & stream, const ColumnWithTypeAndName & w class IColumn; std::ostream & operator<<(std::ostream & stream, const IColumn & what); -class IAST; -std::ostream & operator<<(std::ostream & stream, const IAST & what); - std::ostream & operator<<(std::ostream & stream, const Connection::Packet & what); struct ExpressionAction; diff --git a/dbms/src/Parsers/iostream_debug_helpers.cpp b/dbms/src/Parsers/iostream_debug_helpers.cpp index 61dd08ecddc..66666f0dbfc 100644 --- a/dbms/src/Parsers/iostream_debug_helpers.cpp +++ b/dbms/src/Parsers/iostream_debug_helpers.cpp @@ -1,4 +1,5 @@ #include "iostream_debug_helpers.h" +#include #include #include #include @@ -20,4 +21,12 @@ std::ostream & operator<<(std::ostream & stream, const Expected & what) return stream; } +std::ostream & operator<<(std::ostream & stream, const IAST & what) +{ + stream << "IAST{"; + what.dumpTree(stream); + stream << "}"; + return stream; +} + } diff --git a/dbms/src/Parsers/iostream_debug_helpers.h b/dbms/src/Parsers/iostream_debug_helpers.h index f94c0da5f17..39f52ebcbc2 100644 --- a/dbms/src/Parsers/iostream_debug_helpers.h +++ b/dbms/src/Parsers/iostream_debug_helpers.h @@ -9,6 +9,9 @@ std::ostream & operator<<(std::ostream & stream, const Token & what); struct Expected; std::ostream & operator<<(std::ostream & stream, const Expected & what); +class IAST; +std::ostream & operator<<(std::ostream & stream, const IAST & what); + } #include From 12b57aedf0a2277e6c2abb7236651f0cf9677229 Mon Sep 17 00:00:00 2001 From: chertus Date: Tue, 29 Oct 2019 16:52:05 +0300 Subject: [PATCH 2/6] remove error prone code --- dbms/src/Interpreters/AnalyzedJoin.cpp | 15 --------------- dbms/src/Interpreters/AnalyzedJoin.h | 2 +- 2 files changed, 1 insertion(+), 16 deletions(-) diff --git a/dbms/src/Interpreters/AnalyzedJoin.cpp b/dbms/src/Interpreters/AnalyzedJoin.cpp index b2e967b213c..a11ea893f54 100644 --- a/dbms/src/Interpreters/AnalyzedJoin.cpp +++ b/dbms/src/Interpreters/AnalyzedJoin.cpp @@ -204,21 +204,6 @@ void AnalyzedJoin::addJoinedColumnsAndCorrectNullability(Block & sample_block) c bool make_nullable = join_use_nulls && left_or_full_join; - if (!make_nullable) - { - /// Keys from right table are usually not stored in Join, but copied from the left one. - /// So, if left key is nullable, let's make right key nullable too. - /// Note: for some join types it's not needed and, probably, may be removed. - /// Note: changing this code, take into account the implementation in Join.cpp. - auto it = std::find(key_names_right.begin(), key_names_right.end(), col.name); - if (it != key_names_right.end()) - { - auto pos = it - key_names_right.begin(); - const auto & left_key_name = key_names_left[pos]; - make_nullable = sample_block.getByName(left_key_name).type->isNullable(); - } - } - if (make_nullable && res_type->canBeInsideNullable()) res_type = makeNullable(res_type); diff --git a/dbms/src/Interpreters/AnalyzedJoin.h b/dbms/src/Interpreters/AnalyzedJoin.h index c979b50d3a3..f0b6a8d434c 100644 --- a/dbms/src/Interpreters/AnalyzedJoin.h +++ b/dbms/src/Interpreters/AnalyzedJoin.h @@ -94,7 +94,7 @@ public: void addOnKeys(ASTPtr & left_table_ast, ASTPtr & right_table_ast); bool hasUsing() const { return table_join.using_expression_list != nullptr; } - bool hasOn() const { return !hasUsing(); } + bool hasOn() const { return table_join.on_expression != nullptr; } NameSet getQualifiedColumnsSet() const; NameSet getOriginalColumnsSet() const; From ff4d90f6021667e789e87107da514a3b33f90600 Mon Sep 17 00:00:00 2001 From: Olga Khvostikova Date: Tue, 29 Oct 2019 19:49:37 +0300 Subject: [PATCH 3/6] Add test for greatCircleDist --- dbms/tests/performance/grear_circle_dist.xml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 dbms/tests/performance/grear_circle_dist.xml diff --git a/dbms/tests/performance/grear_circle_dist.xml b/dbms/tests/performance/grear_circle_dist.xml new file mode 100644 index 00000000000..bb26605bd89 --- /dev/null +++ b/dbms/tests/performance/grear_circle_dist.xml @@ -0,0 +1,15 @@ + + once + + + + 1000 + 10000 + + + + + SELECT count() FROM system.numbers WHERE NOT ignore(greatCircleDistance((rand() % 360) * 1. - 180, (number % 150) * 1.2 - 90, (number % 360) + toFloat64(rand()) / 4294967296 - 180, (rand() % 180) * 1. - 90)) + + SELECT count() FROM system.numbers WHERE NOT ignore(greatCircleDistance(55. + toFloat64(rand()) / 4294967296, 37. + toFloat64(rand()) / 4294967296, 55. + toFloat64(rand()) / 4294967296, 37. + toFloat64(rand()) / 4294967296)) + From 4d27e8aa5bc5e2db6973952177de8611ab09c46d Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Tue, 29 Oct 2019 20:33:31 +0300 Subject: [PATCH 4/6] Unpack toolchains into mounted volume (#7534) --- docker/packager/binary/Dockerfile | 4 ---- docker/packager/binary/build.sh | 6 ++++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index c15de71e848..9018afb1153 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -71,14 +71,10 @@ RUN cd cctools-port/cctools && ./configure --prefix=/cctools --with-libtapi=/cct RUN rm -rf cctools-port # Download toolchain for Darwin -RUN mkdir -p /build/cmake/toolchain/darwin-x86_64 RUN wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz -RUN tar xJf MacOSX10.14.sdk.tar.xz -C /build/cmake/toolchain/darwin-x86_64 --strip-components=1 # Download toolchain for ARM -RUN mkdir -p /build/cmake/toolchain/linux-aarch64 RUN wget "https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en" -O gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz -RUN tar xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz -C /build/cmake/toolchain/linux-aarch64 --strip-components=1 COPY build.sh / CMD ["/bin/bash", "/build.sh"] diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index b5c50763b17..115ec06b787 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -2,6 +2,12 @@ set -x -e +mkdir -p build/cmake/toolchain/darwin-x86_64 +tar xJf MacOSX10.14.sdk.tar.xz -C build/cmake/toolchain/darwin-x86_64 --strip-components=1 + +mkdir -p build/cmake/toolchain/linux-aarch64 +tar xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz -C build/cmake/toolchain/linux-aarch64 --strip-components=1 + mkdir -p build/build_docker cd build/build_docker ccache --show-stats ||: From deb2406b306cf4e93d6886446c8f3fc2d7cc8533 Mon Sep 17 00:00:00 2001 From: chertus Date: Tue, 29 Oct 2019 22:39:42 +0300 Subject: [PATCH 5/6] fix JOIN right table keys nullability --- dbms/src/Interpreters/Join.cpp | 66 ++++++++++++++----- dbms/src/Interpreters/join_common.cpp | 16 +++++ dbms/src/Interpreters/join_common.h | 1 + .../00853_join_with_nulls_crash.reference | 16 ++--- 4 files changed, 75 insertions(+), 24 deletions(-) diff --git a/dbms/src/Interpreters/Join.cpp b/dbms/src/Interpreters/Join.cpp index 3267e6a779b..0f068343da9 100644 --- a/dbms/src/Interpreters/Join.cpp +++ b/dbms/src/Interpreters/Join.cpp @@ -37,11 +37,52 @@ namespace ErrorCodes } -/// Converts column to nullable if needed. No backward convertion. +static ColumnPtr filterWithBlanks(ColumnPtr src_column, const IColumn::Filter & filter, bool inverse_filter = false) +{ + ColumnPtr column = src_column->convertToFullColumnIfConst(); + MutableColumnPtr mut_column = column->cloneEmpty(); + mut_column->reserve(column->size()); + + if (inverse_filter) + { + for (size_t row = 0; row < filter.size(); ++row) + { + if (filter[row]) + mut_column->insertDefault(); + else + mut_column->insertFrom(*column, row); + } + } + else + { + for (size_t row = 0; row < filter.size(); ++row) + { + if (filter[row]) + mut_column->insertFrom(*column, row); + else + mut_column->insertDefault(); + } + } + + return mut_column; +} + static ColumnWithTypeAndName correctNullability(ColumnWithTypeAndName && column, bool nullable) { if (nullable) + { JoinCommon::convertColumnToNullable(column); + } + else + { + /// We have to replace values masked by NULLs with defaults. + if (column.column) + if (auto * nullable_column = checkAndGetColumn(*column.column)) + column.column = filterWithBlanks(column.column, nullable_column->getNullMapColumn().getData(), true); + + JoinCommon::removeColumnNullability(column); + } + return std::move(column); } @@ -57,6 +98,9 @@ static ColumnWithTypeAndName correctNullability(ColumnWithTypeAndName && column, column.column = std::move(mutable_column); } } + else + JoinCommon::removeColumnNullability(column); + return std::move(column); } @@ -769,7 +813,7 @@ void Join::joinBlockImpl( for (size_t i = 0; i < existing_columns; ++i) block.safeGetByPosition(i).column = block.safeGetByPosition(i).column->filter(row_filter, -1); - /// Add join key columns from right block if they has different name. + /// Add join key columns from right block if needed. for (size_t i = 0; i < right_table_keys.columns(); ++i) { const auto & right_key = right_table_keys.getByPosition(i); @@ -791,7 +835,7 @@ void Join::joinBlockImpl( null_map_filter.getData().swap(row_filter); const IColumn::Filter & filter = null_map_filter.getData(); - /// Add join key columns from right block if they has different name. + /// Add join key columns from right block if needed. for (size_t i = 0; i < right_table_keys.columns(); ++i) { const auto & right_key = right_table_keys.getByPosition(i); @@ -800,20 +844,10 @@ void Join::joinBlockImpl( if (required_right_keys.count(right_key.name) && !block.has(right_key.name)) { const auto & col = block.getByName(left_name); - ColumnPtr column = col.column->convertToFullColumnIfConst(); - MutableColumnPtr mut_column = column->cloneEmpty(); - mut_column->reserve(column->size()); - - for (size_t row = 0; row < filter.size(); ++row) - { - if (filter[row]) - mut_column->insertFrom(*column, row); - else - mut_column->insertDefault(); - } - bool is_nullable = nullable_right_side || right_key.type->isNullable(); - block.insert(correctNullability({std::move(mut_column), col.type, right_key.name}, is_nullable, null_map_filter)); + + ColumnPtr thin_column = filterWithBlanks(col.column, filter); + block.insert(correctNullability({thin_column, col.type, right_key.name}, is_nullable, null_map_filter)); if constexpr (is_all_join) right_keys_to_replicate.push_back(block.getPositionByName(right_key.name)); diff --git a/dbms/src/Interpreters/join_common.cpp b/dbms/src/Interpreters/join_common.cpp index dad4fbc7301..852867f7775 100644 --- a/dbms/src/Interpreters/join_common.cpp +++ b/dbms/src/Interpreters/join_common.cpp @@ -32,6 +32,22 @@ void convertColumnsToNullable(Block & block, size_t starting_pos) convertColumnToNullable(block.getByPosition(i)); } +/// @warning It assumes that every NULL has default value in nested column (or it does not matter) +void removeColumnNullability(ColumnWithTypeAndName & column) +{ + if (!column.type->isNullable()) + return; + + column.type = static_cast(*column.type).getNestedType(); + if (column.column) + { + auto * nullable_column = checkAndGetColumn(*column.column); + ColumnPtr nested_column = nullable_column->getNestedColumnPtr(); + MutableColumnPtr mutable_column = (*std::move(nested_column)).mutate(); + column.column = std::move(mutable_column); + } +} + ColumnRawPtrs temporaryMaterializeColumns(const Block & block, const Names & names, Columns & materialized) { ColumnRawPtrs ptrs; diff --git a/dbms/src/Interpreters/join_common.h b/dbms/src/Interpreters/join_common.h index 8567ea68f01..85c24515b41 100644 --- a/dbms/src/Interpreters/join_common.h +++ b/dbms/src/Interpreters/join_common.h @@ -15,6 +15,7 @@ namespace JoinCommon void convertColumnToNullable(ColumnWithTypeAndName & column); void convertColumnsToNullable(Block & block, size_t starting_pos = 0); +void removeColumnNullability(ColumnWithTypeAndName & column); ColumnRawPtrs temporaryMaterializeColumns(const Block & block, const Names & names, Columns & materialized); void removeLowCardinalityInplace(Block & block); diff --git a/dbms/tests/queries/0_stateless/00853_join_with_nulls_crash.reference b/dbms/tests/queries/0_stateless/00853_join_with_nulls_crash.reference index 4eafec8a6c9..459b73acdbf 100644 --- a/dbms/tests/queries/0_stateless/00853_join_with_nulls_crash.reference +++ b/dbms/tests/queries/0_stateless/00853_join_with_nulls_crash.reference @@ -1,11 +1,11 @@ -\N test 0 1 Nullable(String) Nullable(String) -bar bar 1 2 Nullable(String) Nullable(String) -\N 0 1 Nullable(String) Nullable(String) -foo \N 2 0 Nullable(String) Nullable(String) -\N test 0 1 Nullable(String) Nullable(String) -bar bar 1 2 Nullable(String) Nullable(String) -\N 0 1 Nullable(String) Nullable(String) -foo \N 2 0 Nullable(String) Nullable(String) +\N test 0 1 Nullable(String) String +bar bar 1 2 Nullable(String) String +\N 0 1 Nullable(String) String +foo 2 0 Nullable(String) String +\N test 0 1 Nullable(String) String +bar bar 1 2 Nullable(String) String +\N 0 1 Nullable(String) String +foo 2 0 Nullable(String) String foo \N 2 0 String Nullable(String) bar bar 1 2 String Nullable(String) test 0 1 String Nullable(String) From 4f2f5cca8479b4902744aefaaee9885fb0fe427b Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Wed, 30 Oct 2019 10:01:53 +0300 Subject: [PATCH 6/6] Add support for cross-compiling to the CPU architecture AARCH64 (#7370) --- .gitignore | 3 + CMakeLists.txt | 6 +- cmake/arch.cmake | 1 + cmake/darwin/default_libs.cmake | 8 +++ cmake/darwin/sdk.cmake | 11 --- cmake/darwin/toolchain-x86_64.cmake | 13 ++++ cmake/linux/default_libs.cmake | 2 +- cmake/linux/toolchain-aarch64.cmake | 20 ++++++ cmake/target.cmake | 67 +++---------------- cmake/toolchain/darwin-x86_64/README.txt | 2 + cmake/toolchain/linux-aarch64/README.txt | 2 + cmake/tools.cmake | 41 ++++++++++++ dbms/src/Common/StackTrace.cpp | 2 +- dbms/src/Common/tests/int_hashes_perf.cpp | 6 +- dbms/src/IO/tests/parse_int_perf.cpp | 2 +- .../Interpreters/tests/hash_map_string.cpp | 4 +- docker/packager/packager | 16 ++++- 17 files changed, 124 insertions(+), 82 deletions(-) delete mode 100644 cmake/darwin/sdk.cmake create mode 100644 cmake/darwin/toolchain-x86_64.cmake create mode 100644 cmake/linux/toolchain-aarch64.cmake create mode 100644 cmake/toolchain/darwin-x86_64/README.txt create mode 100644 cmake/toolchain/linux-aarch64/README.txt create mode 100644 cmake/tools.cmake diff --git a/.gitignore b/.gitignore index 817e333d833..4ec07227e66 100644 --- a/.gitignore +++ b/.gitignore @@ -245,3 +245,6 @@ website/package-lock.json /.ccls-cache /compile_commands.json + +# Toolchains +/cmake/toolchain/* diff --git a/CMakeLists.txt b/CMakeLists.txt index 53021dbe666..986096ba9e8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,10 @@ foreach(policy endforeach() project(ClickHouse) + +include (cmake/arch.cmake) include (cmake/target.cmake) +include (cmake/tools.cmake) # Ignore export() since we don't use it, # but it gets broken with a global targets via link_libraries() @@ -26,8 +29,6 @@ set(CMAKE_LINK_DEPENDS_NO_SHARED 1) # Do not relink all depended targets on .so set(CMAKE_CONFIGURATION_TYPES "RelWithDebInfo;Debug;Release;MinSizeRel" CACHE STRING "" FORCE) set(CMAKE_DEBUG_POSTFIX "d" CACHE STRING "Generate debug library name with a postfix.") # To be consistent with CMakeLists from contrib libs. -include (cmake/arch.cmake) - option(ENABLE_IPO "Enable inter-procedural optimization (aka LTO)" OFF) # need cmake 3.9+ if(ENABLE_IPO) cmake_policy(SET CMP0069 NEW) @@ -230,7 +231,6 @@ include(cmake/dbms_glob_sources.cmake) if (OS_LINUX) include(cmake/linux/default_libs.cmake) elseif (OS_DARWIN) - include(cmake/darwin/sdk.cmake) include(cmake/darwin/default_libs.cmake) endif () diff --git a/cmake/arch.cmake b/cmake/arch.cmake index f339236af64..79fe92c03e5 100644 --- a/cmake/arch.cmake +++ b/cmake/arch.cmake @@ -17,6 +17,7 @@ endif () if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(ppc64le.*|PPC64LE.*)") set (ARCH_PPC64LE 1) + # FIXME: move this check into tools.cmake if (COMPILER_CLANG OR (COMPILER_GCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8)) message(FATAL_ERROR "Only gcc-8 is supported for powerpc architecture") endif () diff --git a/cmake/darwin/default_libs.cmake b/cmake/darwin/default_libs.cmake index e684a10a08f..679fef8808a 100644 --- a/cmake/darwin/default_libs.cmake +++ b/cmake/darwin/default_libs.cmake @@ -11,6 +11,14 @@ message(STATUS "Default libraries: ${DEFAULT_LIBS}") set(CMAKE_CXX_STANDARD_LIBRARIES ${DEFAULT_LIBS}) set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS}) +# Minimal supported SDK version + +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mmacosx-version-min=10.14") +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mmacosx-version-min=10.14") + +set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -mmacosx-version-min=10.14") +set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -mmacosx-version-min=10.14") + # Global libraries add_library(global-libs INTERFACE) diff --git a/cmake/darwin/sdk.cmake b/cmake/darwin/sdk.cmake deleted file mode 100644 index 92a9f8f66e5..00000000000 --- a/cmake/darwin/sdk.cmake +++ /dev/null @@ -1,11 +0,0 @@ -option (SDK_PATH "Path to the SDK to build with" "") - -if (NOT EXISTS "${SDK_PATH}/SDKSettings.plist") - message (FATAL_ERROR "Wrong SDK path provided: ${SDK_PATH}") -endif () - -set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -isysroot ${SDK_PATH} -mmacosx-version-min=10.14") -set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -isysroot ${SDK_PATH} -mmacosx-version-min=10.14") - -set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -isysroot ${SDK_PATH} -mmacosx-version-min=10.14") -set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -isysroot ${SDK_PATH} -mmacosx-version-min=10.14") diff --git a/cmake/darwin/toolchain-x86_64.cmake b/cmake/darwin/toolchain-x86_64.cmake new file mode 100644 index 00000000000..9128311e3bb --- /dev/null +++ b/cmake/darwin/toolchain-x86_64.cmake @@ -0,0 +1,13 @@ +set (CMAKE_SYSTEM_NAME "Darwin") +set (CMAKE_SYSTEM_PROCESSOR "x86_64") +set (CMAKE_C_COMPILER_TARGET "x86_64-apple-darwin") +set (CMAKE_CXX_COMPILER_TARGET "x86_64-apple-darwin") +set (CMAKE_OSX_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/darwin-x86_64") + +set (CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) # disable linkage check - it doesn't work in CMake + +set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) +set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) + +set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) +set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) diff --git a/cmake/linux/default_libs.cmake b/cmake/linux/default_libs.cmake index d1f4730c97e..ef1354628fe 100644 --- a/cmake/linux/default_libs.cmake +++ b/cmake/linux/default_libs.cmake @@ -5,7 +5,7 @@ set (DEFAULT_LIBS "-nodefaultlibs") # We need builtins from Clang's RT even without libcxx - for ubsan+int128. # See https://bugs.llvm.org/show_bug.cgi?id=16404 -if (COMPILER_CLANG) +if (COMPILER_CLANG AND NOT (CMAKE_CROSSCOMPILING AND ARCH_AARCH64)) execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE) else () set (BUILTINS_LIBRARY "-lgcc") diff --git a/cmake/linux/toolchain-aarch64.cmake b/cmake/linux/toolchain-aarch64.cmake new file mode 100644 index 00000000000..e93f0f47faf --- /dev/null +++ b/cmake/linux/toolchain-aarch64.cmake @@ -0,0 +1,20 @@ +set (CMAKE_SYSTEM_NAME "Linux") +set (CMAKE_SYSTEM_PROCESSOR "aarch64") +set (CMAKE_C_COMPILER_TARGET "aarch64-linux-gnu") +set (CMAKE_CXX_COMPILER_TARGET "aarch64-linux-gnu") +set (CMAKE_SYSROOT "${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64/aarch64-linux-gnu/libc") + +# We don't use compiler from toolchain because it's gcc-8, and we provide support only for gcc-9. +set (CMAKE_AR "${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64/bin/aarch64-linux-gnu-ar") + +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --gcc-toolchain=${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64") +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${CMAKE_CURRENT_LIST_DIR}/../toolchain/linux-aarch64") + +set (CMAKE_EXE_LINKER_FLAGS_INIT "-fuse-ld=lld") +set (CMAKE_SHARED_LINKER_FLAGS_INIT "-fuse-ld=lld") + +set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) +set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) + +set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) +set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) diff --git a/cmake/target.cmake b/cmake/target.cmake index 5ce425b82cd..3c6aa225af9 100644 --- a/cmake/target.cmake +++ b/cmake/target.cmake @@ -9,62 +9,8 @@ elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin") add_definitions(-D OS_DARWIN) endif () -if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - set (COMPILER_GCC 1) -elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - set (COMPILER_CLANG 1) -endif () - -if (COMPILER_GCC) - # Require minimum version of gcc - set (GCC_MINIMUM_VERSION 8) - if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${GCC_MINIMUM_VERSION} AND NOT CMAKE_VERSION VERSION_LESS 2.8.9) - message (FATAL_ERROR "GCC version must be at least ${GCC_MINIMUM_VERSION}. For example, if GCC ${GCC_MINIMUM_VERSION} is available under gcc-${GCC_MINIMUM_VERSION}, g++-${GCC_MINIMUM_VERSION} names, do the following: export CC=gcc-${GCC_MINIMUM_VERSION} CXX=g++-${GCC_MINIMUM_VERSION}; rm -rf CMakeCache.txt CMakeFiles; and re run cmake or ./release.") - endif () -elseif (COMPILER_CLANG) - # Require minimum version of clang - set (CLANG_MINIMUM_VERSION 7) - if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION}) - message (FATAL_ERROR "Clang version must be at least ${CLANG_MINIMUM_VERSION}.") - endif () -else () - message (WARNING "You are using an unsupported compiler. Compilation has only been tested with Clang 6+ and GCC 7+.") -endif () - -string(REGEX MATCH "-?[0-9]+(.[0-9]+)?$" COMPILER_POSTFIX ${CMAKE_CXX_COMPILER}) - -if (OS_LINUX) - find_program (LLD_PATH NAMES "lld${COMPILER_POSTFIX}" "lld") - find_program (GOLD_PATH NAMES "ld.gold" "gold") -endif() - -option (LINKER_NAME "Linker name or full path") -if (NOT LINKER_NAME) - if (COMPILER_CLANG AND LLD_PATH) - set (LINKER_NAME "lld") - elseif (GOLD_PATH) - set (LINKER_NAME "gold") - endif () -endif () - -if (LINKER_NAME) - message(STATUS "Using linker: ${LINKER_NAME} (selected from: LLD_PATH=${LLD_PATH}; GOLD_PATH=${GOLD_PATH}; COMPILER_POSTFIX=${COMPILER_POSTFIX})") - set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") - set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") -endif () - if (CMAKE_CROSSCOMPILING) if (OS_DARWIN) - set (CMAKE_SYSTEM_PROCESSOR x86_64) - set (CMAKE_C_COMPILER_TARGET x86_64-apple-darwin) - set (CMAKE_CXX_COMPILER_TARGET x86_64-apple-darwin) - - set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) - set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - - set (HAS_POST_2038_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE) - set (HAS_POST_2038_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE) - # FIXME: broken dependencies set (USE_SNAPPY OFF CACHE INTERNAL "") set (ENABLE_SSL OFF CACHE INTERNAL "") @@ -73,12 +19,19 @@ if (CMAKE_CROSSCOMPILING) set (ENABLE_READLINE OFF CACHE INTERNAL "") set (ENABLE_ICU OFF CACHE INTERNAL "") set (ENABLE_FASTOPS OFF CACHE INTERNAL "") - - message (STATUS "Cross-compiling for Darwin") + elseif (OS_LINUX) + if (ARCH_AARCH64) + # FIXME: broken dependencies + set (ENABLE_PROTOBUF OFF CACHE INTERNAL "") + set (ENABLE_PARQUET OFF CACHE INTERNAL "") + set (ENABLE_MYSQL OFF CACHE INTERNAL "") + endif () else () - message (FATAL_ERROR "Trying to cross-compile to unsupported target: ${CMAKE_SYSTEM_NAME}!") + message (FATAL_ERROR "Trying to cross-compile to unsupported system: ${CMAKE_SYSTEM_NAME}!") endif () # Don't know why but CXX_STANDARD doesn't work for cross-compilation set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") + + message (STATUS "Cross-compiling for target: ${CMAKE_CXX_COMPILE_TARGET}") endif () diff --git a/cmake/toolchain/darwin-x86_64/README.txt b/cmake/toolchain/darwin-x86_64/README.txt new file mode 100644 index 00000000000..4ece240f029 --- /dev/null +++ b/cmake/toolchain/darwin-x86_64/README.txt @@ -0,0 +1,2 @@ +wget https://github.com/phracker/MacOSX-SDKs/releases/download/10.14-beta4/MacOSX10.14.sdk.tar.xz +tar --strip-components=1 xJf MacOSX10.14.sdk.tar.xz diff --git a/cmake/toolchain/linux-aarch64/README.txt b/cmake/toolchain/linux-aarch64/README.txt new file mode 100644 index 00000000000..7146c1d2b9a --- /dev/null +++ b/cmake/toolchain/linux-aarch64/README.txt @@ -0,0 +1,2 @@ +wget https://developer.arm.com/-/media/Files/downloads/gnu-a/8.3-2019.03/binrel/gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz?revision=2e88a73f-d233-4f96-b1f4-d8b36e9bb0b9&la=en -O gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz +tar --strip-components=1 xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz diff --git a/cmake/tools.cmake b/cmake/tools.cmake new file mode 100644 index 00000000000..04e0946ee73 --- /dev/null +++ b/cmake/tools.cmake @@ -0,0 +1,41 @@ +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set (COMPILER_GCC 1) +elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + set (COMPILER_CLANG 1) +endif () + +if (COMPILER_GCC) + # Require minimum version of gcc + set (GCC_MINIMUM_VERSION 8) + if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${GCC_MINIMUM_VERSION} AND NOT CMAKE_VERSION VERSION_LESS 2.8.9) + message (FATAL_ERROR "GCC version must be at least ${GCC_MINIMUM_VERSION}. For example, if GCC ${GCC_MINIMUM_VERSION} is available under gcc-${GCC_MINIMUM_VERSION}, g++-${GCC_MINIMUM_VERSION} names, do the following: export CC=gcc-${GCC_MINIMUM_VERSION} CXX=g++-${GCC_MINIMUM_VERSION}; rm -rf CMakeCache.txt CMakeFiles; and re run cmake or ./release.") + endif () +elseif (COMPILER_CLANG) + # Require minimum version of clang + set (CLANG_MINIMUM_VERSION 7) + if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION}) + message (FATAL_ERROR "Clang version must be at least ${CLANG_MINIMUM_VERSION}.") + endif () +else () + message (WARNING "You are using an unsupported compiler. Compilation has only been tested with Clang 6+ and GCC 7+.") +endif () + +option (LINKER_NAME "Linker name or full path") + +find_program (LLD_PATH NAMES "ld.lld" "lld") +find_program (GOLD_PATH NAMES "ld.gold" "gold") + +if (NOT LINKER_NAME) + if (LLD_PATH) + set (LINKER_NAME "lld") + elseif (GOLD_PATH) + set (LINKER_NAME "gold") + endif () +endif () + +if (LINKER_NAME) + set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") + set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=${LINKER_NAME}") + + message(STATUS "Using custom linker by name: ${LINKER_NAME}") +endif () diff --git a/dbms/src/Common/StackTrace.cpp b/dbms/src/Common/StackTrace.cpp index 072bb55cf57..4ab0847ac18 100644 --- a/dbms/src/Common/StackTrace.cpp +++ b/dbms/src/Common/StackTrace.cpp @@ -30,7 +30,7 @@ std::string signalToErrorMessage(int sig, const siginfo_t & info, const ucontext else error << "Address: " << info.si_addr; -#if defined(__x86_64__) && !defined(__FreeBSD__) && !defined(__APPLE__) +#if defined(__x86_64__) && !defined(__FreeBSD__) && !defined(__APPLE__) && !defined(__arm__) auto err_mask = context.uc_mcontext.gregs[REG_ERR]; if ((err_mask & 0x02)) error << " Access: write."; diff --git a/dbms/src/Common/tests/int_hashes_perf.cpp b/dbms/src/Common/tests/int_hashes_perf.cpp index d5545b1b865..24f3f36a7da 100644 --- a/dbms/src/Common/tests/int_hashes_perf.cpp +++ b/dbms/src/Common/tests/int_hashes_perf.cpp @@ -31,7 +31,7 @@ void setAffinity() static inline ALWAYS_INLINE UInt64 rdtsc() { -#if __x86_64__ +#if defined(__x86_64__) UInt32 a, d; __asm__ volatile ("rdtsc" : "=a" (a), "=d" (d)); return static_cast(a) | (static_cast(d) << 32); @@ -109,7 +109,7 @@ static inline size_t murmurMix(UInt64 x) } -#if __x86_64__ +#if defined(__x86_64__) static inline size_t crc32Hash(UInt64 x) { UInt64 crc = -1ULL; @@ -309,7 +309,7 @@ int main(int argc, char ** argv) if (!method || method == 8) test (n, data.data(), "7: mulShift"); if (!method || method == 9) test(n, data.data(), "8: tabulation"); -#if __x86_64__ +#if defined(__x86_64__) if (!method || method == 10) test (n, data.data(), "9: crc32"); #endif diff --git a/dbms/src/IO/tests/parse_int_perf.cpp b/dbms/src/IO/tests/parse_int_perf.cpp index 02a7a0e5cdb..4e24d89f100 100644 --- a/dbms/src/IO/tests/parse_int_perf.cpp +++ b/dbms/src/IO/tests/parse_int_perf.cpp @@ -17,7 +17,7 @@ UInt64 rdtsc() { -#if __x86_64__ +#if defined(__x86_64__) UInt64 val; __asm__ __volatile__("rdtsc" : "=A" (val) :); return val; diff --git a/dbms/src/Interpreters/tests/hash_map_string.cpp b/dbms/src/Interpreters/tests/hash_map_string.cpp index 61980a614ab..f8335453a96 100644 --- a/dbms/src/Interpreters/tests/hash_map_string.cpp +++ b/dbms/src/Interpreters/tests/hash_map_string.cpp @@ -131,7 +131,7 @@ struct FastHash64 }; -#if __x86_64__ +#if defined(__x86_64__) struct CrapWow { size_t operator() (CompactStringRef x) const @@ -381,7 +381,7 @@ int main(int argc, char ** argv) << std::endl; } -#if __x86_64__ +#if defined(__x86_64__) if (!m || m == 3) { Stopwatch watch; diff --git a/docker/packager/packager b/docker/packager/packager index a29702d5cce..8dd4f108edb 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -107,6 +107,14 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, cache, di result = [] cmake_flags = ['$CMAKE_FLAGS', '-DADD_GDB_INDEX_FOR_GOLD=1'] + is_clang = "clang" in compiler + is_cross_compile = ("darwin" in compiler) or ("aarch64" in compiler) + + # Explicitly use LLD with Clang by default. + # Don't force linker for cross-compilation. + if is_clang and not is_cross_compile: + cmake_flags = ['$CMAKE_FLAGS', '-DLINKER_NAME=lld'] + if compiler.endswith("-darwin"): cc = compiler[:-len("-darwin")] else: @@ -124,9 +132,11 @@ def parse_env_variables(build_type, compiler, sanitizer, package_type, cache, di if "darwin" in compiler: cmake_flags.append("-DCMAKE_AR:FILEPATH=/cctools/bin/x86_64-apple-darwin-ar") cmake_flags.append("-DCMAKE_RANLIB:FILEPATH=/cctools/bin/x86_64-apple-darwin-ranlib") - cmake_flags.append("-DCMAKE_SYSTEM_NAME=Darwin") - cmake_flags.append("-DSDK_PATH=/cctools/MacOSX10.14.sdk") cmake_flags.append("-DLINKER_NAME=/cctools/bin/x86_64-apple-darwin-ld") + cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/darwin/toolchain-x86_64.cmake") + + if "aarch64" in compiler: + cmake_flags.append("-DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-aarch64.cmake") if sanitizer: result.append("SANITIZER={}".format(sanitizer)) @@ -181,7 +191,7 @@ if __name__ == "__main__": parser.add_argument("--clickhouse-repo-path", default="../../") parser.add_argument("--output-dir", required=True) parser.add_argument("--build-type", choices=("debug", ""), default="") - parser.add_argument("--compiler", choices=("clang-6.0", "clang-7", "gcc-7", "clang-8", "clang-8-darwin", "gcc-8", "gcc-9"), default="gcc-7") + parser.add_argument("--compiler", choices=("clang-6.0", "clang-7", "gcc-7", "clang-8", "clang-8-darwin", "clang-8-aarch64", "gcc-8", "gcc-9"), default="gcc-7") parser.add_argument("--sanitizer", choices=("address", "thread", "memory", "undefined", ""), default="") parser.add_argument("--unbundled", action="store_true") parser.add_argument("--split-binary", action="store_true")