thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Updated submodule to llvm12

Browse Source
This commit is contained in:
Maksim Kita 2021-05-22 15:22:38 +03:00
parent b89da7b420
commit 42e91df363
13 changed files with 31 additions and 129 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -504,7 +504,6 @@ include (cmake/find/libuv.cmake) # for amqpcpp and cassandra
include (cmake/find/amqpcpp.cmake)
include (cmake/find/capnp.cmake)
include (cmake/find/llvm.cmake)
include (cmake/find/termcap.cmake) # for external static llvm
include (cmake/find/h3.cmake)
include (cmake/find/libxml2.cmake)
include (cmake/find/brotli.cmake)

91
cmake/find/llvm.cmake
View File

@ -1,98 +1,31 @@
if (APPLE OR SPLIT_SHARED_LIBRARIES OR NOT ARCH_AMD64)
if (APPLE OR SPLIT_SHARED_LIBRARIES OR NOT ARCH_AMD64 OR SANITIZE STREQUAL "undefined")
set (ENABLE_EMBEDDED_COMPILER OFF CACHE INTERNAL "")
endif()
option (ENABLE_EMBEDDED_COMPILER "Enable support for 'compile_expressions' option for query execution" ON)
# Broken in macos. TODO: update clang, re-test, enable on Apple
if (ENABLE_EMBEDDED_COMPILER AND NOT SPLIT_SHARED_LIBRARIES AND ARCH_AMD64 AND NOT (SANITIZE STREQUAL "undefined"))
option (USE_INTERNAL_LLVM_LIBRARY "Use bundled or system LLVM library." ${NOT_UNBUNDLED})
endif()
if (NOT ENABLE_EMBEDDED_COMPILER)
if(USE_INTERNAL_LLVM_LIBRARY)
message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use internal LLVM library with ENABLE_EMBEDDED_COMPILER=OFF")
endif()
set (USE_EMBEDDED_COMPILER 0)
return()
endif()
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm/CMakeLists.txt")
if (USE_INTERNAL_LLVM_LIBRARY)
message (WARNING "submodule contrib/llvm is missing. to fix try run: \n git submodule update --init --recursive")
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't fidd internal LLVM library")
endif()
set (MISSING_INTERNAL_LLVM_LIBRARY 1)
message (${RECONFIGURE_MESSAGE_LEVEL} "submodule /contrib/llvm is missing. to fix try run: \n git submodule update --init --recursive")
endif ()
if (NOT USE_INTERNAL_LLVM_LIBRARY)
set (LLVM_PATHS "/usr/local/lib/llvm" "/usr/lib/llvm")
set (USE_EMBEDDED_COMPILER 1)
foreach(llvm_v 11.1 11)
if (NOT LLVM_FOUND)
find_package (LLVM ${llvm_v} CONFIG PATHS ${LLVM_PATHS})
endif ()
endforeach ()
if (LLVM_FOUND)
# Remove dynamically-linked zlib and libedit from LLVM's dependencies:
set_target_properties(LLVMSupport PROPERTIES INTERFACE_LINK_LIBRARIES "-lpthread;LLVMDemangle;${ZLIB_LIBRARIES}")
set_target_properties(LLVMLineEditor PROPERTIES INTERFACE_LINK_LIBRARIES "LLVMSupport")
option(LLVM_HAS_RTTI "Enable if LLVM was build with RTTI enabled" ON)
set (USE_EMBEDDED_COMPILER 1)
else()
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find system LLVM")
set (USE_EMBEDDED_COMPILER 0)
endif()
if (LLVM_FOUND AND OS_LINUX AND USE_LIBCXX AND NOT FORCE_LLVM_WITH_LIBCXX)
message(WARNING "Option USE_INTERNAL_LLVM_LIBRARY is not set but the LLVM library from OS packages "
"in Linux is incompatible with libc++ ABI. LLVM Will be disabled. Force: -DFORCE_LLVM_WITH_LIBCXX=ON")
message (${RECONFIGURE_MESSAGE_LEVEL} "Unsupported LLVM configuration, cannot enable LLVM")
set (LLVM_FOUND 0)
set (USE_EMBEDDED_COMPILER 0)
endif ()
endif()
if(NOT LLVM_FOUND AND NOT MISSING_INTERNAL_LLVM_LIBRARY)
if (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_CURRENT_BINARY_DIR)
message(WARNING "Option ENABLE_EMBEDDED_COMPILER is set but internal LLVM library cannot build if build directory is the same as source directory.")
set (LLVM_FOUND 0)
set (USE_EMBEDDED_COMPILER 0)
elseif (SPLIT_SHARED_LIBRARIES)
# llvm-tablegen cannot find shared libraries that we build. Probably can be easily fixed.
message(WARNING "Option USE_INTERNAL_LLVM_LIBRARY is not compatible with SPLIT_SHARED_LIBRARIES. Build of LLVM will be disabled.")
set (LLVM_FOUND 0)
set (USE_EMBEDDED_COMPILER 0)
elseif (NOT ARCH_AMD64)
# It's not supported yet, but you can help.
message(WARNING "Option USE_INTERNAL_LLVM_LIBRARY is only available for x86_64. Build of LLVM will be disabled.")
set (LLVM_FOUND 0)
set (USE_EMBEDDED_COMPILER 0)
elseif (SANITIZE STREQUAL "undefined")
# llvm-tblgen, that is used during LLVM build, doesn't work with UBSan.
message(WARNING "Option USE_INTERNAL_LLVM_LIBRARY does not work with UBSan, because 'llvm-tblgen' tool from LLVM has undefined behaviour. Build of LLVM will be disabled.")
set (LLVM_FOUND 0)
set (USE_EMBEDDED_COMPILER 0)
else ()
set (USE_INTERNAL_LLVM_LIBRARY ON)
set (LLVM_FOUND 1)
set (USE_EMBEDDED_COMPILER 1)
set (LLVM_VERSION "9.0.0bundled")
set (LLVM_INCLUDE_DIRS
set (LLVM_FOUND 1)
set (LLVM_VERSION "12.0.0bundled")
set (LLVM_INCLUDE_DIRS
"${ClickHouse_SOURCE_DIR}/contrib/llvm/llvm/include"
"${ClickHouse_BINARY_DIR}/contrib/llvm/llvm/include"
)
set (LLVM_LIBRARY_DIRS "${ClickHouse_BINARY_DIR}/contrib/llvm/llvm")
endif()
endif()
)
set (LLVM_LIBRARY_DIRS "${ClickHouse_BINARY_DIR}/contrib/llvm/llvm")
if (LLVM_FOUND)
message(STATUS "LLVM include Directory: ${LLVM_INCLUDE_DIRS}")
message(STATUS "LLVM library Directory: ${LLVM_LIBRARY_DIRS}")
message(STATUS "LLVM C++ compiler flags: ${LLVM_CXXFLAGS}")
else()
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't enable LLVM")
endif()
message(STATUS "LLVM include Directory: ${LLVM_INCLUDE_DIRS}")
message(STATUS "LLVM library Directory: ${LLVM_LIBRARY_DIRS}")
message(STATUS "LLVM C++ compiler flags: ${LLVM_CXXFLAGS}")
# This list was generated by listing all LLVM libraries, compiling the binary and removing all libraries while it still compiles.
set (REQUIRED_LLVM_LIBRARIES

17
cmake/find/termcap.cmake
View File

@ -1,17 +0,0 @@
if (ENABLE_EMBEDDED_COMPILER AND NOT USE_INTERNAL_LLVM_LIBRARY AND USE_STATIC_LIBRARIES)
find_library (TERMCAP_LIBRARY tinfo)
if (NOT TERMCAP_LIBRARY)
find_library (TERMCAP_LIBRARY ncurses)
endif()
if (NOT TERMCAP_LIBRARY)
find_library (TERMCAP_LIBRARY termcap)
endif()
if (NOT TERMCAP_LIBRARY)
message (FATAL_ERROR "Statically Linking external LLVM requires termcap")
endif()
target_link_libraries(LLVMSupport INTERFACE ${TERMCAP_LIBRARY})
message (STATUS "Using termcap: ${TERMCAP_LIBRARY}")
endif()

5
contrib/CMakeLists.txt vendored
View File

@ -209,11 +209,12 @@ elseif(GTEST_SRC_DIR)
target_compile_definitions(gtest INTERFACE GTEST_HAS_POSIX_RE=0)
endif()
if (USE_EMBEDDED_COMPILER AND USE_INTERNAL_LLVM_LIBRARY)
if (USE_EMBEDDED_COMPILER)
# ld: unknown option: --color-diagnostics
if (APPLE)
set (LINKER_SUPPORTS_COLOR_DIAGNOSTICS 0 CACHE INTERNAL "")
endif ()
set (LLVM_ENABLE_EH 1 CACHE INTERNAL "")
set (LLVM_ENABLE_RTTI 1 CACHE INTERNAL "")
set (LLVM_ENABLE_PIC 0 CACHE INTERNAL "")
@ -228,8 +229,6 @@ if (USE_EMBEDDED_COMPILER AND USE_INTERNAL_LLVM_LIBRARY)
set (CMAKE_CXX_STANDARD ${CMAKE_CXX_STANDARD_bak})
unset (CMAKE_CXX_STANDARD_bak)
target_include_directories(LLVMSupport SYSTEM BEFORE PRIVATE ${ZLIB_INCLUDE_DIR})
endif ()
if (USE_INTERNAL_LIBGSASL_LIBRARY)

2
contrib/llvm vendored

@ -1 +1 @@
Subproject commit cfaf365cf96918999d09d976ec736b4518cf5d02
Subproject commit a7198805de67374eb3fb4c6b89797fa2d1cd7e50

2
src/Core/Settings.h
View File

@ -102,7 +102,7 @@ class IColumn;
\
M(Bool, allow_suspicious_low_cardinality_types, false, "In CREATE TABLE statement allows specifying LowCardinality modifier for types of small fixed size (8 or less). Enabling this may increase merge times and memory consumption.", 0) \
M(Bool, compile_expressions, true, "Compile some scalar functions and operators to native code.", 0) \
M(UInt64, min_count_to_compile_expression, 3, "The number of identical expressions before they are JIT-compiled", 0) \
M(UInt64, min_count_to_compile_expression, 0, "The number of identical expressions before they are JIT-compiled", 0) \
M(UInt64, group_by_two_level_threshold, 100000, "From what number of keys, a two-level aggregation starts. 0 - the threshold is not set.", 0) \
M(UInt64, group_by_two_level_threshold_bytes, 50000000, "From what size of the aggregation state in bytes, a two-level aggregation begins to be used. 0 - the threshold is not set. Two-level aggregation is used when at least one of the thresholds is triggered.", 0) \
M(Bool, distributed_aggregation_memory_efficient, true, "Is the memory-saving mode of distributed aggregation enabled.", 0) \

11
src/DataTypes/Native.h
View File

@ -9,7 +9,6 @@
# include <DataTypes/IDataType.h>
# include <DataTypes/DataTypeNullable.h>
# include <DataTypes/DataTypeFixedString.h>
# include <Columns/ColumnConst.h>
# include <Columns/ColumnNullable.h>
# pragma GCC diagnostic push
@ -41,7 +40,8 @@ static inline llvm::Type * toNativeType(llvm::IRBuilderBase & builder, const IDa
{
const auto & data_type_nullable = static_cast<const DataTypeNullable&>(type);
auto * wrapped = toNativeType(builder, *data_type_nullable.getNestedType());
return wrapped ? llvm::StructType::get(wrapped, /* is null = */ builder.getInt1Ty()) : nullptr;
auto * is_null_type = builder.getInt1Ty();
return wrapped ? llvm::StructType::get(wrapped, is_null_type) : nullptr;
}
/// LLVM doesn't have unsigned types, it has unsigned instructions.
@ -57,11 +57,6 @@ static inline llvm::Type * toNativeType(llvm::IRBuilderBase & builder, const IDa
return builder.getFloatTy();
else if (data_type.isFloat64())
return builder.getDoubleTy();
else if (data_type.isFixedString())
{
const auto & data_type_fixed_string = static_cast<const DataTypeFixedString &>(type);
return llvm::VectorType::get(builder.getInt8Ty(), data_type_fixed_string.getN());
}
return nullptr;
}
@ -76,7 +71,7 @@ static inline bool canBeNativeType(const IDataType & type)
return canBeNativeType(*data_type_nullable.getNestedType());
}
return data_type.isNativeInt() || data_type.isNativeUInt() || data_type.isFloat() || data_type.isFixedString() || data_type.isDate();
return data_type.isNativeInt() || data_type.isNativeUInt() || data_type.isFloat() || data_type.isDate();
}
static inline llvm::Type * toNativeType(llvm::IRBuilderBase & builder, const DataTypePtr & type)

14
src/Interpreters/JIT/compileFunction.cpp
View File

@ -87,7 +87,7 @@ static void compileFunction(llvm::Module & module, const IFunctionBase & functio
for (size_t i = 0; i <= arg_types.size(); ++i)
{
const auto & type = i == arg_types.size() ? function.getResultType() : arg_types[i];
auto * data = b.CreateLoad(b.CreateConstInBoundsGEP1_32(data_type, columns_arg, i));
auto * data = b.CreateLoad(data_type, b.CreateConstInBoundsGEP1_32(data_type, columns_arg, i));
columns[i].data_init = b.CreatePointerCast(b.CreateExtractValue(data, {0}), toNativeType(b, removeNullable(type))->getPointerTo());
columns[i].null_init = type->isNullable() ? b.CreateExtractValue(data, {1}) : nullptr;
}
@ -122,15 +122,19 @@ static void compileFunction(llvm::Module & module, const IFunctionBase & functio
auto & column = columns[i];
auto type = arg_types[i];
auto * value = b.CreateLoad(column.data);
if (!column.null)
auto * value = b.CreateLoad(toNativeType(b, removeNullable(type)), column.data);
if (!type->isNullable())
{
arguments.emplace_back(value);
continue;
}
auto * is_null = b.CreateICmpNE(b.CreateLoad(column.null), b.getInt8(0));
auto * nullable_unitilized = llvm::Constant::getNullValue(toNativeType(b, type));
auto * nullable_type_untyped = toNativeType(b, type);
llvm::StructType * nullable_type = static_cast<llvm::StructType *>(nullable_type_untyped);
llvm::Type * is_null_type = nullable_type->getElementType(1);
auto * is_null = b.CreateICmpNE(b.CreateLoad(is_null_type, column.null), b.getInt8(0));
auto * nullable_unitilized = llvm::Constant::getNullValue(nullable_type_untyped);
auto * nullable_value = b.CreateInsertValue(b.CreateInsertValue(nullable_unitilized, value, {0}), is_null, {1});
arguments.emplace_back(nullable_value);
}

2
src/Interpreters/examples/jit_example.cpp
View File

@ -38,7 +38,7 @@ int main(int argc, char **argv)
// b.CreateCall(func_declaration);
// auto * load_argument = b.CreateLoad(argument);
// auto * load_argument = b.CreateLoad(value_type, argument);
// auto * value = b.CreateAdd(load_argument, load_argument);
// b.CreateRet(value);
// });

5
utils/ci/build-normal.sh
View File

@ -8,11 +8,6 @@ source default-config
mkdir -p "${WORKSPACE}/build"
pushd "${WORKSPACE}/build"
if [[ "${ENABLE_EMBEDDED_COMPILER}" == 1 ]]; then
[[ "$USE_LLVM_LIBRARIES_FROM_SYSTEM" == 0 ]] && CMAKE_FLAGS="$CMAKE_FLAGS -DUSE_INTERNAL_LLVM_LIBRARY=1"
[[ "$USE_LLVM_LIBRARIES_FROM_SYSTEM" != 0 ]] && CMAKE_FLAGS="$CMAKE_FLAGS -DUSE_INTERNAL_LLVM_LIBRARY=0"
fi
cmake -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DENABLE_EMBEDDED_COMPILER=${ENABLE_EMBEDDED_COMPILER} $CMAKE_FLAGS ../sources
[[ "$BUILD_TARGETS" != 'all' ]] && BUILD_TARGETS_STRING="--target $BUILD_TARGETS"

1
utils/ci/default-config
View File

@ -27,7 +27,6 @@ CLANG_SOURCES_BRANCH=trunk # or tags/RELEASE_600/final
GCC_SOURCES_VERSION=latest # or gcc-7.1.0
# install-libraries
USE_LLVM_LIBRARIES_FROM_SYSTEM=0 # 0 or 1
ENABLE_EMBEDDED_COMPILER=1
# build

4
utils/ci/install-libraries.sh
View File

@ -5,7 +5,3 @@ source default-config
./install-os-packages.sh libicu-dev
./install-os-packages.sh libreadline-dev
if [[ "$ENABLE_EMBEDDED_COMPILER" == 1 && "$USE_LLVM_LIBRARIES_FROM_SYSTEM" == 1 ]]; then
./install-os-packages.sh llvm-libs-5.0
fi

1
utils/ci/jobs/quick-build/run.sh
View File

@ -15,7 +15,6 @@ SOURCES_METHOD=local
COMPILER=clang
COMPILER_INSTALL_METHOD=packages
COMPILER_PACKAGE_VERSION=6.0
USE_LLVM_LIBRARIES_FROM_SYSTEM=0
BUILD_METHOD=normal
BUILD_TARGETS=clickhouse
BUILD_TYPE=Debug