diff --git a/cmake/find_llvm.cmake b/cmake/find_llvm.cmake index 618eaadf41a..21ee7f28e4a 100644 --- a/cmake/find_llvm.cmake +++ b/cmake/find_llvm.cmake @@ -1,107 +1,21 @@ option (ENABLE_EMBEDDED_COMPILER "Set to TRUE to enable support for 'compile' option for query execution" 1) if (ENABLE_EMBEDDED_COMPILER) - # Based on source code of YT. - # Authors: Ivan Puzyrevskiy, Alexey Lukyanchikov, Ruslan Savchenko. - - # Find LLVM includes and libraries. - # - # LLVM_VERSION - LLVM version. - # LLVM_INCLUDE_DIRS - Directory containing LLVM headers. - # LLVM_LIBRARY_DIRS - Directory containing LLVM libraries. - # LLVM_CXXFLAGS - C++ compiler flags for files that include LLVM headers. - # LLVM_FOUND - True if LLVM was found. - - # llvm_map_components_to_libraries - Maps LLVM used components to required libraries. - # Usage: llvm_map_components_to_libraries(REQUIRED_LLVM_LIBRARIES core jit interpreter native ...) - if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - set(LLVM_VERSION_POSTFIX "${COMPILER_POSTFIX}" CACHE STRING "") - else() - if (ARCH_FREEBSD) - set(LLVM_VERSION_POSTFIX "50" CACHE STRING "") - else() - set(LLVM_VERSION_POSTFIX "-5.0" CACHE STRING "") - endif() - endif() + find_package(LLVM CONFIG) + else () + find_package(LLVM 5 CONFIG) + endif () - find_program(LLVM_CONFIG_EXECUTABLE - NAMES llvm-config${LLVM_VERSION_POSTFIX} llvm-config llvm-config-devel - PATHS $ENV{LLVM_ROOT}/bin) - - mark_as_advanced(LLVM_CONFIG_EXECUTABLE) - - if(NOT LLVM_CONFIG_EXECUTABLE) - message(WARNING "Cannot find LLVM (looking for `llvm-config${LLVM_VERSION_POSTFIX}`, `llvm-config`, `llvm-config-devel`). Please, provide LLVM_ROOT environment variable.") - else() - set(LLVM_FOUND TRUE) - - execute_process( - COMMAND ${LLVM_CONFIG_EXECUTABLE} --version - OUTPUT_VARIABLE LLVM_VERSION - OUTPUT_STRIP_TRAILING_WHITESPACE) - - if(LLVM_VERSION VERSION_LESS "5") - message(FATAL_ERROR "LLVM 5+ is required. You have ${LLVM_VERSION} (${LLVM_CONFIG_EXECUTABLE})") - endif() - - message(STATUS "LLVM config: ${LLVM_CONFIG_EXECUTABLE}; version: ${LLVM_VERSION}") - - execute_process( - COMMAND ${LLVM_CONFIG_EXECUTABLE} --includedir - OUTPUT_VARIABLE LLVM_INCLUDE_DIRS - OUTPUT_STRIP_TRAILING_WHITESPACE) - - execute_process( - COMMAND ${LLVM_CONFIG_EXECUTABLE} --libdir - OUTPUT_VARIABLE LLVM_LIBRARY_DIRS - OUTPUT_STRIP_TRAILING_WHITESPACE) - - execute_process( - COMMAND ${LLVM_CONFIG_EXECUTABLE} --cxxflags - OUTPUT_VARIABLE LLVM_CXXFLAGS - OUTPUT_STRIP_TRAILING_WHITESPACE) - - execute_process( - COMMAND ${LLVM_CONFIG_EXECUTABLE} --targets-built - OUTPUT_VARIABLE LLVM_TARGETS_BUILT - OUTPUT_STRIP_TRAILING_WHITESPACE) - - string(REPLACE " " ";" LLVM_TARGETS_BUILT "${LLVM_TARGETS_BUILT}") - - if (USE_STATIC_LIBRARIES) - set (LLVM_CONFIG_ADD "--link-static") - endif() - - # Get the link libs we need. - function(llvm_map_components_to_libraries RESULT) - execute_process( - COMMAND ${LLVM_CONFIG_EXECUTABLE} ${LLVM_CONFIG_ADD} --libs ${ARGN} - OUTPUT_VARIABLE _tmp - OUTPUT_STRIP_TRAILING_WHITESPACE) - - string(REPLACE " " ";" _libs_module "${_tmp}") - - #message(STATUS "LLVM Libraries for '${ARGN}': ${_libs_module}") - - execute_process( - COMMAND ${LLVM_CONFIG_EXECUTABLE} --system-libs ${ARGN} - OUTPUT_VARIABLE _libs_system - OUTPUT_STRIP_TRAILING_WHITESPACE) - - string(REPLACE "\n" " " _libs_system "${_libs_system}") - string(REPLACE " " " " _libs_system "${_libs_system}") - string(REPLACE " " ";" _libs_system "${_libs_system}") - - set(${RESULT} ${_libs_module} ${_libs_system} PARENT_SCOPE) - endfunction(llvm_map_components_to_libraries) + if (LLVM_FOUND) + # Remove dynamically-linked zlib and libedit from LLVM's dependencies: + set_target_properties(LLVMSupport PROPERTIES INTERFACE_LINK_LIBRARIES "-lpthread;LLVMDemangle") + set_target_properties(LLVMLineEditor PROPERTIES INTERFACE_LINK_LIBRARIES "LLVMSupport") + message(STATUS "LLVM version: ${LLVM_PACKAGE_VERSION}") message(STATUS "LLVM Include Directory: ${LLVM_INCLUDE_DIRS}") message(STATUS "LLVM Library Directory: ${LLVM_LIBRARY_DIRS}") message(STATUS "LLVM C++ Compiler: ${LLVM_CXXFLAGS}") - endif() - - if (LLVM_FOUND AND LLVM_INCLUDE_DIRS AND LLVM_LIBRARY_DIRS) set (USE_EMBEDDED_COMPILER 1) endif() endif() diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 906897fd0f4..3c99505c622 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -99,6 +99,15 @@ else () install (TARGETS dbms LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT clickhouse) endif () +if (USE_EMBEDDED_COMPILER) + llvm_map_components_to_libnames(REQUIRED_LLVM_LIBRARIES all) + target_link_libraries (dbms ${REQUIRED_LLVM_LIBRARIES}) + target_include_directories (dbms BEFORE PUBLIC ${LLVM_INCLUDE_DIRS}) + # LLVM has a bunch of unused parameters in its header files. + set_source_files_properties(src/Functions/IFunction.cpp PROPERTIES COMPILE_FLAGS "-Wno-unused-parameter") + set_source_files_properties(src/Interpreters/ExpressionJIT.cpp PROPERTIES COMPILE_FLAGS "-Wno-unused-parameter -Wno-non-virtual-dtor") +endif () + if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL") # Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size. diff --git a/dbms/src/Columns/ColumnConst.h b/dbms/src/Columns/ColumnConst.h index 2e11bbdb38b..eeda4dfd03a 100644 --- a/dbms/src/Columns/ColumnConst.h +++ b/dbms/src/Columns/ColumnConst.h @@ -193,6 +193,7 @@ public: bool isFixedAndContiguous() const override { return data->isFixedAndContiguous(); } bool valuesHaveFixedSize() const override { return data->valuesHaveFixedSize(); } size_t sizeOfValueIfFixed() const override { return data->sizeOfValueIfFixed(); } + StringRef getRawData() const override { return data->getRawData(); } /// Not part of the common interface. diff --git a/dbms/src/Columns/ColumnFixedString.h b/dbms/src/Columns/ColumnFixedString.h index cd465a1814d..80b6ccd5456 100644 --- a/dbms/src/Columns/ColumnFixedString.h +++ b/dbms/src/Columns/ColumnFixedString.h @@ -129,7 +129,7 @@ public: bool isFixedAndContiguous() const override { return true; } size_t sizeOfValueIfFixed() const override { return n; } - + StringRef getRawData() const override { return StringRef(chars.data(), chars.size()); } /// Specialized part of interface, not from IColumn. diff --git a/dbms/src/Columns/ColumnVector.h b/dbms/src/Columns/ColumnVector.h index b276b411d2e..b2ab0e9d403 100644 --- a/dbms/src/Columns/ColumnVector.h +++ b/dbms/src/Columns/ColumnVector.h @@ -268,7 +268,7 @@ public: bool isFixedAndContiguous() const override { return true; } size_t sizeOfValueIfFixed() const override { return sizeof(T); } - + StringRef getRawData() const override { return StringRef(reinterpret_cast(data.data()), data.size()); } /** More efficient methods of manipulation - to manipulate with data directly. */ Container & getData() diff --git a/dbms/src/Columns/IColumn.h b/dbms/src/Columns/IColumn.h index 544c8c31165..003e1667350 100644 --- a/dbms/src/Columns/IColumn.h +++ b/dbms/src/Columns/IColumn.h @@ -303,6 +303,9 @@ public: /// Values in column are represented as continuous memory segment of fixed size. Implies valuesHaveFixedSize. virtual bool isFixedAndContiguous() const { return false; } + /// If isFixedAndContiguous, returns the underlying data array, otherwise throws an exception. + virtual StringRef getRawData() const { throw Exception("Column " + getName() + " is not a contiguous block of memory", ErrorCodes::NOT_IMPLEMENTED); } + /// If valuesHaveFixedSize, returns size of value, otherwise throw an exception. virtual size_t sizeOfValueIfFixed() const { throw Exception("Values of column " + getName() + " are not fixed size.", ErrorCodes::CANNOT_GET_SIZE_OF_FIELD); } diff --git a/dbms/src/DataTypes/Native.h b/dbms/src/DataTypes/Native.h new file mode 100644 index 00000000000..d3b7646188e --- /dev/null +++ b/dbms/src/DataTypes/Native.h @@ -0,0 +1,76 @@ +#pragma once + +#include + +#if USE_EMBEDDED_COMPILER + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace DB +{ + +template +static inline bool typeIsEither(const IDataType & type) +{ + return (typeid_cast(&type) || ...); +} + +static inline llvm::Type * toNativeType(llvm::IRBuilderBase & builder, const IDataType & type) +{ + if (auto * nullable = typeid_cast(&type)) + { + auto * wrapped = toNativeType(builder, *nullable->getNestedType()); + return wrapped ? llvm::StructType::get(wrapped, /* is null = */ builder.getInt1Ty()) : nullptr; + } + /// LLVM doesn't have unsigned types, it has unsigned instructions. + if (typeIsEither(type)) + return builder.getInt8Ty(); + if (typeIsEither(type)) + return builder.getInt16Ty(); + if (typeIsEither(type)) + return builder.getInt32Ty(); + if (typeIsEither(type)) + return builder.getInt64Ty(); + if (typeIsEither(type)) + return builder.getInt128Ty(); + if (typeIsEither(type)) + return builder.getFloatTy(); + if (typeIsEither(type)) + return builder.getDoubleTy(); + if (auto * fixed_string = typeid_cast(&type)) + return llvm::VectorType::get(builder.getInt8Ty(), fixed_string->getN()); + return nullptr; +} + +static inline llvm::Type * toNativeType(llvm::IRBuilderBase & builder, const DataTypePtr & type) +{ + return toNativeType(builder, *type); +} + +static inline llvm::Value * castNativeNumber(llvm::IRBuilder<> & builder, llvm::Value * value, llvm::Type * type, bool is_signed) +{ + if (value->getType() == type) + return value; + if (value->getType()->isIntegerTy()) + { + if (type->isIntegerTy()) + return builder.CreateIntCast(value, type, is_signed); + return is_signed ? builder.CreateSIToFP(value, type) : builder.CreateUIToFP(value, type); + } + if (type->isFloatingPointTy()) + return builder.CreateFPCast(value, type); + return is_signed ? builder.CreateFPToSI(value, type) : builder.CreateFPToUI(value, type); +} + +} + +#endif diff --git a/dbms/src/Functions/CMakeLists.txt b/dbms/src/Functions/CMakeLists.txt index 1febb4aa20c..4882710578e 100644 --- a/dbms/src/Functions/CMakeLists.txt +++ b/dbms/src/Functions/CMakeLists.txt @@ -109,3 +109,9 @@ endif () if (ENABLE_TESTS) add_subdirectory (tests) endif () + +if (USE_EMBEDDED_COMPILER) + target_include_directories (clickhouse_functions BEFORE PUBLIC ${LLVM_INCLUDE_DIRS}) + # LLVM has a bunch of unused parameters in its header files. + target_compile_options (clickhouse_functions PRIVATE "-Wno-unused-parameter") +endif () diff --git a/dbms/src/Functions/FunctionHelpers.cpp b/dbms/src/Functions/FunctionHelpers.cpp index 5c2e23248b8..43a0f73cdfc 100644 --- a/dbms/src/Functions/FunctionHelpers.cpp +++ b/dbms/src/Functions/FunctionHelpers.cpp @@ -5,12 +5,16 @@ #include #include #include -#include "FunctionsArithmetic.h" namespace DB { +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; +} + const ColumnConst * checkAndGetColumnConstStringOrFixedString(const IColumn * column) { if (!column->isColumnConst()) diff --git a/dbms/src/Functions/FunctionsArithmetic.h b/dbms/src/Functions/FunctionsArithmetic.h index 16367f0b8b8..4725c8caa74 100644 --- a/dbms/src/Functions/FunctionsArithmetic.h +++ b/dbms/src/Functions/FunctionsArithmetic.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -19,6 +20,10 @@ #include #include +#if USE_EMBEDDED_COMPILER +#include +#endif + namespace DB { @@ -106,6 +111,15 @@ struct PlusImpl /// Next everywhere, static_cast - so that there is no wrong result in expressions of the form Int64 c = UInt32(a) * Int32(-1). return static_cast(a) + b; } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = true; + + static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) + { + return left->getType()->isIntegerTy() ? b.CreateAdd(left, right) : b.CreateFAdd(left, right); + } +#endif }; @@ -119,6 +133,15 @@ struct MultiplyImpl { return static_cast(a) * b; } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = true; + + static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) + { + return left->getType()->isIntegerTy() ? b.CreateMul(left, right) : b.CreateFMul(left, right); + } +#endif }; template @@ -131,6 +154,15 @@ struct MinusImpl { return static_cast(a) - b; } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = true; + + static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) + { + return left->getType()->isIntegerTy() ? b.CreateSub(left, right) : b.CreateFSub(left, right); + } +#endif }; template @@ -143,6 +175,17 @@ struct DivideFloatingImpl { return static_cast(a) / b; } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = true; + + static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) + { + if (left->getType()->isIntegerTy()) + throw Exception("DivideFloatingImpl expected a floating-point type", ErrorCodes::LOGICAL_ERROR); + return b.CreateFDiv(left, right); + } +#endif }; @@ -189,6 +232,10 @@ struct DivideIntegralImpl throwIfDivisionLeadsToFPE(a, b); return a / b; } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = false; /// don't know how to throw from LLVM IR +#endif }; template @@ -201,6 +248,10 @@ struct DivideIntegralOrZeroImpl { return unlikely(divisionLeadsToFPE(a, b)) ? 0 : a / b; } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = false; /// TODO implement the checks +#endif }; template @@ -212,9 +263,12 @@ struct ModuloImpl static inline Result apply(A a, B b) { throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger::Type(a), typename NumberTraits::ToInteger::Type(b)); - return typename NumberTraits::ToInteger::Type(a) - % typename NumberTraits::ToInteger::Type(b); + return typename NumberTraits::ToInteger::Type(a) % typename NumberTraits::ToInteger::Type(b); } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = false; /// don't know how to throw from LLVM IR +#endif }; template @@ -225,9 +279,19 @@ struct BitAndImpl template static inline Result apply(A a, B b) { - return static_cast(a) - & static_cast(b); + return static_cast(a) & static_cast(b); } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = true; + + static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) + { + if (!left->getType()->isIntegerTy()) + throw Exception("BitAndImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); + return b.CreateAnd(left, right); + } +#endif }; template @@ -238,9 +302,19 @@ struct BitOrImpl template static inline Result apply(A a, B b) { - return static_cast(a) - | static_cast(b); + return static_cast(a) | static_cast(b); } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = true; + + static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) + { + if (!left->getType()->isIntegerTy()) + throw Exception("BitOrImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); + return b.CreateOr(left, right); + } +#endif }; template @@ -251,9 +325,19 @@ struct BitXorImpl template static inline Result apply(A a, B b) { - return static_cast(a) - ^ static_cast(b); + return static_cast(a) ^ static_cast(b); } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = true; + + static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) + { + if (!left->getType()->isIntegerTy()) + throw Exception("BitXorImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); + return b.CreateXor(left, right); + } +#endif }; template @@ -264,9 +348,19 @@ struct BitShiftLeftImpl template static inline Result apply(A a, B b) { - return static_cast(a) - << static_cast(b); + return static_cast(a) << static_cast(b); } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = true; + + static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) + { + if (!left->getType()->isIntegerTy()) + throw Exception("BitShiftLeftImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); + return b.CreateShl(left, right); + } +#endif }; template @@ -277,9 +371,19 @@ struct BitShiftRightImpl template static inline Result apply(A a, B b) { - return static_cast(a) - >> static_cast(b); + return static_cast(a) >> static_cast(b); } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = true; + + static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool is_signed) + { + if (!left->getType()->isIntegerTy()) + throw Exception("BitShiftRightImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); + return is_signed ? b.CreateAShr(left, right) : b.CreateLShr(left, right); + } +#endif }; template @@ -293,6 +397,19 @@ struct BitRotateLeftImpl return (static_cast(a) << static_cast(b)) | (static_cast(a) >> ((sizeof(Result) * 8) - static_cast(b))); } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = true; + + static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) + { + if (!left->getType()->isIntegerTy()) + throw Exception("BitRotateLeftImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); + auto * size = llvm::ConstantInt::get(left->getType(), left->getType()->getPrimitiveSizeInBits()); + /// XXX how is this supposed to behave in signed mode? + return b.CreateOr(b.CreateShl(left, right), b.CreateLShr(left, b.CreateSub(size, right))); + } +#endif }; template @@ -306,24 +423,35 @@ struct BitRotateRightImpl return (static_cast(a) >> static_cast(b)) | (static_cast(a) << ((sizeof(Result) * 8) - static_cast(b))); } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = true; + + static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool) + { + if (!left->getType()->isIntegerTy()) + throw Exception("BitRotateRightImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); + auto * size = llvm::ConstantInt::get(left->getType(), left->getType()->getPrimitiveSizeInBits()); + return b.CreateOr(b.CreateLShr(left, right), b.CreateShl(left, b.CreateSub(size, right))); + } +#endif }; - -template -std::enable_if_t, T> toInteger(T x) { return x; } - -template -std::enable_if_t, Int64> toInteger(T x) { return Int64(x); } - template struct BitTestImpl { using ResultType = UInt8; template - static inline Result apply(A a, B b) { return (toInteger(a) >> toInteger(b)) & 1; }; -}; + static inline Result apply(A a, B b) + { + return (typename NumberTraits::ToInteger::Type(a) >> typename NumberTraits::ToInteger::Type(b)) & 1; + }; +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = false; /// TODO +#endif +}; template struct LeastBaseImpl @@ -336,6 +464,18 @@ struct LeastBaseImpl /** gcc 4.9.2 successfully vectorizes a loop from this function. */ return static_cast(a) < static_cast(b) ? static_cast(a) : static_cast(b); } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = true; + + static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool is_signed) + { + if (!left->getType()->isIntegerTy()) + /// XXX minnum is basically fmin(), it may or may not match whatever apply() does + return b.CreateMinNum(left, right); + return b.CreateSelect(is_signed ? b.CreateICmpSLT(left, right) : b.CreateICmpULT(left, right), left, right); + } +#endif }; template @@ -349,6 +489,10 @@ struct LeastSpecialImpl static_assert(std::is_same_v, "ResultType != Result"); return accurate::lessOp(a, b) ? static_cast(a) : static_cast(b); } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = false; /// ??? +#endif }; template @@ -365,6 +509,19 @@ struct GreatestBaseImpl { return static_cast(a) > static_cast(b) ? static_cast(a) : static_cast(b); } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = true; + + static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool is_signed) + { + if (!left->getType()->isIntegerTy()) + /// XXX maxnum is basically fmax(), it may or may not match whatever apply() does + /// XXX CreateMaxNum is broken on LLVM 5.0 and 6.0 (generates minnum instead; fixed in 7) + return b.CreateBinaryIntrinsic(llvm::Intrinsic::maxnum, left, right); + return b.CreateSelect(is_signed ? b.CreateICmpSGT(left, right) : b.CreateICmpUGT(left, right), left, right); + } +#endif }; template @@ -378,6 +535,10 @@ struct GreatestSpecialImpl static_assert(std::is_same_v, "ResultType != Result"); return accurate::greaterOp(a, b) ? static_cast(a) : static_cast(b); } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = false; /// ??? +#endif }; template @@ -393,6 +554,15 @@ struct NegateImpl { return -static_cast(a); } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = true; + + static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * arg, bool) + { + return arg->getType()->isIntegerTy() ? b.CreateNeg(arg) : b.CreateFNeg(arg); + } +#endif }; template @@ -404,6 +574,17 @@ struct BitNotImpl { return ~static_cast(a); } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = true; + + static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * arg, bool) + { + if (!arg->getType()->isIntegerTy()) + throw Exception("BitNotImpl expected an integral type", ErrorCodes::LOGICAL_ERROR); + return b.CreateNot(arg); + } +#endif }; template @@ -420,6 +601,10 @@ struct AbsImpl else if constexpr (std::is_floating_point_v) return static_cast(std::abs(a)); } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = false; /// special type handling, some other time +#endif }; template @@ -436,6 +621,10 @@ struct GCDImpl typename NumberTraits::ToInteger::Type(a), typename NumberTraits::ToInteger::Type(b)); } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = false; /// exceptions (and a non-trivial algorithm) +#endif }; template @@ -452,6 +641,10 @@ struct LCMImpl typename NumberTraits::ToInteger::Type(a), typename NumberTraits::ToInteger::Type(b)); } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = false; /// exceptions (and a non-trivial algorithm) +#endif }; template @@ -463,6 +656,10 @@ struct IntExp2Impl { return intExp2(a); } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = false; /// library function +#endif }; template @@ -474,6 +671,10 @@ struct IntExp10Impl { return intExp10(a); } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = false; /// library function +#endif }; /// Used to indicate undefined operation @@ -745,6 +946,43 @@ public: if (!valid) throw Exception(getName() + "'s arguments do not match the expected data types", ErrorCodes::LOGICAL_ERROR); } + +#if USE_EMBEDDED_COMPILER + bool isCompilableImpl(const DataTypes & arguments) const override + { + return castBothTypes(arguments[0].get(), arguments[1].get(), [&](const auto & left, const auto & right) + { + using LeftDataType = std::decay_t; + using RightDataType = std::decay_t; + using ResultDataType = typename BinaryOperationTraits::ResultDataType; + using OpSpec = Op; + return !std::is_same_v && OpSpec::compilable; + }); + } + + llvm::Value * compileImpl(llvm::IRBuilderBase & builder, const DataTypes & types, ValuePlaceholders values) const override + { + llvm::Value * result = nullptr; + castBothTypes(types[0].get(), types[1].get(), [&](const auto & left, const auto & right) + { + using LeftDataType = std::decay_t; + using RightDataType = std::decay_t; + using ResultDataType = typename BinaryOperationTraits::ResultDataType; + using OpSpec = Op; + if constexpr (!std::is_same_v && OpSpec::compilable) + { + auto & b = static_cast &>(builder); + auto * type = toNativeType(b, ResultDataType{}); + auto * lval = castNativeNumber(b, values[0](), type, std::is_signed_v); + auto * rval = castNativeNumber(b, values[1](), type, std::is_signed_v); + result = OpSpec::compile(b, lval, rval, std::is_signed_v); + return true; + } + return false; + }); + return result; + } +#endif }; @@ -821,6 +1059,35 @@ public: throw Exception(getName() + "'s argument does not match the expected data type", ErrorCodes::LOGICAL_ERROR); } +#if USE_EMBEDDED_COMPILER + bool isCompilableImpl(const DataTypes & arguments) const override + { + return castType(arguments[0].get(), [&](const auto & type) + { + return Op::FieldType>::compilable; + }); + } + + llvm::Value * compileImpl(llvm::IRBuilderBase & builder, const DataTypes & types, ValuePlaceholders values) const override + { + llvm::Value * result = nullptr; + castType(types[0].get(), [&](const auto & type) + { + using T0 = typename std::decay_t::FieldType; + using T1 = typename Op::ResultType; + if constexpr (Op::compilable) + { + auto & b = static_cast &>(builder); + auto * v = castNativeNumber(b, values[0](), toNativeType(b, DataTypeNumber{}), std::is_signed_v); + result = Op::compile(b, v, std::is_signed_v); + return true; + } + return false; + }); + return result; + } +#endif + bool hasInformationAboutMonotonicity() const override { return FunctionUnaryArithmeticMonotonicity::has(); diff --git a/dbms/src/Functions/FunctionsLogical.h b/dbms/src/Functions/FunctionsLogical.h index cc11e598f96..5d3982e95de 100644 --- a/dbms/src/Functions/FunctionsLogical.h +++ b/dbms/src/Functions/FunctionsLogical.h @@ -12,6 +12,10 @@ #include #include +#if USE_EMBEDDED_COMPILER +#include +#endif + namespace DB { @@ -31,65 +35,71 @@ namespace ErrorCodes * For example, 1 OR NULL returns 1, not NULL. */ - struct AndImpl { - static inline bool isSaturable() + static inline constexpr bool isSaturable() { return true; } - static inline bool isSaturatedValue(bool a) + static inline constexpr bool isSaturatedValue(bool a) { return !a; } - static inline bool apply(bool a, bool b) + static inline constexpr bool apply(bool a, bool b) { return a && b; } - static inline bool specialImplementationForNulls() { return false; } + static inline constexpr bool specialImplementationForNulls() { return false; } }; struct OrImpl { - static inline bool isSaturable() + static inline constexpr bool isSaturable() { return true; } - static inline bool isSaturatedValue(bool a) + static inline constexpr bool isSaturatedValue(bool a) { return a; } - static inline bool apply(bool a, bool b) + static inline constexpr bool apply(bool a, bool b) { return a || b; } - static inline bool specialImplementationForNulls() { return true; } + static inline constexpr bool specialImplementationForNulls() { return true; } }; struct XorImpl { - static inline bool isSaturable() + static inline constexpr bool isSaturable() { return false; } - static inline bool isSaturatedValue(bool) + static inline constexpr bool isSaturatedValue(bool) { return false; } - static inline bool apply(bool a, bool b) + static inline constexpr bool apply(bool a, bool b) { return a != b; } - static inline bool specialImplementationForNulls() { return false; } + static inline constexpr bool specialImplementationForNulls() { return false; } + +#if USE_EMBEDDED_COMPILER + static inline llvm::Value * apply(llvm::IRBuilder<> & builder, llvm::Value * a, llvm::Value * b) + { + return builder.CreateXor(a, b); + } +#endif }; template @@ -101,6 +111,13 @@ struct NotImpl { return !a; } + +#if USE_EMBEDDED_COMPILER + static inline llvm::Value * apply(llvm::IRBuilder<> & builder, llvm::Value * a) + { + return builder.CreateNot(a); + } +#endif }; @@ -172,6 +189,20 @@ struct AssociativeOperationImpl }; +#if USE_EMBEDDED_COMPILER +static llvm::Value * isNativeTrueValue(llvm::IRBuilder<> & b, const DataTypePtr & type, llvm::Value * x) +{ + if (type->isNullable()) + { + auto * subexpr = isNativeTrueValue(b, removeNullable(type), b.CreateExtractValue(x, {0})); + return b.CreateAnd(b.CreateNot(b.CreateExtractValue(x, {1})), subexpr); + } + auto * zero = llvm::Constant::getNullValue(x->getType()); + return x->getType()->isIntegerTy() ? b.CreateICmpNE(x, zero) : b.CreateFCmpONE(x, zero); /// QNaN -> false +} +#endif + + template class FunctionAnyArityLogical : public IFunction { @@ -364,6 +395,44 @@ public: block.getByPosition(result).column = std::move(col_res); } + +#if USE_EMBEDDED_COMPILER + bool isCompilableImpl(const DataTypes &) const override { return true; } + + llvm::Value * compileImpl(llvm::IRBuilderBase & builder, const DataTypes & types, ValuePlaceholders values) const override + { + auto & b = static_cast &>(builder); + if constexpr (!Impl::isSaturable()) + { + auto * result = isNativeTrueValue(b, types[0], values[0]()); + for (size_t i = 1; i < types.size(); i++) + result = Impl::apply(b, result, isNativeTrueValue(b, types[i], values[i]())); + return b.CreateSelect(result, b.getInt8(1), b.getInt8(0)); + } + constexpr bool breakOnTrue = Impl::isSaturatedValue(true); + auto * next = b.GetInsertBlock(); + auto * stop = llvm::BasicBlock::Create(next->getContext(), "", next->getParent()); + b.SetInsertPoint(stop); + auto * phi = b.CreatePHI(b.getInt8Ty(), values.size()); + for (size_t i = 0; i < types.size(); i++) + { + b.SetInsertPoint(next); + auto * value = values[i](); + auto * truth = isNativeTrueValue(b, types[i], value); + if (!types[i]->equals(DataTypeUInt8{})) + value = b.CreateSelect(truth, b.getInt8(1), b.getInt8(0)); + phi->addIncoming(value, b.GetInsertBlock()); + if (i + 1 < types.size()) + { + next = llvm::BasicBlock::Create(next->getContext(), "", next->getParent()); + b.CreateCondBr(truth, breakOnTrue ? stop : next, breakOnTrue ? next : stop); + } + } + b.CreateBr(stop); + b.SetInsertPoint(stop); + return phi; + } +#endif }; @@ -430,6 +499,16 @@ public: + " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); } + +#if USE_EMBEDDED_COMPILER + bool isCompilableImpl(const DataTypes &) const override { return true; } + + llvm::Value * compileImpl(llvm::IRBuilderBase & builder, const DataTypes & types, ValuePlaceholders values) const override + { + auto & b = static_cast &>(builder); + return b.CreateSelect(Impl::apply(b, isNativeTrueValue(b, types[0], values[0]())), b.getInt8(1), b.getInt8(0)); + } +#endif }; diff --git a/dbms/src/Functions/FunctionsRound.h b/dbms/src/Functions/FunctionsRound.h index ac07c0e0740..89fad9fbf1c 100644 --- a/dbms/src/Functions/FunctionsRound.h +++ b/dbms/src/Functions/FunctionsRound.h @@ -93,6 +93,10 @@ struct RoundToExp2Impl { return roundDownToPowerOfTwo(x); } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = false; +#endif }; @@ -120,6 +124,10 @@ struct RoundDurationImpl : (x < 36000 ? 18000 : 36000)))))))))))))); } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = false; +#endif }; template @@ -137,6 +145,10 @@ struct RoundAgeImpl : (x < 55 ? 45 : 55))))); } + +#if USE_EMBEDDED_COMPILER + static constexpr bool compilable = false; +#endif }; diff --git a/dbms/src/Functions/IFunction.cpp b/dbms/src/Functions/IFunction.cpp index 4f89811d1e7..45deb9aceeb 100644 --- a/dbms/src/Functions/IFunction.cpp +++ b/dbms/src/Functions/IFunction.cpp @@ -1,16 +1,22 @@ -#include -#include -#include -#include -#include -#include -#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include #include +#if USE_EMBEDDED_COMPILER +#include +#endif + namespace DB { @@ -259,4 +265,71 @@ DataTypePtr FunctionBuilderImpl::getReturnType(const ColumnsWithTypeAndName & ar return getReturnTypeImpl(arguments); } + +#if USE_EMBEDDED_COMPILER + +static std::optional removeNullables(const DataTypes & types) +{ + for (const auto & type : types) + { + if (!typeid_cast(type.get())) + continue; + DataTypes filtered; + for (const auto & type : types) + filtered.emplace_back(removeNullable(type)); + return filtered; + } + return {}; +} + +bool IFunction::isCompilable(const DataTypes & arguments) const +{ + if (useDefaultImplementationForNulls()) + if (auto denulled = removeNullables(arguments)) + return isCompilableImpl(*denulled); + return isCompilableImpl(arguments); +} + +llvm::Value * IFunction::compile(llvm::IRBuilderBase & builder, const DataTypes & arguments, ValuePlaceholders values) const +{ + if (useDefaultImplementationForNulls()) + { + if (auto denulled = removeNullables(arguments)) + { + /// FIXME: when only one column is nullable, this can actually be slower than the non-jitted version + /// because this involves copying the null map while `wrapInNullable` reuses it. + auto & b = static_cast &>(builder); + auto * fail = llvm::BasicBlock::Create(b.GetInsertBlock()->getContext(), "", b.GetInsertBlock()->getParent()); + auto * join = llvm::BasicBlock::Create(b.GetInsertBlock()->getContext(), "", b.GetInsertBlock()->getParent()); + auto * zero = llvm::Constant::getNullValue(toNativeType(b, makeNullable(getReturnTypeImpl(*denulled)))); + for (size_t i = 0; i < arguments.size(); i++) + { + if (!arguments[i]->isNullable()) + continue; + /// Would be nice to evaluate all this lazily, but that'd change semantics: if only unevaluated + /// arguments happen to contain NULLs, the return value would not be NULL, though it should be. + auto * value = values[i](); + auto * ok = llvm::BasicBlock::Create(b.GetInsertBlock()->getContext(), "", b.GetInsertBlock()->getParent()); + b.CreateCondBr(b.CreateExtractValue(value, {1}), fail, ok); + b.SetInsertPoint(ok); + values[i] = [value = b.CreateExtractValue(value, {0})]() { return value; }; + } + auto * result = b.CreateInsertValue(zero, compileImpl(builder, *denulled, std::move(values)), {0}); + auto * result_block = b.GetInsertBlock(); + b.CreateBr(join); + b.SetInsertPoint(fail); + auto * null = b.CreateInsertValue(zero, b.getTrue(), {1}); + b.CreateBr(join); + b.SetInsertPoint(join); + auto * phi = b.CreatePHI(result->getType(), 2); + phi->addIncoming(result, result_block); + phi->addIncoming(null, fail); + return phi; + } + } + return compileImpl(builder, arguments, std::move(values)); +} + +#endif + } diff --git a/dbms/src/Functions/IFunction.h b/dbms/src/Functions/IFunction.h index db67e1f4c23..230c9624954 100644 --- a/dbms/src/Functions/IFunction.h +++ b/dbms/src/Functions/IFunction.h @@ -2,6 +2,7 @@ #include +#include #include #include #include @@ -9,6 +10,14 @@ #include +namespace llvm +{ + class LLVMContext; + class Value; + class IRBuilderBase; +} + + namespace DB { @@ -70,6 +79,8 @@ private: size_t input_rows_count); }; +using ValuePlaceholders = std::vector>; + /// Function with known arguments and return type. class IFunctionBase { @@ -92,6 +103,25 @@ public: return prepare(block)->execute(block, arguments, result, input_rows_count); } +#if USE_EMBEDDED_COMPILER + + virtual bool isCompilable() const { return false; } + + /** Produce LLVM IR code that operates on scalar values. See `toNativeType` in DataTypes/Native.h + * for supported value types and how they map to LLVM types. + * + * NOTE: the builder is actually guaranteed to be exactly `llvm::IRBuilder<>`, so you may safely + * downcast it to that type. This method is specified with `IRBuilderBase` because forward-declaring + * templates with default arguments is impossible and including LLVM in such a generic header + * as this one is a major pain. + */ + virtual llvm::Value * compile(llvm::IRBuilderBase & /*builder*/, ValuePlaceholders /*values*/) const + { + throw Exception(getName() + " is not JIT-compilable", ErrorCodes::NOT_IMPLEMENTED); + } + +#endif + /** Should we evaluate this function while constant folding, if arguments are constants? * Usually this is true. Notable counterexample is function 'sleep'. * If we will call it during query analysis, we will sleep extra amount of time. @@ -261,7 +291,6 @@ public: using PreparedFunctionImpl::execute; using FunctionBuilderImpl::getReturnTypeImpl; using FunctionBuilderImpl::getLambdaArgumentTypesImpl; - using FunctionBuilderImpl::getReturnType; PreparedFunctionPtr prepare(const Block & /*sample_block*/) const final @@ -269,17 +298,51 @@ public: throw Exception("prepare is not implemented for IFunction", ErrorCodes::NOT_IMPLEMENTED); } +#if USE_EMBEDDED_COMPILER + + bool isCompilable() const final + { + throw Exception("isCompilable without explicit types is not implemented for IFunction", ErrorCodes::NOT_IMPLEMENTED); + } + + llvm::Value * compile(llvm::IRBuilderBase & /*builder*/, ValuePlaceholders /*values*/) const final + { + throw Exception("compile without explicit types is not implemented for IFunction", ErrorCodes::NOT_IMPLEMENTED); + } + +#endif + const DataTypes & getArgumentTypes() const final { throw Exception("getArgumentTypes is not implemented for IFunction", ErrorCodes::NOT_IMPLEMENTED); } - const DataTypePtr & getReturnType() const override + const DataTypePtr & getReturnType() const final { throw Exception("getReturnType is not implemented for IFunction", ErrorCodes::NOT_IMPLEMENTED); } +#if USE_EMBEDDED_COMPILER + + bool isCompilable(const DataTypes & arguments) const; + + llvm::Value * compile(llvm::IRBuilderBase &, const DataTypes & arguments, ValuePlaceholders values) const; + +#endif + protected: + +#if USE_EMBEDDED_COMPILER + + virtual bool isCompilableImpl(const DataTypes &) const { return false; } + + virtual llvm::Value * compileImpl(llvm::IRBuilderBase &, const DataTypes &, ValuePlaceholders) const + { + throw Exception(getName() + " is not JIT-compilable", ErrorCodes::NOT_IMPLEMENTED); + } + +#endif + FunctionBasePtr buildImpl(const ColumnsWithTypeAndName & /*arguments*/, const DataTypePtr & /*return_type*/) const final { throw Exception("buildImpl is not implemented for IFunction", ErrorCodes::NOT_IMPLEMENTED); @@ -319,6 +382,14 @@ public: const DataTypes & getArgumentTypes() const override { return arguments; } const DataTypePtr & getReturnType() const override { return return_type; } +#if USE_EMBEDDED_COMPILER + + bool isCompilable() const override { return function->isCompilable(arguments); } + + llvm::Value * compile(llvm::IRBuilderBase & builder, ValuePlaceholders values) const override { return function->compile(builder, arguments, std::move(values)); } + +#endif + PreparedFunctionPtr prepare(const Block & /*sample_block*/) const override { return std::make_shared(function); } bool isSuitableForConstantFolding() const override { return function->isSuitableForConstantFolding(); } diff --git a/dbms/src/Interpreters/ExpressionActions.cpp b/dbms/src/Interpreters/ExpressionActions.cpp index acd45f3c98e..8efd319d647 100644 --- a/dbms/src/Interpreters/ExpressionActions.cpp +++ b/dbms/src/Interpreters/ExpressionActions.cpp @@ -1,5 +1,7 @@ +#include #include #include +#include #include #include #include @@ -754,6 +756,13 @@ void ExpressionActions::finalize(const Names & output_columns) final_columns.insert(name); } +#if USE_EMBEDDED_COMPILER + /// This has to be done before removing redundant actions and inserting REMOVE_COLUMNs + /// because inlining may change dependency sets. + if (settings.compile_expressions) + compileFunctions(actions, output_columns, sample_block); +#endif + /// Which columns are needed to perform actions from the current to the last. NameSet needed_columns = final_columns; /// Which columns nobody will touch from the current action to the last. @@ -936,7 +945,7 @@ void ExpressionActions::finalize(const Names & output_columns) std::cerr << action.toString() << "\n"; std::cerr << "\n";*/ - optimize(); + optimizeArrayJoin(); checkLimits(sample_block); } @@ -961,11 +970,6 @@ std::string ExpressionActions::dumpActions() const return ss.str(); } -void ExpressionActions::optimize() -{ - optimizeArrayJoin(); -} - void ExpressionActions::optimizeArrayJoin() { const size_t NONE = actions.size(); diff --git a/dbms/src/Interpreters/ExpressionActions.h b/dbms/src/Interpreters/ExpressionActions.h index 7a021d1a864..8da5fe2a279 100644 --- a/dbms/src/Interpreters/ExpressionActions.h +++ b/dbms/src/Interpreters/ExpressionActions.h @@ -215,8 +215,6 @@ private: void addImpl(ExpressionAction action, Names & new_names); - /// Try to improve something without changing the lists of input and output columns. - void optimize(); /// Move all arrayJoin as close as possible to the end. void optimizeArrayJoin(); }; diff --git a/dbms/src/Interpreters/ExpressionJIT.cpp b/dbms/src/Interpreters/ExpressionJIT.cpp new file mode 100644 index 00000000000..161345f569c --- /dev/null +++ b/dbms/src/Interpreters/ExpressionJIT.cpp @@ -0,0 +1,525 @@ +#include + +#if USE_EMBEDDED_COMPILER + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +namespace +{ + struct ColumnData + { + const char * data = nullptr; + const char * null = nullptr; + size_t stride; + }; + + struct ColumnDataPlaceholder + { + llvm::Value * data_init; /// first row + llvm::Value * null_init; + llvm::Value * stride; + llvm::PHINode * data; /// current row + llvm::PHINode * null; + }; +} + +static ColumnData getColumnData(const IColumn * column) +{ + ColumnData result; + const bool is_const = column->isColumnConst(); + if (is_const) + column = &reinterpret_cast(column)->getDataColumn(); + if (auto * nullable = typeid_cast(column)) + { + result.null = nullable->getNullMapColumn().getRawData().data; + column = &nullable->getNestedColumn(); + } + result.data = column->getRawData().data; + result.stride = is_const ? 0 : column->sizeOfValueIfFixed(); + return result; +} + +static void applyFunction(IFunctionBase & function, Field & value) +{ + const auto & type = function.getArgumentTypes().at(0); + Block block = {{ type->createColumnConst(1, value), type, "x" }, { nullptr, function.getReturnType(), "y" }}; + function.execute(block, {0}, 1, 1); + block.safeGetByPosition(1).column->get(0, value); +} + +struct LLVMContext +{ + llvm::LLVMContext context; + std::shared_ptr module; + std::unique_ptr machine; + llvm::orc::RTDyldObjectLinkingLayer objectLayer; + llvm::orc::IRCompileLayer compileLayer; + llvm::DataLayout layout; + llvm::IRBuilder<> builder; + std::unordered_map symbols; + + LLVMContext() + : module(std::make_shared("jit", context)) + , machine(llvm::EngineBuilder().selectTarget()) + , objectLayer([]() { return std::make_shared(); }) + , compileLayer(objectLayer, llvm::orc::SimpleCompiler(*machine)) + , layout(machine->createDataLayout()) + , builder(context) + { + module->setDataLayout(layout); + module->setTargetTriple(machine->getTargetTriple().getTriple()); + } + + void finalize() + { + if (!module->size()) + return; + llvm::PassManagerBuilder builder; + llvm::legacy::FunctionPassManager fpm(module.get()); + builder.OptLevel = 3; + builder.SLPVectorize = true; + builder.LoopVectorize = true; + builder.RerollLoops = true; + builder.VerifyInput = true; + builder.VerifyOutput = true; + builder.populateFunctionPassManager(fpm); + for (auto & function : *module) + fpm.run(function); + llvm::cantFail(compileLayer.addModule(module, std::make_shared())); + for (const auto & function : *module) + { + std::string mangledName; + llvm::raw_string_ostream mangledNameStream(mangledName); + llvm::Mangler::getNameWithPrefix(mangledNameStream, function.getName(), layout); + if (auto symbol = compileLayer.findSymbol(mangledNameStream.str(), false).getAddress()) + symbols[function.getName()] = reinterpret_cast(*symbol); + } + } +}; + +class LLVMPreparedFunction : public PreparedFunctionImpl +{ + std::string name; + std::shared_ptr context; + const void * function; + +public: + LLVMPreparedFunction(std::string name_, std::shared_ptr context) + : name(std::move(name_)), context(context), function(context->symbols.at(name)) + {} + + String getName() const override { return name; } + + bool useDefaultImplementationForNulls() const override { return false; } + + bool useDefaultImplementationForConstants() const override { return true; } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t block_size) override + { + auto col_res = block.getByPosition(result).type->createColumn()->cloneResized(block_size); + if (block_size) + { + std::vector columns(arguments.size() + 1); + for (size_t i = 0; i < arguments.size(); i++) + { + auto * column = block.getByPosition(arguments[i]).column.get(); + if (!column) + throw Exception("column " + block.getByPosition(arguments[i]).name + " is missing", ErrorCodes::LOGICAL_ERROR); + columns[i] = getColumnData(column); + } + columns[arguments.size()] = getColumnData(col_res.get()); + reinterpret_cast(function)(block_size, columns.data()); + } + block.getByPosition(result).column = std::move(col_res); + }; +}; + +static void compileFunction(std::shared_ptr & context, const IFunctionBase & f) +{ + auto & arg_types = f.getArgumentTypes(); + auto & b = context->builder; + auto * size_type = b.getIntNTy(sizeof(size_t) * 8); + auto * data_type = llvm::StructType::get(b.getInt8PtrTy(), b.getInt8PtrTy(), size_type); + auto * func_type = llvm::FunctionType::get(b.getVoidTy(), { size_type, data_type->getPointerTo() }, /*isVarArg=*/false); + auto * func = llvm::Function::Create(func_type, llvm::Function::ExternalLinkage, f.getName(), context->module.get()); + auto args = func->args().begin(); + llvm::Value * counter_arg = &*args++; + llvm::Value * columns_arg = &*args++; + + auto * entry = llvm::BasicBlock::Create(b.getContext(), "entry", func); + b.SetInsertPoint(entry); + std::vector columns(arg_types.size() + 1); + for (size_t i = 0; i <= arg_types.size(); i++) + { + auto & type = i == arg_types.size() ? f.getReturnType() : arg_types[i]; + auto * data = b.CreateLoad(b.CreateConstInBoundsGEP1_32(data_type, columns_arg, i)); + columns[i].data_init = b.CreatePointerCast(b.CreateExtractValue(data, {0}), toNativeType(b, removeNullable(type))->getPointerTo()); + columns[i].null_init = type->isNullable() ? b.CreateExtractValue(data, {1}) : nullptr; + columns[i].stride = b.CreateExtractValue(data, {2}); + } + + /// assume nonzero initial value in `counter_arg` + auto * loop = llvm::BasicBlock::Create(b.getContext(), "loop", func); + b.CreateBr(loop); + b.SetInsertPoint(loop); + auto * counter_phi = b.CreatePHI(counter_arg->getType(), 2); + counter_phi->addIncoming(counter_arg, entry); + for (auto & col : columns) + { + col.data = b.CreatePHI(col.data_init->getType(), 2); + col.data->addIncoming(col.data_init, entry); + if (col.null_init) + { + col.null = b.CreatePHI(col.null_init->getType(), 2); + col.null->addIncoming(col.null_init, entry); + } + } + ValuePlaceholders arguments(arg_types.size()); + for (size_t i = 0; i < arguments.size(); i++) + { + arguments[i] = [&b, &col = columns[i], &type = arg_types[i]]() -> llvm::Value * + { + auto * value = b.CreateLoad(col.data); + if (!col.null) + return value; + auto * is_null = b.CreateICmpNE(b.CreateLoad(col.null), b.getInt8(0)); + auto * nullable = llvm::Constant::getNullValue(toNativeType(b, type)); + return b.CreateInsertValue(b.CreateInsertValue(nullable, value, {0}), is_null, {1}); + }; + } + auto * result = f.compile(b, std::move(arguments)); + if (columns.back().null) + { + b.CreateStore(b.CreateExtractValue(result, {0}), columns.back().data); + b.CreateStore(b.CreateSelect(b.CreateExtractValue(result, {1}), b.getInt8(1), b.getInt8(0)), columns.back().null); + } + else + { + b.CreateStore(result, columns.back().data); + } + auto * cur_block = b.GetInsertBlock(); + for (auto & col : columns) + { + /// currently, stride is either 0 or size of native type + auto * is_const = b.CreateICmpEQ(col.stride, llvm::ConstantInt::get(size_type, 0)); + col.data->addIncoming(b.CreateSelect(is_const, col.data, b.CreateConstInBoundsGEP1_32(nullptr, col.data, 1)), cur_block); + if (col.null) + col.null->addIncoming(b.CreateSelect(is_const, col.null, b.CreateConstInBoundsGEP1_32(nullptr, col.null, 1)), cur_block); + } + counter_phi->addIncoming(b.CreateSub(counter_phi, llvm::ConstantInt::get(size_type, 1)), cur_block); + + auto * end = llvm::BasicBlock::Create(b.getContext(), "end", func); + b.CreateCondBr(b.CreateICmpNE(counter_phi, llvm::ConstantInt::get(size_type, 1)), loop, end); + b.SetInsertPoint(end); + b.CreateRetVoid(); +} + +static llvm::Constant * getNativeValue(llvm::Type * type, const IColumn & column, size_t i) +{ + if (!type) + return nullptr; + if (auto * constant = typeid_cast(&column)) + return getNativeValue(type, constant->getDataColumn(), 0); + if (auto * nullable = typeid_cast(&column)) + { + auto * value = getNativeValue(type->getContainedType(0), nullable->getNestedColumn(), i); + auto * is_null = llvm::ConstantInt::get(type->getContainedType(1), nullable->isNullAt(i)); + return value ? llvm::ConstantStruct::get(static_cast(type), value, is_null) : nullptr; + } + if (type->isFloatTy()) + return llvm::ConstantFP::get(type, static_cast &>(column).getElement(i)); + if (type->isDoubleTy()) + return llvm::ConstantFP::get(type, static_cast &>(column).getElement(i)); + if (type->isIntegerTy()) + return llvm::ConstantInt::get(type, column.getUInt(i)); + /// TODO: if (type->isVectorTy()) + return nullptr; +} + +/// Same as IFunctionBase::compile, but also for constants and input columns. +using CompilableExpression = std::function; + +static CompilableExpression subexpression(ColumnPtr c, DataTypePtr type) +{ + return [=](llvm::IRBuilderBase & b, const ValuePlaceholders &) { return getNativeValue(toNativeType(b, type), *c, 0); }; +} + +static CompilableExpression subexpression(size_t i) +{ + return [=](llvm::IRBuilderBase &, const ValuePlaceholders & inputs) { return inputs[i](); }; +} + +static CompilableExpression subexpression(const IFunctionBase & f, std::vector args) +{ + return [&, args = std::move(args)](llvm::IRBuilderBase & builder, const ValuePlaceholders & inputs) + { + ValuePlaceholders input; + for (const auto & arg : args) + input.push_back([&]() { return arg(builder, inputs); }); + auto * result = f.compile(builder, input); + if (result->getType() != toNativeType(builder, f.getReturnType())) + throw Exception("function " + f.getName() + " generated an llvm::Value of invalid type", ErrorCodes::LOGICAL_ERROR); + return result; + }; +} + +class LLVMFunction : public IFunctionBase +{ + std::string name; + Names arg_names; + DataTypes arg_types; + std::shared_ptr context; + std::vector originals; + std::unordered_map subexpressions; + +public: + LLVMFunction(const ExpressionActions::Actions & actions, std::shared_ptr context, const Block & sample_block) + : name(actions.back().result_name), context(context) + { + for (const auto & c : sample_block) + /// TODO: implement `getNativeValue` for all types & replace the check with `c.column && toNativeType(...)` + if (c.column && getNativeValue(toNativeType(context->builder, c.type), *c.column, 0)) + subexpressions[c.name] = subexpression(c.column, c.type); + for (const auto & action : actions) + { + const auto & names = action.argument_names; + const auto & types = action.function->getArgumentTypes(); + std::vector args; + for (size_t i = 0; i < names.size(); i++) + { + auto inserted = subexpressions.emplace(names[i], subexpression(arg_names.size())); + if (inserted.second) + { + arg_names.push_back(names[i]); + arg_types.push_back(types[i]); + } + args.push_back(inserted.first->second); + } + subexpressions[action.result_name] = subexpression(*action.function, std::move(args)); + originals.push_back(action.function); + } + compileFunction(context, *this); + } + + bool isCompilable() const override { return true; } + + llvm::Value * compile(llvm::IRBuilderBase & builder, ValuePlaceholders values) const override { return subexpressions.at(name)(builder, values); } + + String getName() const override { return name; } + + const Names & getArgumentNames() const { return arg_names; } + + const DataTypes & getArgumentTypes() const override { return arg_types; } + + const DataTypePtr & getReturnType() const override { return originals.back()->getReturnType(); } + + PreparedFunctionPtr prepare(const Block &) const override { return std::make_shared(name, context); } + + bool isDeterministic() override + { + for (const auto & f : originals) + if (!f->isDeterministic()) + return false; + return true; + } + + bool isDeterministicInScopeOfQuery() override + { + for (const auto & f : originals) + if (!f->isDeterministicInScopeOfQuery()) + return false; + return true; + } + + bool isSuitableForConstantFolding() const override + { + for (const auto & f : originals) + if (!f->isSuitableForConstantFolding()) + return false; + return true; + } + + bool isInjective(const Block & sample_block) override + { + for (const auto & f : originals) + if (!f->isInjective(sample_block)) + return false; + return true; + } + + bool hasInformationAboutMonotonicity() const override + { + for (const auto & f : originals) + if (!f->hasInformationAboutMonotonicity()) + return false; + return true; + } + + Monotonicity getMonotonicityForRange(const IDataType & type, const Field & left, const Field & right) const override + { + const IDataType * type_ = &type; + Field left_ = left; + Field right_ = right; + Monotonicity result(true, true, true); + /// monotonicity is only defined for unary functions, so the chain must describe a sequence of nested calls + for (size_t i = 0; i < originals.size(); i++) + { + Monotonicity m = originals[i]->getMonotonicityForRange(*type_, left_, right_); + if (!m.is_monotonic) + return m; + result.is_positive ^= !m.is_positive; + result.is_always_monotonic &= m.is_always_monotonic; + if (i + 1 < originals.size()) + { + if (left_ != Field()) + applyFunction(*originals[i], left_); + if (right_ != Field()) + applyFunction(*originals[i], right_); + if (!m.is_positive) + std::swap(left_, right_); + type_ = originals[i]->getReturnType().get(); + } + } + return result; + } +}; + +static bool isCompilable(llvm::IRBuilderBase & builder, const IFunctionBase& function) +{ + if (!toNativeType(builder, function.getReturnType())) + return false; + for (const auto & type : function.getArgumentTypes()) + if (!toNativeType(builder, type)) + return false; + return function.isCompilable(); +} + +void compileFunctions(ExpressionActions::Actions & actions, const Names & output_columns, const Block & sample_block) +{ + auto context = std::make_shared(); + /// an empty optional is a poisoned value prohibiting the column's producer from being removed + /// (which it could be, if it was inlined into every dependent function). + std::unordered_map>> current_dependents; + for (const auto & name : output_columns) + current_dependents[name].emplace(); + /// a snapshot of each compilable function's dependents at the time of its execution. + std::vector>> dependents(actions.size()); + for (size_t i = actions.size(); i--;) + { + switch (actions[i].type) + { + case ExpressionAction::REMOVE_COLUMN: + current_dependents.erase(actions[i].source_name); + /// poison every other column used after this point so that inlining chains do not cross it. + for (auto & dep : current_dependents) + dep.second.emplace(); + break; + + case ExpressionAction::PROJECT: + current_dependents.clear(); + for (const auto & proj : actions[i].projection) + current_dependents[proj.first].emplace(); + break; + + case ExpressionAction::ADD_COLUMN: + case ExpressionAction::COPY_COLUMN: + case ExpressionAction::ARRAY_JOIN: + case ExpressionAction::JOIN: + { + Names columns = actions[i].getNeededColumns(); + for (const auto & column : columns) + current_dependents[column].emplace(); + break; + } + + case ExpressionAction::APPLY_FUNCTION: + { + dependents[i] = current_dependents[actions[i].result_name]; + const bool compilable = isCompilable(context->builder, *actions[i].function); + for (const auto & name : actions[i].argument_names) + { + if (compilable) + current_dependents[name].emplace(i); + else + current_dependents[name].emplace(); + } + break; + } + } + } + + std::vector fused(actions.size()); + for (size_t i = 0; i < actions.size(); i++) + { + if (actions[i].type != ExpressionAction::APPLY_FUNCTION || !isCompilable(context->builder, *actions[i].function)) + continue; + fused[i].push_back(actions[i]); + if (dependents[i].find({}) != dependents[i].end()) + { + /// the result of compiling one function in isolation is pretty much the same as its `execute` method. + if (fused[i].size() == 1) + continue; + auto fn = std::make_shared(std::move(fused[i]), context, sample_block); + actions[i].function = fn; + actions[i].argument_names = fn->getArgumentNames(); + continue; + } + /// TODO: determine whether it's profitable to inline the function if there's more than one dependent. + for (const auto & dep : dependents[i]) + fused[*dep].insert(fused[*dep].end(), fused[i].begin(), fused[i].end()); + } + context->finalize(); +} + +} + + +namespace +{ + struct LLVMTargetInitializer + { + LLVMTargetInitializer() + { + llvm::InitializeNativeTarget(); + llvm::InitializeNativeTargetAsmPrinter(); + } + } llvmInitializer; +} + +#endif diff --git a/dbms/src/Interpreters/ExpressionJIT.h b/dbms/src/Interpreters/ExpressionJIT.h new file mode 100644 index 00000000000..5a7a39c9e21 --- /dev/null +++ b/dbms/src/Interpreters/ExpressionJIT.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include + +#if USE_EMBEDDED_COMPILER + +namespace DB +{ + +/// For each APPLY_FUNCTION action, try to compile the function to native code; if the only uses of a compilable +/// function's result are as arguments to other compilable functions, inline it and leave the now-redundant action as-is. +void compileFunctions(ExpressionActions::Actions & actions, const Names & output_columns, const Block & sample_block); + +} + +#endif diff --git a/dbms/src/Interpreters/Settings.h b/dbms/src/Interpreters/Settings.h index b1442d65c0f..9696e7866bf 100644 --- a/dbms/src/Interpreters/Settings.h +++ b/dbms/src/Interpreters/Settings.h @@ -65,6 +65,7 @@ struct Settings M(SettingFloat, totals_auto_threshold, 0.5, "The threshold for totals_mode = 'auto'.") \ \ M(SettingBool, compile, false, "Whether query compilation is enabled.") \ + M(SettingBool, compile_expressions, true, "Compile some scalar functions and operators to native code.") \ M(SettingUInt64, min_count_to_compile, 3, "The number of structurally identical queries before they are compiled.") \ M(SettingUInt64, group_by_two_level_threshold, 100000, "From what number of keys, a two-level aggregation starts. 0 - the threshold is not set.") \ M(SettingUInt64, group_by_two_level_threshold_bytes, 100000000, "From what size of the aggregation state in bytes, a two-level aggregation begins to be used. 0 - the threshold is not set. Two-level aggregation is used when at least one of the thresholds is triggered.") \ diff --git a/dbms/src/Server/Compiler-5.0.0/CMakeLists.txt b/dbms/src/Server/Compiler-5.0.0/CMakeLists.txt index 5151c183bcf..e72e3f6753b 100644 --- a/dbms/src/Server/Compiler-5.0.0/CMakeLists.txt +++ b/dbms/src/Server/Compiler-5.0.0/CMakeLists.txt @@ -8,12 +8,7 @@ add_library(clickhouse-compiler-lib target_compile_options(clickhouse-compiler-lib PRIVATE -fno-rtti -fno-exceptions -g0) -llvm_map_components_to_libraries(REQUIRED_LLVM_LIBRARIES all) - -# We link statically with zlib, and LLVM (sometimes) tries to bring its own dependency. -list(REMOVE_ITEM REQUIRED_LLVM_LIBRARIES "-lz") -# Wrong library in freebsd: -list(REMOVE_ITEM REQUIRED_LLVM_LIBRARIES "-l/usr/lib/libexecinfo.so") +llvm_map_components_to_libnames(REQUIRED_LLVM_LIBRARIES all) message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}") diff --git a/dbms/src/Server/Compiler-6.0.0/CMakeLists.txt b/dbms/src/Server/Compiler-6.0.0/CMakeLists.txt index 4c07363abb5..a66af8bbc7a 100644 --- a/dbms/src/Server/Compiler-6.0.0/CMakeLists.txt +++ b/dbms/src/Server/Compiler-6.0.0/CMakeLists.txt @@ -8,12 +8,7 @@ add_library(clickhouse-compiler-lib target_compile_options(clickhouse-compiler-lib PRIVATE -fno-rtti -fno-exceptions -g0) -llvm_map_components_to_libraries(REQUIRED_LLVM_LIBRARIES all) - -# We link statically with zlib, and LLVM (sometimes) tries to bring its own dependency. -list(REMOVE_ITEM REQUIRED_LLVM_LIBRARIES "-lz") -# Wrong library in freebsd: -list(REMOVE_ITEM REQUIRED_LLVM_LIBRARIES "-l/usr/lib/libexecinfo.so") +llvm_map_components_to_libnames(REQUIRED_LLVM_LIBRARIES all) message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}") @@ -24,7 +19,7 @@ target_include_directories(clickhouse-compiler-lib PRIVATE ${LLVM_INCLUDE_DIRS}) target_link_libraries(clickhouse-compiler-lib PRIVATE clangBasic clangCodeGen clangDriver -clangFrontend +clangFrontend clangFrontendTool clangRewriteFrontend clangARCMigrate clangStaticAnalyzerFrontend clangParse clangSerialization clangSema clangEdit clangStaticAnalyzerCheckers