thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 00:30:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'llvm-jit' of https://github.com/pyos/ClickHouse into pyos-llvm-jit

Browse Source
This commit is contained in:
Alexey Milovidov 2018-05-06 12:22:42 +03:00
commit 53f03a4909
21 changed files with 1212 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

104
cmake/find_llvm.cmake
View File

@ -1,107 +1,21 @@
option (ENABLE_EMBEDDED_COMPILER "Set to TRUE to enable support for 'compile' option for query execution" 1)
if (ENABLE_EMBEDDED_COMPILER)
# Based on source code of YT.
# Authors: Ivan Puzyrevskiy, Alexey Lukyanchikov, Ruslan Savchenko.
# Find LLVM includes and libraries.
#
# LLVM_VERSION - LLVM version.
# LLVM_INCLUDE_DIRS - Directory containing LLVM headers.
# LLVM_LIBRARY_DIRS - Directory containing LLVM libraries.
# LLVM_CXXFLAGS - C++ compiler flags for files that include LLVM headers.
# LLVM_FOUND - True if LLVM was found.
# llvm_map_components_to_libraries - Maps LLVM used components to required libraries.
# Usage: llvm_map_components_to_libraries(REQUIRED_LLVM_LIBRARIES core jit interpreter native ...)
if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
set(LLVM_VERSION_POSTFIX "${COMPILER_POSTFIX}" CACHE STRING "")
else()
if (ARCH_FREEBSD)
set(LLVM_VERSION_POSTFIX "50" CACHE STRING "")
else()
set(LLVM_VERSION_POSTFIX "-5.0" CACHE STRING "")
endif()
endif()
find_package(LLVM CONFIG)
else ()
find_package(LLVM 5 CONFIG)
endif ()
find_program(LLVM_CONFIG_EXECUTABLE
NAMES llvm-config${LLVM_VERSION_POSTFIX} llvm-config llvm-config-devel
PATHS $ENV{LLVM_ROOT}/bin)
mark_as_advanced(LLVM_CONFIG_EXECUTABLE)
if(NOT LLVM_CONFIG_EXECUTABLE)
message(WARNING "Cannot find LLVM (looking for `llvm-config${LLVM_VERSION_POSTFIX}`, `llvm-config`, `llvm-config-devel`). Please, provide LLVM_ROOT environment variable.")
else()
set(LLVM_FOUND TRUE)
execute_process(
COMMAND ${LLVM_CONFIG_EXECUTABLE} --version
OUTPUT_VARIABLE LLVM_VERSION
OUTPUT_STRIP_TRAILING_WHITESPACE)
if(LLVM_VERSION VERSION_LESS "5")
message(FATAL_ERROR "LLVM 5+ is required. You have ${LLVM_VERSION} (${LLVM_CONFIG_EXECUTABLE})")
endif()
message(STATUS "LLVM config: ${LLVM_CONFIG_EXECUTABLE}; version: ${LLVM_VERSION}")
execute_process(
COMMAND ${LLVM_CONFIG_EXECUTABLE} --includedir
OUTPUT_VARIABLE LLVM_INCLUDE_DIRS
OUTPUT_STRIP_TRAILING_WHITESPACE)
execute_process(
COMMAND ${LLVM_CONFIG_EXECUTABLE} --libdir
OUTPUT_VARIABLE LLVM_LIBRARY_DIRS
OUTPUT_STRIP_TRAILING_WHITESPACE)
execute_process(
COMMAND ${LLVM_CONFIG_EXECUTABLE} --cxxflags
OUTPUT_VARIABLE LLVM_CXXFLAGS
OUTPUT_STRIP_TRAILING_WHITESPACE)
execute_process(
COMMAND ${LLVM_CONFIG_EXECUTABLE} --targets-built
OUTPUT_VARIABLE LLVM_TARGETS_BUILT
OUTPUT_STRIP_TRAILING_WHITESPACE)
string(REPLACE " " ";" LLVM_TARGETS_BUILT "${LLVM_TARGETS_BUILT}")
if (USE_STATIC_LIBRARIES)
set (LLVM_CONFIG_ADD "--link-static")
endif()
# Get the link libs we need.
function(llvm_map_components_to_libraries RESULT)
execute_process(
COMMAND ${LLVM_CONFIG_EXECUTABLE} ${LLVM_CONFIG_ADD} --libs ${ARGN}
OUTPUT_VARIABLE _tmp
OUTPUT_STRIP_TRAILING_WHITESPACE)
string(REPLACE " " ";" _libs_module "${_tmp}")
#message(STATUS "LLVM Libraries for '${ARGN}': ${_libs_module}")
execute_process(
COMMAND ${LLVM_CONFIG_EXECUTABLE} --system-libs ${ARGN}
OUTPUT_VARIABLE _libs_system
OUTPUT_STRIP_TRAILING_WHITESPACE)
string(REPLACE "\n" " " _libs_system "${_libs_system}")
string(REPLACE " " " " _libs_system "${_libs_system}")
string(REPLACE " " ";" _libs_system "${_libs_system}")
set(${RESULT} ${_libs_module} ${_libs_system} PARENT_SCOPE)
endfunction(llvm_map_components_to_libraries)
if (LLVM_FOUND)
# Remove dynamically-linked zlib and libedit from LLVM's dependencies:
set_target_properties(LLVMSupport PROPERTIES INTERFACE_LINK_LIBRARIES "-lpthread;LLVMDemangle")
set_target_properties(LLVMLineEditor PROPERTIES INTERFACE_LINK_LIBRARIES "LLVMSupport")
message(STATUS "LLVM version: ${LLVM_PACKAGE_VERSION}")
message(STATUS "LLVM Include Directory: ${LLVM_INCLUDE_DIRS}")
message(STATUS "LLVM Library Directory: ${LLVM_LIBRARY_DIRS}")
message(STATUS "LLVM C++ Compiler: ${LLVM_CXXFLAGS}")
endif()
if (LLVM_FOUND AND LLVM_INCLUDE_DIRS AND LLVM_LIBRARY_DIRS)
set (USE_EMBEDDED_COMPILER 1)
endif()
endif()

9
dbms/CMakeLists.txt
View File

@ -99,6 +99,15 @@ else ()
install (TARGETS dbms LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} COMPONENT clickhouse)
endif ()
if (USE_EMBEDDED_COMPILER)
llvm_map_components_to_libnames(REQUIRED_LLVM_LIBRARIES all)
target_link_libraries (dbms ${REQUIRED_LLVM_LIBRARIES})
target_include_directories (dbms BEFORE PUBLIC ${LLVM_INCLUDE_DIRS})
# LLVM has a bunch of unused parameters in its header files.
set_source_files_properties(src/Functions/IFunction.cpp PROPERTIES COMPILE_FLAGS "-Wno-unused-parameter")
set_source_files_properties(src/Interpreters/ExpressionJIT.cpp PROPERTIES COMPILE_FLAGS "-Wno-unused-parameter -Wno-non-virtual-dtor")
endif ()
if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL")
# Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size.

1
dbms/src/Columns/ColumnConst.h
View File

@ -193,6 +193,7 @@ public:
bool isFixedAndContiguous() const override { return data->isFixedAndContiguous(); }
bool valuesHaveFixedSize() const override { return data->valuesHaveFixedSize(); }
size_t sizeOfValueIfFixed() const override { return data->sizeOfValueIfFixed(); }
StringRef getRawData() const override { return data->getRawData(); }
/// Not part of the common interface.

2
dbms/src/Columns/ColumnFixedString.h
View File

@ -129,7 +129,7 @@ public:
bool isFixedAndContiguous() const override { return true; }
size_t sizeOfValueIfFixed() const override { return n; }
StringRef getRawData() const override { return StringRef(chars.data(), chars.size()); }
/// Specialized part of interface, not from IColumn.

2
dbms/src/Columns/ColumnVector.h
View File

@ -268,7 +268,7 @@ public:
bool isFixedAndContiguous() const override { return true; }
size_t sizeOfValueIfFixed() const override { return sizeof(T); }
StringRef getRawData() const override { return StringRef(reinterpret_cast<const char*>(data.data()), data.size()); }
/** More efficient methods of manipulation - to manipulate with data directly. */
Container & getData()

3
dbms/src/Columns/IColumn.h
View File

@ -303,6 +303,9 @@ public:
/// Values in column are represented as continuous memory segment of fixed size. Implies valuesHaveFixedSize.
virtual bool isFixedAndContiguous() const { return false; }
/// If isFixedAndContiguous, returns the underlying data array, otherwise throws an exception.
virtual StringRef getRawData() const { throw Exception("Column " + getName() + " is not a contiguous block of memory", ErrorCodes::NOT_IMPLEMENTED); }
/// If valuesHaveFixedSize, returns size of value, otherwise throw an exception.
virtual size_t sizeOfValueIfFixed() const { throw Exception("Values of column " + getName() + " are not fixed size.", ErrorCodes::CANNOT_GET_SIZE_OF_FIELD); }

76
dbms/src/DataTypes/Native.h Normal file
View File

@ -0,0 +1,76 @@
#pragma once
#include <Common/config.h>
#if USE_EMBEDDED_COMPILER
#include <Common/typeid_cast.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypeFixedString.h>
#include <DataTypes/DataTypeInterval.h>
#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeUUID.h>
#include <llvm/IR/IRBuilder.h>
namespace DB
{
template <typename... Ts>
static inline bool typeIsEither(const IDataType & type)
{
return (typeid_cast<const Ts *>(&type) || ...);
}
static inline llvm::Type * toNativeType(llvm::IRBuilderBase & builder, const IDataType & type)
{
if (auto * nullable = typeid_cast<const DataTypeNullable *>(&type))
{
auto * wrapped = toNativeType(builder, *nullable->getNestedType());
return wrapped ? llvm::StructType::get(wrapped, /* is null = */ builder.getInt1Ty()) : nullptr;
}
/// LLVM doesn't have unsigned types, it has unsigned instructions.
if (typeIsEither<DataTypeInt8, DataTypeUInt8>(type))
return builder.getInt8Ty();
if (typeIsEither<DataTypeInt16, DataTypeUInt16, DataTypeDate>(type))
return builder.getInt16Ty();
if (typeIsEither<DataTypeInt32, DataTypeUInt32, DataTypeDateTime>(type))
return builder.getInt32Ty();
if (typeIsEither<DataTypeInt64, DataTypeUInt64, DataTypeInterval>(type))
return builder.getInt64Ty();
if (typeIsEither<DataTypeUUID>(type))
return builder.getInt128Ty();
if (typeIsEither<DataTypeFloat32>(type))
return builder.getFloatTy();
if (typeIsEither<DataTypeFloat64>(type))
return builder.getDoubleTy();
if (auto * fixed_string = typeid_cast<const DataTypeFixedString *>(&type))
return llvm::VectorType::get(builder.getInt8Ty(), fixed_string->getN());
return nullptr;
}
static inline llvm::Type * toNativeType(llvm::IRBuilderBase & builder, const DataTypePtr & type)
{
return toNativeType(builder, *type);
}
static inline llvm::Value * castNativeNumber(llvm::IRBuilder<> & builder, llvm::Value * value, llvm::Type * type, bool is_signed)
{
if (value->getType() == type)
return value;
if (value->getType()->isIntegerTy())
{
if (type->isIntegerTy())
return builder.CreateIntCast(value, type, is_signed);
return is_signed ? builder.CreateSIToFP(value, type) : builder.CreateUIToFP(value, type);
}
if (type->isFloatingPointTy())
return builder.CreateFPCast(value, type);
return is_signed ? builder.CreateFPToSI(value, type) : builder.CreateFPToUI(value, type);
}
}
#endif

6
dbms/src/Functions/CMakeLists.txt
View File

@ -109,3 +109,9 @@ endif ()
if (ENABLE_TESTS)
add_subdirectory (tests)
endif ()
if (USE_EMBEDDED_COMPILER)
target_include_directories (clickhouse_functions BEFORE PUBLIC ${LLVM_INCLUDE_DIRS})
# LLVM has a bunch of unused parameters in its header files.
target_compile_options (clickhouse_functions PRIVATE "-Wno-unused-parameter")
endif ()

6
dbms/src/Functions/FunctionHelpers.cpp
View File

@ -5,12 +5,16 @@
#include <Columns/ColumnNullable.h>
#include <DataTypes/DataTypeNullable.h>
#include <IO/WriteHelpers.h>
#include "FunctionsArithmetic.h"
namespace DB
{
namespace ErrorCodes
{
extern const int ILLEGAL_COLUMN;
}
const ColumnConst * checkAndGetColumnConstStringOrFixedString(const IColumn * column)
{
if (!column->isColumnConst())

309
dbms/src/Functions/FunctionsArithmetic.h
View File

@ -4,6 +4,7 @@
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypeInterval.h>
#include <DataTypes/Native.h>
#include <Columns/ColumnVector.h>
#include <Columns/ColumnConst.h>
#include <Functions/IFunction.h>
@ -19,6 +20,10 @@
#include <common/intExp.h>
#include <boost/math/common_factor.hpp>
#if USE_EMBEDDED_COMPILER
#include <llvm/IR/IRBuilder.h>
#endif
namespace DB
{
@ -106,6 +111,15 @@ struct PlusImpl
/// Next everywhere, static_cast - so that there is no wrong result in expressions of the form Int64 c = UInt32(a) * Int32(-1).
return static_cast<Result>(a) + b;
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = true;
static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool)
{
return left->getType()->isIntegerTy() ? b.CreateAdd(left, right) : b.CreateFAdd(left, right);
}
#endif
};
@ -119,6 +133,15 @@ struct MultiplyImpl
{
return static_cast<Result>(a) * b;
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = true;
static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool)
{
return left->getType()->isIntegerTy() ? b.CreateMul(left, right) : b.CreateFMul(left, right);
}
#endif
};
template <typename A, typename B>
@ -131,6 +154,15 @@ struct MinusImpl
{
return static_cast<Result>(a) - b;
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = true;
static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool)
{
return left->getType()->isIntegerTy() ? b.CreateSub(left, right) : b.CreateFSub(left, right);
}
#endif
};
template <typename A, typename B>
@ -143,6 +175,17 @@ struct DivideFloatingImpl
{
return static_cast<Result>(a) / b;
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = true;
static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool)
{
if (left->getType()->isIntegerTy())
throw Exception("DivideFloatingImpl expected a floating-point type", ErrorCodes::LOGICAL_ERROR);
return b.CreateFDiv(left, right);
}
#endif
};
@ -189,6 +232,10 @@ struct DivideIntegralImpl
throwIfDivisionLeadsToFPE(a, b);
return a / b;
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = false; /// don't know how to throw from LLVM IR
#endif
};
template <typename A, typename B>
@ -201,6 +248,10 @@ struct DivideIntegralOrZeroImpl
{
return unlikely(divisionLeadsToFPE(a, b)) ? 0 : a / b;
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = false; /// TODO implement the checks
#endif
};
template <typename A, typename B>
@ -212,9 +263,12 @@ struct ModuloImpl
static inline Result apply(A a, B b)
{
throwIfDivisionLeadsToFPE(typename NumberTraits::ToInteger<A>::Type(a), typename NumberTraits::ToInteger<B>::Type(b));
return typename NumberTraits::ToInteger<A>::Type(a)
% typename NumberTraits::ToInteger<B>::Type(b);
return typename NumberTraits::ToInteger<A>::Type(a) % typename NumberTraits::ToInteger<B>::Type(b);
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = false; /// don't know how to throw from LLVM IR
#endif
};
template <typename A, typename B>
@ -225,9 +279,19 @@ struct BitAndImpl
template <typename Result = ResultType>
static inline Result apply(A a, B b)
{
return static_cast<Result>(a)
& static_cast<Result>(b);
return static_cast<Result>(a) & static_cast<Result>(b);
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = true;
static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool)
{
if (!left->getType()->isIntegerTy())
throw Exception("BitAndImpl expected an integral type", ErrorCodes::LOGICAL_ERROR);
return b.CreateAnd(left, right);
}
#endif
};
template <typename A, typename B>
@ -238,9 +302,19 @@ struct BitOrImpl
template <typename Result = ResultType>
static inline Result apply(A a, B b)
{
return static_cast<Result>(a)
| static_cast<Result>(b);
return static_cast<Result>(a) | static_cast<Result>(b);
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = true;
static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool)
{
if (!left->getType()->isIntegerTy())
throw Exception("BitOrImpl expected an integral type", ErrorCodes::LOGICAL_ERROR);
return b.CreateOr(left, right);
}
#endif
};
template <typename A, typename B>
@ -251,9 +325,19 @@ struct BitXorImpl
template <typename Result = ResultType>
static inline Result apply(A a, B b)
{
return static_cast<Result>(a)
^ static_cast<Result>(b);
return static_cast<Result>(a) ^ static_cast<Result>(b);
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = true;
static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool)
{
if (!left->getType()->isIntegerTy())
throw Exception("BitXorImpl expected an integral type", ErrorCodes::LOGICAL_ERROR);
return b.CreateXor(left, right);
}
#endif
};
template <typename A, typename B>
@ -264,9 +348,19 @@ struct BitShiftLeftImpl
template <typename Result = ResultType>
static inline Result apply(A a, B b)
{
return static_cast<Result>(a)
<< static_cast<Result>(b);
return static_cast<Result>(a) << static_cast<Result>(b);
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = true;
static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool)
{
if (!left->getType()->isIntegerTy())
throw Exception("BitShiftLeftImpl expected an integral type", ErrorCodes::LOGICAL_ERROR);
return b.CreateShl(left, right);
}
#endif
};
template <typename A, typename B>
@ -277,9 +371,19 @@ struct BitShiftRightImpl
template <typename Result = ResultType>
static inline Result apply(A a, B b)
{
return static_cast<Result>(a)
>> static_cast<Result>(b);
return static_cast<Result>(a) >> static_cast<Result>(b);
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = true;
static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool is_signed)
{
if (!left->getType()->isIntegerTy())
throw Exception("BitShiftRightImpl expected an integral type", ErrorCodes::LOGICAL_ERROR);
return is_signed ? b.CreateAShr(left, right) : b.CreateLShr(left, right);
}
#endif
};
template <typename A, typename B>
@ -293,6 +397,19 @@ struct BitRotateLeftImpl
return (static_cast<Result>(a) << static_cast<Result>(b))
| (static_cast<Result>(a) >> ((sizeof(Result) * 8) - static_cast<Result>(b)));
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = true;
static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool)
{
if (!left->getType()->isIntegerTy())
throw Exception("BitRotateLeftImpl expected an integral type", ErrorCodes::LOGICAL_ERROR);
auto * size = llvm::ConstantInt::get(left->getType(), left->getType()->getPrimitiveSizeInBits());
/// XXX how is this supposed to behave in signed mode?
return b.CreateOr(b.CreateShl(left, right), b.CreateLShr(left, b.CreateSub(size, right)));
}
#endif
};
template <typename A, typename B>
@ -306,24 +423,35 @@ struct BitRotateRightImpl
return (static_cast<Result>(a) >> static_cast<Result>(b))
| (static_cast<Result>(a) << ((sizeof(Result) * 8) - static_cast<Result>(b)));
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = true;
static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool)
{
if (!left->getType()->isIntegerTy())
throw Exception("BitRotateRightImpl expected an integral type", ErrorCodes::LOGICAL_ERROR);
auto * size = llvm::ConstantInt::get(left->getType(), left->getType()->getPrimitiveSizeInBits());
return b.CreateOr(b.CreateLShr(left, right), b.CreateShl(left, b.CreateSub(size, right)));
}
#endif
};
template <typename T>
std::enable_if_t<std::is_integral_v<T>, T> toInteger(T x) { return x; }
template <typename T>
std::enable_if_t<std::is_floating_point_v<T>, Int64> toInteger(T x) { return Int64(x); }
template <typename A, typename B>
struct BitTestImpl
{
using ResultType = UInt8;
template <typename Result = ResultType>
static inline Result apply(A a, B b) { return (toInteger(a) >> toInteger(b)) & 1; };
};
static inline Result apply(A a, B b)
{
return (typename NumberTraits::ToInteger<A>::Type(a) >> typename NumberTraits::ToInteger<B>::Type(b)) & 1;
};
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = false; /// TODO
#endif
};
template <typename A, typename B>
struct LeastBaseImpl
@ -336,6 +464,18 @@ struct LeastBaseImpl
/** gcc 4.9.2 successfully vectorizes a loop from this function. */
return static_cast<Result>(a) < static_cast<Result>(b) ? static_cast<Result>(a) : static_cast<Result>(b);
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = true;
static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool is_signed)
{
if (!left->getType()->isIntegerTy())
/// XXX minnum is basically fmin(), it may or may not match whatever apply() does
return b.CreateMinNum(left, right);
return b.CreateSelect(is_signed ? b.CreateICmpSLT(left, right) : b.CreateICmpULT(left, right), left, right);
}
#endif
};
template <typename A, typename B>
@ -349,6 +489,10 @@ struct LeastSpecialImpl
static_assert(std::is_same_v<Result, ResultType>, "ResultType != Result");
return accurate::lessOp(a, b) ? static_cast<Result>(a) : static_cast<Result>(b);
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = false; /// ???
#endif
};
template <typename A, typename B>
@ -365,6 +509,19 @@ struct GreatestBaseImpl
{
return static_cast<Result>(a) > static_cast<Result>(b) ? static_cast<Result>(a) : static_cast<Result>(b);
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = true;
static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * left, llvm::Value * right, bool is_signed)
{
if (!left->getType()->isIntegerTy())
/// XXX maxnum is basically fmax(), it may or may not match whatever apply() does
/// XXX CreateMaxNum is broken on LLVM 5.0 and 6.0 (generates minnum instead; fixed in 7)
return b.CreateBinaryIntrinsic(llvm::Intrinsic::maxnum, left, right);
return b.CreateSelect(is_signed ? b.CreateICmpSGT(left, right) : b.CreateICmpUGT(left, right), left, right);
}
#endif
};
template <typename A, typename B>
@ -378,6 +535,10 @@ struct GreatestSpecialImpl
static_assert(std::is_same_v<Result, ResultType>, "ResultType != Result");
return accurate::greaterOp(a, b) ? static_cast<Result>(a) : static_cast<Result>(b);
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = false; /// ???
#endif
};
template <typename A, typename B>
@ -393,6 +554,15 @@ struct NegateImpl
{
return -static_cast<ResultType>(a);
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = true;
static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * arg, bool)
{
return arg->getType()->isIntegerTy() ? b.CreateNeg(arg) : b.CreateFNeg(arg);
}
#endif
};
template <typename A>
@ -404,6 +574,17 @@ struct BitNotImpl
{
return ~static_cast<ResultType>(a);
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = true;
static inline llvm::Value * compile(llvm::IRBuilder<> & b, llvm::Value * arg, bool)
{
if (!arg->getType()->isIntegerTy())
throw Exception("BitNotImpl expected an integral type", ErrorCodes::LOGICAL_ERROR);
return b.CreateNot(arg);
}
#endif
};
template <typename A>
@ -420,6 +601,10 @@ struct AbsImpl
else if constexpr (std::is_floating_point_v<A>)
return static_cast<ResultType>(std::abs(a));
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = false; /// special type handling, some other time
#endif
};
template <typename A, typename B>
@ -436,6 +621,10 @@ struct GCDImpl
typename NumberTraits::ToInteger<Result>::Type(a),
typename NumberTraits::ToInteger<Result>::Type(b));
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = false; /// exceptions (and a non-trivial algorithm)
#endif
};
template <typename A, typename B>
@ -452,6 +641,10 @@ struct LCMImpl
typename NumberTraits::ToInteger<Result>::Type(a),
typename NumberTraits::ToInteger<Result>::Type(b));
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = false; /// exceptions (and a non-trivial algorithm)
#endif
};
template <typename A>
@ -463,6 +656,10 @@ struct IntExp2Impl
{
return intExp2(a);
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = false; /// library function
#endif
};
template <typename A>
@ -474,6 +671,10 @@ struct IntExp10Impl
{
return intExp10(a);
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = false; /// library function
#endif
};
/// Used to indicate undefined operation
@ -745,6 +946,43 @@ public:
if (!valid)
throw Exception(getName() + "'s arguments do not match the expected data types", ErrorCodes::LOGICAL_ERROR);
}
#if USE_EMBEDDED_COMPILER
bool isCompilableImpl(const DataTypes & arguments) const override
{
return castBothTypes(arguments[0].get(), arguments[1].get(), [&](const auto & left, const auto & right)
{
using LeftDataType = std::decay_t<decltype(left)>;
using RightDataType = std::decay_t<decltype(right)>;
using ResultDataType = typename BinaryOperationTraits<Op, LeftDataType, RightDataType>::ResultDataType;
using OpSpec = Op<typename LeftDataType::FieldType, typename RightDataType::FieldType>;
return !std::is_same_v<ResultDataType, InvalidType> && OpSpec::compilable;
});
}
llvm::Value * compileImpl(llvm::IRBuilderBase & builder, const DataTypes & types, ValuePlaceholders values) const override
{
llvm::Value * result = nullptr;
castBothTypes(types[0].get(), types[1].get(), [&](const auto & left, const auto & right)
{
using LeftDataType = std::decay_t<decltype(left)>;
using RightDataType = std::decay_t<decltype(right)>;
using ResultDataType = typename BinaryOperationTraits<Op, LeftDataType, RightDataType>::ResultDataType;
using OpSpec = Op<typename LeftDataType::FieldType, typename RightDataType::FieldType>;
if constexpr (!std::is_same_v<ResultDataType, InvalidType> && OpSpec::compilable)
{
auto & b = static_cast<llvm::IRBuilder<> &>(builder);
auto * type = toNativeType(b, ResultDataType{});
auto * lval = castNativeNumber(b, values[0](), type, std::is_signed_v<typename LeftDataType::FieldType>);
auto * rval = castNativeNumber(b, values[1](), type, std::is_signed_v<typename RightDataType::FieldType>);
result = OpSpec::compile(b, lval, rval, std::is_signed_v<typename ResultDataType::FieldType>);
return true;
}
return false;
});
return result;
}
#endif
};
@ -821,6 +1059,35 @@ public:
throw Exception(getName() + "'s argument does not match the expected data type", ErrorCodes::LOGICAL_ERROR);
}
#if USE_EMBEDDED_COMPILER
bool isCompilableImpl(const DataTypes & arguments) const override
{
return castType(arguments[0].get(), [&](const auto & type)
{
return Op<typename std::decay_t<decltype(type)>::FieldType>::compilable;
});
}
llvm::Value * compileImpl(llvm::IRBuilderBase & builder, const DataTypes & types, ValuePlaceholders values) const override
{
llvm::Value * result = nullptr;
castType(types[0].get(), [&](const auto & type)
{
using T0 = typename std::decay_t<decltype(type)>::FieldType;
using T1 = typename Op<T0>::ResultType;
if constexpr (Op<T1>::compilable)
{
auto & b = static_cast<llvm::IRBuilder<> &>(builder);
auto * v = castNativeNumber(b, values[0](), toNativeType(b, DataTypeNumber<T1>{}), std::is_signed_v<T0>);
result = Op<T0>::compile(b, v, std::is_signed_v<T1>);
return true;
}
return false;
});
return result;
}
#endif
bool hasInformationAboutMonotonicity() const override
{
return FunctionUnaryArithmeticMonotonicity<Name>::has();

105
dbms/src/Functions/FunctionsLogical.h
View File

@ -12,6 +12,10 @@
#include <Functions/FunctionHelpers.h>
#include <type_traits>
#if USE_EMBEDDED_COMPILER
#include <llvm/IR/IRBuilder.h>
#endif
namespace DB
{
@ -31,65 +35,71 @@ namespace ErrorCodes
* For example, 1 OR NULL returns 1, not NULL.
*/
struct AndImpl
{
static inline bool isSaturable()
static inline constexpr bool isSaturable()
{
return true;
}
static inline bool isSaturatedValue(bool a)
static inline constexpr bool isSaturatedValue(bool a)
{
return !a;
}
static inline bool apply(bool a, bool b)
static inline constexpr bool apply(bool a, bool b)
{
return a && b;
}
static inline bool specialImplementationForNulls() { return false; }
static inline constexpr bool specialImplementationForNulls() { return false; }
};
struct OrImpl
{
static inline bool isSaturable()
static inline constexpr bool isSaturable()
{
return true;
}
static inline bool isSaturatedValue(bool a)
static inline constexpr bool isSaturatedValue(bool a)
{
return a;
}
static inline bool apply(bool a, bool b)
static inline constexpr bool apply(bool a, bool b)
{
return a || b;
}
static inline bool specialImplementationForNulls() { return true; }
static inline constexpr bool specialImplementationForNulls() { return true; }
};
struct XorImpl
{
static inline bool isSaturable()
static inline constexpr bool isSaturable()
{
return false;
}
static inline bool isSaturatedValue(bool)
static inline constexpr bool isSaturatedValue(bool)
{
return false;
}
static inline bool apply(bool a, bool b)
static inline constexpr bool apply(bool a, bool b)
{
return a != b;
}
static inline bool specialImplementationForNulls() { return false; }
static inline constexpr bool specialImplementationForNulls() { return false; }
#if USE_EMBEDDED_COMPILER
static inline llvm::Value * apply(llvm::IRBuilder<> & builder, llvm::Value * a, llvm::Value * b)
{
return builder.CreateXor(a, b);
}
#endif
};
template <typename A>
@ -101,6 +111,13 @@ struct NotImpl
{
return !a;
}
#if USE_EMBEDDED_COMPILER
static inline llvm::Value * apply(llvm::IRBuilder<> & builder, llvm::Value * a)
{
return builder.CreateNot(a);
}
#endif
};
@ -172,6 +189,20 @@ struct AssociativeOperationImpl<Op, 1>
};
#if USE_EMBEDDED_COMPILER
static llvm::Value * isNativeTrueValue(llvm::IRBuilder<> & b, const DataTypePtr & type, llvm::Value * x)
{
if (type->isNullable())
{
auto * subexpr = isNativeTrueValue(b, removeNullable(type), b.CreateExtractValue(x, {0}));
return b.CreateAnd(b.CreateNot(b.CreateExtractValue(x, {1})), subexpr);
}
auto * zero = llvm::Constant::getNullValue(x->getType());
return x->getType()->isIntegerTy() ? b.CreateICmpNE(x, zero) : b.CreateFCmpONE(x, zero); /// QNaN -> false
}
#endif
template <typename Impl, typename Name>
class FunctionAnyArityLogical : public IFunction
{
@ -364,6 +395,44 @@ public:
block.getByPosition(result).column = std::move(col_res);
}
#if USE_EMBEDDED_COMPILER
bool isCompilableImpl(const DataTypes &) const override { return true; }
llvm::Value * compileImpl(llvm::IRBuilderBase & builder, const DataTypes & types, ValuePlaceholders values) const override
{
auto & b = static_cast<llvm::IRBuilder<> &>(builder);
if constexpr (!Impl::isSaturable())
{
auto * result = isNativeTrueValue(b, types[0], values[0]());
for (size_t i = 1; i < types.size(); i++)
result = Impl::apply(b, result, isNativeTrueValue(b, types[i], values[i]()));
return b.CreateSelect(result, b.getInt8(1), b.getInt8(0));
}
constexpr bool breakOnTrue = Impl::isSaturatedValue(true);
auto * next = b.GetInsertBlock();
auto * stop = llvm::BasicBlock::Create(next->getContext(), "", next->getParent());
b.SetInsertPoint(stop);
auto * phi = b.CreatePHI(b.getInt8Ty(), values.size());
for (size_t i = 0; i < types.size(); i++)
{
b.SetInsertPoint(next);
auto * value = values[i]();
auto * truth = isNativeTrueValue(b, types[i], value);
if (!types[i]->equals(DataTypeUInt8{}))
value = b.CreateSelect(truth, b.getInt8(1), b.getInt8(0));
phi->addIncoming(value, b.GetInsertBlock());
if (i + 1 < types.size())
{
next = llvm::BasicBlock::Create(next->getContext(), "", next->getParent());
b.CreateCondBr(truth, breakOnTrue ? stop : next, breakOnTrue ? next : stop);
}
}
b.CreateBr(stop);
b.SetInsertPoint(stop);
return phi;
}
#endif
};
@ -430,6 +499,16 @@ public:
+ " of argument of function " + getName(),
ErrorCodes::ILLEGAL_COLUMN);
}
#if USE_EMBEDDED_COMPILER
bool isCompilableImpl(const DataTypes &) const override { return true; }
llvm::Value * compileImpl(llvm::IRBuilderBase & builder, const DataTypes & types, ValuePlaceholders values) const override
{
auto & b = static_cast<llvm::IRBuilder<> &>(builder);
return b.CreateSelect(Impl<UInt8>::apply(b, isNativeTrueValue(b, types[0], values[0]())), b.getInt8(1), b.getInt8(0));
}
#endif
};

12
dbms/src/Functions/FunctionsRound.h
View File

@ -93,6 +93,10 @@ struct RoundToExp2Impl
{
return roundDownToPowerOfTwo<T>(x);
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = false;
#endif
};
@ -120,6 +124,10 @@ struct RoundDurationImpl
: (x < 36000 ? 18000
: 36000))))))))))))));
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = false;
#endif
};
template <typename A>
@ -137,6 +145,10 @@ struct RoundAgeImpl
: (x < 55 ? 45
: 55)))));
}
#if USE_EMBEDDED_COMPILER
static constexpr bool compilable = false;
#endif
};

87
dbms/src/Functions/IFunction.cpp
View File

@ -1,16 +1,22 @@
#include <Functions/IFunction.h>
#include <Functions/FunctionHelpers.h>
#include <Columns/ColumnNullable.h>
#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/DataTypeNothing.h>
#include <Columns/ColumnConst.h>
#include <Interpreters/ExpressionActions.h>
#include <Common/config.h>
#include <Common/typeid_cast.h>
#include <Columns/ColumnConst.h>
#include <Columns/ColumnNullable.h>
#include <DataTypes/DataTypeNothing.h>
#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/Native.h>
#include <Functions/FunctionHelpers.h>
#include <Functions/IFunction.h>
#include <Interpreters/ExpressionActions.h>
#include <ext/range.h>
#include <ext/collection_cast.h>
#include <cstdlib>
#include <memory>
#if USE_EMBEDDED_COMPILER
#include <llvm/IR/IRBuilder.h>
#endif
namespace DB
{
@ -259,4 +265,71 @@ DataTypePtr FunctionBuilderImpl::getReturnType(const ColumnsWithTypeAndName & ar
return getReturnTypeImpl(arguments);
}
#if USE_EMBEDDED_COMPILER
static std::optional<DataTypes> removeNullables(const DataTypes & types)
{
for (const auto & type : types)
{
if (!typeid_cast<const DataTypeNullable *>(type.get()))
continue;
DataTypes filtered;
for (const auto & type : types)
filtered.emplace_back(removeNullable(type));
return filtered;
}
return {};
}
bool IFunction::isCompilable(const DataTypes & arguments) const
{
if (useDefaultImplementationForNulls())
if (auto denulled = removeNullables(arguments))
return isCompilableImpl(*denulled);
return isCompilableImpl(arguments);
}
llvm::Value * IFunction::compile(llvm::IRBuilderBase & builder, const DataTypes & arguments, ValuePlaceholders values) const
{
if (useDefaultImplementationForNulls())
{
if (auto denulled = removeNullables(arguments))
{
/// FIXME: when only one column is nullable, this can actually be slower than the non-jitted version
/// because this involves copying the null map while `wrapInNullable` reuses it.
auto & b = static_cast<llvm::IRBuilder<> &>(builder);
auto * fail = llvm::BasicBlock::Create(b.GetInsertBlock()->getContext(), "", b.GetInsertBlock()->getParent());
auto * join = llvm::BasicBlock::Create(b.GetInsertBlock()->getContext(), "", b.GetInsertBlock()->getParent());
auto * zero = llvm::Constant::getNullValue(toNativeType(b, makeNullable(getReturnTypeImpl(*denulled))));
for (size_t i = 0; i < arguments.size(); i++)
{
if (!arguments[i]->isNullable())
continue;
/// Would be nice to evaluate all this lazily, but that'd change semantics: if only unevaluated
/// arguments happen to contain NULLs, the return value would not be NULL, though it should be.
auto * value = values[i]();
auto * ok = llvm::BasicBlock::Create(b.GetInsertBlock()->getContext(), "", b.GetInsertBlock()->getParent());
b.CreateCondBr(b.CreateExtractValue(value, {1}), fail, ok);
b.SetInsertPoint(ok);
values[i] = [value = b.CreateExtractValue(value, {0})]() { return value; };
}
auto * result = b.CreateInsertValue(zero, compileImpl(builder, *denulled, std::move(values)), {0});
auto * result_block = b.GetInsertBlock();
b.CreateBr(join);
b.SetInsertPoint(fail);
auto * null = b.CreateInsertValue(zero, b.getTrue(), {1});
b.CreateBr(join);
b.SetInsertPoint(join);
auto * phi = b.CreatePHI(result->getType(), 2);
phi->addIncoming(result, result_block);
phi->addIncoming(null, fail);
return phi;
}
}
return compileImpl(builder, arguments, std::move(values));
}
#endif
}

75
dbms/src/Functions/IFunction.h
View File

@ -2,6 +2,7 @@
#include <memory>
#include <Common/config.h>
#include <Core/Names.h>
#include <Core/Field.h>
#include <Core/Block.h>
@ -9,6 +10,14 @@
#include <DataTypes/IDataType.h>
namespace llvm
{
class LLVMContext;
class Value;
class IRBuilderBase;
}
namespace DB
{
@ -70,6 +79,8 @@ private:
size_t input_rows_count);
};
using ValuePlaceholders = std::vector<std::function<llvm::Value * ()>>;
/// Function with known arguments and return type.
class IFunctionBase
{
@ -92,6 +103,25 @@ public:
return prepare(block)->execute(block, arguments, result, input_rows_count);
}
#if USE_EMBEDDED_COMPILER
virtual bool isCompilable() const { return false; }
/** Produce LLVM IR code that operates on scalar values. See `toNativeType` in DataTypes/Native.h
* for supported value types and how they map to LLVM types.
*
* NOTE: the builder is actually guaranteed to be exactly `llvm::IRBuilder<>`, so you may safely
* downcast it to that type. This method is specified with `IRBuilderBase` because forward-declaring
* templates with default arguments is impossible and including LLVM in such a generic header
* as this one is a major pain.
*/
virtual llvm::Value * compile(llvm::IRBuilderBase & /*builder*/, ValuePlaceholders /*values*/) const
{
throw Exception(getName() + " is not JIT-compilable", ErrorCodes::NOT_IMPLEMENTED);
}
#endif
/** Should we evaluate this function while constant folding, if arguments are constants?
* Usually this is true. Notable counterexample is function 'sleep'.
* If we will call it during query analysis, we will sleep extra amount of time.
@ -261,7 +291,6 @@ public:
using PreparedFunctionImpl::execute;
using FunctionBuilderImpl::getReturnTypeImpl;
using FunctionBuilderImpl::getLambdaArgumentTypesImpl;
using FunctionBuilderImpl::getReturnType;
PreparedFunctionPtr prepare(const Block & /*sample_block*/) const final
@ -269,17 +298,51 @@ public:
throw Exception("prepare is not implemented for IFunction", ErrorCodes::NOT_IMPLEMENTED);
}
#if USE_EMBEDDED_COMPILER
bool isCompilable() const final
{
throw Exception("isCompilable without explicit types is not implemented for IFunction", ErrorCodes::NOT_IMPLEMENTED);
}
llvm::Value * compile(llvm::IRBuilderBase & /*builder*/, ValuePlaceholders /*values*/) const final
{
throw Exception("compile without explicit types is not implemented for IFunction", ErrorCodes::NOT_IMPLEMENTED);
}
#endif
const DataTypes & getArgumentTypes() const final
{
throw Exception("getArgumentTypes is not implemented for IFunction", ErrorCodes::NOT_IMPLEMENTED);
}
const DataTypePtr & getReturnType() const override
const DataTypePtr & getReturnType() const final
{
throw Exception("getReturnType is not implemented for IFunction", ErrorCodes::NOT_IMPLEMENTED);
}
#if USE_EMBEDDED_COMPILER
bool isCompilable(const DataTypes & arguments) const;
llvm::Value * compile(llvm::IRBuilderBase &, const DataTypes & arguments, ValuePlaceholders values) const;
#endif
protected:
#if USE_EMBEDDED_COMPILER
virtual bool isCompilableImpl(const DataTypes &) const { return false; }
virtual llvm::Value * compileImpl(llvm::IRBuilderBase &, const DataTypes &, ValuePlaceholders) const
{
throw Exception(getName() + " is not JIT-compilable", ErrorCodes::NOT_IMPLEMENTED);
}
#endif
FunctionBasePtr buildImpl(const ColumnsWithTypeAndName & /*arguments*/, const DataTypePtr & /*return_type*/) const final
{
throw Exception("buildImpl is not implemented for IFunction", ErrorCodes::NOT_IMPLEMENTED);
@ -319,6 +382,14 @@ public:
const DataTypes & getArgumentTypes() const override { return arguments; }
const DataTypePtr & getReturnType() const override { return return_type; }
#if USE_EMBEDDED_COMPILER
bool isCompilable() const override { return function->isCompilable(arguments); }
llvm::Value * compile(llvm::IRBuilderBase & builder, ValuePlaceholders values) const override { return function->compile(builder, arguments, std::move(values)); }
#endif
PreparedFunctionPtr prepare(const Block & /*sample_block*/) const override { return std::make_shared<DefaultExecutable>(function); }
bool isSuitableForConstantFolding() const override { return function->isSuitableForConstantFolding(); }

16
dbms/src/Interpreters/ExpressionActions.cpp
View File

@ -1,5 +1,7 @@
#include <Common/config.h>
#include <Common/ProfileEvents.h>
#include <Interpreters/ExpressionActions.h>
#include <Interpreters/ExpressionJIT.h>
#include <Interpreters/Join.h>
#include <Columns/ColumnsNumber.h>
#include <Columns/ColumnArray.h>
@ -754,6 +756,13 @@ void ExpressionActions::finalize(const Names & output_columns)
final_columns.insert(name);
}
#if USE_EMBEDDED_COMPILER
/// This has to be done before removing redundant actions and inserting REMOVE_COLUMNs
/// because inlining may change dependency sets.
if (settings.compile_expressions)
compileFunctions(actions, output_columns, sample_block);
#endif
/// Which columns are needed to perform actions from the current to the last.
NameSet needed_columns = final_columns;
/// Which columns nobody will touch from the current action to the last.
@ -936,7 +945,7 @@ void ExpressionActions::finalize(const Names & output_columns)
std::cerr << action.toString() << "\n";
std::cerr << "\n";*/
optimize();
optimizeArrayJoin();
checkLimits(sample_block);
}
@ -961,11 +970,6 @@ std::string ExpressionActions::dumpActions() const
return ss.str();
}
void ExpressionActions::optimize()
{
optimizeArrayJoin();
}
void ExpressionActions::optimizeArrayJoin()
{
const size_t NONE = actions.size();

2
dbms/src/Interpreters/ExpressionActions.h
View File

@ -215,8 +215,6 @@ private:
void addImpl(ExpressionAction action, Names & new_names);
/// Try to improve something without changing the lists of input and output columns.
void optimize();
/// Move all arrayJoin as close as possible to the end.
void optimizeArrayJoin();
};

525
dbms/src/Interpreters/ExpressionJIT.cpp Normal file
View File

@ -0,0 +1,525 @@
#include <Interpreters/ExpressionJIT.h>
#if USE_EMBEDDED_COMPILER
#include <Columns/ColumnConst.h>
#include <Columns/ColumnNullable.h>
#include <Columns/ColumnVector.h>
#include <Common/typeid_cast.h>
#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/Native.h>
#include <Functions/IFunction.h>
#include <llvm/IR/BasicBlock.h>
#include <llvm/IR/DataLayout.h>
#include <llvm/IR/DerivedTypes.h>
#include <llvm/IR/Function.h>
#include <llvm/IR/IRBuilder.h>
#include <llvm/IR/LLVMContext.h>
#include <llvm/IR/Mangler.h>
#include <llvm/IR/Module.h>
#include <llvm/IR/Type.h>
#include <llvm/ExecutionEngine/ExecutionEngine.h>
#include <llvm/ExecutionEngine/JITSymbol.h>
#include <llvm/ExecutionEngine/SectionMemoryManager.h>
#include <llvm/ExecutionEngine/Orc/CompileUtils.h>
#include <llvm/ExecutionEngine/Orc/IRCompileLayer.h>
#include <llvm/ExecutionEngine/Orc/NullResolver.h>
#include <llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h>
#include <llvm/Target/TargetMachine.h>
#include <llvm/Support/TargetSelect.h>
#include <llvm/Transforms/IPO/PassManagerBuilder.h>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
}
namespace
{
struct ColumnData
{
const char * data = nullptr;
const char * null = nullptr;
size_t stride;
};
struct ColumnDataPlaceholder
{
llvm::Value * data_init; /// first row
llvm::Value * null_init;
llvm::Value * stride;
llvm::PHINode * data; /// current row
llvm::PHINode * null;
};
}
static ColumnData getColumnData(const IColumn * column)
{
ColumnData result;
const bool is_const = column->isColumnConst();
if (is_const)
column = &reinterpret_cast<const ColumnConst *>(column)->getDataColumn();
if (auto * nullable = typeid_cast<const ColumnNullable *>(column))
{
result.null = nullable->getNullMapColumn().getRawData().data;
column = &nullable->getNestedColumn();
}
result.data = column->getRawData().data;
result.stride = is_const ? 0 : column->sizeOfValueIfFixed();
return result;
}
static void applyFunction(IFunctionBase & function, Field & value)
{
const auto & type = function.getArgumentTypes().at(0);
Block block = {{ type->createColumnConst(1, value), type, "x" }, { nullptr, function.getReturnType(), "y" }};
function.execute(block, {0}, 1, 1);
block.safeGetByPosition(1).column->get(0, value);
}
struct LLVMContext
{
llvm::LLVMContext context;
std::shared_ptr<llvm::Module> module;
std::unique_ptr<llvm::TargetMachine> machine;
llvm::orc::RTDyldObjectLinkingLayer objectLayer;
llvm::orc::IRCompileLayer<decltype(objectLayer), llvm::orc::SimpleCompiler> compileLayer;
llvm::DataLayout layout;
llvm::IRBuilder<> builder;
std::unordered_map<std::string, const void *> symbols;
LLVMContext()
: module(std::make_shared<llvm::Module>("jit", context))
, machine(llvm::EngineBuilder().selectTarget())
, objectLayer([]() { return std::make_shared<llvm::SectionMemoryManager>(); })
, compileLayer(objectLayer, llvm::orc::SimpleCompiler(*machine))
, layout(machine->createDataLayout())
, builder(context)
{
module->setDataLayout(layout);
module->setTargetTriple(machine->getTargetTriple().getTriple());
}
void finalize()
{
if (!module->size())
return;
llvm::PassManagerBuilder builder;
llvm::legacy::FunctionPassManager fpm(module.get());
builder.OptLevel = 3;
builder.SLPVectorize = true;
builder.LoopVectorize = true;
builder.RerollLoops = true;
builder.VerifyInput = true;
builder.VerifyOutput = true;
builder.populateFunctionPassManager(fpm);
for (auto & function : *module)
fpm.run(function);
llvm::cantFail(compileLayer.addModule(module, std::make_shared<llvm::orc::NullResolver>()));
for (const auto & function : *module)
{
std::string mangledName;
llvm::raw_string_ostream mangledNameStream(mangledName);
llvm::Mangler::getNameWithPrefix(mangledNameStream, function.getName(), layout);
if (auto symbol = compileLayer.findSymbol(mangledNameStream.str(), false).getAddress())
symbols[function.getName()] = reinterpret_cast<const void *>(*symbol);
}
}
};
class LLVMPreparedFunction : public PreparedFunctionImpl
{
std::string name;
std::shared_ptr<LLVMContext> context;
const void * function;
public:
LLVMPreparedFunction(std::string name_, std::shared_ptr<LLVMContext> context)
: name(std::move(name_)), context(context), function(context->symbols.at(name))
{}
String getName() const override { return name; }
bool useDefaultImplementationForNulls() const override { return false; }
bool useDefaultImplementationForConstants() const override { return true; }
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t block_size) override
{
auto col_res = block.getByPosition(result).type->createColumn()->cloneResized(block_size);
if (block_size)
{
std::vector<ColumnData> columns(arguments.size() + 1);
for (size_t i = 0; i < arguments.size(); i++)
{
auto * column = block.getByPosition(arguments[i]).column.get();
if (!column)
throw Exception("column " + block.getByPosition(arguments[i]).name + " is missing", ErrorCodes::LOGICAL_ERROR);
columns[i] = getColumnData(column);
}
columns[arguments.size()] = getColumnData(col_res.get());
reinterpret_cast<void (*) (size_t, ColumnData *)>(function)(block_size, columns.data());
}
block.getByPosition(result).column = std::move(col_res);
};
};
static void compileFunction(std::shared_ptr<LLVMContext> & context, const IFunctionBase & f)
{
auto & arg_types = f.getArgumentTypes();
auto & b = context->builder;
auto * size_type = b.getIntNTy(sizeof(size_t) * 8);
auto * data_type = llvm::StructType::get(b.getInt8PtrTy(), b.getInt8PtrTy(), size_type);
auto * func_type = llvm::FunctionType::get(b.getVoidTy(), { size_type, data_type->getPointerTo() }, /*isVarArg=*/false);
auto * func = llvm::Function::Create(func_type, llvm::Function::ExternalLinkage, f.getName(), context->module.get());
auto args = func->args().begin();
llvm::Value * counter_arg = &*args++;
llvm::Value * columns_arg = &*args++;
auto * entry = llvm::BasicBlock::Create(b.getContext(), "entry", func);
b.SetInsertPoint(entry);
std::vector<ColumnDataPlaceholder> columns(arg_types.size() + 1);
for (size_t i = 0; i <= arg_types.size(); i++)
{
auto & type = i == arg_types.size() ? f.getReturnType() : arg_types[i];
auto * data = b.CreateLoad(b.CreateConstInBoundsGEP1_32(data_type, columns_arg, i));
columns[i].data_init = b.CreatePointerCast(b.CreateExtractValue(data, {0}), toNativeType(b, removeNullable(type))->getPointerTo());
columns[i].null_init = type->isNullable() ? b.CreateExtractValue(data, {1}) : nullptr;
columns[i].stride = b.CreateExtractValue(data, {2});
}
/// assume nonzero initial value in `counter_arg`
auto * loop = llvm::BasicBlock::Create(b.getContext(), "loop", func);
b.CreateBr(loop);
b.SetInsertPoint(loop);
auto * counter_phi = b.CreatePHI(counter_arg->getType(), 2);
counter_phi->addIncoming(counter_arg, entry);
for (auto & col : columns)
{
col.data = b.CreatePHI(col.data_init->getType(), 2);
col.data->addIncoming(col.data_init, entry);
if (col.null_init)
{
col.null = b.CreatePHI(col.null_init->getType(), 2);
col.null->addIncoming(col.null_init, entry);
}
}
ValuePlaceholders arguments(arg_types.size());
for (size_t i = 0; i < arguments.size(); i++)
{
arguments[i] = [&b, &col = columns[i], &type = arg_types[i]]() -> llvm::Value *
{
auto * value = b.CreateLoad(col.data);
if (!col.null)
return value;
auto * is_null = b.CreateICmpNE(b.CreateLoad(col.null), b.getInt8(0));
auto * nullable = llvm::Constant::getNullValue(toNativeType(b, type));
return b.CreateInsertValue(b.CreateInsertValue(nullable, value, {0}), is_null, {1});
};
}
auto * result = f.compile(b, std::move(arguments));
if (columns.back().null)
{
b.CreateStore(b.CreateExtractValue(result, {0}), columns.back().data);
b.CreateStore(b.CreateSelect(b.CreateExtractValue(result, {1}), b.getInt8(1), b.getInt8(0)), columns.back().null);
}
else
{
b.CreateStore(result, columns.back().data);
}
auto * cur_block = b.GetInsertBlock();
for (auto & col : columns)
{
/// currently, stride is either 0 or size of native type
auto * is_const = b.CreateICmpEQ(col.stride, llvm::ConstantInt::get(size_type, 0));
col.data->addIncoming(b.CreateSelect(is_const, col.data, b.CreateConstInBoundsGEP1_32(nullptr, col.data, 1)), cur_block);
if (col.null)
col.null->addIncoming(b.CreateSelect(is_const, col.null, b.CreateConstInBoundsGEP1_32(nullptr, col.null, 1)), cur_block);
}
counter_phi->addIncoming(b.CreateSub(counter_phi, llvm::ConstantInt::get(size_type, 1)), cur_block);
auto * end = llvm::BasicBlock::Create(b.getContext(), "end", func);
b.CreateCondBr(b.CreateICmpNE(counter_phi, llvm::ConstantInt::get(size_type, 1)), loop, end);
b.SetInsertPoint(end);
b.CreateRetVoid();
}
static llvm::Constant * getNativeValue(llvm::Type * type, const IColumn & column, size_t i)
{
if (!type)
return nullptr;
if (auto * constant = typeid_cast<const ColumnConst *>(&column))
return getNativeValue(type, constant->getDataColumn(), 0);
if (auto * nullable = typeid_cast<const ColumnNullable *>(&column))
{
auto * value = getNativeValue(type->getContainedType(0), nullable->getNestedColumn(), i);
auto * is_null = llvm::ConstantInt::get(type->getContainedType(1), nullable->isNullAt(i));
return value ? llvm::ConstantStruct::get(static_cast<llvm::StructType *>(type), value, is_null) : nullptr;
}
if (type->isFloatTy())
return llvm::ConstantFP::get(type, static_cast<const ColumnVector<Float32> &>(column).getElement(i));
if (type->isDoubleTy())
return llvm::ConstantFP::get(type, static_cast<const ColumnVector<Float64> &>(column).getElement(i));
if (type->isIntegerTy())
return llvm::ConstantInt::get(type, column.getUInt(i));
/// TODO: if (type->isVectorTy())
return nullptr;
}
/// Same as IFunctionBase::compile, but also for constants and input columns.
using CompilableExpression = std::function<llvm::Value * (llvm::IRBuilderBase &, const ValuePlaceholders &)>;
static CompilableExpression subexpression(ColumnPtr c, DataTypePtr type)
{
return [=](llvm::IRBuilderBase & b, const ValuePlaceholders &) { return getNativeValue(toNativeType(b, type), *c, 0); };
}
static CompilableExpression subexpression(size_t i)
{
return [=](llvm::IRBuilderBase &, const ValuePlaceholders & inputs) { return inputs[i](); };
}
static CompilableExpression subexpression(const IFunctionBase & f, std::vector<CompilableExpression> args)
{
return [&, args = std::move(args)](llvm::IRBuilderBase & builder, const ValuePlaceholders & inputs)
{
ValuePlaceholders input;
for (const auto & arg : args)
input.push_back([&]() { return arg(builder, inputs); });
auto * result = f.compile(builder, input);
if (result->getType() != toNativeType(builder, f.getReturnType()))
throw Exception("function " + f.getName() + " generated an llvm::Value of invalid type", ErrorCodes::LOGICAL_ERROR);
return result;
};
}
class LLVMFunction : public IFunctionBase
{
std::string name;
Names arg_names;
DataTypes arg_types;
std::shared_ptr<LLVMContext> context;
std::vector<FunctionBasePtr> originals;
std::unordered_map<StringRef, CompilableExpression> subexpressions;
public:
LLVMFunction(const ExpressionActions::Actions & actions, std::shared_ptr<LLVMContext> context, const Block & sample_block)
: name(actions.back().result_name), context(context)
{
for (const auto & c : sample_block)
/// TODO: implement `getNativeValue` for all types & replace the check with `c.column && toNativeType(...)`
if (c.column && getNativeValue(toNativeType(context->builder, c.type), *c.column, 0))
subexpressions[c.name] = subexpression(c.column, c.type);
for (const auto & action : actions)
{
const auto & names = action.argument_names;
const auto & types = action.function->getArgumentTypes();
std::vector<CompilableExpression> args;
for (size_t i = 0; i < names.size(); i++)
{
auto inserted = subexpressions.emplace(names[i], subexpression(arg_names.size()));
if (inserted.second)
{
arg_names.push_back(names[i]);
arg_types.push_back(types[i]);
}
args.push_back(inserted.first->second);
}
subexpressions[action.result_name] = subexpression(*action.function, std::move(args));
originals.push_back(action.function);
}
compileFunction(context, *this);
}
bool isCompilable() const override { return true; }
llvm::Value * compile(llvm::IRBuilderBase & builder, ValuePlaceholders values) const override { return subexpressions.at(name)(builder, values); }
String getName() const override { return name; }
const Names & getArgumentNames() const { return arg_names; }
const DataTypes & getArgumentTypes() const override { return arg_types; }
const DataTypePtr & getReturnType() const override { return originals.back()->getReturnType(); }
PreparedFunctionPtr prepare(const Block &) const override { return std::make_shared<LLVMPreparedFunction>(name, context); }
bool isDeterministic() override
{
for (const auto & f : originals)
if (!f->isDeterministic())
return false;
return true;
}
bool isDeterministicInScopeOfQuery() override
{
for (const auto & f : originals)
if (!f->isDeterministicInScopeOfQuery())
return false;
return true;
}
bool isSuitableForConstantFolding() const override
{
for (const auto & f : originals)
if (!f->isSuitableForConstantFolding())
return false;
return true;
}
bool isInjective(const Block & sample_block) override
{
for (const auto & f : originals)
if (!f->isInjective(sample_block))
return false;
return true;
}
bool hasInformationAboutMonotonicity() const override
{
for (const auto & f : originals)
if (!f->hasInformationAboutMonotonicity())
return false;
return true;
}
Monotonicity getMonotonicityForRange(const IDataType & type, const Field & left, const Field & right) const override
{
const IDataType * type_ = &type;
Field left_ = left;
Field right_ = right;
Monotonicity result(true, true, true);
/// monotonicity is only defined for unary functions, so the chain must describe a sequence of nested calls
for (size_t i = 0; i < originals.size(); i++)
{
Monotonicity m = originals[i]->getMonotonicityForRange(*type_, left_, right_);
if (!m.is_monotonic)
return m;
result.is_positive ^= !m.is_positive;
result.is_always_monotonic &= m.is_always_monotonic;
if (i + 1 < originals.size())
{
if (left_ != Field())
applyFunction(*originals[i], left_);
if (right_ != Field())
applyFunction(*originals[i], right_);
if (!m.is_positive)
std::swap(left_, right_);
type_ = originals[i]->getReturnType().get();
}
}
return result;
}
};
static bool isCompilable(llvm::IRBuilderBase & builder, const IFunctionBase& function)
{
if (!toNativeType(builder, function.getReturnType()))
return false;
for (const auto & type : function.getArgumentTypes())
if (!toNativeType(builder, type))
return false;
return function.isCompilable();
}
void compileFunctions(ExpressionActions::Actions & actions, const Names & output_columns, const Block & sample_block)
{
auto context = std::make_shared<LLVMContext>();
/// an empty optional is a poisoned value prohibiting the column's producer from being removed
/// (which it could be, if it was inlined into every dependent function).
std::unordered_map<std::string, std::unordered_set<std::optional<size_t>>> current_dependents;
for (const auto & name : output_columns)
current_dependents[name].emplace();
/// a snapshot of each compilable function's dependents at the time of its execution.
std::vector<std::unordered_set<std::optional<size_t>>> dependents(actions.size());
for (size_t i = actions.size(); i--;)
{
switch (actions[i].type)
{
case ExpressionAction::REMOVE_COLUMN:
current_dependents.erase(actions[i].source_name);
/// poison every other column used after this point so that inlining chains do not cross it.
for (auto & dep : current_dependents)
dep.second.emplace();
break;
case ExpressionAction::PROJECT:
current_dependents.clear();
for (const auto & proj : actions[i].projection)
current_dependents[proj.first].emplace();
break;
case ExpressionAction::ADD_COLUMN:
case ExpressionAction::COPY_COLUMN:
case ExpressionAction::ARRAY_JOIN:
case ExpressionAction::JOIN:
{
Names columns = actions[i].getNeededColumns();
for (const auto & column : columns)
current_dependents[column].emplace();
break;
}
case ExpressionAction::APPLY_FUNCTION:
{
dependents[i] = current_dependents[actions[i].result_name];
const bool compilable = isCompilable(context->builder, *actions[i].function);
for (const auto & name : actions[i].argument_names)
{
if (compilable)
current_dependents[name].emplace(i);
else
current_dependents[name].emplace();
}
break;
}
}
}
std::vector<ExpressionActions::Actions> fused(actions.size());
for (size_t i = 0; i < actions.size(); i++)
{
if (actions[i].type != ExpressionAction::APPLY_FUNCTION || !isCompilable(context->builder, *actions[i].function))
continue;
fused[i].push_back(actions[i]);
if (dependents[i].find({}) != dependents[i].end())
{
/// the result of compiling one function in isolation is pretty much the same as its `execute` method.
if (fused[i].size() == 1)
continue;
auto fn = std::make_shared<LLVMFunction>(std::move(fused[i]), context, sample_block);
actions[i].function = fn;
actions[i].argument_names = fn->getArgumentNames();
continue;
}
/// TODO: determine whether it's profitable to inline the function if there's more than one dependent.
for (const auto & dep : dependents[i])
fused[*dep].insert(fused[*dep].end(), fused[i].begin(), fused[i].end());
}
context->finalize();
}
}
namespace
{
struct LLVMTargetInitializer
{
LLVMTargetInitializer()
{
llvm::InitializeNativeTarget();
llvm::InitializeNativeTargetAsmPrinter();
}
} llvmInitializer;
}
#endif

17
dbms/src/Interpreters/ExpressionJIT.h Normal file
View File

@ -0,0 +1,17 @@
#pragma once
#include <Common/config.h>
#include <Interpreters/ExpressionActions.h>
#if USE_EMBEDDED_COMPILER
namespace DB
{
/// For each APPLY_FUNCTION action, try to compile the function to native code; if the only uses of a compilable
/// function's result are as arguments to other compilable functions, inline it and leave the now-redundant action as-is.
void compileFunctions(ExpressionActions::Actions & actions, const Names & output_columns, const Block & sample_block);
}
#endif

1
dbms/src/Interpreters/Settings.h
View File

@ -65,6 +65,7 @@ struct Settings
M(SettingFloat, totals_auto_threshold, 0.5, "The threshold for totals_mode = 'auto'.") \
\
M(SettingBool, compile, false, "Whether query compilation is enabled.") \
M(SettingBool, compile_expressions, true, "Compile some scalar functions and operators to native code.") \
M(SettingUInt64, min_count_to_compile, 3, "The number of structurally identical queries before they are compiled.") \
M(SettingUInt64, group_by_two_level_threshold, 100000, "From what number of keys, a two-level aggregation starts. 0 - the threshold is not set.") \
M(SettingUInt64, group_by_two_level_threshold_bytes, 100000000, "From what size of the aggregation state in bytes, a two-level aggregation begins to be used. 0 - the threshold is not set. Two-level aggregation is used when at least one of the thresholds is triggered.") \

7
dbms/src/Server/Compiler-5.0.0/CMakeLists.txt
View File

@ -8,12 +8,7 @@ add_library(clickhouse-compiler-lib
target_compile_options(clickhouse-compiler-lib PRIVATE -fno-rtti -fno-exceptions -g0)
llvm_map_components_to_libraries(REQUIRED_LLVM_LIBRARIES all)
# We link statically with zlib, and LLVM (sometimes) tries to bring its own dependency.
list(REMOVE_ITEM REQUIRED_LLVM_LIBRARIES "-lz")
# Wrong library in freebsd:
list(REMOVE_ITEM REQUIRED_LLVM_LIBRARIES "-l/usr/lib/libexecinfo.so")
llvm_map_components_to_libnames(REQUIRED_LLVM_LIBRARIES all)
message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}")

9
dbms/src/Server/Compiler-6.0.0/CMakeLists.txt
View File

@ -8,12 +8,7 @@ add_library(clickhouse-compiler-lib
target_compile_options(clickhouse-compiler-lib PRIVATE -fno-rtti -fno-exceptions -g0)
llvm_map_components_to_libraries(REQUIRED_LLVM_LIBRARIES all)
# We link statically with zlib, and LLVM (sometimes) tries to bring its own dependency.
list(REMOVE_ITEM REQUIRED_LLVM_LIBRARIES "-lz")
# Wrong library in freebsd:
list(REMOVE_ITEM REQUIRED_LLVM_LIBRARIES "-l/usr/lib/libexecinfo.so")
llvm_map_components_to_libnames(REQUIRED_LLVM_LIBRARIES all)
message(STATUS "Using LLVM ${LLVM_VERSION}: ${LLVM_INCLUDE_DIRS} : ${REQUIRED_LLVM_LIBRARIES}")
@ -24,7 +19,7 @@ target_include_directories(clickhouse-compiler-lib PRIVATE ${LLVM_INCLUDE_DIRS})
target_link_libraries(clickhouse-compiler-lib PRIVATE
clangBasic clangCodeGen clangDriver
clangFrontend
clangFrontend
clangFrontendTool
clangRewriteFrontend clangARCMigrate clangStaticAnalyzerFrontend
clangParse clangSerialization clangSema clangEdit clangStaticAnalyzerCheckers