ClickHouse/src/Common/TargetSpecific.h

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

351 lines
12 KiB
C++
Raw Normal View History

2020-03-30 16:36:02 +00:00
#pragma once
2021-10-02 07:13:14 +00:00
#include <base/types.h>
2020-05-15 10:10:34 +00:00
2020-05-18 17:07:36 +00:00
/* This file contains macros and helpers for writing platform-dependent code.
2020-06-21 18:47:16 +00:00
*
* Macros DECLARE_<Arch>_SPECIFIC_CODE will wrap code inside it into the
2020-05-18 17:07:36 +00:00
* namespace TargetSpecific::<Arch> and enable Arch-specific compile options.
* Thus, it's allowed to call functions inside these namespaces only after
2020-06-21 18:47:16 +00:00
* checking platform in runtime (see isArchSupported() below).
2020-05-18 17:07:36 +00:00
*
* If compiler is not gcc/clang or target isn't x86_64 or ENABLE_MULTITARGET_CODE
2020-08-08 00:47:03 +00:00
* was set to OFF in cmake, all code inside these macros will be removed and
2020-09-19 16:42:36 +00:00
* USE_MULTITARGET_CODE will be set to 0. Use #if USE_MULTITARGET_CODE whenever you
* use anything from this namespaces.
2020-06-21 18:47:16 +00:00
*
2020-05-18 17:07:36 +00:00
* For similarities there is a macros DECLARE_DEFAULT_CODE, which wraps code
2020-08-08 00:47:03 +00:00
* into the namespace TargetSpecific::Default but doesn't specify any additional
* copile options. Functions and classes inside this macros are available regardless
* of USE_MUTLITARGE_CODE.
2020-06-21 18:47:16 +00:00
*
* Example of usage:
2020-06-21 18:47:16 +00:00
*
2020-05-18 17:07:36 +00:00
* DECLARE_DEFAULT_CODE (
* int funcImpl() {
* return 1;
* }
* ) // DECLARE_DEFAULT_CODE
2020-06-21 18:47:16 +00:00
*
2020-05-18 17:07:36 +00:00
* DECLARE_AVX2_SPECIFIC_CODE (
* int funcImpl() {
* return 2;
* }
* ) // DECLARE_DEFAULT_CODE
2020-06-21 18:47:16 +00:00
*
2020-05-18 17:07:36 +00:00
* int func() {
* #if USE_MULTITARGET_CODE
2020-06-21 18:47:16 +00:00
* if (isArchSupported(TargetArch::AVX2))
2021-04-12 20:02:16 +00:00
* return TargetSpecific::AVX2::funcImpl();
* #endif
2021-04-12 20:02:16 +00:00
* return TargetSpecific::Default::funcImpl();
2020-06-21 18:47:16 +00:00
* }
*
2020-05-18 17:07:36 +00:00
* Sometimes code may benefit from compiling with different options.
* For these purposes use DECLARE_MULTITARGET_CODE macros. It will create a copy
* of the code for every supported target and compile it with different options.
2021-04-12 20:02:16 +00:00
* These copies are available via TargetSpecific namespaces described above.
2020-06-21 18:47:16 +00:00
*
* Inside every TargetSpecific namespace there is a constexpr variable BuildArch,
2020-05-18 17:07:36 +00:00
* which indicates the target platform for current code.
2020-06-21 18:47:16 +00:00
*
2020-05-18 17:07:36 +00:00
* Example:
2020-06-21 18:47:16 +00:00
*
2020-05-18 17:07:36 +00:00
* DECLARE_MULTITARGET_CODE(
* int funcImpl(int size, ...) {
* int iteration_size = 1;
* if constexpr (BuildArch == TargetArch::SSE42)
* iteration_size = 2
* else if constexpr (BuildArch == TargetArch::AVX || BuildArch == TargetArch::AVX2)
* iteration_size = 4;
* for (int i = 0; i < size; i += iteration_size)
* ...
* }
* ) // DECLARE_MULTITARGET_CODE
*
* // All target-specific and default implementations are available here via
* TargetSpecific::<Arch>::funcImpl. Use runtime detection to choose one.
2020-05-18 17:07:36 +00:00
*
* If you want to write IFunction or IExecutableFuncionImpl with several implementations
* see PerformanceAdaptors.h.
2020-05-18 17:07:36 +00:00
*/
namespace DB
2020-03-30 16:36:02 +00:00
{
2020-05-16 06:59:08 +00:00
enum class TargetArch : UInt32
{
2020-05-18 17:07:36 +00:00
Default = 0, /// Without any additional compiler options.
SSE42 = (1 << 0), /// SSE4.2
2020-05-15 12:00:20 +00:00
AVX = (1 << 1),
AVX2 = (1 << 2),
AVX512F = (1 << 3),
AVX512BW = (1 << 4),
AVX512VBMI = (1 << 5),
2020-03-30 16:36:02 +00:00
};
/// Runtime detection.
2020-06-21 18:47:16 +00:00
bool isArchSupported(TargetArch arch);
2020-06-21 18:47:16 +00:00
String toString(TargetArch arch);
2020-05-15 10:10:34 +00:00
#ifndef ENABLE_MULTITARGET_CODE
# define ENABLE_MULTITARGET_CODE 0
#endif
#if ENABLE_MULTITARGET_CODE && defined(__GNUC__) && defined(__x86_64__)
/// NOLINTNEXTLINE
#define USE_MULTITARGET_CODE 1
2020-04-14 15:46:53 +00:00
#if defined(__clang__)
2022-07-15 07:01:16 +00:00
#define AVX512VBMI_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi")))
2022-06-02 09:19:46 +00:00
#define AVX512BW_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw")))
#define AVX512_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f")))
#define AVX2_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2")))
#define AVX_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx"))
#define SSE42_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt")))
#define DEFAULT_FUNCTION_SPECIFIC_ATTRIBUTE
2022-07-15 07:01:16 +00:00
# define BEGIN_AVX512VBMI_SPECIFIC_CODE \
_Pragma("clang attribute push(__attribute__((target(\"sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi\"))),apply_to=function)")
2022-06-02 09:19:46 +00:00
# define BEGIN_AVX512BW_SPECIFIC_CODE \
_Pragma("clang attribute push(__attribute__((target(\"sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw\"))),apply_to=function)")
2020-05-18 17:07:36 +00:00
# define BEGIN_AVX512F_SPECIFIC_CODE \
_Pragma("clang attribute push(__attribute__((target(\"sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f\"))),apply_to=function)")
2020-05-18 17:07:36 +00:00
# define BEGIN_AVX2_SPECIFIC_CODE \
_Pragma("clang attribute push(__attribute__((target(\"sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2\"))),apply_to=function)")
2020-05-18 17:07:36 +00:00
# define BEGIN_AVX_SPECIFIC_CODE \
_Pragma("clang attribute push(__attribute__((target(\"sse,sse2,sse3,ssse3,sse4,popcnt,avx\"))),apply_to=function)")
2020-05-18 17:07:36 +00:00
# define BEGIN_SSE42_SPECIFIC_CODE \
_Pragma("clang attribute push(__attribute__((target(\"sse,sse2,sse3,ssse3,sse4,popcnt\"))),apply_to=function)")
2020-05-18 17:07:36 +00:00
# define END_TARGET_SPECIFIC_CODE \
_Pragma("clang attribute pop")
/* Clang shows warning when there aren't any objects to apply pragma.
* To prevent this warning we define this function inside every macros with pragmas.
*/
2021-08-25 20:34:53 +00:00
# define DUMMY_FUNCTION_DEFINITION [[maybe_unused]] void _dummy_function_definition();
#else
2022-07-15 07:01:16 +00:00
#define AVX512VBMI_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi,tune=native")))
2022-06-02 09:19:46 +00:00
#define AVX512BW_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,tune=native")))
#define AVX512_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,tune=native")))
#define AVX2_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,tune=native")))
#define AVX_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt,avx,tune=native")))
2022-07-15 07:01:16 +00:00
#define SSE42_FUNCTION_SPECIFIC_ATTRIBUTE __attribute__((target("sse,sse2,sse3,ssse3,sse4,popcnt",tune=native)))
#define DEFAULT_FUNCTION_SPECIFIC_ATTRIBUTE
2022-07-15 07:01:16 +00:00
# define BEGIN_AVX512VBMI_SPECIFIC_CODE \
_Pragma("GCC push_options") \
_Pragma("GCC target(\"sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,avx512vl,avx512vbmi,tune=native\")")
2022-06-02 09:19:46 +00:00
# define BEGIN_AVX512BW_SPECIFIC_CODE \
_Pragma("GCC push_options") \
_Pragma("GCC target(\"sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,avx512bw,tune=native\")")
2020-05-15 12:00:20 +00:00
# define BEGIN_AVX512F_SPECIFIC_CODE \
2020-03-30 16:36:02 +00:00
_Pragma("GCC push_options") \
_Pragma("GCC target(\"sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,avx512f,tune=native\")")
2020-03-30 16:36:02 +00:00
# define BEGIN_AVX2_SPECIFIC_CODE \
_Pragma("GCC push_options") \
_Pragma("GCC target(\"sse,sse2,sse3,ssse3,sse4,popcnt,avx,avx2,tune=native\")")
2020-03-30 16:36:02 +00:00
# define BEGIN_AVX_SPECIFIC_CODE \
_Pragma("GCC push_options") \
_Pragma("GCC target(\"sse,sse2,sse3,ssse3,sse4,popcnt,avx,tune=native\")")
# define BEGIN_SSE42_SPECIFIC_CODE \
2020-03-30 16:36:02 +00:00
_Pragma("GCC push_options") \
_Pragma("GCC target(\"sse,sse2,sse3,ssse3,sse4,popcnt,tune=native\")")
2020-03-30 16:36:02 +00:00
# define END_TARGET_SPECIFIC_CODE \
_Pragma("GCC pop_options")
2020-05-18 17:07:36 +00:00
/* GCC doesn't show such warning, we don't need to define anything.
*/
# define DUMMY_FUNCTION_DEFINITION
2020-03-30 16:36:02 +00:00
#endif
#define DECLARE_SSE42_SPECIFIC_CODE(...) \
BEGIN_SSE42_SPECIFIC_CODE \
namespace TargetSpecific::SSE42 { \
2020-05-18 17:07:36 +00:00
DUMMY_FUNCTION_DEFINITION \
using namespace DB::TargetSpecific::SSE42; \
2020-03-30 16:36:02 +00:00
__VA_ARGS__ \
} \
END_TARGET_SPECIFIC_CODE
2020-04-02 13:48:14 +00:00
#define DECLARE_AVX_SPECIFIC_CODE(...) \
2020-03-30 16:36:02 +00:00
BEGIN_AVX_SPECIFIC_CODE \
namespace TargetSpecific::AVX { \
2020-05-18 17:07:36 +00:00
DUMMY_FUNCTION_DEFINITION \
using namespace DB::TargetSpecific::AVX; \
2020-03-30 16:36:02 +00:00
__VA_ARGS__ \
} \
END_TARGET_SPECIFIC_CODE
2020-04-02 13:48:14 +00:00
#define DECLARE_AVX2_SPECIFIC_CODE(...) \
2020-03-30 16:36:02 +00:00
BEGIN_AVX2_SPECIFIC_CODE \
namespace TargetSpecific::AVX2 { \
2020-05-18 17:07:36 +00:00
DUMMY_FUNCTION_DEFINITION \
using namespace DB::TargetSpecific::AVX2; \
2020-03-30 16:36:02 +00:00
__VA_ARGS__ \
} \
END_TARGET_SPECIFIC_CODE
2020-05-15 12:00:20 +00:00
#define DECLARE_AVX512F_SPECIFIC_CODE(...) \
BEGIN_AVX512F_SPECIFIC_CODE \
namespace TargetSpecific::AVX512F { \
2020-05-18 17:07:36 +00:00
DUMMY_FUNCTION_DEFINITION \
2020-05-15 12:00:20 +00:00
using namespace DB::TargetSpecific::AVX512F; \
2020-03-30 16:36:02 +00:00
__VA_ARGS__ \
} \
END_TARGET_SPECIFIC_CODE
2022-06-02 09:19:46 +00:00
#define DECLARE_AVX512BW_SPECIFIC_CODE(...) \
BEGIN_AVX512BW_SPECIFIC_CODE \
namespace TargetSpecific::AVX512BW { \
DUMMY_FUNCTION_DEFINITION \
using namespace DB::TargetSpecific::AVX512BW; \
__VA_ARGS__ \
} \
END_TARGET_SPECIFIC_CODE
2022-07-15 07:01:16 +00:00
#define DECLARE_AVX512VBMI_SPECIFIC_CODE(...) \
BEGIN_AVX512VBMI_SPECIFIC_CODE \
namespace TargetSpecific::AVX512VBMI { \
DUMMY_FUNCTION_DEFINITION \
using namespace DB::TargetSpecific::AVX512VBMI; \
__VA_ARGS__ \
} \
END_TARGET_SPECIFIC_CODE
#else
#define USE_MULTITARGET_CODE 0
/* Multitarget code is disabled, just delete target-specific code.
*/
#define DECLARE_SSE42_SPECIFIC_CODE(...)
#define DECLARE_AVX_SPECIFIC_CODE(...)
#define DECLARE_AVX2_SPECIFIC_CODE(...)
#define DECLARE_AVX512F_SPECIFIC_CODE(...)
2022-06-02 09:19:46 +00:00
#define DECLARE_AVX512BW_SPECIFIC_CODE(...)
2022-07-15 07:01:16 +00:00
#define DECLARE_AVX512VBMI_SPECIFIC_CODE(...)
#endif
#define DECLARE_DEFAULT_CODE(...) \
namespace TargetSpecific::Default { \
using namespace DB::TargetSpecific::Default; \
__VA_ARGS__ \
}
/// NOLINTNEXTLINE
2020-04-02 13:48:14 +00:00
#define DECLARE_MULTITARGET_CODE(...) \
2020-05-15 12:00:20 +00:00
DECLARE_DEFAULT_CODE (__VA_ARGS__) \
DECLARE_SSE42_SPECIFIC_CODE (__VA_ARGS__) \
2020-05-15 12:00:20 +00:00
DECLARE_AVX_SPECIFIC_CODE (__VA_ARGS__) \
DECLARE_AVX2_SPECIFIC_CODE (__VA_ARGS__) \
2022-06-02 09:19:46 +00:00
DECLARE_AVX512F_SPECIFIC_CODE(__VA_ARGS__) \
DECLARE_AVX512BW_SPECIFIC_CODE(__VA_ARGS__) \
DECLARE_AVX512VBMI_SPECIFIC_CODE(__VA_ARGS__)
2020-03-30 16:36:02 +00:00
DECLARE_DEFAULT_CODE(
constexpr auto BuildArch = TargetArch::Default; /// NOLINT
2020-03-30 16:36:02 +00:00
) // DECLARE_DEFAULT_CODE
DECLARE_SSE42_SPECIFIC_CODE(
constexpr auto BuildArch = TargetArch::SSE42; /// NOLINT
) // DECLARE_SSE42_SPECIFIC_CODE
2020-03-30 16:36:02 +00:00
DECLARE_AVX_SPECIFIC_CODE(
constexpr auto BuildArch = TargetArch::AVX; /// NOLINT
2020-03-30 16:36:02 +00:00
) // DECLARE_AVX_SPECIFIC_CODE
DECLARE_AVX2_SPECIFIC_CODE(
constexpr auto BuildArch = TargetArch::AVX2; /// NOLINT
2020-03-30 16:36:02 +00:00
) // DECLARE_AVX2_SPECIFIC_CODE
2020-05-15 12:00:20 +00:00
DECLARE_AVX512F_SPECIFIC_CODE(
constexpr auto BuildArch = TargetArch::AVX512F; /// NOLINT
2020-05-15 12:00:20 +00:00
) // DECLARE_AVX512F_SPECIFIC_CODE
2020-03-30 16:36:02 +00:00
2022-06-02 09:19:46 +00:00
DECLARE_AVX512BW_SPECIFIC_CODE(
constexpr auto BuildArch = TargetArch::AVX512BW; /// NOLINT
) // DECLARE_AVX512BW_SPECIFIC_CODE
2022-07-15 07:01:16 +00:00
DECLARE_AVX512VBMI_SPECIFIC_CODE(
constexpr auto BuildArch = TargetArch::AVX512VBMI; /// NOLINT
) // DECLARE_AVX512VBMI_SPECIFIC_CODE
/** Runtime Dispatch helpers for class members.
*
* Example of usage:
*
* class TestClass
* {
* public:
* MULTITARGET_FUNCTION_AVX2_SSE42(
* MULTITARGET_FUNCTION_HEADER(int), testFunctionImpl, MULTITARGET_FUNCTION_BODY((int value)
* {
* return value;
* })
* )
*
* void testFunction(int value) {
* if (isArchSupported(TargetArch::AVX2))
* {
* testFunctionImplAVX2(value);
* }
* else if (isArchSupported(TargetArch::SSE42))
* {
* testFunctionImplSSE42(value);
* }
* else
* {
* testFunction(value);
* }
* }
*};
*
*/
/// Function header
#define MULTITARGET_FUNCTION_HEADER(...) __VA_ARGS__
/// Function body
#define MULTITARGET_FUNCTION_BODY(...) __VA_ARGS__
#if ENABLE_MULTITARGET_CODE && defined(__GNUC__) && defined(__x86_64__)
/// NOLINTNEXTLINE
#define MULTITARGET_FUNCTION_AVX2_SSE42(FUNCTION_HEADER, name, FUNCTION_BODY) \
FUNCTION_HEADER \
\
AVX2_FUNCTION_SPECIFIC_ATTRIBUTE \
name##AVX2 \
FUNCTION_BODY \
\
FUNCTION_HEADER \
\
2022-05-17 14:59:11 +00:00
SSE42_FUNCTION_SPECIFIC_ATTRIBUTE \
name##SSE42 \
FUNCTION_BODY \
\
FUNCTION_HEADER \
\
name \
FUNCTION_BODY \
#else
/// NOLINTNEXTLINE
#define MULTITARGET_FUNCTION_AVX2_SSE42(FUNCTION_HEADER, name, FUNCTION_BODY) \
FUNCTION_HEADER \
\
name \
FUNCTION_BODY \
#endif
2020-05-16 06:59:08 +00:00
}