ClickHouse/utils/memcpy-bench/memcpy-bench.cpp

967 lines
34 KiB
C++
Raw Normal View History

#ifdef HAS_RESERVED_IDENTIFIER
2021-08-25 22:22:38 +00:00
#pragma clang diagnostic ignored "-Wreserved-identifier"
#endif
2021-03-08 09:04:52 +00:00
#include <memory>
#include <cstddef>
2021-03-14 16:52:51 +00:00
#include <stdexcept>
2021-03-08 09:04:52 +00:00
#include <string>
#include <random>
#include <iostream>
#include <iomanip>
#include <thread>
#include <dlfcn.h>
#include <pcg_random.hpp>
2021-10-02 07:13:14 +00:00
#include <base/defines.h>
2021-03-08 09:04:52 +00:00
#include <Common/Stopwatch.h>
#include <emmintrin.h>
#include <immintrin.h>
2021-03-14 16:52:51 +00:00
#include <boost/program_options.hpp>
2021-03-08 09:04:52 +00:00
template <typename F, typename MemcpyImpl>
void NO_INLINE loop(uint8_t * dst, uint8_t * src, size_t size, F && chunk_size_distribution, MemcpyImpl && impl)
{
while (size)
{
size_t bytes_to_copy = std::min<size_t>(size, chunk_size_distribution());
impl(dst, src, bytes_to_copy);
dst += bytes_to_copy;
src += bytes_to_copy;
size -= bytes_to_copy;
2021-03-15 08:16:15 +00:00
/// Execute at least one SSE instruction as a penalty after running AVX code.
__asm__ __volatile__ ("pxor %%xmm15, %%xmm15" ::: "xmm15");
2021-03-08 09:04:52 +00:00
}
}
using RNG = pcg32_fast;
template <size_t N>
size_t generatorUniform(RNG & rng) { return rng() % N; };
template <typename F, typename MemcpyImpl>
2021-03-14 16:52:51 +00:00
uint64_t test(uint8_t * dst, uint8_t * src, size_t size, size_t iterations, size_t num_threads, F && generator, MemcpyImpl && impl, const char * name)
2021-03-08 09:04:52 +00:00
{
Stopwatch watch;
std::vector<std::thread> threads;
threads.reserve(num_threads);
for (size_t thread_num = 0; thread_num < num_threads; ++thread_num)
{
size_t begin = size * thread_num / num_threads;
size_t end = size * (thread_num + 1) / num_threads;
threads.emplace_back([begin, end, iterations, &src, &dst, &generator, &impl]
{
for (size_t iteration = 0; iteration < iterations; ++iteration)
{
loop(
iteration % 2 ? &src[begin] : &dst[begin],
iteration % 2 ? &dst[begin] : &src[begin],
end - begin,
[rng = RNG(), &generator]() mutable { return generator(rng); },
std::forward<MemcpyImpl>(impl));
}
});
}
for (auto & thread : threads)
thread.join();
2021-03-14 16:52:51 +00:00
uint64_t elapsed_ns = watch.elapsed();
2021-03-08 09:04:52 +00:00
/// Validation
for (size_t i = 0; i < size; ++i)
2021-03-15 08:16:15 +00:00
if (dst[i] != uint8_t(i))
throw std::logic_error("Incorrect result");
2021-03-08 09:04:52 +00:00
2021-03-14 16:52:51 +00:00
std::cout << name;
return elapsed_ns;
2021-03-08 09:04:52 +00:00
}
using memcpy_type = void * (*)(const void * __restrict, void * __restrict, size_t);
static void * memcpy_erms(void * dst, const void * src, size_t size)
{
asm volatile (
"rep movsb"
: "=D"(dst), "=S"(src), "=c"(size)
: "0"(dst), "1"(src), "2"(size)
: "memory");
return dst;
}
2021-03-14 16:52:51 +00:00
static void * memcpy_trivial(void * __restrict dst_, const void * __restrict src_, size_t size)
{
char * __restrict dst = reinterpret_cast<char * __restrict>(dst_);
const char * __restrict src = reinterpret_cast<const char * __restrict>(src_);
void * ret = dst;
while (size > 0)
{
*dst = *src;
++dst;
++src;
--size;
}
return ret;
}
2021-03-08 09:04:52 +00:00
extern "C" void * memcpy_jart(void * dst, const void * src, size_t size);
extern "C" void MemCpy(void * dst, const void * src, size_t size);
2021-03-14 16:52:51 +00:00
void * memcpy_fast_sse(void * dst, const void * src, size_t size);
void * memcpy_fast_avx(void * dst, const void * src, size_t size);
void * memcpy_tiny(void * dst, const void * src, size_t size);
2021-03-08 09:04:52 +00:00
static void * memcpySSE2(void * __restrict destination, const void * __restrict source, size_t size)
{
unsigned char *dst = reinterpret_cast<unsigned char *>(destination);
const unsigned char *src = reinterpret_cast<const unsigned char *>(source);
size_t padding;
// small memory copy
if (size <= 16)
return memcpy_tiny(dst, src, size);
// align destination to 16 bytes boundary
padding = (16 - (reinterpret_cast<size_t>(dst) & 15)) & 15;
if (padding > 0)
{
__m128i head = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src));
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head);
dst += padding;
src += padding;
size -= padding;
}
// medium size copy
__m128i c0;
for (; size >= 16; size -= 16)
{
c0 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src));
src += 16;
_mm_store_si128((reinterpret_cast<__m128i*>(dst)), c0);
dst += 16;
}
memcpy_tiny(dst, src, size);
return destination;
}
static void * memcpySSE2Unrolled2(void * __restrict destination, const void * __restrict source, size_t size)
{
unsigned char *dst = reinterpret_cast<unsigned char *>(destination);
const unsigned char *src = reinterpret_cast<const unsigned char *>(source);
size_t padding;
// small memory copy
if (size <= 32)
return memcpy_tiny(dst, src, size);
// align destination to 16 bytes boundary
padding = (16 - (reinterpret_cast<size_t>(dst) & 15)) & 15;
if (padding > 0)
{
__m128i head = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src));
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head);
dst += padding;
src += padding;
size -= padding;
}
// medium size copy
__m128i c0, c1;
for (; size >= 32; size -= 32)
{
c0 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 0);
c1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 1);
src += 32;
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 0), c0);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 1), c1);
dst += 32;
}
memcpy_tiny(dst, src, size);
return destination;
}
static void * memcpySSE2Unrolled4(void * __restrict destination, const void * __restrict source, size_t size)
{
unsigned char *dst = reinterpret_cast<unsigned char *>(destination);
const unsigned char *src = reinterpret_cast<const unsigned char *>(source);
size_t padding;
// small memory copy
if (size <= 64)
return memcpy_tiny(dst, src, size);
// align destination to 16 bytes boundary
padding = (16 - (reinterpret_cast<size_t>(dst) & 15)) & 15;
if (padding > 0)
{
__m128i head = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src));
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head);
dst += padding;
src += padding;
size -= padding;
}
// medium size copy
__m128i c0, c1, c2, c3;
for (; size >= 64; size -= 64)
{
c0 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 0);
c1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 1);
c2 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 2);
c3 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 3);
src += 64;
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 0), c0);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 1), c1);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 2), c2);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 3), c3);
dst += 64;
}
memcpy_tiny(dst, src, size);
return destination;
}
static void * memcpySSE2Unrolled8(void * __restrict destination, const void * __restrict source, size_t size)
{
unsigned char *dst = reinterpret_cast<unsigned char *>(destination);
const unsigned char *src = reinterpret_cast<const unsigned char *>(source);
size_t padding;
// small memory copy
if (size <= 128)
return memcpy_tiny(dst, src, size);
// align destination to 16 bytes boundary
padding = (16 - (reinterpret_cast<size_t>(dst) & 15)) & 15;
if (padding > 0)
{
__m128i head = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src));
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head);
dst += padding;
src += padding;
size -= padding;
}
// medium size copy
__m128i c0, c1, c2, c3, c4, c5, c6, c7;
for (; size >= 128; size -= 128)
{
c0 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 0);
c1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 1);
c2 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 2);
c3 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 3);
c4 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 4);
c5 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 5);
c6 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 6);
c7 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 7);
src += 128;
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 0), c0);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 1), c1);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 2), c2);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 3), c3);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 4), c4);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 5), c5);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 6), c6);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 7), c7);
dst += 128;
}
memcpy_tiny(dst, src, size);
return destination;
}
//static __attribute__((__always_inline__, __target__("sse2")))
2021-04-05 14:37:04 +00:00
__attribute__((__always_inline__)) inline void
memcpy_my_medium_sse(uint8_t * __restrict & dst, const uint8_t * __restrict & src, size_t & size)
2021-03-08 09:04:52 +00:00
{
/// Align destination to 16 bytes boundary.
size_t padding = (16 - (reinterpret_cast<size_t>(dst) & 15)) & 15;
if (padding > 0)
{
__m128i head = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src));
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head);
dst += padding;
src += padding;
size -= padding;
}
/// Aligned unrolled copy.
__m128i c0, c1, c2, c3, c4, c5, c6, c7;
while (size >= 128)
{
c0 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 0);
c1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 1);
c2 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 2);
c3 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 3);
c4 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 4);
c5 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 5);
c6 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 6);
c7 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 7);
src += 128;
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 0), c0);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 1), c1);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 2), c2);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 3), c3);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 4), c4);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 5), c5);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 6), c6);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 7), c7);
dst += 128;
size -= 128;
}
}
__attribute__((__target__("avx")))
void memcpy_my_medium_avx(uint8_t * __restrict & __restrict dst, const uint8_t * __restrict & __restrict src, size_t & __restrict size)
{
size_t padding = (32 - (reinterpret_cast<size_t>(dst) & 31)) & 31;
if (padding > 0)
{
__m256i head = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(src));
2021-03-14 16:52:51 +00:00
_mm256_storeu_si256(reinterpret_cast<__m256i*>(dst), head);
2021-03-08 09:04:52 +00:00
dst += padding;
src += padding;
size -= padding;
}
__m256i c0, c1, c2, c3, c4, c5, c6, c7;
while (size >= 256)
{
c0 = _mm256_loadu_si256((reinterpret_cast<const __m256i*>(src)) + 0);
c1 = _mm256_loadu_si256((reinterpret_cast<const __m256i*>(src)) + 1);
c2 = _mm256_loadu_si256((reinterpret_cast<const __m256i*>(src)) + 2);
c3 = _mm256_loadu_si256((reinterpret_cast<const __m256i*>(src)) + 3);
c4 = _mm256_loadu_si256((reinterpret_cast<const __m256i*>(src)) + 4);
c5 = _mm256_loadu_si256((reinterpret_cast<const __m256i*>(src)) + 5);
c6 = _mm256_loadu_si256((reinterpret_cast<const __m256i*>(src)) + 6);
c7 = _mm256_loadu_si256((reinterpret_cast<const __m256i*>(src)) + 7);
src += 256;
_mm256_store_si256(((reinterpret_cast<__m256i*>(dst)) + 0), c0);
_mm256_store_si256(((reinterpret_cast<__m256i*>(dst)) + 1), c1);
_mm256_store_si256(((reinterpret_cast<__m256i*>(dst)) + 2), c2);
_mm256_store_si256(((reinterpret_cast<__m256i*>(dst)) + 3), c3);
_mm256_store_si256(((reinterpret_cast<__m256i*>(dst)) + 4), c4);
_mm256_store_si256(((reinterpret_cast<__m256i*>(dst)) + 5), c5);
_mm256_store_si256(((reinterpret_cast<__m256i*>(dst)) + 6), c6);
_mm256_store_si256(((reinterpret_cast<__m256i*>(dst)) + 7), c7);
dst += 256;
size -= 256;
}
}
bool have_avx = true;
2021-03-08 09:04:52 +00:00
static uint8_t * memcpy_my(uint8_t * __restrict dst, const uint8_t * __restrict src, size_t size)
{
uint8_t * ret = dst;
tail:
if (size <= 16)
{
if (size >= 8)
{
__builtin_memcpy(dst + size - 8, src + size - 8, 8);
__builtin_memcpy(dst, src, 8);
}
else if (size >= 4)
{
__builtin_memcpy(dst + size - 4, src + size - 4, 4);
__builtin_memcpy(dst, src, 4);
}
else if (size >= 2)
{
__builtin_memcpy(dst + size - 2, src + size - 2, 2);
__builtin_memcpy(dst, src, 2);
}
else if (size >= 1)
{
*dst = *src;
}
}
else if (have_avx)
{
2021-03-08 10:05:18 +00:00
if (size <= 32)
{
__builtin_memcpy(dst, src, 8);
__builtin_memcpy(dst + 8, src + 8, 8);
dst += 16;
src += 16;
size -= 16;
goto tail;
}
2021-03-08 09:04:52 +00:00
if (size <= 256)
{
__asm__(
"vmovups -0x20(%[s],%[size],1), %%ymm0\n"
"vmovups %%ymm0, -0x20(%[d],%[size],1)\n"
: [d]"+r"(dst), [s]"+r"(src)
: [size]"r"(size)
: "ymm0", "memory");
while (size > 32)
{
__asm__(
"vmovups (%[s]), %%ymm0\n"
"vmovups %%ymm0, (%[d])\n"
: [d]"+r"(dst), [s]"+r"(src)
:
: "ymm0", "memory");
dst += 32;
src += 32;
size -= 32;
}
}
else
{
size_t padding = (32 - (reinterpret_cast<size_t>(dst) & 31)) & 31;
if (padding > 0)
{
__asm__(
"vmovups (%[s]), %%ymm0\n"
"vmovups %%ymm0, (%[d])\n"
: [d]"+r"(dst), [s]"+r"(src)
:
: "ymm0", "memory");
dst += padding;
src += padding;
size -= padding;
}
while (size >= 256)
{
__asm__(
"vmovups (%[s]), %%ymm0\n"
"vmovups 0x20(%[s]), %%ymm1\n"
"vmovups 0x40(%[s]), %%ymm2\n"
"vmovups 0x60(%[s]), %%ymm3\n"
"vmovups 0x80(%[s]), %%ymm4\n"
"vmovups 0xa0(%[s]), %%ymm5\n"
"vmovups 0xc0(%[s]), %%ymm6\n"
"vmovups 0xe0(%[s]), %%ymm7\n"
"add $0x100,%[s]\n"
"vmovaps %%ymm0, (%[d])\n"
"vmovaps %%ymm1, 0x20(%[d])\n"
"vmovaps %%ymm2, 0x40(%[d])\n"
"vmovaps %%ymm3, 0x60(%[d])\n"
"vmovaps %%ymm4, 0x80(%[d])\n"
"vmovaps %%ymm5, 0xa0(%[d])\n"
"vmovaps %%ymm6, 0xc0(%[d])\n"
"vmovaps %%ymm7, 0xe0(%[d])\n"
"add $0x100, %[d]\n"
: [d]"+r"(dst), [s]"+r"(src)
:
: "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "memory");
size -= 256;
}
goto tail;
}
}
else
{
if (size <= 128)
{
_mm_storeu_si128(reinterpret_cast<__m128i *>(dst + size - 16), _mm_loadu_si128(reinterpret_cast<const __m128i *>(src + size - 16)));
while (size > 16)
{
_mm_storeu_si128(reinterpret_cast<__m128i *>(dst), _mm_loadu_si128(reinterpret_cast<const __m128i *>(src)));
dst += 16;
src += 16;
size -= 16;
}
}
else
{
/// Align destination to 16 bytes boundary.
size_t padding = (16 - (reinterpret_cast<size_t>(dst) & 15)) & 15;
if (padding > 0)
{
__m128i head = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src));
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head);
dst += padding;
src += padding;
size -= padding;
}
/// Aligned unrolled copy.
__m128i c0, c1, c2, c3, c4, c5, c6, c7;
while (size >= 128)
{
c0 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 0);
c1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 1);
c2 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 2);
c3 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 3);
c4 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 4);
c5 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 5);
c6 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 6);
c7 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 7);
src += 128;
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 0), c0);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 1), c1);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 2), c2);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 3), c3);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 4), c4);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 5), c5);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 6), c6);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 7), c7);
dst += 128;
size -= 128;
}
goto tail;
}
}
return ret;
}
static uint8_t * memcpy_my2(uint8_t * __restrict dst, const uint8_t * __restrict src, size_t size)
{
uint8_t * ret = dst;
if (size <= 16)
{
if (size >= 8)
{
__builtin_memcpy(dst + size - 8, src + size - 8, 8);
__builtin_memcpy(dst, src, 8);
}
else if (size >= 4)
{
__builtin_memcpy(dst + size - 4, src + size - 4, 4);
__builtin_memcpy(dst, src, 4);
}
else if (size >= 2)
{
__builtin_memcpy(dst + size - 2, src + size - 2, 2);
__builtin_memcpy(dst, src, 2);
}
else if (size >= 1)
{
*dst = *src;
}
}
2021-03-15 22:13:27 +00:00
else if (size <= 128)
{
2021-03-15 22:13:27 +00:00
_mm_storeu_si128(reinterpret_cast<__m128i *>(dst + size - 16), _mm_loadu_si128(reinterpret_cast<const __m128i *>(src + size - 16)));
2021-03-15 22:13:27 +00:00
while (size > 16)
{
2021-03-15 22:13:27 +00:00
_mm_storeu_si128(reinterpret_cast<__m128i *>(dst), _mm_loadu_si128(reinterpret_cast<const __m128i *>(src)));
dst += 16;
src += 16;
size -= 16;
}
}
else if (size < 30000 || !have_avx)
{
/// Align destination to 16 bytes boundary.
size_t padding = (16 - (reinterpret_cast<size_t>(dst) & 15)) & 15;
2021-03-15 22:13:27 +00:00
if (padding > 0)
{
__m128i head = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src));
_mm_storeu_si128(reinterpret_cast<__m128i*>(dst), head);
dst += padding;
src += padding;
size -= padding;
}
2021-03-15 22:13:27 +00:00
/// Aligned unrolled copy.
__m128i c0, c1, c2, c3, c4, c5, c6, c7;
2021-03-15 22:13:27 +00:00
while (size >= 128)
{
c0 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 0);
c1 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 1);
c2 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 2);
c3 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 3);
c4 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 4);
c5 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 5);
c6 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 6);
c7 = _mm_loadu_si128(reinterpret_cast<const __m128i*>(src) + 7);
src += 128;
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 0), c0);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 1), c1);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 2), c2);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 3), c3);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 4), c4);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 5), c5);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 6), c6);
_mm_store_si128((reinterpret_cast<__m128i*>(dst) + 7), c7);
dst += 128;
size -= 128;
}
2021-03-15 22:13:27 +00:00
_mm_storeu_si128(reinterpret_cast<__m128i *>(dst + size - 16), _mm_loadu_si128(reinterpret_cast<const __m128i *>(src + size - 16)));
2021-03-15 22:13:27 +00:00
while (size > 16)
{
_mm_storeu_si128(reinterpret_cast<__m128i *>(dst), _mm_loadu_si128(reinterpret_cast<const __m128i *>(src)));
dst += 16;
src += 16;
size -= 16;
}
2021-03-15 22:13:27 +00:00
}
else
{
size_t padding = (32 - (reinterpret_cast<size_t>(dst) & 31)) & 31;
if (padding > 0)
{
2021-03-15 22:13:27 +00:00
__asm__(
"vmovups (%[s]), %%ymm0\n"
"vmovups %%ymm0, (%[d])\n"
: [d]"+r"(dst), [s]"+r"(src)
:
: "ymm0", "memory");
2021-03-15 22:13:27 +00:00
dst += padding;
src += padding;
size -= padding;
}
2021-03-15 22:13:27 +00:00
while (size >= 512)
{
__asm__(
"vmovups (%[s]), %%ymm0\n"
"vmovups 0x20(%[s]), %%ymm1\n"
"vmovups 0x40(%[s]), %%ymm2\n"
"vmovups 0x60(%[s]), %%ymm3\n"
"vmovups 0x80(%[s]), %%ymm4\n"
"vmovups 0xa0(%[s]), %%ymm5\n"
"vmovups 0xc0(%[s]), %%ymm6\n"
"vmovups 0xe0(%[s]), %%ymm7\n"
"vmovups 0x100(%[s]), %%ymm8\n"
"vmovups 0x120(%[s]), %%ymm9\n"
"vmovups 0x140(%[s]), %%ymm10\n"
"vmovups 0x160(%[s]), %%ymm11\n"
"vmovups 0x180(%[s]), %%ymm12\n"
"vmovups 0x1a0(%[s]), %%ymm13\n"
"vmovups 0x1c0(%[s]), %%ymm14\n"
"vmovups 0x1e0(%[s]), %%ymm15\n"
"add $0x200, %[s]\n"
"sub $0x200, %[size]\n"
"vmovaps %%ymm0, (%[d])\n"
"vmovaps %%ymm1, 0x20(%[d])\n"
"vmovaps %%ymm2, 0x40(%[d])\n"
"vmovaps %%ymm3, 0x60(%[d])\n"
"vmovaps %%ymm4, 0x80(%[d])\n"
"vmovaps %%ymm5, 0xa0(%[d])\n"
"vmovaps %%ymm6, 0xc0(%[d])\n"
"vmovaps %%ymm7, 0xe0(%[d])\n"
"vmovaps %%ymm8, 0x100(%[d])\n"
"vmovaps %%ymm9, 0x120(%[d])\n"
"vmovaps %%ymm10, 0x140(%[d])\n"
"vmovaps %%ymm11, 0x160(%[d])\n"
"vmovaps %%ymm12, 0x180(%[d])\n"
"vmovaps %%ymm13, 0x1a0(%[d])\n"
"vmovaps %%ymm14, 0x1c0(%[d])\n"
"vmovaps %%ymm15, 0x1e0(%[d])\n"
"add $0x200, %[d]\n"
: [d]"+r"(dst), [s]"+r"(src), [size]"+r"(size)
:
: "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7",
"ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15",
"memory");
}
2021-03-15 22:13:27 +00:00
/*while (size >= 256)
{
__asm__(
"vmovups (%[s]), %%ymm0\n"
"vmovups 0x20(%[s]), %%ymm1\n"
"vmovups 0x40(%[s]), %%ymm2\n"
"vmovups 0x60(%[s]), %%ymm3\n"
"vmovups 0x80(%[s]), %%ymm4\n"
"vmovups 0xa0(%[s]), %%ymm5\n"
"vmovups 0xc0(%[s]), %%ymm6\n"
"vmovups 0xe0(%[s]), %%ymm7\n"
"add $0x100,%[s]\n"
"vmovaps %%ymm0, (%[d])\n"
"vmovaps %%ymm1, 0x20(%[d])\n"
"vmovaps %%ymm2, 0x40(%[d])\n"
"vmovaps %%ymm3, 0x60(%[d])\n"
"vmovaps %%ymm4, 0x80(%[d])\n"
"vmovaps %%ymm5, 0xa0(%[d])\n"
"vmovaps %%ymm6, 0xc0(%[d])\n"
"vmovaps %%ymm7, 0xe0(%[d])\n"
"add $0x100, %[d]\n"
: [d]"+r"(dst), [s]"+r"(src)
:
: "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7", "memory");
2021-03-15 22:13:27 +00:00
size -= 256;
}*/
2021-03-15 22:13:27 +00:00
/*while (size > 128)
{
__asm__(
2021-03-15 22:13:27 +00:00
"vmovups (%[s]), %%ymm0\n"
"vmovups 0x20(%[s]), %%ymm1\n"
"vmovups 0x40(%[s]), %%ymm2\n"
"vmovups 0x60(%[s]), %%ymm3\n"
"add $0x80, %[s]\n"
"sub $0x80, %[size]\n"
"vmovaps %%ymm0, (%[d])\n"
"vmovaps %%ymm1, 0x20(%[d])\n"
"vmovaps %%ymm2, 0x40(%[d])\n"
"vmovaps %%ymm3, 0x60(%[d])\n"
"add $0x80, %[d]\n"
: [d]"+r"(dst), [s]"+r"(src), [size]"+r"(size)
:
: "ymm0", "ymm1", "ymm2", "ymm3", "memory");
}*/
__asm__(
"vmovups -0x20(%[s],%[size],1), %%ymm0\n"
"vmovups %%ymm0, -0x20(%[d],%[size],1)\n"
: [d]"+r"(dst), [s]"+r"(src)
: [size]"r"(size)
: "ymm0", "memory");
while (size > 32)
{
__asm__(
"vmovups (%[s]), %%ymm0\n"
"vmovups %%ymm0, (%[d])\n"
: [d]"+r"(dst), [s]"+r"(src)
2021-03-15 22:13:27 +00:00
:
: "ymm0", "memory");
2021-03-15 22:13:27 +00:00
dst += 32;
src += 32;
size -= 32;
}
2021-03-15 22:13:27 +00:00
__asm__ __volatile__ ("vzeroupper"
::: "ymm0", "ymm1", "ymm2", "ymm3", "ymm4", "ymm5", "ymm6", "ymm7",
"ymm8", "ymm9", "ymm10", "ymm11", "ymm12", "ymm13", "ymm14", "ymm15");
}
return ret;
}
2021-03-14 16:52:51 +00:00
extern "C" void * __memcpy_erms(void * __restrict destination, const void * __restrict source, size_t size);
extern "C" void * __memcpy_sse2_unaligned(void * __restrict destination, const void * __restrict source, size_t size);
extern "C" void * __memcpy_ssse3(void * __restrict destination, const void * __restrict source, size_t size);
extern "C" void * __memcpy_ssse3_back(void * __restrict destination, const void * __restrict source, size_t size);
extern "C" void * __memcpy_avx_unaligned(void * __restrict destination, const void * __restrict source, size_t size);
extern "C" void * __memcpy_avx_unaligned_erms(void * __restrict destination, const void * __restrict source, size_t size);
extern "C" void * __memcpy_avx512_unaligned(void * __restrict destination, const void * __restrict source, size_t size);
extern "C" void * __memcpy_avx512_unaligned_erms(void * __restrict destination, const void * __restrict source, size_t size);
extern "C" void * __memcpy_avx512_no_vzeroupper(void * __restrict destination, const void * __restrict source, size_t size);
#define VARIANT(N, NAME) \
if (memcpy_variant == N) \
return test(dst, src, size, iterations, num_threads, std::forward<F>(generator), NAME, #NAME);
2021-03-08 09:04:52 +00:00
template <typename F>
2021-03-14 16:52:51 +00:00
uint64_t dispatchMemcpyVariants(size_t memcpy_variant, uint8_t * dst, uint8_t * src, size_t size, size_t iterations, size_t num_threads, F && generator)
2021-03-08 09:04:52 +00:00
{
2021-03-14 16:52:51 +00:00
memcpy_type memcpy_libc_old = reinterpret_cast<memcpy_type>(dlsym(RTLD_NEXT, "memcpy"));
VARIANT(1, memcpy)
VARIANT(2, memcpy_trivial)
VARIANT(3, memcpy_libc_old)
VARIANT(4, memcpy_erms)
VARIANT(5, MemCpy)
VARIANT(6, memcpySSE2)
VARIANT(7, memcpySSE2Unrolled2)
VARIANT(8, memcpySSE2Unrolled4)
VARIANT(9, memcpySSE2Unrolled8)
VARIANT(10, memcpy_fast_sse)
VARIANT(11, memcpy_fast_avx)
VARIANT(12, memcpy_my)
VARIANT(13, memcpy_my2)
2021-03-14 16:52:51 +00:00
VARIANT(21, __memcpy_erms)
VARIANT(22, __memcpy_sse2_unaligned)
VARIANT(23, __memcpy_ssse3)
VARIANT(24, __memcpy_ssse3_back)
VARIANT(25, __memcpy_avx_unaligned)
VARIANT(26, __memcpy_avx_unaligned_erms)
VARIANT(27, __memcpy_avx512_unaligned)
VARIANT(28, __memcpy_avx512_unaligned_erms)
VARIANT(29, __memcpy_avx512_no_vzeroupper)
return 0;
2021-03-08 09:04:52 +00:00
}
2021-03-14 16:52:51 +00:00
uint64_t dispatchVariants(
size_t memcpy_variant, size_t generator_variant, uint8_t * dst, uint8_t * src, size_t size, size_t iterations, size_t num_threads)
2021-03-08 09:04:52 +00:00
{
if (generator_variant == 1)
2021-03-14 16:52:51 +00:00
return dispatchMemcpyVariants(memcpy_variant, dst, src, size, iterations, num_threads, generatorUniform<16>);
2021-03-08 09:04:52 +00:00
if (generator_variant == 2)
2021-03-14 16:52:51 +00:00
return dispatchMemcpyVariants(memcpy_variant, dst, src, size, iterations, num_threads, generatorUniform<256>);
2021-03-08 09:04:52 +00:00
if (generator_variant == 3)
2021-03-14 16:52:51 +00:00
return dispatchMemcpyVariants(memcpy_variant, dst, src, size, iterations, num_threads, generatorUniform<4096>);
2021-03-08 09:04:52 +00:00
if (generator_variant == 4)
2021-03-14 16:52:51 +00:00
return dispatchMemcpyVariants(memcpy_variant, dst, src, size, iterations, num_threads, generatorUniform<65536>);
2021-03-08 09:04:52 +00:00
if (generator_variant == 5)
2021-03-14 16:52:51 +00:00
return dispatchMemcpyVariants(memcpy_variant, dst, src, size, iterations, num_threads, generatorUniform<1048576>);
return 0;
2021-03-08 09:04:52 +00:00
}
int main(int argc, char ** argv)
{
2021-03-14 16:52:51 +00:00
boost::program_options::options_description desc("Allowed options");
desc.add_options()("help,h", "produce help message")
("size", boost::program_options::value<size_t>()->default_value(1000000), "Bytes to copy on every iteration")
("iterations", boost::program_options::value<size_t>(), "Number of iterations")
("threads", boost::program_options::value<size_t>()->default_value(1), "Number of copying threads")
("distribution", boost::program_options::value<size_t>()->default_value(4), "Distribution of chunk sizes to perform copy")
("variant", boost::program_options::value<size_t>(), "Variant of memcpy implementation")
("tsv", "Print result in tab-separated format")
;
boost::program_options::variables_map options;
boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options);
if (options.count("help") || !options.count("variant"))
{
std::cout << R"(Usage:
for size in 4096 16384 50000 65536 100000 1000000 10000000 100000000; do
for threads in 1 2 4 $(($(nproc) / 2)) $(nproc); do
for distribution in 1 2 3 4 5; do
for variant in {1..13} {21..29}; do
2021-03-14 16:52:51 +00:00
for i in {1..10}; do
./memcpy-bench --tsv --size $size --variant $variant --threads $threads --distribution $distribution;
done;
done;
done;
done;
done | tee result.tsv
2021-03-15 08:18:11 +00:00
clickhouse-local --structure '
name String,
size UInt64,
iterations UInt64,
threads UInt16,
generator UInt8,
memcpy UInt8,
elapsed UInt64
' --query "
SELECT
size, name,
avg(1000 * elapsed / size / iterations) AS s,
count() AS c
FROM table
GROUP BY size, name
ORDER BY size ASC, s DESC
" --output-format PrettyCompact < result.tsv
2021-03-14 16:52:51 +00:00
)" << std::endl;
std::cout << desc << std::endl;
return 1;
}
2021-03-08 09:04:52 +00:00
2021-03-14 16:52:51 +00:00
size_t size = options["size"].as<size_t>();
size_t num_threads = options["threads"].as<size_t>();
size_t memcpy_variant = options["variant"].as<size_t>();
size_t generator_variant = options["distribution"].as<size_t>();
2021-03-08 09:04:52 +00:00
2021-03-14 16:52:51 +00:00
size_t iterations;
if (options.count("iterations"))
{
iterations = options["iterations"].as<size_t>();
}
else
{
2021-03-15 08:16:15 +00:00
iterations = 10000000000ULL / size;
2021-03-08 09:04:52 +00:00
2021-03-14 16:52:51 +00:00
if (generator_variant == 1)
iterations /= 10;
}
2021-03-08 09:04:52 +00:00
std::unique_ptr<uint8_t[]> src(new uint8_t[size]);
std::unique_ptr<uint8_t[]> dst(new uint8_t[size]);
/// Fill src with some pattern for validation.
for (size_t i = 0; i < size; ++i)
src[i] = i;
/// Fill dst to avoid page faults.
memset(dst.get(), 0, size);
2021-03-14 16:52:51 +00:00
uint64_t elapsed_ns = dispatchVariants(memcpy_variant, generator_variant, dst.get(), src.get(), size, iterations, num_threads);
std::cout << std::fixed << std::setprecision(3);
if (options.count("tsv"))
{
std::cout
<< '\t' << size
<< '\t' << iterations
<< '\t' << num_threads
<< '\t' << generator_variant
<< '\t' << memcpy_variant
<< '\t' << elapsed_ns
<< '\n';
}
else
{
2021-03-15 22:13:27 +00:00
std::cout << ": " << num_threads << " threads, " << "size: " << size << ", distribution " << generator_variant
<< ", processed in " << (elapsed_ns / 1e9) << " sec, " << (size * iterations * 1.0 / elapsed_ns) << " GB/sec\n";
2021-03-14 16:52:51 +00:00
}
2021-03-08 09:04:52 +00:00
return 0;
}