ClickHouse/utils/memcpy-bench/glibc/memmove-vec-unaligned-erms.S

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

560 lines
17 KiB
ArmAsm
Raw Normal View History

2021-03-14 16:52:51 +00:00
/* memmove/memcpy/mempcpy with unaligned load/store and rep movsb
Copyright (C) 2016-2020 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<https://www.gnu.org/licenses/>. */
/* memmove/memcpy/mempcpy is implemented as:
1. Use overlapping load and store to avoid branch.
2. Load all sources into registers and store them together to avoid
possible address overlap between source and destination.
3. If size is 8 * VEC_SIZE or less, load all sources into registers
and store them together.
4. If address of destination > address of source, backward copy
4 * VEC_SIZE at a time with unaligned load and aligned store.
Load the first 4 * VEC and last VEC before the loop and store
them after the loop to support overlapping addresses.
5. Otherwise, forward copy 4 * VEC_SIZE at a time with unaligned
load and aligned store. Load the last 4 * VEC and first VEC
before the loop and store them after the loop to support
overlapping addresses.
6. If size >= __x86_shared_non_temporal_threshold and there is no
overlap between destination and source, use non-temporal store
instead of aligned store. */
#include "sysdep.h"
#ifndef MEMCPY_SYMBOL
2021-03-14 20:22:12 +00:00
# define MEMCPY_SYMBOL(p,s) MEMMOVE_SYMBOL(p, s)
2021-03-14 16:52:51 +00:00
#endif
#ifndef MEMPCPY_SYMBOL
2021-03-14 20:22:12 +00:00
# define MEMPCPY_SYMBOL(p,s) MEMMOVE_SYMBOL(p, s)
2021-03-14 16:52:51 +00:00
#endif
#ifndef MEMMOVE_CHK_SYMBOL
2021-03-14 20:22:12 +00:00
# define MEMMOVE_CHK_SYMBOL(p,s) MEMMOVE_SYMBOL(p, s)
2021-03-14 16:52:51 +00:00
#endif
#ifndef VZEROUPPER
# if VEC_SIZE > 16
# define VZEROUPPER vzeroupper
# else
# define VZEROUPPER
# endif
#endif
#ifndef PREFETCH
# define PREFETCH(addr) prefetcht0 addr
#endif
/* Assume 64-byte prefetch size. */
#ifndef PREFETCH_SIZE
# define PREFETCH_SIZE 64
#endif
#define PREFETCHED_LOAD_SIZE (VEC_SIZE * 4)
#if PREFETCH_SIZE == 64
# if PREFETCHED_LOAD_SIZE == PREFETCH_SIZE
# define PREFETCH_ONE_SET(dir, base, offset) \
2021-03-14 20:22:12 +00:00
PREFETCH ((offset)base)
2021-03-14 16:52:51 +00:00
# elif PREFETCHED_LOAD_SIZE == 2 * PREFETCH_SIZE
# define PREFETCH_ONE_SET(dir, base, offset) \
2021-03-14 20:22:12 +00:00
PREFETCH ((offset)base); \
PREFETCH ((offset + dir * PREFETCH_SIZE)base)
2021-03-14 16:52:51 +00:00
# elif PREFETCHED_LOAD_SIZE == 4 * PREFETCH_SIZE
# define PREFETCH_ONE_SET(dir, base, offset) \
2021-03-14 20:22:12 +00:00
PREFETCH ((offset)base); \
PREFETCH ((offset + dir * PREFETCH_SIZE)base); \
PREFETCH ((offset + dir * PREFETCH_SIZE * 2)base); \
PREFETCH ((offset + dir * PREFETCH_SIZE * 3)base)
2021-03-14 16:52:51 +00:00
# else
# error Unsupported PREFETCHED_LOAD_SIZE!
# endif
#else
# error Unsupported PREFETCH_SIZE!
#endif
#ifndef SECTION
# error SECTION is not defined!
#endif
2021-03-14 20:22:12 +00:00
.section SECTION(.text),"ax",@progbits
2021-03-14 16:52:51 +00:00
#if defined SHARED
ENTRY (MEMMOVE_CHK_SYMBOL (__mempcpy_chk, unaligned))
2021-03-14 20:22:12 +00:00
cmp %RDX_LP, %RCX_LP
jb HIDDEN_JUMPTARGET (__chk_fail)
2021-03-14 16:52:51 +00:00
END (MEMMOVE_CHK_SYMBOL (__mempcpy_chk, unaligned))
#endif
ENTRY (MEMPCPY_SYMBOL (__mempcpy, unaligned))
2021-03-14 20:22:12 +00:00
mov %RDI_LP, %RAX_LP
add %RDX_LP, %RAX_LP
jmp L(start)
2021-03-14 16:52:51 +00:00
END (MEMPCPY_SYMBOL (__mempcpy, unaligned))
#if defined SHARED
ENTRY (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned))
2021-03-14 20:22:12 +00:00
cmp %RDX_LP, %RCX_LP
jb HIDDEN_JUMPTARGET (__chk_fail)
2021-03-14 16:52:51 +00:00
END (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned))
#endif
ENTRY (MEMMOVE_SYMBOL (__memmove, unaligned))
2021-03-14 20:22:12 +00:00
movq %rdi, %rax
2021-03-14 16:52:51 +00:00
L(start):
# ifdef __ILP32__
2021-03-14 20:22:12 +00:00
/* Clear the upper 32 bits. */
movl %edx, %edx
2021-03-14 16:52:51 +00:00
# endif
2021-03-14 20:22:12 +00:00
cmp $VEC_SIZE, %RDX_LP
jb L(less_vec)
cmp $(VEC_SIZE * 2), %RDX_LP
ja L(more_2x_vec)
2021-03-14 16:52:51 +00:00
#if !defined USE_MULTIARCH
L(last_2x_vec):
#endif
2021-03-14 20:22:12 +00:00
/* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */
VMOVU (%rsi), %VEC(0)
VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(1)
VMOVU %VEC(0), (%rdi)
VMOVU %VEC(1), -VEC_SIZE(%rdi,%rdx)
VZEROUPPER
2021-03-14 16:52:51 +00:00
#if !defined USE_MULTIARCH
L(nop):
#endif
2021-03-14 20:22:12 +00:00
ret
2021-03-14 16:52:51 +00:00
#if defined USE_MULTIARCH
END (MEMMOVE_SYMBOL (__memmove, unaligned))
# if VEC_SIZE == 16
ENTRY (__mempcpy_chk_erms)
2021-03-14 20:22:12 +00:00
cmp %RDX_LP, %RCX_LP
jb HIDDEN_JUMPTARGET (__chk_fail)
2021-03-14 16:52:51 +00:00
END (__mempcpy_chk_erms)
/* Only used to measure performance of REP MOVSB. */
ENTRY (__mempcpy_erms)
2021-03-14 20:22:12 +00:00
mov %RDI_LP, %RAX_LP
/* Skip zero length. */
test %RDX_LP, %RDX_LP
jz 2f
add %RDX_LP, %RAX_LP
jmp L(start_movsb)
2021-03-14 16:52:51 +00:00
END (__mempcpy_erms)
ENTRY (__memmove_chk_erms)
2021-03-14 20:22:12 +00:00
cmp %RDX_LP, %RCX_LP
jb HIDDEN_JUMPTARGET (__chk_fail)
2021-03-14 16:52:51 +00:00
END (__memmove_chk_erms)
ENTRY (__memmove_erms)
2021-03-14 20:22:12 +00:00
movq %rdi, %rax
/* Skip zero length. */
test %RDX_LP, %RDX_LP
jz 2f
2021-03-14 16:52:51 +00:00
L(start_movsb):
2021-03-14 20:22:12 +00:00
mov %RDX_LP, %RCX_LP
cmp %RSI_LP, %RDI_LP
jb 1f
/* Source == destination is less common. */
je 2f
lea (%rsi,%rcx), %RDX_LP
cmp %RDX_LP, %RDI_LP
jb L(movsb_backward)
2021-03-14 16:52:51 +00:00
1:
2021-03-14 20:22:12 +00:00
rep movsb
2021-03-14 16:52:51 +00:00
2:
2021-03-14 20:22:12 +00:00
ret
2021-03-14 16:52:51 +00:00
L(movsb_backward):
2021-03-14 20:22:12 +00:00
leaq -1(%rdi,%rcx), %rdi
leaq -1(%rsi,%rcx), %rsi
std
rep movsb
cld
ret
2021-03-14 16:52:51 +00:00
END (__memmove_erms)
strong_alias (__memmove_erms, __memcpy_erms)
strong_alias (__memmove_chk_erms, __memcpy_chk_erms)
# endif
# ifdef SHARED
ENTRY (MEMMOVE_CHK_SYMBOL (__mempcpy_chk, unaligned_erms))
2021-03-14 20:22:12 +00:00
cmp %RDX_LP, %RCX_LP
jb HIDDEN_JUMPTARGET (__chk_fail)
2021-03-14 16:52:51 +00:00
END (MEMMOVE_CHK_SYMBOL (__mempcpy_chk, unaligned_erms))
# endif
ENTRY (MEMMOVE_SYMBOL (__mempcpy, unaligned_erms))
2021-03-14 20:22:12 +00:00
mov %RDI_LP, %RAX_LP
add %RDX_LP, %RAX_LP
jmp L(start_erms)
2021-03-14 16:52:51 +00:00
END (MEMMOVE_SYMBOL (__mempcpy, unaligned_erms))
# ifdef SHARED
ENTRY (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned_erms))
2021-03-14 20:22:12 +00:00
cmp %RDX_LP, %RCX_LP
jb HIDDEN_JUMPTARGET (__chk_fail)
2021-03-14 16:52:51 +00:00
END (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned_erms))
# endif
ENTRY (MEMMOVE_SYMBOL (__memmove, unaligned_erms))
2021-03-14 20:22:12 +00:00
movq %rdi, %rax
2021-03-14 16:52:51 +00:00
L(start_erms):
# ifdef __ILP32__
2021-03-14 20:22:12 +00:00
/* Clear the upper 32 bits. */
movl %edx, %edx
2021-03-14 16:52:51 +00:00
# endif
2021-03-14 20:22:12 +00:00
cmp $VEC_SIZE, %RDX_LP
jb L(less_vec)
cmp $(VEC_SIZE * 2), %RDX_LP
ja L(movsb_more_2x_vec)
2021-03-14 16:52:51 +00:00
L(last_2x_vec):
2021-03-14 20:22:12 +00:00
/* From VEC and to 2 * VEC. No branch when size == VEC_SIZE. */
VMOVU (%rsi), %VEC(0)
VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(1)
VMOVU %VEC(0), (%rdi)
VMOVU %VEC(1), -VEC_SIZE(%rdi,%rdx)
2021-03-14 16:52:51 +00:00
L(return):
2021-03-14 20:22:12 +00:00
VZEROUPPER
ret
2021-03-14 16:52:51 +00:00
L(movsb):
2021-03-14 20:22:12 +00:00
cmp $SHARED_NON_TEMPORAL_THRESHOLD, %RDX_LP
jae L(more_8x_vec)
cmpq %rsi, %rdi
jb 1f
/* Source == destination is less common. */
je L(nop)
leaq (%rsi,%rdx), %r9
cmpq %r9, %rdi
/* Avoid slow backward REP MOVSB. */
jb L(more_8x_vec_backward)
2021-03-14 16:52:51 +00:00
1:
2021-03-14 20:22:12 +00:00
mov %RDX_LP, %RCX_LP
rep movsb
2021-03-14 16:52:51 +00:00
L(nop):
2021-03-14 20:22:12 +00:00
ret
2021-03-14 16:52:51 +00:00
#endif
L(less_vec):
2021-03-14 20:22:12 +00:00
/* Less than 1 VEC. */
2021-03-14 16:52:51 +00:00
#if VEC_SIZE != 16 && VEC_SIZE != 32 && VEC_SIZE != 64
# error Unsupported VEC_SIZE!
#endif
#if VEC_SIZE > 32
2021-03-14 20:22:12 +00:00
cmpb $32, %dl
jae L(between_32_63)
2021-03-14 16:52:51 +00:00
#endif
#if VEC_SIZE > 16
2021-03-14 20:22:12 +00:00
cmpb $16, %dl
jae L(between_16_31)
2021-03-14 16:52:51 +00:00
#endif
2021-03-14 20:22:12 +00:00
cmpb $8, %dl
jae L(between_8_15)
cmpb $4, %dl
jae L(between_4_7)
cmpb $1, %dl
ja L(between_2_3)
jb 1f
movzbl (%rsi), %ecx
movb %cl, (%rdi)
2021-03-14 16:52:51 +00:00
1:
2021-03-14 20:22:12 +00:00
ret
2021-03-14 16:52:51 +00:00
#if VEC_SIZE > 32
L(between_32_63):
2021-03-14 20:22:12 +00:00
/* From 32 to 63. No branch when size == 32. */
vmovdqu (%rsi), %ymm0
vmovdqu -32(%rsi,%rdx), %ymm1
vmovdqu %ymm0, (%rdi)
vmovdqu %ymm1, -32(%rdi,%rdx)
VZEROUPPER
ret
2021-03-14 16:52:51 +00:00
#endif
#if VEC_SIZE > 16
2021-03-14 20:22:12 +00:00
/* From 16 to 31. No branch when size == 16. */
2021-03-14 16:52:51 +00:00
L(between_16_31):
2021-03-14 20:22:12 +00:00
vmovdqu (%rsi), %xmm0
vmovdqu -16(%rsi,%rdx), %xmm1
vmovdqu %xmm0, (%rdi)
vmovdqu %xmm1, -16(%rdi,%rdx)
ret
2021-03-14 16:52:51 +00:00
#endif
L(between_8_15):
2021-03-14 20:22:12 +00:00
/* From 8 to 15. No branch when size == 8. */
movq -8(%rsi,%rdx), %rcx
movq (%rsi), %rsi
movq %rcx, -8(%rdi,%rdx)
movq %rsi, (%rdi)
ret
2021-03-14 16:52:51 +00:00
L(between_4_7):
2021-03-14 20:22:12 +00:00
/* From 4 to 7. No branch when size == 4. */
movl -4(%rsi,%rdx), %ecx
movl (%rsi), %esi
movl %ecx, -4(%rdi,%rdx)
movl %esi, (%rdi)
ret
2021-03-14 16:52:51 +00:00
L(between_2_3):
2021-03-14 20:22:12 +00:00
/* From 2 to 3. No branch when size == 2. */
movzwl -2(%rsi,%rdx), %ecx
movzwl (%rsi), %esi
movw %cx, -2(%rdi,%rdx)
movw %si, (%rdi)
ret
2021-03-14 16:52:51 +00:00
#if defined USE_MULTIARCH
L(movsb_more_2x_vec):
2021-03-14 20:22:12 +00:00
cmp $REP_MOSB_THRESHOLD, %RDX_LP
ja L(movsb)
2021-03-14 16:52:51 +00:00
#endif
L(more_2x_vec):
2021-03-14 20:22:12 +00:00
/* More than 2 * VEC and there may be overlap between destination
and source. */
cmpq $(VEC_SIZE * 8), %rdx
ja L(more_8x_vec)
cmpq $(VEC_SIZE * 4), %rdx
jb L(last_4x_vec)
/* Copy from 4 * VEC to 8 * VEC, inclusively. */
VMOVU (%rsi), %VEC(0)
VMOVU VEC_SIZE(%rsi), %VEC(1)
VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2)
VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3)
VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(4)
VMOVU -(VEC_SIZE * 2)(%rsi,%rdx), %VEC(5)
VMOVU -(VEC_SIZE * 3)(%rsi,%rdx), %VEC(6)
VMOVU -(VEC_SIZE * 4)(%rsi,%rdx), %VEC(7)
VMOVU %VEC(0), (%rdi)
VMOVU %VEC(1), VEC_SIZE(%rdi)
VMOVU %VEC(2), (VEC_SIZE * 2)(%rdi)
VMOVU %VEC(3), (VEC_SIZE * 3)(%rdi)
VMOVU %VEC(4), -VEC_SIZE(%rdi,%rdx)
VMOVU %VEC(5), -(VEC_SIZE * 2)(%rdi,%rdx)
VMOVU %VEC(6), -(VEC_SIZE * 3)(%rdi,%rdx)
VMOVU %VEC(7), -(VEC_SIZE * 4)(%rdi,%rdx)
VZEROUPPER
ret
2021-03-14 16:52:51 +00:00
L(last_4x_vec):
2021-03-14 20:22:12 +00:00
/* Copy from 2 * VEC to 4 * VEC. */
VMOVU (%rsi), %VEC(0)
VMOVU VEC_SIZE(%rsi), %VEC(1)
VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(2)
VMOVU -(VEC_SIZE * 2)(%rsi,%rdx), %VEC(3)
VMOVU %VEC(0), (%rdi)
VMOVU %VEC(1), VEC_SIZE(%rdi)
VMOVU %VEC(2), -VEC_SIZE(%rdi,%rdx)
VMOVU %VEC(3), -(VEC_SIZE * 2)(%rdi,%rdx)
VZEROUPPER
ret
2021-03-14 16:52:51 +00:00
L(more_8x_vec):
2021-03-14 20:22:12 +00:00
cmpq %rsi, %rdi
ja L(more_8x_vec_backward)
/* Source == destination is less common. */
je L(nop)
/* Load the first VEC and last 4 * VEC to support overlapping
addresses. */
VMOVU (%rsi), %VEC(4)
VMOVU -VEC_SIZE(%rsi, %rdx), %VEC(5)
VMOVU -(VEC_SIZE * 2)(%rsi, %rdx), %VEC(6)
VMOVU -(VEC_SIZE * 3)(%rsi, %rdx), %VEC(7)
VMOVU -(VEC_SIZE * 4)(%rsi, %rdx), %VEC(8)
/* Save start and stop of the destination buffer. */
movq %rdi, %r11
leaq -VEC_SIZE(%rdi, %rdx), %rcx
/* Align destination for aligned stores in the loop. Compute
how much destination is misaligned. */
movq %rdi, %r8
andq $(VEC_SIZE - 1), %r8
/* Get the negative of offset for alignment. */
subq $VEC_SIZE, %r8
/* Adjust source. */
subq %r8, %rsi
/* Adjust destination which should be aligned now. */
subq %r8, %rdi
/* Adjust length. */
addq %r8, %rdx
2021-03-14 16:52:51 +00:00
#if (defined USE_MULTIARCH || VEC_SIZE == 16)
2021-03-14 20:22:12 +00:00
/* Check non-temporal store threshold. */
cmp $SHARED_NON_TEMPORAL_THRESHOLD, %RDX_LP
ja L(large_forward)
2021-03-14 16:52:51 +00:00
#endif
L(loop_4x_vec_forward):
2021-03-14 20:22:12 +00:00
/* Copy 4 * VEC a time forward. */
VMOVU (%rsi), %VEC(0)
VMOVU VEC_SIZE(%rsi), %VEC(1)
VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2)
VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3)
addq $(VEC_SIZE * 4), %rsi
subq $(VEC_SIZE * 4), %rdx
VMOVA %VEC(0), (%rdi)
VMOVA %VEC(1), VEC_SIZE(%rdi)
VMOVA %VEC(2), (VEC_SIZE * 2)(%rdi)
VMOVA %VEC(3), (VEC_SIZE * 3)(%rdi)
addq $(VEC_SIZE * 4), %rdi
cmpq $(VEC_SIZE * 4), %rdx
ja L(loop_4x_vec_forward)
/* Store the last 4 * VEC. */
VMOVU %VEC(5), (%rcx)
VMOVU %VEC(6), -VEC_SIZE(%rcx)
VMOVU %VEC(7), -(VEC_SIZE * 2)(%rcx)
VMOVU %VEC(8), -(VEC_SIZE * 3)(%rcx)
/* Store the first VEC. */
VMOVU %VEC(4), (%r11)
VZEROUPPER
ret
2021-03-14 16:52:51 +00:00
L(more_8x_vec_backward):
2021-03-14 20:22:12 +00:00
/* Load the first 4 * VEC and last VEC to support overlapping
addresses. */
VMOVU (%rsi), %VEC(4)
VMOVU VEC_SIZE(%rsi), %VEC(5)
VMOVU (VEC_SIZE * 2)(%rsi), %VEC(6)
VMOVU (VEC_SIZE * 3)(%rsi), %VEC(7)
VMOVU -VEC_SIZE(%rsi,%rdx), %VEC(8)
/* Save stop of the destination buffer. */
leaq -VEC_SIZE(%rdi, %rdx), %r11
/* Align destination end for aligned stores in the loop. Compute
how much destination end is misaligned. */
leaq -VEC_SIZE(%rsi, %rdx), %rcx
movq %r11, %r9
movq %r11, %r8
andq $(VEC_SIZE - 1), %r8
/* Adjust source. */
subq %r8, %rcx
/* Adjust the end of destination which should be aligned now. */
subq %r8, %r9
/* Adjust length. */
subq %r8, %rdx
2021-03-14 16:52:51 +00:00
#if (defined USE_MULTIARCH || VEC_SIZE == 16)
2021-03-14 20:22:12 +00:00
/* Check non-temporal store threshold. */
cmp $SHARED_NON_TEMPORAL_THRESHOLD, %RDX_LP
ja L(large_backward)
2021-03-14 16:52:51 +00:00
#endif
L(loop_4x_vec_backward):
2021-03-14 20:22:12 +00:00
/* Copy 4 * VEC a time backward. */
VMOVU (%rcx), %VEC(0)
VMOVU -VEC_SIZE(%rcx), %VEC(1)
VMOVU -(VEC_SIZE * 2)(%rcx), %VEC(2)
VMOVU -(VEC_SIZE * 3)(%rcx), %VEC(3)
subq $(VEC_SIZE * 4), %rcx
subq $(VEC_SIZE * 4), %rdx
VMOVA %VEC(0), (%r9)
VMOVA %VEC(1), -VEC_SIZE(%r9)
VMOVA %VEC(2), -(VEC_SIZE * 2)(%r9)
VMOVA %VEC(3), -(VEC_SIZE * 3)(%r9)
subq $(VEC_SIZE * 4), %r9
cmpq $(VEC_SIZE * 4), %rdx
ja L(loop_4x_vec_backward)
/* Store the first 4 * VEC. */
VMOVU %VEC(4), (%rdi)
VMOVU %VEC(5), VEC_SIZE(%rdi)
VMOVU %VEC(6), (VEC_SIZE * 2)(%rdi)
VMOVU %VEC(7), (VEC_SIZE * 3)(%rdi)
/* Store the last VEC. */
VMOVU %VEC(8), (%r11)
VZEROUPPER
ret
2021-03-14 16:52:51 +00:00
#if (defined USE_MULTIARCH || VEC_SIZE == 16)
L(large_forward):
2021-03-14 20:22:12 +00:00
/* Don't use non-temporal store if there is overlap between
destination and source since destination may be in cache
when source is loaded. */
leaq (%rdi, %rdx), %r10
cmpq %r10, %rsi
jb L(loop_4x_vec_forward)
2021-03-14 16:52:51 +00:00
L(loop_large_forward):
2021-03-14 20:22:12 +00:00
/* Copy 4 * VEC a time forward with non-temporal stores. */
PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE * 2)
PREFETCH_ONE_SET (1, (%rsi), PREFETCHED_LOAD_SIZE * 3)
VMOVU (%rsi), %VEC(0)
VMOVU VEC_SIZE(%rsi), %VEC(1)
VMOVU (VEC_SIZE * 2)(%rsi), %VEC(2)
VMOVU (VEC_SIZE * 3)(%rsi), %VEC(3)
addq $PREFETCHED_LOAD_SIZE, %rsi
subq $PREFETCHED_LOAD_SIZE, %rdx
VMOVNT %VEC(0), (%rdi)
VMOVNT %VEC(1), VEC_SIZE(%rdi)
VMOVNT %VEC(2), (VEC_SIZE * 2)(%rdi)
VMOVNT %VEC(3), (VEC_SIZE * 3)(%rdi)
addq $PREFETCHED_LOAD_SIZE, %rdi
cmpq $PREFETCHED_LOAD_SIZE, %rdx
ja L(loop_large_forward)
sfence
/* Store the last 4 * VEC. */
VMOVU %VEC(5), (%rcx)
VMOVU %VEC(6), -VEC_SIZE(%rcx)
VMOVU %VEC(7), -(VEC_SIZE * 2)(%rcx)
VMOVU %VEC(8), -(VEC_SIZE * 3)(%rcx)
/* Store the first VEC. */
VMOVU %VEC(4), (%r11)
VZEROUPPER
ret
2021-03-14 16:52:51 +00:00
L(large_backward):
2021-03-14 20:22:12 +00:00
/* Don't use non-temporal store if there is overlap between
destination and source since destination may be in cache
when source is loaded. */
leaq (%rcx, %rdx), %r10
cmpq %r10, %r9
jb L(loop_4x_vec_backward)
2021-03-14 16:52:51 +00:00
L(loop_large_backward):
2021-03-14 20:22:12 +00:00
/* Copy 4 * VEC a time backward with non-temporal stores. */
PREFETCH_ONE_SET (-1, (%rcx), -PREFETCHED_LOAD_SIZE * 2)
PREFETCH_ONE_SET (-1, (%rcx), -PREFETCHED_LOAD_SIZE * 3)
VMOVU (%rcx), %VEC(0)
VMOVU -VEC_SIZE(%rcx), %VEC(1)
VMOVU -(VEC_SIZE * 2)(%rcx), %VEC(2)
VMOVU -(VEC_SIZE * 3)(%rcx), %VEC(3)
subq $PREFETCHED_LOAD_SIZE, %rcx
subq $PREFETCHED_LOAD_SIZE, %rdx
VMOVNT %VEC(0), (%r9)
VMOVNT %VEC(1), -VEC_SIZE(%r9)
VMOVNT %VEC(2), -(VEC_SIZE * 2)(%r9)
VMOVNT %VEC(3), -(VEC_SIZE * 3)(%r9)
subq $PREFETCHED_LOAD_SIZE, %r9
cmpq $PREFETCHED_LOAD_SIZE, %rdx
ja L(loop_large_backward)
sfence
/* Store the first 4 * VEC. */
VMOVU %VEC(4), (%rdi)
VMOVU %VEC(5), VEC_SIZE(%rdi)
VMOVU %VEC(6), (VEC_SIZE * 2)(%rdi)
VMOVU %VEC(7), (VEC_SIZE * 3)(%rdi)
/* Store the last VEC. */
VMOVU %VEC(8), (%r11)
VZEROUPPER
ret
2021-03-14 16:52:51 +00:00
#endif
END (MEMMOVE_SYMBOL (__memmove, unaligned_erms))
#if 1
# ifdef USE_MULTIARCH
strong_alias (MEMMOVE_SYMBOL (__memmove, unaligned_erms),
2021-03-14 20:22:12 +00:00
MEMMOVE_SYMBOL (__memcpy, unaligned_erms))
2021-03-14 16:52:51 +00:00
# ifdef SHARED
strong_alias (MEMMOVE_SYMBOL (__memmove_chk, unaligned_erms),
2021-03-14 20:22:12 +00:00
MEMMOVE_SYMBOL (__memcpy_chk, unaligned_erms))
2021-03-14 16:52:51 +00:00
# endif
# endif
# ifdef SHARED
strong_alias (MEMMOVE_CHK_SYMBOL (__memmove_chk, unaligned),
2021-03-14 20:22:12 +00:00
MEMMOVE_CHK_SYMBOL (__memcpy_chk, unaligned))
2021-03-14 16:52:51 +00:00
# endif
#endif
strong_alias (MEMMOVE_SYMBOL (__memmove, unaligned),
2021-03-14 20:22:12 +00:00
MEMCPY_SYMBOL (__memcpy, unaligned))