diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index f24ba444203..8149623ffce 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -307,6 +308,11 @@ int Server::main(const std::vector & /*args*/) { if (config().getBool("mlock_executable", false)) { + LOG_DEBUG(log, "Will remap executable in memory."); + remapExecutable(); + LOG_DEBUG(log, "The code in memory has been successfully remapped."); + +/* if (hasLinuxCapability(CAP_IPC_LOCK)) { LOG_TRACE(log, "Will mlockall to prevent executable memory from being paged out. It may take a few seconds."); @@ -321,7 +327,7 @@ int Server::main(const std::vector & /*args*/) " It could happen due to incorrect ClickHouse package installation." " You could resolve the problem manually with 'sudo setcap cap_ipc_lock=+ep {}'." " Note that it will not work on 'nosuid' mounted filesystems.", executable_path); - } + }*/ } } #endif diff --git a/src/Common/remapExecutable.cpp b/src/Common/remapExecutable.cpp new file mode 100644 index 00000000000..f7f353a83c6 --- /dev/null +++ b/src/Common/remapExecutable.cpp @@ -0,0 +1,213 @@ +#if defined(__linux__) && defined(__amd64__) && defined(__SSE2__) + +#include +#include +#include + +#include + +#include + +#include +#include +#include +#include +#include + +#include "remapExecutable.h" + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int CANNOT_ALLOCATE_MEMORY; +} + + +namespace +{ + +uintptr_t readAddressHex(DB::ReadBuffer & in) +{ + uintptr_t res = 0; + while (!in.eof()) + { + if (isHexDigit(*in.position())) + { + res *= 16; + res += unhex(*in.position()); + ++in.position(); + } + else + break; + } + return res; +} + + +/** Find the address and size of the mapped memory region pointed by ptr. + */ +std::pair getMappedArea(void * ptr) +{ + using namespace DB; + + uintptr_t uintptr = reinterpret_cast(ptr); + ReadBufferFromFile in("/proc/self/maps"); + + while (!in.eof()) + { + uintptr_t begin = readAddressHex(in); + assertChar('-', in); + uintptr_t end = readAddressHex(in); + skipToNextLineOrEOF(in); + + if (begin <= uintptr && uintptr < end) + return {reinterpret_cast(begin), end - begin}; + } + + throw Exception("Cannot find mapped area for pointer", ErrorCodes::LOGICAL_ERROR); +} + + +__attribute__((__noinline__)) int64_t our_syscall(...) +{ + __asm__ __volatile__ (R"( + movq %%rdi,%%rax; + movq %%rsi,%%rdi; + movq %%rdx,%%rsi; + movq %%rcx,%%rdx; + movq %%r8,%%r10; + movq %%r9,%%r8; + movq 8(%%rsp),%%r9; + syscall; + ret + )" : : : "memory"); + return 0; +} + + +__attribute__((__always_inline__)) void our_memcpy(char * __restrict dst, const char * __restrict src, ssize_t n) +{ + while (n > 0) + { + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), + _mm_loadu_si128(reinterpret_cast(src))); + + dst += 16; + src += 16; + n -= 16; + } +} + + +__attribute__((__noinline__)) void remapToHugeStep3(void * scratch, size_t size, size_t offset) +{ + /// The function should not use the stack, otherwise various optimizations, including "omit-frame-pointer" may break the code. + + /// Unmap the scratch area. + our_syscall(SYS_munmap, scratch, size); + + /** The return address of this function is pointing to scratch area (because it was called from there). + * But the scratch area no longer exists. We should correct the return address by subtracting the offset. + */ + __asm__ __volatile__("subq %0, 8(%%rsp)" : : "r"(offset) : "memory"); +} + + +__attribute__((__noinline__)) void remapToHugeStep2(void * begin, size_t size, void * scratch) +{ + /** Unmap old memory region with the code of our program. + * Our instruction pointer is located inside scratch area and this function can execute after old code is unmapped. + * But it cannot call any other functions because they are not available at usual addresses + * - that's why we have to use "our_syscall" and "our_memcpy" functions. + * (Relative addressing may continue to work but we should not assume that). + */ + + int64_t offset = reinterpret_cast(scratch) - reinterpret_cast(begin); + int64_t (*syscall_func)(...) = reinterpret_cast(reinterpret_cast(our_syscall) + offset); + + //char dot = '.'; + //syscall_func(SYS_write, 2, &dot, 1); + + int64_t munmap_res = syscall_func(SYS_munmap, begin, size); + if (munmap_res != 0) + return; + + //syscall_func(SYS_write, 2, &dot, 1); + + /// Map new anonymous memory region in place of old region with code. + + int64_t mmap_res = syscall_func(SYS_mmap, begin, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (-1 == mmap_res) + syscall_func(SYS_exit, 1); + //syscall_func(SYS_write, 2, &dot, 1); + + /// As the memory region is anonymous, we can do madvise with MADV_HUGEPAGE. + + syscall_func(SYS_madvise, begin, size, MADV_HUGEPAGE); + //syscall_func(SYS_write, 2, &dot, 1); + + /// Copy the code from scratch area to the old memory location. + + our_memcpy(reinterpret_cast(begin), reinterpret_cast(scratch), size); + //syscall_func(SYS_write, 2, &dot, 1); + + /// Make the memory area with the code executable and non-writable. + + syscall_func(SYS_mprotect, begin, size, PROT_READ | PROT_EXEC); + //syscall_func(SYS_write, 2, &dot, 1); + + /** Step 3 function should unmap the scratch area. + * The currently executed code is located in the scratch area and cannot be removed here. + * We have to call another function and use its address from the original location (not in scratch area). + * To do it, we obtain it's pointer and call by pointer. + */ + + void(* volatile step3)(void*, size_t, size_t) = remapToHugeStep3; + step3(scratch, size, offset); +} + + +__attribute__((__noinline__)) void remapToHugeStep1(void * begin, size_t size) +{ + /// Allocate scratch area and copy the code there. + + void * scratch = mmap(nullptr, size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (MAP_FAILED == scratch) + throwFromErrno(fmt::format("Cannot mmap {} bytes", size), ErrorCodes::CANNOT_ALLOCATE_MEMORY); + + memcpy(scratch, begin, size); + + /// Offset to the scratch area from previous location. + + int64_t offset = reinterpret_cast(scratch) - reinterpret_cast(begin); + + /// Jump to the next function inside the scratch area. + + reinterpret_cast(reinterpret_cast(remapToHugeStep2) + offset)(begin, size, scratch); +} + +} + + +void remapExecutable() +{ + auto [begin, size] = getMappedArea(reinterpret_cast(remapExecutable)); + remapToHugeStep1(begin, size); +} + +} + +#else + +namespace DB +{ + +void remapExecutable() {} + +} + +#endif diff --git a/src/Common/remapExecutable.h b/src/Common/remapExecutable.h new file mode 100644 index 00000000000..7acb61f13bd --- /dev/null +++ b/src/Common/remapExecutable.h @@ -0,0 +1,7 @@ +namespace DB +{ + +/// This function tries to reallocate the code of the running program in a more efficient way. +void remapExecutable(); + +}