diff --git a/cmake/limit_jobs.cmake b/cmake/limit_jobs.cmake
index 17d8dd42a2c..8e48fc9b9d8 100644
--- a/cmake/limit_jobs.cmake
+++ b/cmake/limit_jobs.cmake
@@ -42,19 +42,9 @@ endif ()
 # But use 2 parallel jobs, since:
 # - this is what llvm does
 # - and I've verfied that lld-11 does not use all available CPU time (in peak) while linking one binary
-if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND ENABLE_THINLTO)
-    if (ARCH_AARCH64)
-        # aarch64 builds start to often fail with OOMs (reason not yet clear), for now let's limit the concurrency
-        message(STATUS "ThinLTO provides its own parallel linking - limiting parallel link jobs to 1.")
-        set (PARALLEL_LINK_JOBS 1)
-        if (LINKER_NAME MATCHES "lld")
-            math(EXPR LTO_JOBS ${NUMBER_OF_LOGICAL_CORES}/4)
-            set (CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_EXE_LINKER_FLAGS_RELWITHDEBINFO} -Wl,--thinlto-jobs=${LTO_JOBS}")
-        endif()
-    elseif (PARALLEL_LINK_JOBS GREATER 2)
-        message(STATUS "ThinLTO provides its own parallel linking - limiting parallel link jobs to 2.")
-        set (PARALLEL_LINK_JOBS 2)
-    endif ()
+if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND ENABLE_THINLTO AND PARALLEL_LINK_JOBS GREATER 2)
+    message(STATUS "ThinLTO provides its own parallel linking - limiting parallel link jobs to 2.")
+    set (PARALLEL_LINK_JOBS 2)
 endif()
 
 message(STATUS "Building sub-tree with ${PARALLEL_COMPILE_JOBS} compile jobs and ${PARALLEL_LINK_JOBS} linker jobs (system: ${NUMBER_OF_LOGICAL_CORES} cores, ${TOTAL_PHYSICAL_MEMORY} MB RAM, 'OFF' means the native core count).")