Merge pull request #3820 from aa022/dev

ARM64 RandomX JIT: dataset prefetch + non-temporal loads (+~8% on M4 base)

src/crypto/common/VirtualMemory_unix.cpp

@@ -74,6 +74,11 @@
 #endif
 
 
+#ifndef MADV_COLLAPSE
+#   define MADV_COLLAPSE 25
+#endif
+
+
 #if defined(XMRIG_OS_LINUX) || (!defined(XMRIG_OS_APPLE) && !defined(XMRIG_OS_FREEBSD))
 static inline int hugePagesFlag(size_t size)
 {
@@ -278,8 +283,9 @@ bool xmrig::VirtualMemory::allocateOneGbPagesMemory()
 
 bool xmrig::VirtualMemory::adviseLargePages(void *p, size_t size)
 {
-#   ifdef XMRIG_OS_LINUX
-    return (madvise(p, size, MADV_HUGEPAGE) == 0);
+#   if defined(XMRIG_OS_ANDROID) || defined(XMRIG_OS_LINUX)
+    // MADV_COLLAPSE works even if /sys/kernel/mm/transparent_hugepage/enabled is set to "never", but only on Linux 6.1+
+    return (madvise(p, size, MADV_COLLAPSE) == 0) || (madvise(p, size, MADV_HUGEPAGE) == 0);
 #   else
     return false;
 #   endif

src/crypto/randomx/jit_compiler_a64.cpp

@@ -141,7 +141,7 @@ void JitCompilerA64::generateProgram(Program& program, ProgramConfiguration& con
 
 	codePos = PrologueSize;
 	literalPos = ImulRcpLiteralsEnd;
-	num32bitLiterals = 0;
+	num32bitLiterals = 64; // effectively disabled because it's slower than plain movn/movz+movk
 
 	for (uint32_t i = 0; i < RegistersCount; ++i)
 		reg_changed_offset[i] = codePos;
@@ -237,7 +237,7 @@ void JitCompilerA64::generateProgramLight(Program& program, ProgramConfiguration
 
 	codePos = PrologueSize;
 	literalPos = ImulRcpLiteralsEnd;
-	num32bitLiterals = 0;
+	num32bitLiterals = 64; // effectively disabled because it's slower than plain movn/movz+movk
 
 	for (uint32_t i = 0; i < RegistersCount; ++i)
 		reg_changed_offset[i] = codePos;
@@ -488,13 +488,31 @@ void JitCompilerA64::emitMovImmediate(uint32_t dst, uint32_t imm, uint8_t* code,
 {
 	uint32_t k = codePos;
 
+	// 196606 different values can be encoded with a single instruction, the rest requires smov/umov load, or movn/movz+movk pair
 	if (imm < (1 << 16))
 	{
+		// Sign-extended 64-bit value: 0x000000000000xxxx
 		// movz tmp_reg, imm32 (16 low bits)
 		emit32(ARMV8A::MOVZ | dst | (imm << 5), code, k);
 	}
+	else if ((imm >> 16) == 0xFFFF) {
+		// Sign-extended 64-bit value: 0xFFFFFFFFFFFFxxxx
+		// movn tmp_reg, ~imm32 (16 low bits)
+		emit32(ARMV8A::MOVN | dst | ((~imm & 0xFFFF) << 5), code, k);
+	}
+	else if (((imm & 0xFFFF) == 0xFFFF) && (static_cast<int32_t>(imm) < 0)) {
+		// Sign-extended 64-bit value: 0xFFFFFFFFxxxxFFFF
+		// movn tmp_reg, ~imm32 (16 high bits)
+		emit32(ARMV8A::MOVN | dst | (1 << 21) | ((~imm >> 16) << 5), code, k);
+	}
+	else if (((imm & 0xFFFF) == 0) && (static_cast<int32_t>(imm) >= 0)) {
+		// Sign-extended 64-bit value: 0x00000000xxxx0000
+		// movz tmp_reg, imm32 (16 high bits)
+		emit32(ARMV8A::MOVZ | dst | (1 << 21) | ((imm >> 16) << 5), code, k);
+	}
 	else
 	{
+		// Full sign-extended 64-bit value: 0x00000000xxxxxxxx or 0xFFFFFFFFxxxxxxxx
 		if (num32bitLiterals < 64)
 		{
 			if (static_cast<int32_t>(imm) < 0)
@@ -611,18 +629,17 @@ void JitCompilerA64::emitMemLoad(uint32_t dst, uint32_t src, Instruction& instr,
 	else
 	{
 		imm = (imm & ScratchpadL3Mask) >> 3;
-		if (imm)
+		if (imm < 4096) {
+			// ldr tmp_reg, [x2, #imm*8]
+			emit32(0xf9400040 | tmp_reg | (imm << 10), code, k);
+		}
+		else
 		{
 			emitMovImmediate(tmp_reg, imm, code, k);
 
 			// ldr tmp_reg, [x2, tmp_reg, lsl 3]
 			emit32(0xf8607840 | tmp_reg | (tmp_reg << 16), code, k);
 		}
-		else
-		{
-			// ldr tmp_reg, [x2]
-			emit32(0xf9400040 | tmp_reg, code, k);
-		}
 	}
 
 	codePos = k;

src/crypto/randomx/jit_compiler_a64_static.S

@@ -303,7 +303,7 @@ DECL(randomx_program_aarch64_cacheline_align_mask1):
 	add	x20, x20, x1
 
 	# Prefetch dataset data
-	prfm	pldl2strm, [x20]
+	prfm	pldl1strm, [x20]
 
 DECL(randomx_program_aarch64_cacheline_align_mask2):
 	# Actual mask will be inserted by JIT compiler
@@ -312,16 +312,16 @@ DECL(randomx_program_aarch64_cacheline_align_mask2):
 
 DECL(randomx_program_aarch64_xor_with_dataset_line):
 	# xor integer registers with dataset data
-	ldp	x20, x19, [x10]
+	ldnp	x20, x19, [x10]
 	eor	x4, x4, x20
 	eor	x5, x5, x19
-	ldp	x20, x19, [x10, 16]
+	ldnp	x20, x19, [x10, 16]
 	eor	x6, x6, x20
 	eor	x7, x7, x19
-	ldp	x20, x19, [x10, 32]
+	ldnp	x20, x19, [x10, 32]
 	eor	x12, x12, x20
 	eor	x13, x13, x19
-	ldp	x20, x19, [x10, 48]
+	ldnp	x20, x19, [x10, 48]
 	eor	x14, x14, x20
 	eor	x15, x15, x19