avx512: tweak zmm16-zmm31 register clearing

* cipher/asm-common-amd64.h (spec_stop_avx512): Clear ymm16
before and after vpopcntb.
* cipher/camellia-gfni-avx512-amd64.S (clear_zmm16_zmm31): Clear
YMM16-YMM31 registers instead of XMM16-XMM31.
* cipher/chacha20-amd64-avx512.S (clear_zmm16_zmm31): Likewise.
* cipher/keccak-amd64-avx512.S (clear_regs): Likewise.
(clear_avx512_4regs): Clear all 4 registers with XOR.
* cipher/cipher-gcm-intel-pclmul.c (_gcry_ghash_intel_pclmul)
(_gcry_polyval_intel_pclmul): Clear YMM16-YMM19 registers instead of
ZMM16-ZMM19.
* cipher/poly1305-amd64-avx512.S (POLY1305_BLOCKS): Clear YMM16-YMM31
registers after vector processing instead of XMM16-XMM31.
* cipher/sha512-avx512-amd64.S
(_gcry_sha512_transform_amd64_avx512): Likewise.
--

Clear zmm16-zmm31 registers with 256bit XOR instead of 128bit
as this is better for AMD Zen4. Also clear xmm16 register
after vpopcnt in avx512 spec-stop so we do not leave any zmm
register state which might end up unnecessarily using CPU
resources.

Signed-off-by: Jussi Kivilinna <jussi.kivilinna@iki.fi>

7 files changed, 39 insertions, 37 deletions

diff --git a/cipher/asm-common-amd64.h b/cipher/asm-common-amd64.h
index cd93abc3..d9bbc01b 100644
--- a/cipher/asm-common-amd64.h
+++ b/cipher/asm-common-amd64.h
@@ -195,11 +195,13 @@
  * available on newer CPUs that do not suffer from significant frequency
  * drop when 512-bit vectors are utilized. */
 #define spec_stop_avx512 \
-	vpxord %xmm16, %xmm16, %xmm16; \
-	vpopcntb %xmm16, %xmm16; /* Supported only by newer AVX512 CPUs. */
+	vpxord %ymm16, %ymm16, %ymm16; \
+	vpopcntb %xmm16, %xmm16; /* Supported only by newer AVX512 CPUs. */ \
+	vpxord %ymm16, %ymm16, %ymm16;
 
 #define spec_stop_avx512_intel_syntax \
-	vpxord xmm16, xmm16, xmm16; \
-	vpopcntb xmm16, xmm16; /* Supported only by newer AVX512 CPUs. */
+	vpxord ymm16, ymm16, ymm16; \
+	vpopcntb xmm16, xmm16; /* Supported only by newer AVX512 CPUs. */ \
+	vpxord ymm16, ymm16, ymm16;
 
 #endif /* GCRY_ASM_COMMON_AMD64_H */
diff --git a/cipher/camellia-gfni-avx512-amd64.S b/cipher/camellia-gfni-avx512-amd64.S
index bddad804..14725b4a 100644
--- a/cipher/camellia-gfni-avx512-amd64.S
+++ b/cipher/camellia-gfni-avx512-amd64.S
@@ -99,10 +99,10 @@
 	vpxord v3, v3, v3
 
 #define clear_zmm16_zmm31() \
-	clear_vec4(%xmm16, %xmm20, %xmm24, %xmm28); \
-	clear_vec4(%xmm17, %xmm21, %xmm25, %xmm29); \
-	clear_vec4(%xmm18, %xmm22, %xmm26, %xmm30); \
-	clear_vec4(%xmm19, %xmm23, %xmm27, %xmm31)
+	clear_vec4(%ymm16, %ymm20, %ymm24, %ymm28); \
+	clear_vec4(%ymm17, %ymm21, %ymm25, %ymm29); \
+	clear_vec4(%ymm18, %ymm22, %ymm26, %ymm30); \
+	clear_vec4(%ymm19, %ymm23, %ymm27, %ymm31)
 
 #define clear_regs() \
 	kxorq %k1, %k1, %k1; \
diff --git a/cipher/chacha20-amd64-avx512.S b/cipher/chacha20-amd64-avx512.S
index 544e7cdc..4b183528 100644
--- a/cipher/chacha20-amd64-avx512.S
+++ b/cipher/chacha20-amd64-avx512.S
@@ -205,10 +205,10 @@
 	vpxord v3, v3, v3;
 
 #define clear_zmm16_zmm31() \
-	clear_vec4(%xmm16, %xmm20, %xmm24, %xmm28); \
-	clear_vec4(%xmm17, %xmm21, %xmm25, %xmm29); \
-	clear_vec4(%xmm18, %xmm22, %xmm26, %xmm30); \
-	clear_vec4(%xmm19, %xmm23, %xmm27, %xmm31);
+	clear_vec4(%ymm16, %ymm20, %ymm24, %ymm28); \
+	clear_vec4(%ymm17, %ymm21, %ymm25, %ymm29); \
+	clear_vec4(%ymm18, %ymm22, %ymm26, %ymm30); \
+	clear_vec4(%ymm19, %ymm23, %ymm27, %ymm31);
 
 /**********************************************************************
   16-way (zmm), 8-way (ymm), 4-way (xmm) chacha20
diff --git a/cipher/cipher-gcm-intel-pclmul.c b/cipher/cipher-gcm-intel-pclmul.c
index ec00df09..391cbe6f 100644
--- a/cipher/cipher-gcm-intel-pclmul.c
+++ b/cipher/cipher-gcm-intel-pclmul.c
@@ -1560,10 +1560,10 @@ _gcry_ghash_intel_pclmul (gcry_cipher_hd_t c, byte *result, const byte *buf,
 	    }
 
 	  asm volatile ("vmovdqa %%xmm15, %%xmm7\n\t"
-			"vpxorq %%zmm16, %%zmm16, %%zmm16\n\t"
-			"vpxorq %%zmm17, %%zmm17, %%zmm17\n\t"
-			"vpxorq %%zmm18, %%zmm18, %%zmm18\n\t"
-			"vpxorq %%zmm19, %%zmm19, %%zmm19\n\t"
+			"vpxorq %%ymm16, %%ymm16, %%ymm16\n\t"
+			"vpxorq %%ymm17, %%ymm17, %%ymm17\n\t"
+			"vpxorq %%ymm18, %%ymm18, %%ymm18\n\t"
+			"vpxorq %%ymm19, %%ymm19, %%ymm19\n\t"
 			:
 			:
 			: "memory" );
@@ -1838,15 +1838,15 @@ _gcry_polyval_intel_pclmul (gcry_cipher_hd_t c, byte *result, const byte *buf,
 	    }
 
 	  asm volatile ("vpxor %%xmm7, %%xmm7, %%xmm7\n\t"
-			"vpxorq %%zmm16, %%zmm16, %%zmm16\n\t"
-			"vpxorq %%zmm17, %%zmm17, %%zmm17\n\t"
-			"vpxorq %%zmm18, %%zmm18, %%zmm18\n\t"
-			"vpxorq %%zmm19, %%zmm19, %%zmm19\n\t"
+			"vpxorq %%ymm16, %%ymm16, %%ymm16\n\t"
+			"vpxorq %%ymm17, %%ymm17, %%ymm17\n\t"
+			"vpxorq %%ymm18, %%ymm18, %%ymm18\n\t"
+			"vpxorq %%ymm19, %%ymm19, %%ymm19\n\t"
 			:
 			:
 			: "memory" );
 	}
-#endif
+#endif /* GCM_USE_INTEL_VPCLMUL_AVX512 */
 
       if (nblocks >= 16)
 	{
diff --git a/cipher/keccak-amd64-avx512.S b/cipher/keccak-amd64-avx512.S
index 58b4150f..b1fc7b64 100644
--- a/cipher/keccak-amd64-avx512.S
+++ b/cipher/keccak-amd64-avx512.S
@@ -160,14 +160,14 @@
 
 /* Misc helper macros. */
 #define clear_avx512_4regs(a, b, c, d) \
-	eor(a, a, a); vmovdqa64 a, b; vmovdqa64 a, c; vmovdqa64 a, d;
+	eor(a, a, a); eor(b, b, b); eor(c, c, c); eor(d, d, d);
 
 #define clear_regs() \
 	vzeroall; /* xmm0-xmm15 */ \
-	clear_avx512_4regs(%xmm16, %xmm17, %xmm18, %xmm19); \
-	clear_avx512_4regs(%xmm20, %xmm21, %xmm22, %xmm23); \
-	clear_avx512_4regs(%xmm24, %xmm25, %xmm26, %xmm27); \
-	clear_avx512_4regs(%xmm28, %xmm29, %xmm30, %xmm31);
+	clear_avx512_4regs(%ymm16, %ymm17, %ymm18, %ymm19); \
+	clear_avx512_4regs(%ymm20, %ymm21, %ymm22, %ymm23); \
+	clear_avx512_4regs(%ymm24, %ymm25, %ymm26, %ymm27); \
+	clear_avx512_4regs(%ymm28, %ymm29, %ymm30, %ymm31);
 
 ELF(.type	KeccakF1600_ce,@function)
 .align	64, 0xcc
diff --git a/cipher/poly1305-amd64-avx512.S b/cipher/poly1305-amd64-avx512.S
index 6622861f..9beed8ad 100644
--- a/cipher/poly1305-amd64-avx512.S
+++ b/cipher/poly1305-amd64-avx512.S
@@ -1425,10 +1425,10 @@ ELF(.size _gcry_poly1305_avx512_consts,.-_gcry_poly1305_avx512_consts)
 	vmovdqa64 [rsp + STACK_r_save + 64*5], zmm0; \
 	\
 	vzeroall; \
-	clear_zmm(xmm16); clear_zmm(xmm20); clear_zmm(xmm24); clear_zmm(xmm28); \
-	clear_zmm(xmm17); clear_zmm(xmm21); clear_zmm(xmm25); clear_zmm(xmm29); \
-	clear_zmm(xmm18); clear_zmm(xmm22); clear_zmm(xmm26); clear_zmm(xmm30); \
-	clear_zmm(xmm19); clear_zmm(xmm23); clear_zmm(xmm27); clear_zmm(xmm31); \
+	clear_zmm(ymm16); clear_zmm(ymm20); clear_zmm(ymm24); clear_zmm(ymm28); \
+	clear_zmm(ymm17); clear_zmm(ymm21); clear_zmm(ymm25); clear_zmm(ymm29); \
+	clear_zmm(ymm18); clear_zmm(ymm22); clear_zmm(ymm26); clear_zmm(ymm30); \
+	clear_zmm(ymm19); clear_zmm(ymm23); clear_zmm(ymm27); clear_zmm(ymm31); \
 	\
 .L_final_loop: \
 	cmp     LEN, POLY1305_BLOCK_SIZE; \
diff --git a/cipher/sha512-avx512-amd64.S b/cipher/sha512-avx512-amd64.S
index 65475422..431fb3e9 100644
--- a/cipher/sha512-avx512-amd64.S
+++ b/cipher/sha512-avx512-amd64.S
@@ -384,13 +384,13 @@ _gcry_sha512_transform_amd64_avx512:
 	vmovdqa	[rsp + frame_XFER + 1*32], ymm0 /* burn stack */
 	vmovdqa	[rsp + frame_XFER + 2*32], ymm0 /* burn stack */
 	vmovdqa	[rsp + frame_XFER + 3*32], ymm0 /* burn stack */
-	clear_reg(xmm16);
-	clear_reg(xmm17);
-	clear_reg(xmm18);
-	clear_reg(xmm19);
-	clear_reg(xmm20);
-	clear_reg(xmm21);
-	clear_reg(xmm22);
+	clear_reg(ymm16);
+	clear_reg(ymm17);
+	clear_reg(ymm18);
+	clear_reg(ymm19);
+	clear_reg(ymm20);
+	clear_reg(ymm21);
+	clear_reg(ymm22);
 
 	/* Restore Stack Pointer */
 	mov	rsp, RSP_SAVE