Add straight-line speculation hardening for amd64 and i386 assembly

* cipher/asm-common-amd64.h (ret_spec_stop): New.
* cipher/arcfour-amd64.S: Use 'ret_spec_stop' for 'ret' instruction.
* cipher/blake2b-amd64-avx2.S: Likewise.
* cipher/blake2s-amd64-avx.S: Likewise.
* cipher/blowfish-amd64.S: Likewise.
* cipher/camellia-aesni-avx-amd64.S: Likewise.
* cipher/camellia-aesni-avx2-amd64.h: Likewise.
* cipher/cast5-amd64.S: Likewise.
* cipher/chacha20-amd64-avx2.S: Likewise.
* cipher/chacha20-amd64-ssse3.S: Likewise.
* cipher/des-amd64.S: Likewise.
* cipher/rijndael-aarch64.S: Likewise.
* cipher/rijndael-amd64.S: Likewise.
* cipher/rijndael-ssse3-amd64-asm.S: Likewise.
* cipher/rijndael-vaes-avx2-amd64.S: Likewise.
* cipher/salsa20-amd64.S: Likewise.
* cipher/serpent-avx2-amd64.S: Likewise.
* cipher/serpent-sse2-amd64.S: Likewise.
* cipher/sha1-avx-amd64.S: Likewise.
* cipher/sha1-avx-bmi2-amd64.S: Likewise.
* cipher/sha1-avx2-bmi2-amd64.S: Likewise.
* cipher/sha1-ssse3-amd64.S: Likewise.
* cipher/sha256-avx-amd64.S: Likewise.
* cipher/sha256-avx2-bmi2-amd64.S: Likewise.
* cipher/sha256-ssse3-amd64.S: Likewise.
* cipher/sha512-avx-amd64.S: Likewise.
* cipher/sha512-avx2-bmi2-amd64.S: Likewise.
* cipher/sha512-ssse3-amd64.S: Likewise.
* cipher/sm3-avx-bmi2-amd64.S: Likewise.
* cipher/sm4-aesni-avx-amd64.S: Likewise.
* cipher/sm4-aesni-avx2-amd64.S: Likewise.
* cipher/twofish-amd64.S: Likewise.
* cipher/twofish-avx2-amd64.S: Likewise.
* cipher/whirlpool-sse2-amd64.S: Likewise.
* mpi/amd64/func_abi.h (CFI_*): Remove, include from "asm-common-amd64.h"
instead.
(FUNC_EXIT): Use 'ret_spec_stop' for 'ret' instruction.
* mpi/asm-common-amd64.h: New.
* mpi/i386/mpih-add1.S: Use 'ret_spec_stop' for 'ret' instruction.
* mpi/i386/mpih-lshift.S: Likewise.
* mpi/i386/mpih-mul1.S: Likewise.
* mpi/i386/mpih-mul2.S: Likewise.
* mpi/i386/mpih-mul3.S: Likewise.
* mpi/i386/mpih-rshift.S: Likewise.
* mpi/i386/mpih-sub1.S: Likewise.
* mpi/i386/syntax.h (ret_spec_stop): New.
--

Signed-off-by: Jussi Kivilinna <jussi.kivilinna@iki.fi>

10 files changed, 42 insertions, 34 deletions

diff --git a/mpi/amd64/func_abi.h b/mpi/amd64/func_abi.h
index a60363e4..c3f2d026 100644
--- a/mpi/amd64/func_abi.h
+++ b/mpi/amd64/func_abi.h
@@ -1,28 +1,6 @@
 #include <config.h>
 
-#ifdef __x86_64__
-#ifdef HAVE_GCC_ASM_CFI_DIRECTIVES
-# define CFI_STARTPROC()            .cfi_startproc
-# define CFI_ENDPROC()              .cfi_endproc
-# define CFI_ADJUST_CFA_OFFSET(off) .cfi_adjust_cfa_offset off
-# define CFI_REL_OFFSET(reg,off)    .cfi_rel_offset reg, off
-# define CFI_RESTORE(reg)           .cfi_restore reg
-
-# define CFI_PUSH(reg) \
-	CFI_ADJUST_CFA_OFFSET(8); CFI_REL_OFFSET(reg, 0)
-# define CFI_POP(reg) \
-	CFI_ADJUST_CFA_OFFSET(-8); CFI_RESTORE(reg)
-#else
-# define CFI_STARTPROC()
-# define CFI_ENDPROC()
-# define CFI_ADJUST_CFA_OFFSET(off)
-# define CFI_REL_OFFSET(reg,off)
-# define CFI_RESTORE(reg)
-
-# define CFI_PUSH(reg)
-# define CFI_POP(reg)
-#endif
-#endif
+#include "asm-common-amd64.h"
 
 #ifdef USE_MS_ABI
  /* Store registers and move four first input arguments from MS ABI to
@@ -44,13 +22,13 @@
 	CFI_POP(%rdi); \
 	popq %rsi; \
 	CFI_POP(%rsi); \
-	ret; \
+	ret_spec_stop; \
 	CFI_ENDPROC();
 #else
  #define FUNC_ENTRY() \
 	CFI_STARTPROC();
 
  #define FUNC_EXIT() \
-	ret; \
+	ret_spec_stop; \
 	CFI_ENDPROC();
 #endif
diff --git a/mpi/asm-common-amd64.h b/mpi/asm-common-amd64.h
new file mode 100644
index 00000000..ad0e8e62
--- /dev/null
+++ b/mpi/asm-common-amd64.h
@@ -0,0 +1,26 @@
+/* asm-common-amd64.h  -  Common macros for AMD64 assembly
+ *
+ * Copyright (C) 2022 Jussi Kivilinna <jussi.kivilinna@iki.fi>
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Libgcrypt is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * Libgcrypt is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef MPI_ASM_COMMON_AMD64_H
+#define MPI_ASM_COMMON_AMD64_H
+
+#include "../cipher/asm-common-amd64.h"
+
+#endif /* MPI_ASM_COMMON_AMD64_H */
diff --git a/mpi/i386/mpih-add1.S b/mpi/i386/mpih-add1.S
index de78a0cb..95a75890 100644
--- a/mpi/i386/mpih-add1.S
+++ b/mpi/i386/mpih-add1.S
@@ -156,6 +156,6 @@ Loop:	movl	(%esi),%eax
 	CFI_POP(%esi)
 	popl %edi
 	CFI_POP(%edi)
-	ret
+	ret_spec_stop
 	CFI_ENDPROC()
 
diff --git a/mpi/i386/mpih-lshift.S b/mpi/i386/mpih-lshift.S
index 55da0678..3404cf55 100644
--- a/mpi/i386/mpih-lshift.S
+++ b/mpi/i386/mpih-lshift.S
@@ -86,7 +86,7 @@ L1:	movl	(%esi,%edx,4),%eax
 	popl	%ebx
 	popl	%esi
 	popl	%edi
-	ret
+	ret_spec_stop
 
 Lend:	shll	%cl,%ebx		/* compute least significant limb */
 	movl	%ebx,(%edi)		/* store it */
@@ -97,6 +97,6 @@ Lend:	shll	%cl,%ebx		/* compute least significant limb */
 	CFI_POP(%esi)
 	popl	%edi
 	CFI_POP(%edi)
-	ret
+	ret_spec_stop
 	CFI_ENDPROC()
 
diff --git a/mpi/i386/mpih-mul1.S b/mpi/i386/mpih-mul1.S
index 9679ea62..a672d052 100644
--- a/mpi/i386/mpih-mul1.S
+++ b/mpi/i386/mpih-mul1.S
@@ -89,6 +89,6 @@ Loop:
 	CFI_POP(%esi)
 	INSN1(pop,l	,R(edi))
 	CFI_POP(%edi)
-	ret
+	ret_spec_stop
 	CFI_ENDPROC()
 
diff --git a/mpi/i386/mpih-mul2.S b/mpi/i386/mpih-mul2.S
index fe4129c4..e09c3f7c 100644
--- a/mpi/i386/mpih-mul2.S
+++ b/mpi/i386/mpih-mul2.S
@@ -91,6 +91,6 @@ Loop:
 	CFI_POP(%esi)
 	INSN1(pop,l	,R(edi))
 	CFI_POP(%edi)
-	ret
+	ret_spec_stop
 	CFI_ENDPROC()
 
diff --git a/mpi/i386/mpih-mul3.S b/mpi/i386/mpih-mul3.S
index 87577d54..4112c699 100644
--- a/mpi/i386/mpih-mul3.S
+++ b/mpi/i386/mpih-mul3.S
@@ -91,6 +91,6 @@ Loop:
 	CFI_POP(%esi)
 	INSN1(pop,l	,R(edi))
 	CFI_POP(%edi)
-	ret
+	ret_spec_stop
 	CFI_ENDPROC()
 
diff --git a/mpi/i386/mpih-rshift.S b/mpi/i386/mpih-rshift.S
index 35a8201f..5d34696c 100644
--- a/mpi/i386/mpih-rshift.S
+++ b/mpi/i386/mpih-rshift.S
@@ -89,7 +89,7 @@ L2:	movl	(%esi,%edx,4),%eax
 	popl	%ebx
 	popl	%esi
 	popl	%edi
-	ret
+	ret_spec_stop
 
 Lend2:	shrl	%cl,%ebx		/* compute most significant limb */
 	movl	%ebx,(%edi)		/* store it */
@@ -100,6 +100,6 @@ Lend2:	shrl	%cl,%ebx		/* compute most significant limb */
 	CFI_POP(%esi)
 	popl	%edi
 	CFI_POP(%edi)
-	ret
+	ret_spec_stop
 	CFI_ENDPROC()
 
diff --git a/mpi/i386/mpih-sub1.S b/mpi/i386/mpih-sub1.S
index 2bdc1438..49477ae3 100644
--- a/mpi/i386/mpih-sub1.S
+++ b/mpi/i386/mpih-sub1.S
@@ -157,6 +157,6 @@ Loop:	movl	(%esi),%eax
 	CFI_POP(%esi)
 	popl %edi
 	CFI_POP(%edi)
-	ret
+	ret_spec_stop
 	CFI_ENDPROC()
 
diff --git a/mpi/i386/syntax.h b/mpi/i386/syntax.h
index dd300319..af4d9e80 100644
--- a/mpi/i386/syntax.h
+++ b/mpi/i386/syntax.h
@@ -92,3 +92,7 @@
 #undef ALIGN
 #define ALIGN(log) .align log,0x90
 #endif
+
+/* 'ret' instruction replacement for straight-line speculation mitigation */
+#define ret_spec_stop \
+	ret; int3;