rijndael: add x86_64 VAES/AVX2 accelerated implementation

* cipher/Makefile.am: Add 'rijndael-vaes.c' and
'rijndael-vaes-avx2-amd64.S'.
* cipher/rijndael-internal.h (USE_VAES): New.
* cipher/rijndael-vaes-avx2-amd64.S: New.
* cipher/rijndael-vaes.c: New.
* cipher/rijndael.c (_gcry_aes_vaes_cfb_dec, _gcry_aes_vaes_cbc_dec)
(_gcry_aes_vaes_ctr_enc, _gcry_aes_vaes_ocb_crypt)
(_gcry_aes_vaes_xts_crypt): New.
(do_setkey) [USE_VAES]: Add detection for VAES.
(selftest_ctr_128, selftest_cbc_128, selftest_cfb_128)
[USE_VAES]: Increase number of selftest blocks.
* configure.ac: Add 'rijndael-vaes.lo' and
'rijndael-vaes-avx2-amd64.lo'.
--

Patch adds VAES/AVX2 accelerated implementation for CBC-decryption,
CFB-decryption, CTR-encryption, OCB-en/decryption and XTS-en/decryption.

Benchmarks on AMD Ryzen 5800X:

Before:
 AES            |  nanosecs/byte   mebibytes/sec   cycles/byte  auto Mhz
        CBC dec |     0.067 ns/B     14314 MiB/s     0.323 c/B      4850
        CFB dec |     0.067 ns/B     14322 MiB/s     0.323 c/B      4850
        CTR enc |     0.066 ns/B     14429 MiB/s     0.321 c/B      4850
        CTR dec |     0.066 ns/B     14433 MiB/s     0.320 c/B      4850
        XTS enc |     0.087 ns/B     10910 MiB/s     0.424 c/B      4850
        XTS dec |     0.088 ns/B     10856 MiB/s     0.426 c/B      4850
        OCB enc |     0.070 ns/B     13633 MiB/s     0.339 c/B      4850
        OCB dec |     0.069 ns/B     13911 MiB/s     0.332 c/B      4850

After (XTS ~1.7x faster, others ~1.9x faster):
 AES            |  nanosecs/byte   mebibytes/sec   cycles/byte  auto Mhz
        CBC dec |     0.034 ns/B     28159 MiB/s     0.164 c/B      4850
        CFB dec |     0.034 ns/B     27955 MiB/s     0.165 c/B      4850
        CTR enc |     0.034 ns/B     28214 MiB/s     0.164 c/B      4850
        CTR dec |     0.034 ns/B     28146 MiB/s     0.164 c/B      4850
        XTS enc |     0.051 ns/B     18539 MiB/s     0.249 c/B      4850
        XTS dec |     0.051 ns/B     18655 MiB/s     0.248 c/B      4850
       GCM auth |     0.088 ns/B     10817 MiB/s     0.428 c/B      4850
        OCB enc |     0.037 ns/B     25824 MiB/s     0.179 c/B      4850
        OCB dec |     0.038 ns/B     25359 MiB/s     0.182 c/B      4850

Signed-off-by: Jussi Kivilinna <jussi.kivilinna@iki.fi>

1 files changed, 176 insertions, 0 deletions

diff --git a/cipher/rijndael-vaes.c b/cipher/rijndael-vaes.c
new file mode 100644
index 00000000..56afce17
--- /dev/null
+++ b/cipher/rijndael-vaes.c
@@ -0,0 +1,176 @@
+/* VAES/AVX2 accelerated AES for Libgcrypt
+ * Copyright (C) 2021 Jussi Kivilinna <jussi.kivilinna@iki.fi>
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Libgcrypt is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * Libgcrypt is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; if not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+#include <config.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "types.h"  /* for byte and u32 typedefs */
+#include "g10lib.h"
+#include "cipher.h"
+#include "bufhelp.h"
+#include "cipher-selftest.h"
+#include "rijndael-internal.h"
+#include "./cipher-internal.h"
+
+
+#ifdef USE_VAES
+
+
+# ifdef HAVE_COMPATIBLE_GCC_WIN64_PLATFORM_AS
+#  define ASM_FUNC_ABI __attribute__((sysv_abi))
+# else
+#  define ASM_FUNC_ABI
+# endif
+
+
+extern void _gcry_aes_aesni_prepare_decryption(RIJNDAEL_context *ctx);
+
+
+extern void _gcry_vaes_avx2_cbc_dec_amd64 (const void *keysched,
+					   unsigned char *iv,
+					   void *outbuf_arg,
+					   const void *inbuf_arg,
+					   size_t nblocks,
+					   unsigned int nrounds) ASM_FUNC_ABI;
+
+extern void _gcry_vaes_avx2_cfb_dec_amd64 (const void *keysched,
+					   unsigned char *iv,
+					   void *outbuf_arg,
+					   const void *inbuf_arg,
+					   size_t nblocks,
+					   unsigned int nrounds) ASM_FUNC_ABI;
+
+extern void _gcry_vaes_avx2_ctr_enc_amd64 (const void *keysched,
+					   unsigned char *ctr,
+					   void *outbuf_arg,
+					   const void *inbuf_arg,
+					   size_t nblocks,
+					   unsigned int nrounds) ASM_FUNC_ABI;
+
+extern void _gcry_vaes_avx2_ocb_crypt_amd64 (const void *keysched,
+					     unsigned int blkn,
+					     void *outbuf_arg,
+					     const void *inbuf_arg,
+					     size_t nblocks,
+					     unsigned int nrounds,
+					     unsigned char *offset,
+					     unsigned char *checksum,
+					     unsigned char *L_table,
+					     int encrypt) ASM_FUNC_ABI;
+
+extern void _gcry_vaes_avx2_xts_crypt_amd64 (const void *keysched,
+					     unsigned char *tweak,
+					     void *outbuf_arg,
+					     const void *inbuf_arg,
+					     size_t nblocks,
+					     unsigned int nrounds,
+					     int encrypt) ASM_FUNC_ABI;
+
+
+void
+_gcry_aes_vaes_cbc_dec (void *context, unsigned char *iv,
+			void *outbuf, const void *inbuf,
+			size_t nblocks)
+{
+  RIJNDAEL_context *ctx = context;
+  const void *keysched = ctx->keyschdec32;
+  unsigned int nrounds = ctx->rounds;
+
+  if (!ctx->decryption_prepared)
+    {
+      _gcry_aes_aesni_prepare_decryption (ctx);
+      ctx->decryption_prepared = 1;
+    }
+
+  _gcry_vaes_avx2_cbc_dec_amd64 (keysched, iv, outbuf, inbuf, nblocks, nrounds);
+}
+
+void
+_gcry_aes_vaes_cfb_dec (void *context, unsigned char *iv,
+			void *outbuf, const void *inbuf,
+			size_t nblocks)
+{
+  RIJNDAEL_context *ctx = context;
+  const void *keysched = ctx->keyschenc32;
+  unsigned int nrounds = ctx->rounds;
+
+  _gcry_vaes_avx2_cfb_dec_amd64 (keysched, iv, outbuf, inbuf, nblocks, nrounds);
+}
+
+void
+_gcry_aes_vaes_ctr_enc (void *context, unsigned char *iv,
+			void *outbuf, const void *inbuf,
+			size_t nblocks)
+{
+  RIJNDAEL_context *ctx = context;
+  const void *keysched = ctx->keyschenc32;
+  unsigned int nrounds = ctx->rounds;
+
+  _gcry_vaes_avx2_ctr_enc_amd64 (keysched, iv, outbuf, inbuf, nblocks, nrounds);
+}
+
+size_t
+_gcry_aes_vaes_ocb_crypt (gcry_cipher_hd_t c, void *outbuf_arg,
+			  const void *inbuf_arg, size_t nblocks,
+			  int encrypt)
+{
+  RIJNDAEL_context *ctx = (void *)&c->context.c;
+  const void *keysched = encrypt ? ctx->keyschenc32 : ctx->keyschdec32;
+  unsigned char *outbuf = outbuf_arg;
+  const unsigned char *inbuf = inbuf_arg;
+  unsigned int nrounds = ctx->rounds;
+  u64 blkn = c->u_mode.ocb.data_nblocks;
+
+  if (!encrypt && !ctx->decryption_prepared)
+    {
+      _gcry_aes_aesni_prepare_decryption (ctx);
+      ctx->decryption_prepared = 1;
+    }
+
+  c->u_mode.ocb.data_nblocks = blkn + nblocks;
+
+  _gcry_vaes_avx2_ocb_crypt_amd64 (keysched, (unsigned int)blkn, outbuf, inbuf,
+				   nblocks, nrounds, c->u_iv.iv, c->u_ctr.ctr,
+				   c->u_mode.ocb.L[0], encrypt);
+
+  return 0;
+}
+
+void
+_gcry_aes_vaes_xts_crypt (void *context, unsigned char *tweak,
+			  void *outbuf, const void *inbuf,
+			  size_t nblocks, int encrypt)
+{
+  RIJNDAEL_context *ctx = context;
+  const void *keysched = encrypt ? ctx->keyschenc32 : ctx->keyschdec32;
+  unsigned int nrounds = ctx->rounds;
+
+  if (!encrypt && !ctx->decryption_prepared)
+    {
+      _gcry_aes_aesni_prepare_decryption (ctx);
+      ctx->decryption_prepared = 1;
+    }
+
+  _gcry_vaes_avx2_xts_crypt_amd64 (keysched, tweak, outbuf, inbuf, nblocks,
+				   nrounds, encrypt);
+}
+
+#endif /* USE_VAES */