Add bulk AES-GCM acceleration for s390x/zSeries

* cipher/Makefile.am: Add 'asm-inline-s390x.h'.
* cipher/asm-inline-s390x.h: New.
* cipher/cipher-gcm.c [GCM_USE_S390X_CRYPTO] (ghash_s390x_kimd): New.
(setupM) [GCM_USE_S390X_CRYPTO]: Add setup for s390x GHASH function.
* cipher/cipher-internal.h (GCM_USE_S390X_CRYPTO): New.
* cipher/rijndael-s390x.c (u128_t, km_functions_e): Move to
'asm-inline-s390x.h'.
(aes_s390x_gcm_crypt): New.
(_gcry_aes_s390x_setup_acceleration): Use 'km_function_to_mask'; Add
setup for GCM bulk function.
--

This patch adds zSeries acceleration for GHASH and AES-GCM.

Benchmarks (z15, 5.2Ghz):

Before:
 AES            |  nanosecs/byte   mebibytes/sec   cycles/byte
        GCM enc |      2.64 ns/B     361.6 MiB/s     13.71 c/B
        GCM dec |      2.64 ns/B     361.3 MiB/s     13.72 c/B
       GCM auth |      2.58 ns/B     370.1 MiB/s     13.40 c/B

After:
 AES            |  nanosecs/byte   mebibytes/sec   cycles/byte
        GCM enc |     0.059 ns/B     16066 MiB/s     0.309 c/B
        GCM dec |     0.059 ns/B     16114 MiB/s     0.308 c/B
       GCM auth |     0.057 ns/B     16747 MiB/s     0.296 c/B

Signed-off-by: Jussi Kivilinna <jussi.kivilinna@iki.fi>

5 files changed, 213 insertions, 31 deletions

diff --git a/cipher/Makefile.am b/cipher/Makefile.am
index 6260654b..c445e590 100644
--- a/cipher/Makefile.am
+++ b/cipher/Makefile.am
@@ -71,6 +71,7 @@ libcipher_la_SOURCES = \
 EXTRA_libcipher_la_SOURCES = \
 	asm-common-aarch64.h \
 	asm-common-amd64.h \
+	asm-inline-s390x.h \
 	asm-poly1305-aarch64.h \
 	asm-poly1305-amd64.h \
 	arcfour.c arcfour-amd64.S \
diff --git a/cipher/asm-inline-s390x.h b/cipher/asm-inline-s390x.h
new file mode 100644
index 00000000..82007531
--- /dev/null
+++ b/cipher/asm-inline-s390x.h
@@ -0,0 +1,114 @@
+/* asm-inline-s390x.h  -  Common macros for zSeries inline assembly
+ *
+ * Copyright (C) 2020 Jussi Kivilinna <jussi.kivilinna@iki.fi>
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Libgcrypt is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * Libgcrypt is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef GCRY_ASM_INLINE_S390X_H
+#define GCRY_ASM_INLINE_S390X_H
+
+#include <config.h>
+
+typedef unsigned int u128_t __attribute__ ((mode (TI)));
+
+enum kmxx_functions_e
+{
+  KM_FUNCTION_AES_128 = 18,
+  KM_FUNCTION_AES_192 = 19,
+  KM_FUNCTION_AES_256 = 20,
+  KM_FUNCTION_XTS_AES_128 = 50,
+  KM_FUNCTION_XTS_AES_256 = 52,
+
+  KMID_FUNCTION_SHA1 = 1,
+  KMID_FUNCTION_SHA256 = 2,
+  KMID_FUNCTION_SHA512 = 3,
+  KMID_FUNCTION_SHA3_224 = 32,
+  KMID_FUNCTION_SHA3_256 = 33,
+  KMID_FUNCTION_SHA3_384 = 34,
+  KMID_FUNCTION_SHA3_512 = 35,
+  KMID_FUNCTION_SHAKE128 = 36,
+  KMID_FUNCTION_SHAKE256 = 37,
+  KMID_FUNCTION_GHASH = 65,
+};
+
+enum kmxx_function_flags_e
+{
+  KM_ENCRYPT  = 0 << 7,
+  KM_DECRYPT  = 1 << 7,
+
+  KMF_LCFB_16 = 16 << 24,
+
+  KMA_LPC     = 1 << 8,
+  KMA_LAAD    = 1 << 9,
+  KMA_HS      = 1 << 10,
+};
+
+static inline u128_t km_function_to_mask(enum kmxx_functions_e func)
+{
+  return (u128_t)1 << (127 - func);
+}
+
+static inline u128_t kimd_query(void)
+{
+  static u128_t function_codes = 0;
+  static int initialized = 0;
+  register unsigned long reg0 asm("0") = 0;
+  register void *reg1 asm("1") = &function_codes;
+  u128_t r1;
+
+  if (initialized)
+    return function_codes;
+
+  asm volatile ("0: .insn rre,0xb93e << 16, 0, %[r1]\n\t"
+		"   brc 1,0b\n\t"
+		: [r1] "=a" (r1)
+		: [reg0] "r" (reg0), [reg1] "r" (reg1)
+		: "cc", "memory");
+
+  initialized = 1;
+  return function_codes;
+}
+
+static inline void kimd_execute(unsigned int func, void *param_block,
+				const void *src, size_t src_len)
+{
+  register unsigned long reg0 asm("0") = func;
+  register byte *reg1 asm("1") = param_block;
+  u128_t r1 = ((u128_t)(uintptr_t)src << 64) | (u64)src_len;
+
+  asm volatile ("0: .insn rre,0xb93e << 16, 0, %[r1]\n\t"
+		"   brc 1,0b\n\t"
+		: [r1] "+a" (r1)
+		: [func] "r" (reg0), [param_ptr] "r" (reg1)
+		: "cc", "memory");
+}
+
+static inline void klmd_execute(unsigned int func, void *param_block,
+				const void *src, size_t src_len)
+{
+  register unsigned long reg0 asm("0") = func;
+  register byte *reg1 asm("1") = param_block;
+  u128_t r1 = ((u128_t)(uintptr_t)src << 64) | (u64)src_len;
+
+  asm volatile ("0: .insn rre,0xb93f << 16, 0, %[r1]\n\t"
+		"   brc 1,0b\n\t"
+		: [r1] "+a" (r1)
+		: [func] "r" (reg0), [param_ptr] "r" (reg1)
+		: "cc", "memory");
+}
+
+#endif /* GCRY_ASM_INLINE_S390X_H */
diff --git a/cipher/cipher-gcm.c b/cipher/cipher-gcm.c
index c8669311..7aad1277 100644
--- a/cipher/cipher-gcm.c
+++ b/cipher/cipher-gcm.c
@@ -89,6 +89,26 @@ ghash_armv7_neon (gcry_cipher_hd_t c, byte *result, const byte *buf,
 }
 #endif /* GCM_USE_ARM_NEON */
 
+#ifdef GCM_USE_S390X_CRYPTO
+#include "asm-inline-s390x.h"
+
+static unsigned int
+ghash_s390x_kimd (gcry_cipher_hd_t c, byte *result, const byte *buf,
+		  size_t nblocks)
+{
+  u128_t params[2];
+
+  memcpy (&params[0], result, 16);
+  memcpy (&params[1], c->u_mode.gcm.u_ghash_key.key, 16);
+
+  kimd_execute (KMID_FUNCTION_GHASH, &params, buf, nblocks * 16);
+
+  memcpy (result, &params[0], 16);
+  wipememory (params, sizeof(params));
+  return 0;
+}
+#endif /* GCM_USE_S390X_CRYPTO*/
+
 
 #ifdef GCM_USE_TABLES
 static struct
@@ -522,10 +542,13 @@ ghash_internal (gcry_cipher_hd_t c, byte *result, const byte *buf,
 static void
 setupM (gcry_cipher_hd_t c)
 {
-#if defined(GCM_USE_INTEL_PCLMUL) || defined(GCM_USE_ARM_PMULL)
+#if defined(GCM_USE_INTEL_PCLMUL) || defined(GCM_USE_ARM_PMULL) || \
+    defined(GCM_USE_S390X_CRYPTO)
   unsigned int features = _gcry_get_hw_features ();
 #endif
 
+  c->u_mode.gcm.ghash_fn = NULL;
+
   if (0)
     ;
 #ifdef GCM_USE_INTEL_PCLMUL
@@ -549,7 +572,17 @@ setupM (gcry_cipher_hd_t c)
       ghash_setup_armv7_neon (c);
     }
 #endif
-  else
+#ifdef GCM_USE_S390X_CRYPTO
+  else if (features & HWF_S390X_MSA)
+    {
+      if (kimd_query () & km_function_to_mask (KMID_FUNCTION_GHASH))
+	{
+	  c->u_mode.gcm.ghash_fn = ghash_s390x_kimd;
+	}
+    }
+#endif
+
+  if (c->u_mode.gcm.ghash_fn == NULL)
     {
       c->u_mode.gcm.ghash_fn = ghash_internal;
       fillM (c);
diff --git a/cipher/cipher-internal.h b/cipher/cipher-internal.h
index d63b659a..59b36ce7 100644
--- a/cipher/cipher-internal.h
+++ b/cipher/cipher-internal.h
@@ -96,6 +96,12 @@
 #endif
 #endif /* GCM_USE_ARM_NEON */
 
+/* GCM_USE_S390X_CRYPTO indicates whether to enable zSeries code. */
+#undef GCM_USE_S390X_CRYPTO
+#if defined(HAVE_GCC_INLINE_ASM_S390X)
+# define GCM_USE_S390X_CRYPTO 1
+#endif /* GCM_USE_S390X_CRYPTO */
+
 typedef unsigned int (*ghash_fn_t) (gcry_cipher_hd_t c, byte *result,
                                     const byte *buf, size_t nblocks);
 
diff --git a/cipher/rijndael-s390x.c b/cipher/rijndael-s390x.c
index 7b40b8a8..5ab019f9 100644
--- a/cipher/rijndael-s390x.c
+++ b/cipher/rijndael-s390x.c
@@ -25,25 +25,7 @@
 
 #ifdef USE_S390X_CRYPTO
 
-typedef unsigned int u128_t __attribute__ ((mode (TI)));
-
-enum km_functions_e
-{
-  KM_FUNCTION_AES_128 = 18,
-  KM_FUNCTION_AES_192 = 19,
-  KM_FUNCTION_AES_256 = 20,
-  KM_FUNCTION_XTS_AES_128 = 50,
-  KM_FUNCTION_XTS_AES_256 = 52,
-
-  KM_ENCRYPT = 0x00,
-  KM_DECRYPT = 0x80,
-
-  KMF_LCFB_16 = 16 << 24,
-
-  KMA_LPC  = 1 << 8,
-  KMA_LAAD = 1 << 9,
-  KMA_HS   = 1 << 10,
-};
+#include "asm-inline-s390x.h"
 
 #define ALWAYS_INLINE inline __attribute__((always_inline))
 #define NO_INLINE __attribute__((noinline))
@@ -453,6 +435,45 @@ static void aes_s390x_ctr128_enc(void *context, unsigned char *ctr,
   wipememory (&params, sizeof(params));
 }
 
+static size_t aes_s390x_gcm_crypt(gcry_cipher_hd_t c, void *outbuf_arg,
+				  const void *inbuf_arg, size_t nblocks,
+				  int encrypt)
+{
+  RIJNDAEL_context *ctx = (void *)&c->context.c;
+  byte *out = outbuf_arg;
+  const byte *in = inbuf_arg;
+  byte *ctr = c->u_ctr.ctr;
+  unsigned int function;
+  struct aes_s390x_gcm_params_s params;
+
+  function = ctx->kma_func | (encrypt ? KM_ENCRYPT : KM_DECRYPT)
+	      | KMA_HS | KMA_LAAD;
+
+  /* Prepare parameter block. */
+  memset (&params.reserved, 0, sizeof(params.reserved));
+  buf_put_be32 (&params.counter_value, buf_get_be32(ctr + 12) - 1);
+  memcpy (&params.tag, c->u_mode.gcm.u_tag.tag, 16);
+  memcpy (&params.hash_subkey, c->u_mode.gcm.u_ghash_key.key, 16);
+  params.total_aad_length = 0;
+  params.total_cipher_length = 0;
+  memcpy (&params.initial_counter_value, ctr, 12);
+  params.initial_counter_value[3] = params.counter_value;
+  memcpy (&params.key, ctx->keyschenc, 32);
+
+  /* Update counter (CTR32). */
+  buf_put_be32(ctr + 12, buf_get_be32(ctr + 12) + nblocks);
+
+  /* Perform KMA-GCM. */
+  kma_execute (function, &params, out, in, nblocks * BLOCKSIZE, NULL, 0);
+
+  /* Update tag. */
+  memcpy (c->u_mode.gcm.u_tag.tag, &params.tag, 16);
+
+  wipememory (&params, sizeof(params));
+
+  return 0;
+}
+
 static void aes_s390x_xts_crypt(void *context, unsigned char *tweak,
 				void *outbuf_arg, const void *inbuf_arg,
 				size_t nblocks, int encrypt)
@@ -1014,20 +1035,20 @@ int _gcry_aes_s390x_setup_acceleration(RIJNDAEL_context *ctx,
     case 16:
       func = KM_FUNCTION_AES_128;
       func_xts = KM_FUNCTION_XTS_AES_128;
-      func_mask = (u128_t)1 << (127 - KM_FUNCTION_AES_128);
-      func_xts_mask = (u128_t)1 << (127 - KM_FUNCTION_XTS_AES_128);
+      func_mask = km_function_to_mask(KM_FUNCTION_AES_128);
+      func_xts_mask = km_function_to_mask(KM_FUNCTION_XTS_AES_128);
       break;
     case 24:
       func = KM_FUNCTION_AES_192;
       func_xts = 0;
-      func_mask = (u128_t)1 << (127 - KM_FUNCTION_AES_192);
-      func_xts_mask = 0;
+      func_mask = km_function_to_mask(KM_FUNCTION_AES_192);
+      func_xts_mask = 0; /* XTS-AES192 not available. */
       break;
     case 32:
       func = KM_FUNCTION_AES_256;
       func_xts = KM_FUNCTION_XTS_AES_256;
-      func_mask = (u128_t)1 << (127 - KM_FUNCTION_AES_256);
-      func_xts_mask = (u128_t)1 << (127 - KM_FUNCTION_AES_256);
+      func_mask = km_function_to_mask(KM_FUNCTION_AES_256);
+      func_xts_mask = km_function_to_mask(KM_FUNCTION_AES_256);
       break;
     }
 
@@ -1079,6 +1100,11 @@ int _gcry_aes_s390x_setup_acceleration(RIJNDAEL_context *ctx,
       bulk_ops->cfb_dec = aes_s390x_cfb128_dec;
     }
 
+  if (ctx->km_func_xts)
+    {
+      bulk_ops->xts_crypt = aes_s390x_xts_crypt;
+    }
+
   if (ctx->kmc_func)
     {
       if(ctx->kmac_func)
@@ -1103,11 +1129,13 @@ int _gcry_aes_s390x_setup_acceleration(RIJNDAEL_context *ctx,
   if (ctx->kma_func)
     {
       bulk_ops->ctr_enc = aes_s390x_ctr128_enc;
-    }
 
-  if (ctx->km_func_xts)
-    {
-      bulk_ops->xts_crypt = aes_s390x_xts_crypt;
+      if (kimd_query () & km_function_to_mask (KMID_FUNCTION_GHASH))
+	{
+	  /* KIMD based GHASH implementation is required with AES-GCM
+	   * acceleration. */
+	  bulk_ops->gcm_crypt = aes_s390x_gcm_crypt;
+	}
     }
 
   return 1;