diff options
| -rw-r--r-- | cipher/Makefile.am | 1 | ||||
| -rw-r--r-- | cipher/asm-inline-s390x.h | 114 | ||||
| -rw-r--r-- | cipher/cipher-gcm.c | 37 | ||||
| -rw-r--r-- | cipher/cipher-internal.h | 6 | ||||
| -rw-r--r-- | cipher/rijndael-s390x.c | 86 |
5 files changed, 213 insertions, 31 deletions
diff --git a/cipher/Makefile.am b/cipher/Makefile.am index 6260654b..c445e590 100644 --- a/cipher/Makefile.am +++ b/cipher/Makefile.am @@ -71,6 +71,7 @@ libcipher_la_SOURCES = \ EXTRA_libcipher_la_SOURCES = \ asm-common-aarch64.h \ asm-common-amd64.h \ + asm-inline-s390x.h \ asm-poly1305-aarch64.h \ asm-poly1305-amd64.h \ arcfour.c arcfour-amd64.S \ diff --git a/cipher/asm-inline-s390x.h b/cipher/asm-inline-s390x.h new file mode 100644 index 00000000..82007531 --- /dev/null +++ b/cipher/asm-inline-s390x.h @@ -0,0 +1,114 @@ +/* asm-inline-s390x.h - Common macros for zSeries inline assembly + * + * Copyright (C) 2020 Jussi Kivilinna <jussi.kivilinna@iki.fi> + * + * This file is part of Libgcrypt. + * + * Libgcrypt is free software; you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as + * published by the Free Software Foundation; either version 2.1 of + * the License, or (at your option) any later version. + * + * Libgcrypt is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this program; if not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef GCRY_ASM_INLINE_S390X_H +#define GCRY_ASM_INLINE_S390X_H + +#include <config.h> + +typedef unsigned int u128_t __attribute__ ((mode (TI))); + +enum kmxx_functions_e +{ + KM_FUNCTION_AES_128 = 18, + KM_FUNCTION_AES_192 = 19, + KM_FUNCTION_AES_256 = 20, + KM_FUNCTION_XTS_AES_128 = 50, + KM_FUNCTION_XTS_AES_256 = 52, + + KMID_FUNCTION_SHA1 = 1, + KMID_FUNCTION_SHA256 = 2, + KMID_FUNCTION_SHA512 = 3, + KMID_FUNCTION_SHA3_224 = 32, + KMID_FUNCTION_SHA3_256 = 33, + KMID_FUNCTION_SHA3_384 = 34, + KMID_FUNCTION_SHA3_512 = 35, + KMID_FUNCTION_SHAKE128 = 36, + KMID_FUNCTION_SHAKE256 = 37, + KMID_FUNCTION_GHASH = 65, +}; + +enum kmxx_function_flags_e +{ + KM_ENCRYPT = 0 << 7, + KM_DECRYPT = 1 << 7, + + KMF_LCFB_16 = 16 << 24, + + KMA_LPC = 1 << 8, + KMA_LAAD = 1 << 9, + KMA_HS = 1 << 10, +}; + +static inline u128_t km_function_to_mask(enum kmxx_functions_e func) +{ + return (u128_t)1 << (127 - func); +} + +static inline u128_t kimd_query(void) +{ + static u128_t function_codes = 0; + static int initialized = 0; + register unsigned long reg0 asm("0") = 0; + register void *reg1 asm("1") = &function_codes; + u128_t r1; + + if (initialized) + return function_codes; + + asm volatile ("0: .insn rre,0xb93e << 16, 0, %[r1]\n\t" + " brc 1,0b\n\t" + : [r1] "=a" (r1) + : [reg0] "r" (reg0), [reg1] "r" (reg1) + : "cc", "memory"); + + initialized = 1; + return function_codes; +} + +static inline void kimd_execute(unsigned int func, void *param_block, + const void *src, size_t src_len) +{ + register unsigned long reg0 asm("0") = func; + register byte *reg1 asm("1") = param_block; + u128_t r1 = ((u128_t)(uintptr_t)src << 64) | (u64)src_len; + + asm volatile ("0: .insn rre,0xb93e << 16, 0, %[r1]\n\t" + " brc 1,0b\n\t" + : [r1] "+a" (r1) + : [func] "r" (reg0), [param_ptr] "r" (reg1) + : "cc", "memory"); +} + +static inline void klmd_execute(unsigned int func, void *param_block, + const void *src, size_t src_len) +{ + register unsigned long reg0 asm("0") = func; + register byte *reg1 asm("1") = param_block; + u128_t r1 = ((u128_t)(uintptr_t)src << 64) | (u64)src_len; + + asm volatile ("0: .insn rre,0xb93f << 16, 0, %[r1]\n\t" + " brc 1,0b\n\t" + : [r1] "+a" (r1) + : [func] "r" (reg0), [param_ptr] "r" (reg1) + : "cc", "memory"); +} + +#endif /* GCRY_ASM_INLINE_S390X_H */ diff --git a/cipher/cipher-gcm.c b/cipher/cipher-gcm.c index c8669311..7aad1277 100644 --- a/cipher/cipher-gcm.c +++ b/cipher/cipher-gcm.c @@ -89,6 +89,26 @@ ghash_armv7_neon (gcry_cipher_hd_t c, byte *result, const byte *buf, } #endif /* GCM_USE_ARM_NEON */ +#ifdef GCM_USE_S390X_CRYPTO +#include "asm-inline-s390x.h" + +static unsigned int +ghash_s390x_kimd (gcry_cipher_hd_t c, byte *result, const byte *buf, + size_t nblocks) +{ + u128_t params[2]; + + memcpy (¶ms[0], result, 16); + memcpy (¶ms[1], c->u_mode.gcm.u_ghash_key.key, 16); + + kimd_execute (KMID_FUNCTION_GHASH, ¶ms, buf, nblocks * 16); + + memcpy (result, ¶ms[0], 16); + wipememory (params, sizeof(params)); + return 0; +} +#endif /* GCM_USE_S390X_CRYPTO*/ + #ifdef GCM_USE_TABLES static struct @@ -522,10 +542,13 @@ ghash_internal (gcry_cipher_hd_t c, byte *result, const byte *buf, static void setupM (gcry_cipher_hd_t c) { -#if defined(GCM_USE_INTEL_PCLMUL) || defined(GCM_USE_ARM_PMULL) +#if defined(GCM_USE_INTEL_PCLMUL) || defined(GCM_USE_ARM_PMULL) || \ + defined(GCM_USE_S390X_CRYPTO) unsigned int features = _gcry_get_hw_features (); #endif + c->u_mode.gcm.ghash_fn = NULL; + if (0) ; #ifdef GCM_USE_INTEL_PCLMUL @@ -549,7 +572,17 @@ setupM (gcry_cipher_hd_t c) ghash_setup_armv7_neon (c); } #endif - else +#ifdef GCM_USE_S390X_CRYPTO + else if (features & HWF_S390X_MSA) + { + if (kimd_query () & km_function_to_mask (KMID_FUNCTION_GHASH)) + { + c->u_mode.gcm.ghash_fn = ghash_s390x_kimd; + } + } +#endif + + if (c->u_mode.gcm.ghash_fn == NULL) { c->u_mode.gcm.ghash_fn = ghash_internal; fillM (c); diff --git a/cipher/cipher-internal.h b/cipher/cipher-internal.h index d63b659a..59b36ce7 100644 --- a/cipher/cipher-internal.h +++ b/cipher/cipher-internal.h @@ -96,6 +96,12 @@ #endif #endif /* GCM_USE_ARM_NEON */ +/* GCM_USE_S390X_CRYPTO indicates whether to enable zSeries code. */ +#undef GCM_USE_S390X_CRYPTO +#if defined(HAVE_GCC_INLINE_ASM_S390X) +# define GCM_USE_S390X_CRYPTO 1 +#endif /* GCM_USE_S390X_CRYPTO */ + typedef unsigned int (*ghash_fn_t) (gcry_cipher_hd_t c, byte *result, const byte *buf, size_t nblocks); diff --git a/cipher/rijndael-s390x.c b/cipher/rijndael-s390x.c index 7b40b8a8..5ab019f9 100644 --- a/cipher/rijndael-s390x.c +++ b/cipher/rijndael-s390x.c @@ -25,25 +25,7 @@ #ifdef USE_S390X_CRYPTO -typedef unsigned int u128_t __attribute__ ((mode (TI))); - -enum km_functions_e -{ - KM_FUNCTION_AES_128 = 18, - KM_FUNCTION_AES_192 = 19, - KM_FUNCTION_AES_256 = 20, - KM_FUNCTION_XTS_AES_128 = 50, - KM_FUNCTION_XTS_AES_256 = 52, - - KM_ENCRYPT = 0x00, - KM_DECRYPT = 0x80, - - KMF_LCFB_16 = 16 << 24, - - KMA_LPC = 1 << 8, - KMA_LAAD = 1 << 9, - KMA_HS = 1 << 10, -}; +#include "asm-inline-s390x.h" #define ALWAYS_INLINE inline __attribute__((always_inline)) #define NO_INLINE __attribute__((noinline)) @@ -453,6 +435,45 @@ static void aes_s390x_ctr128_enc(void *context, unsigned char *ctr, wipememory (¶ms, sizeof(params)); } +static size_t aes_s390x_gcm_crypt(gcry_cipher_hd_t c, void *outbuf_arg, + const void *inbuf_arg, size_t nblocks, + int encrypt) +{ + RIJNDAEL_context *ctx = (void *)&c->context.c; + byte *out = outbuf_arg; + const byte *in = inbuf_arg; + byte *ctr = c->u_ctr.ctr; + unsigned int function; + struct aes_s390x_gcm_params_s params; + + function = ctx->kma_func | (encrypt ? KM_ENCRYPT : KM_DECRYPT) + | KMA_HS | KMA_LAAD; + + /* Prepare parameter block. */ + memset (¶ms.reserved, 0, sizeof(params.reserved)); + buf_put_be32 (¶ms.counter_value, buf_get_be32(ctr + 12) - 1); + memcpy (¶ms.tag, c->u_mode.gcm.u_tag.tag, 16); + memcpy (¶ms.hash_subkey, c->u_mode.gcm.u_ghash_key.key, 16); + params.total_aad_length = 0; + params.total_cipher_length = 0; + memcpy (¶ms.initial_counter_value, ctr, 12); + params.initial_counter_value[3] = params.counter_value; + memcpy (¶ms.key, ctx->keyschenc, 32); + + /* Update counter (CTR32). */ + buf_put_be32(ctr + 12, buf_get_be32(ctr + 12) + nblocks); + + /* Perform KMA-GCM. */ + kma_execute (function, ¶ms, out, in, nblocks * BLOCKSIZE, NULL, 0); + + /* Update tag. */ + memcpy (c->u_mode.gcm.u_tag.tag, ¶ms.tag, 16); + + wipememory (¶ms, sizeof(params)); + + return 0; +} + static void aes_s390x_xts_crypt(void *context, unsigned char *tweak, void *outbuf_arg, const void *inbuf_arg, size_t nblocks, int encrypt) @@ -1014,20 +1035,20 @@ int _gcry_aes_s390x_setup_acceleration(RIJNDAEL_context *ctx, case 16: func = KM_FUNCTION_AES_128; func_xts = KM_FUNCTION_XTS_AES_128; - func_mask = (u128_t)1 << (127 - KM_FUNCTION_AES_128); - func_xts_mask = (u128_t)1 << (127 - KM_FUNCTION_XTS_AES_128); + func_mask = km_function_to_mask(KM_FUNCTION_AES_128); + func_xts_mask = km_function_to_mask(KM_FUNCTION_XTS_AES_128); break; case 24: func = KM_FUNCTION_AES_192; func_xts = 0; - func_mask = (u128_t)1 << (127 - KM_FUNCTION_AES_192); - func_xts_mask = 0; + func_mask = km_function_to_mask(KM_FUNCTION_AES_192); + func_xts_mask = 0; /* XTS-AES192 not available. */ break; case 32: func = KM_FUNCTION_AES_256; func_xts = KM_FUNCTION_XTS_AES_256; - func_mask = (u128_t)1 << (127 - KM_FUNCTION_AES_256); - func_xts_mask = (u128_t)1 << (127 - KM_FUNCTION_AES_256); + func_mask = km_function_to_mask(KM_FUNCTION_AES_256); + func_xts_mask = km_function_to_mask(KM_FUNCTION_AES_256); break; } @@ -1079,6 +1100,11 @@ int _gcry_aes_s390x_setup_acceleration(RIJNDAEL_context *ctx, bulk_ops->cfb_dec = aes_s390x_cfb128_dec; } + if (ctx->km_func_xts) + { + bulk_ops->xts_crypt = aes_s390x_xts_crypt; + } + if (ctx->kmc_func) { if(ctx->kmac_func) @@ -1103,11 +1129,13 @@ int _gcry_aes_s390x_setup_acceleration(RIJNDAEL_context *ctx, if (ctx->kma_func) { bulk_ops->ctr_enc = aes_s390x_ctr128_enc; - } - if (ctx->km_func_xts) - { - bulk_ops->xts_crypt = aes_s390x_xts_crypt; + if (kimd_query () & km_function_to_mask (KMID_FUNCTION_GHASH)) + { + /* KIMD based GHASH implementation is required with AES-GCM + * acceleration. */ + bulk_ops->gcm_crypt = aes_s390x_gcm_crypt; + } } return 1; |