Add POWER9 little-endian variant of PPC AES implementation

* configure.ac: Add 'rijndael-ppc9le.lo'.
* cipher/Makefile.am: Add 'rijndael-ppc9le.c', 'rijndael-ppc-common.h'
and 'rijndael-ppc-functions.h'.
* cipher/rijndael-internal.h (USE_PPC_CRYPTO_WITH_PPC9LE): New.
(RIJNDAEL_context_s): Add 'use_ppc9le_crypto'.
* cipher/rijndael.c (_gcry_aes_ppc9le_encrypt)
(_gcry_aes_ppc9le_decrypt, _gcry_aes_ppc9le_cfb_enc)
(_gcry_aes_ppc9le_cfb_dec, _gcry_aes_ppc9le_ctr_enc)
(_gcry_aes_ppc9le_cbc_enc, _gcry_aes_ppc9le_cbc_dec)
(_gcry_aes_ppc9le_ocb_crypt, _gcry_aes_ppc9le_ocb_auth)
(_gcry_aes_ppc9le_xts_crypt): New.
(do_setkey, _gcry_aes_cfb_enc, _gcry_aes_cbc_enc)
(_gcry_aes_ctr_enc, _gcry_aes_cfb_dec, _gcry_aes_cbc_dec)
(_gcry_aes_ocb_crypt, _gcry_aes_ocb_auth, _gcry_aes_xts_crypt)
[USE_PPC_CRYPTO_WITH_PPC9LE]: New.
* cipher/rijndael-ppc.c: Split common code to headers
'rijndael-ppc-common.h' and 'rijndael-ppc-functions.h'.
* cipher/rijndael-ppc-common.h: Split from 'rijndael-ppc.c'.
(asm_add_uint64, asm_sra_int64, asm_swap_uint64_halfs): New.
* cipher/rijndael-ppc-functions.h: Split from 'rijndael-ppc.c'.
(CFB_ENC_FUNC, CBC_ENC_FUNC): Unroll loop by 2.
(XTS_CRYPT_FUNC, GEN_TWEAK): Tweak generation without vperm
instruction.
* cipher/rijndael-ppc9le.c: New.
--

Provide POWER9 little-endian optimized variant of PPC vcrypto AES
implementation. This implementation uses 'lxvb16x' and 'stxvb16x'
instructions to load/store vectors directly in big-endian order.

Benchmark on POWER9 (~3.8Ghz):

Before:
 AES            |  nanosecs/byte   mebibytes/sec   cycles/byte
        CBC enc |      1.04 ns/B     918.7 MiB/s      3.94 c/B
        CBC dec |     0.222 ns/B      4292 MiB/s     0.844 c/B
        CFB enc |      1.04 ns/B     916.9 MiB/s      3.95 c/B
        CFB dec |     0.224 ns/B      4252 MiB/s     0.852 c/B
        CTR enc |     0.226 ns/B      4218 MiB/s     0.859 c/B
        CTR dec |     0.225 ns/B      4233 MiB/s     0.856 c/B
        XTS enc |     0.500 ns/B      1907 MiB/s      1.90 c/B
        XTS dec |     0.494 ns/B      1932 MiB/s      1.88 c/B
        OCB enc |     0.288 ns/B      3312 MiB/s      1.09 c/B
        OCB dec |     0.292 ns/B      3266 MiB/s      1.11 c/B
       OCB auth |     0.267 ns/B      3567 MiB/s      1.02 c/B

After (ctr & ocb & cbc-dec & cfb-dec ~15% and xts ~8% faster):
 AES            |  nanosecs/byte   mebibytes/sec   cycles/byte
        CBC enc |      1.04 ns/B     914.2 MiB/s      3.96 c/B
        CBC dec |     0.191 ns/B      4984 MiB/s     0.727 c/B
        CFB enc |      1.03 ns/B     930.0 MiB/s      3.90 c/B
        CFB dec |     0.194 ns/B      4906 MiB/s     0.739 c/B
        CTR enc |     0.196 ns/B      4868 MiB/s     0.744 c/B
        CTR dec |     0.197 ns/B      4834 MiB/s     0.750 c/B
        XTS enc |     0.460 ns/B      2075 MiB/s      1.75 c/B
        XTS dec |     0.455 ns/B      2097 MiB/s      1.73 c/B
        OCB enc |     0.250 ns/B      3812 MiB/s     0.951 c/B
        OCB dec |     0.253 ns/B      3764 MiB/s     0.963 c/B
       OCB auth |     0.232 ns/B      4106 MiB/s     0.883 c/B

Signed-off-by: Jussi Kivilinna <jussi.kivilinna@iki.fi>

1 files changed, 116 insertions, 0 deletions

diff --git a/cipher/rijndael.c b/cipher/rijndael.c
index ebd1a11a..a1c4cfc1 100644
--- a/cipher/rijndael.c
+++ b/cipher/rijndael.c
@@ -239,6 +239,43 @@ extern void _gcry_aes_ppc8_xts_crypt (void *context, unsigned char *tweak,
 				      size_t nblocks, int encrypt);
 #endif /*USE_PPC_CRYPTO*/
 
+#ifdef USE_PPC_CRYPTO_WITH_PPC9LE
+/* Power9 little-endian crypto implementations of AES */
+extern unsigned int _gcry_aes_ppc9le_encrypt(const RIJNDAEL_context *ctx,
+					    unsigned char *dst,
+					    const unsigned char *src);
+extern unsigned int _gcry_aes_ppc9le_decrypt(const RIJNDAEL_context *ctx,
+					    unsigned char *dst,
+					    const unsigned char *src);
+
+extern void _gcry_aes_ppc9le_cfb_enc (void *context, unsigned char *iv,
+				      void *outbuf_arg, const void *inbuf_arg,
+				      size_t nblocks);
+extern void _gcry_aes_ppc9le_cbc_enc (void *context, unsigned char *iv,
+				      void *outbuf_arg, const void *inbuf_arg,
+				      size_t nblocks, int cbc_mac);
+extern void _gcry_aes_ppc9le_ctr_enc (void *context, unsigned char *ctr,
+				      void *outbuf_arg, const void *inbuf_arg,
+				      size_t nblocks);
+extern void _gcry_aes_ppc9le_cfb_dec (void *context, unsigned char *iv,
+				      void *outbuf_arg, const void *inbuf_arg,
+				      size_t nblocks);
+extern void _gcry_aes_ppc9le_cbc_dec (void *context, unsigned char *iv,
+				      void *outbuf_arg, const void *inbuf_arg,
+				      size_t nblocks);
+
+extern size_t _gcry_aes_ppc9le_ocb_crypt (gcry_cipher_hd_t c, void *outbuf_arg,
+					  const void *inbuf_arg, size_t nblocks,
+					  int encrypt);
+extern size_t _gcry_aes_ppc9le_ocb_auth (gcry_cipher_hd_t c,
+					const void *abuf_arg, size_t nblocks);
+
+extern void _gcry_aes_ppc9le_xts_crypt (void *context, unsigned char *tweak,
+					void *outbuf_arg,
+					const void *inbuf_arg,
+					size_t nblocks, int encrypt);
+#endif /*USE_PPC_CRYPTO_WITH_PPC9LE*/
+
 static unsigned int do_encrypt (const RIJNDAEL_context *ctx, unsigned char *bx,
                                 const unsigned char *ax);
 static unsigned int do_decrypt (const RIJNDAEL_context *ctx, unsigned char *bx,
@@ -384,6 +421,9 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen,
 #ifdef USE_PPC_CRYPTO
   ctx->use_ppc_crypto = 0;
 #endif
+#ifdef USE_PPC_CRYPTO_WITH_PPC9LE
+  ctx->use_ppc9le_crypto = 0;
+#endif
 
   if (0)
     {
@@ -464,6 +504,28 @@ do_setkey (RIJNDAEL_context *ctx, const byte *key, const unsigned keylen,
         }
     }
 #endif
+#ifdef USE_PPC_CRYPTO_WITH_PPC9LE
+  else if ((hwfeatures & HWF_PPC_VCRYPTO) && (hwfeatures & HWF_PPC_ARCH_3_00))
+    {
+      ctx->encrypt_fn = _gcry_aes_ppc9le_encrypt;
+      ctx->decrypt_fn = _gcry_aes_ppc9le_decrypt;
+      ctx->prefetch_enc_fn = NULL;
+      ctx->prefetch_dec_fn = NULL;
+      ctx->use_ppc_crypto = 1; /* same key-setup as USE_PPC_CRYPTO */
+      ctx->use_ppc9le_crypto = 1;
+      if (hd)
+        {
+          hd->bulk.cfb_enc = _gcry_aes_ppc9le_cfb_enc;
+          hd->bulk.cfb_dec = _gcry_aes_ppc9le_cfb_dec;
+          hd->bulk.cbc_enc = _gcry_aes_ppc9le_cbc_enc;
+          hd->bulk.cbc_dec = _gcry_aes_ppc9le_cbc_dec;
+          hd->bulk.ctr_enc = _gcry_aes_ppc9le_ctr_enc;
+          hd->bulk.ocb_crypt = _gcry_aes_ppc9le_ocb_crypt;
+          hd->bulk.ocb_auth = _gcry_aes_ppc9le_ocb_auth;
+          hd->bulk.xts_crypt = _gcry_aes_ppc9le_xts_crypt;
+        }
+    }
+#endif
 #ifdef USE_PPC_CRYPTO
   else if (hwfeatures & HWF_PPC_VCRYPTO)
     {
@@ -924,6 +986,13 @@ _gcry_aes_cfb_enc (void *context, unsigned char *iv,
       return;
     }
 #endif /*USE_ARM_CE*/
+#ifdef USE_PPC_CRYPTO_WITH_PPC9LE
+  else if (ctx->use_ppc9le_crypto)
+    {
+      _gcry_aes_ppc9le_cfb_enc (ctx, iv, outbuf, inbuf, nblocks);
+      return;
+    }
+#endif /*USE_PPC_CRYPTO_WITH_PPC9LE*/
 #ifdef USE_PPC_CRYPTO
   else if (ctx->use_ppc_crypto)
     {
@@ -992,6 +1061,13 @@ _gcry_aes_cbc_enc (void *context, unsigned char *iv,
       return;
     }
 #endif /*USE_ARM_CE*/
+#ifdef USE_PPC_CRYPTO_WITH_PPC9LE
+  else if (ctx->use_ppc9le_crypto)
+    {
+      _gcry_aes_ppc9le_cbc_enc (ctx, iv, outbuf, inbuf, nblocks, cbc_mac);
+      return;
+    }
+#endif /*USE_PPC_CRYPTO_WITH_PPC9LE*/
 #ifdef USE_PPC_CRYPTO
   else if (ctx->use_ppc_crypto)
     {
@@ -1067,6 +1143,13 @@ _gcry_aes_ctr_enc (void *context, unsigned char *ctr,
       return;
     }
 #endif /*USE_ARM_CE*/
+#ifdef USE_PPC_CRYPTO_WITH_PPC9LE
+  else if (ctx->use_ppc9le_crypto)
+    {
+      _gcry_aes_ppc9le_ctr_enc (ctx, ctr, outbuf, inbuf, nblocks);
+      return;
+    }
+#endif /*USE_PPC_CRYPTO_WITH_PPC9LE*/
 #ifdef USE_PPC_CRYPTO
   else if (ctx->use_ppc_crypto)
     {
@@ -1317,6 +1400,13 @@ _gcry_aes_cfb_dec (void *context, unsigned char *iv,
       return;
     }
 #endif /*USE_ARM_CE*/
+#ifdef USE_PPC_CRYPTO_WITH_PPC9LE
+  else if (ctx->use_ppc9le_crypto)
+    {
+      _gcry_aes_ppc9le_cfb_dec (ctx, iv, outbuf, inbuf, nblocks);
+      return;
+    }
+#endif /*USE_PPC_CRYPTO_WITH_PPC9LE*/
 #ifdef USE_PPC_CRYPTO
   else if (ctx->use_ppc_crypto)
     {
@@ -1382,6 +1472,13 @@ _gcry_aes_cbc_dec (void *context, unsigned char *iv,
       return;
     }
 #endif /*USE_ARM_CE*/
+#ifdef USE_PPC_CRYPTO_WITH_PPC9LE
+  else if (ctx->use_ppc9le_crypto)
+    {
+      _gcry_aes_ppc9le_cbc_dec (ctx, iv, outbuf, inbuf, nblocks);
+      return;
+    }
+#endif /*USE_PPC_CRYPTO_WITH_PPC9LE*/
 #ifdef USE_PPC_CRYPTO
   else if (ctx->use_ppc_crypto)
     {
@@ -1450,6 +1547,12 @@ _gcry_aes_ocb_crypt (gcry_cipher_hd_t c, void *outbuf_arg,
       return _gcry_aes_armv8_ce_ocb_crypt (c, outbuf, inbuf, nblocks, encrypt);
     }
 #endif /*USE_ARM_CE*/
+#ifdef USE_PPC_CRYPTO_WITH_PPC9LE
+  else if (ctx->use_ppc9le_crypto)
+    {
+      return _gcry_aes_ppc9le_ocb_crypt (c, outbuf, inbuf, nblocks, encrypt);
+    }
+#endif /*USE_PPC_CRYPTO_WITH_PPC9LE*/
 #ifdef USE_PPC_CRYPTO
   else if (ctx->use_ppc_crypto)
     {
@@ -1550,6 +1653,12 @@ _gcry_aes_ocb_auth (gcry_cipher_hd_t c, const void *abuf_arg, size_t nblocks)
       return _gcry_aes_armv8_ce_ocb_auth (c, abuf, nblocks);
     }
 #endif /*USE_ARM_CE*/
+#ifdef USE_PPC_CRYPTO_WITH_PPC9LE
+  else if (ctx->use_ppc9le_crypto)
+    {
+      return _gcry_aes_ppc9le_ocb_auth (c, abuf, nblocks);
+    }
+#endif /*USE_PPC_CRYPTO_WITH_PPC9LE*/
 #ifdef USE_PPC_CRYPTO
   else if (ctx->use_ppc_crypto)
     {
@@ -1619,6 +1728,13 @@ _gcry_aes_xts_crypt (void *context, unsigned char *tweak,
       return;
     }
 #endif /*USE_ARM_CE*/
+#ifdef USE_PPC_CRYPTO_WITH_PPC9LE
+  else if (ctx->use_ppc9le_crypto)
+    {
+      _gcry_aes_ppc9le_xts_crypt (ctx, tweak, outbuf, inbuf, nblocks, encrypt);
+      return;
+    }
+#endif /*USE_PPC_CRYPTO_WITH_PPC9LE*/
 #ifdef USE_PPC_CRYPTO
   else if (ctx->use_ppc_crypto)
     {