Add bulk function interface for GCM mode

* cipher/cipher-gcm.c (do_ghash_buf): Proper handling for the case
where 'unused' gets filled to full blocksize.
(gcm_crypt_inner): New.
(_gcry_cipher_gcm_encrypt, _gcry_cipher_gcm_decrypt): Use
'gcm_crypt_inner'.
* cipher/cipher-internal.h (cipher_bulk_ops_t): Add 'gcm_crypt'.
--

Signed-off-by: Jussi Kivilinna <jussi.kivilinna@iki.fi>

1 files changed, 77 insertions, 48 deletions

diff --git a/cipher/cipher-gcm.c b/cipher/cipher-gcm.c
index 194e2ec9..c8669311 100644
--- a/cipher/cipher-gcm.c
+++ b/cipher/cipher-gcm.c
@@ -648,8 +648,10 @@ do_ghash_buf(gcry_cipher_hd_t c, byte *hash, const byte *buf,
         }
       if (!buflen)
         {
-          if (!do_padding)
-            break;
+          if (!do_padding && unused < blocksize)
+	    {
+	      break;
+	    }
 
 	  n = blocksize - unused;
 	  if (n > 0)
@@ -757,13 +759,83 @@ gcm_ctr_encrypt (gcry_cipher_hd_t c, byte *outbuf, size_t outbuflen,
 }
 
 
+static gcry_err_code_t
+gcm_crypt_inner (gcry_cipher_hd_t c, byte *outbuf, size_t outbuflen,
+		 const byte *inbuf, size_t inbuflen, int encrypt)
+{
+  gcry_err_code_t err;
+
+  while (inbuflen)
+    {
+      size_t currlen = inbuflen;
+
+      /* Use a bulk method if available.  */
+      if (c->bulk.gcm_crypt)
+	{
+	  /* Bulk method requires that there is no cached data. */
+	  if (inbuflen >= GCRY_GCM_BLOCK_LEN && c->u_mode.gcm.mac_unused == 0)
+	    {
+	      size_t nblks = inbuflen / GCRY_GCM_BLOCK_LEN;
+	      size_t nleft;
+	      size_t ndone;
+
+	      nleft = c->bulk.gcm_crypt (c, outbuf, inbuf, nblks, encrypt);
+	      ndone = nblks - nleft;
+
+	      inbuf += ndone * GCRY_GCM_BLOCK_LEN;
+	      outbuf += ndone * GCRY_GCM_BLOCK_LEN;
+	      inbuflen -= ndone * GCRY_GCM_BLOCK_LEN;
+	      outbuflen -= ndone * GCRY_GCM_BLOCK_LEN;
+
+	      if (inbuflen == 0)
+		break;
+
+	      currlen = inbuflen;
+	    }
+	  else if (c->u_mode.gcm.mac_unused > 0
+	           && inbuflen >= GCRY_GCM_BLOCK_LEN
+			  + (16 - c->u_mode.gcm.mac_unused))
+	    {
+	      /* Handle just enough data so that cache is depleted, and on
+	       * next loop iteration use bulk method. */
+	      currlen = 16 - c->u_mode.gcm.mac_unused;
+
+	      gcry_assert(currlen);
+	    }
+	}
+
+      /* Since checksumming is done after/before encryption/decryption,
+       * process input in 24KiB chunks to keep data loaded in L1 cache for
+       * checksumming/decryption. */
+      if (currlen > 24 * 1024)
+	currlen = 24 * 1024;
+
+      if (!encrypt)
+	do_ghash_buf(c, c->u_mode.gcm.u_tag.tag, inbuf, currlen, 0);
+
+      err = gcm_ctr_encrypt(c, outbuf, outbuflen, inbuf, currlen);
+      if (err != 0)
+	return err;
+
+      if (encrypt)
+	do_ghash_buf(c, c->u_mode.gcm.u_tag.tag, outbuf, currlen, 0);
+
+      outbuf += currlen;
+      inbuf += currlen;
+      outbuflen -= currlen;
+      inbuflen -= currlen;
+    }
+
+  return 0;
+}
+
+
 gcry_err_code_t
 _gcry_cipher_gcm_encrypt (gcry_cipher_hd_t c,
                           byte *outbuf, size_t outbuflen,
                           const byte *inbuf, size_t inbuflen)
 {
   static const unsigned char zerobuf[MAX_BLOCKSIZE];
-  gcry_err_code_t err;
 
   if (c->spec->blocksize != GCRY_GCM_BLOCK_LEN)
     return GPG_ERR_CIPHER_ALGO;
@@ -796,28 +868,7 @@ _gcry_cipher_gcm_encrypt (gcry_cipher_hd_t c,
       return GPG_ERR_INV_LENGTH;
     }
 
-  while (inbuflen)
-    {
-      size_t currlen = inbuflen;
-
-      /* Since checksumming is done after encryption, process input in 24KiB
-       * chunks to keep data loaded in L1 cache for checksumming. */
-      if (currlen > 24 * 1024)
-	currlen = 24 * 1024;
-
-      err = gcm_ctr_encrypt(c, outbuf, outbuflen, inbuf, currlen);
-      if (err != 0)
-	return err;
-
-      do_ghash_buf(c, c->u_mode.gcm.u_tag.tag, outbuf, currlen, 0);
-
-      outbuf += currlen;
-      inbuf += currlen;
-      outbuflen -= currlen;
-      inbuflen -= currlen;
-    }
-
-  return 0;
+  return gcm_crypt_inner (c, outbuf, outbuflen, inbuf, inbuflen, 1);
 }
 
 
@@ -827,7 +878,6 @@ _gcry_cipher_gcm_decrypt (gcry_cipher_hd_t c,
                           const byte *inbuf, size_t inbuflen)
 {
   static const unsigned char zerobuf[MAX_BLOCKSIZE];
-  gcry_err_code_t err;
 
   if (c->spec->blocksize != GCRY_GCM_BLOCK_LEN)
     return GPG_ERR_CIPHER_ALGO;
@@ -857,28 +907,7 @@ _gcry_cipher_gcm_decrypt (gcry_cipher_hd_t c,
       return GPG_ERR_INV_LENGTH;
     }
 
-  while (inbuflen)
-    {
-      size_t currlen = inbuflen;
-
-      /* Since checksumming is done before decryption, process input in
-       * 24KiB chunks to keep data loaded in L1 cache for decryption. */
-      if (currlen > 24 * 1024)
-	currlen = 24 * 1024;
-
-      do_ghash_buf(c, c->u_mode.gcm.u_tag.tag, inbuf, currlen, 0);
-
-      err = gcm_ctr_encrypt(c, outbuf, outbuflen, inbuf, currlen);
-      if (err)
-	return err;
-
-      outbuf += currlen;
-      inbuf += currlen;
-      outbuflen -= currlen;
-      inbuflen -= currlen;
-    }
-
-  return 0;
+  return gcm_crypt_inner (c, outbuf, outbuflen, inbuf, inbuflen, 0);
 }