Add ARMv8/AArch32 Crypto Extension implementation of SHA-1

* cipher/Makefile.am: Add 'sha1-armv8-aarch32-ce.S'.
* cipher/sha1-armv7-neon.S (_gcry_sha1_transform_armv7_neon): Add
missing size.
* cipher/sha1-armv8-aarch32-ce.S: New.
* cipher/sha1.c (USE_ARM_CE): New.
(sha1_init): Check features for HWF_ARM_SHA1.
[USE_ARM_CE] (_gcry_sha1_transform_armv8_ce): New.
(transform) [USE_ARM_CE]: Use ARMv8 CE implementation if HW supports
it.
* cipher/sha1.h (SHA1_CONTEXT): Add 'use_arm_ce'.
* configure.ac: Add 'sha1-armv8-aarch32-ce.lo'.
--

Benchmark on Cortex-A53 (1152 Mhz):

Before (SHA-1 NEON):

                 |  nanosecs/byte   mebibytes/sec   cycles/byte
  SHA1           |      6.62 ns/B     144.2 MiB/s      7.62 c/B

After (~3.8x faster):

                 |  nanosecs/byte   mebibytes/sec   cycles/byte
  SHA1           |      1.73 ns/B     552.2 MiB/s      1.99 c/B

Signed-off-by: Jussi Kivilinna <jussi.kivilinna@iki.fi>

6 files changed, 247 insertions, 3 deletions

diff --git a/cipher/Makefile.am b/cipher/Makefile.am
index f60338a6..571673ea 100644
--- a/cipher/Makefile.am
+++ b/cipher/Makefile.am
@@ -87,7 +87,7 @@ scrypt.c \
 seed.c \
 serpent.c serpent-sse2-amd64.S serpent-avx2-amd64.S serpent-armv7-neon.S \
 sha1.c sha1-ssse3-amd64.S sha1-avx-amd64.S sha1-avx-bmi2-amd64.S \
-  sha1-armv7-neon.S \
+  sha1-armv7-neon.S sha1-armv8-aarch32-ce.S \
 sha256.c sha256-ssse3-amd64.S sha256-avx-amd64.S sha256-avx2-bmi2-amd64.S \
 sha512.c sha512-ssse3-amd64.S sha512-avx-amd64.S sha512-avx2-bmi2-amd64.S \
   sha512-armv7-neon.S sha512-arm.S \
diff --git a/cipher/sha1-armv7-neon.S b/cipher/sha1-armv7-neon.S
index f314d8e2..61cc541c 100644
--- a/cipher/sha1-armv7-neon.S
+++ b/cipher/sha1-armv7-neon.S
@@ -521,5 +521,6 @@ _gcry_sha1_transform_armv7_neon:
 .Ldo_nothing:
   mov r0, #0;
   bx lr
+.size _gcry_sha1_transform_armv7_neon,.-_gcry_sha1_transform_armv7_neon;
 
 #endif
diff --git a/cipher/sha1-armv8-aarch32-ce.S b/cipher/sha1-armv8-aarch32-ce.S
new file mode 100644
index 00000000..b0bc5ffe
--- /dev/null
+++ b/cipher/sha1-armv8-aarch32-ce.S
@@ -0,0 +1,219 @@
+/* sha1-armv8-aarch32-ce.S - ARM/CE accelerated SHA-1 transform function
+ * Copyright (C) 2016 Jussi Kivilinna <jussi.kivilinna@iki.fi>
+ *
+ * This file is part of Libgcrypt.
+ *
+ * Libgcrypt is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2.1 of
+ * the License, or (at your option) any later version.
+ *
+ * Libgcrypt is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <config.h>
+
+#if defined(HAVE_ARM_ARCH_V6) && defined(__ARMEL__) && \
+    defined(HAVE_COMPATIBLE_GCC_ARM_PLATFORM_AS) && \
+    defined(HAVE_GCC_INLINE_ASM_AARCH32_CRYPTO) && defined(USE_SHA1)
+
+.syntax unified
+.fpu crypto-neon-fp-armv8
+.arm
+
+.text
+
+#ifdef __PIC__
+#  define GET_DATA_POINTER(reg, name, rtmp) \
+		ldr reg, 1f; \
+		ldr rtmp, 2f; \
+		b 3f; \
+	1:	.word _GLOBAL_OFFSET_TABLE_-(3f+8); \
+	2:	.word name(GOT); \
+	3:	add reg, pc, reg; \
+		ldr reg, [reg, rtmp];
+#else
+#  define GET_DATA_POINTER(reg, name, rtmp) ldr reg, =name
+#endif
+
+
+/* Constants */
+
+#define K1  0x5A827999
+#define K2  0x6ED9EBA1
+#define K3  0x8F1BBCDC
+#define K4  0xCA62C1D6
+.align 4
+gcry_sha1_aarch32_ce_K_VEC:
+.LK_VEC:
+.LK1:	.long K1, K1, K1, K1
+.LK2:	.long K2, K2, K2, K2
+.LK3:	.long K3, K3, K3, K3
+.LK4:	.long K4, K4, K4, K4
+
+
+/* Register macros */
+
+#define qH4    q0
+#define sH4    s0
+#define qH0123 q1
+
+#define qABCD q2
+#define qE0   q3
+#define qE1   q4
+
+#define qT0   q5
+#define qT1   q6
+
+#define qW0 q8
+#define qW1 q9
+#define qW2 q10
+#define qW3 q11
+
+#define qK1 q12
+#define qK2 q13
+#define qK3 q14
+#define qK4 q15
+
+
+/* Round macros */
+
+#define _(...) /*_*/
+#define do_add(dst, src0, src1) vadd.u32 dst, src0, src1;
+#define do_sha1su0(w0,w1,w2) sha1su0.32 w0,w1,w2;
+#define do_sha1su1(w0,w3) sha1su1.32 w0,w3;
+
+#define do_rounds(f, e0, e1, t, k, w0, w1, w2, w3, add_fn, sha1su0_fn, sha1su1_fn) \
+        sha1su1_fn( w3, w2     ); \
+        sha1h.32    e0, qABCD; \
+        sha1##f.32  qABCD, e1, t; \
+        add_fn(     t, w2, k   ); \
+        sha1su0_fn( w0, w1, w2 );
+
+
+/* Other functional macros */
+
+#define CLEAR_REG(reg) veor reg, reg;
+
+
+/*
+ * unsigned int
+ * _gcry_sha1_transform_armv8_ce (void *ctx, const unsigned char *data,
+ *                                size_t nblks)
+ */
+.align 3
+.globl _gcry_sha1_transform_armv8_ce
+.type  _gcry_sha1_transform_armv8_ce,%function;
+_gcry_sha1_transform_armv8_ce:
+  /* input:
+   *	r0: ctx, CTX
+   *	r1: data (64*nblks bytes)
+   *	r2: nblks
+   */
+
+  cmp r2, #0;
+  push {r4,lr};
+  beq .Ldo_nothing;
+
+  vpush {q4-q7};
+
+  GET_DATA_POINTER(r4, .LK_VEC, lr);
+
+  veor qH4, qH4
+  vld1.32 {qH0123}, [r0]    /* load h0,h1,h2,h3 */
+
+  vld1.32 {qK1-qK2}, [r4]!  /* load K1,K2 */
+  vldr sH4, [r0, #16]       /* load h4 */
+  vld1.32 {qK3-qK4}, [r4]   /* load K3,K4 */
+
+  vld1.8 {qW0-qW1}, [r1]!
+  vmov qABCD, qH0123
+  vld1.8 {qW2-qW3}, [r1]!
+
+  vrev32.8 qW0, qW0
+  vrev32.8 qW1, qW1
+  vrev32.8 qW2, qW2
+  do_add(qT0, qW0, qK1)
+  vrev32.8 qW3, qW3
+  do_add(qT1, qW1, qK1)
+
+.Loop:
+  do_rounds(c, qE1, qH4, qT0, qK1, qW0, qW1, qW2, qW3, do_add, do_sha1su0, _)
+  subs r2, r2, #1
+  do_rounds(c, qE0, qE1, qT1, qK1, qW1, qW2, qW3, qW0, do_add, do_sha1su0, do_sha1su1)
+  do_rounds(c, qE1, qE0, qT0, qK1, qW2, qW3, qW0, qW1, do_add, do_sha1su0, do_sha1su1)
+  do_rounds(c, qE0, qE1, qT1, qK2, qW3, qW0, qW1, qW2, do_add, do_sha1su0, do_sha1su1)
+  do_rounds(c, qE1, qE0, qT0, qK2, qW0, qW1, qW2, qW3, do_add, do_sha1su0, do_sha1su1)
+
+  do_rounds(p, qE0, qE1, qT1, qK2, qW1, qW2, qW3, qW0, do_add, do_sha1su0, do_sha1su1)
+  do_rounds(p, qE1, qE0, qT0, qK2, qW2, qW3, qW0, qW1, do_add, do_sha1su0, do_sha1su1)
+  do_rounds(p, qE0, qE1, qT1, qK2, qW3, qW0, qW1, qW2, do_add, do_sha1su0, do_sha1su1)
+  do_rounds(p, qE1, qE0, qT0, qK3, qW0, qW1, qW2, qW3, do_add, do_sha1su0, do_sha1su1)
+  do_rounds(p, qE0, qE1, qT1, qK3, qW1, qW2, qW3, qW0, do_add, do_sha1su0, do_sha1su1)
+
+  do_rounds(m, qE1, qE0, qT0, qK3, qW2, qW3, qW0, qW1, do_add, do_sha1su0, do_sha1su1)
+  do_rounds(m, qE0, qE1, qT1, qK3, qW3, qW0, qW1, qW2, do_add, do_sha1su0, do_sha1su1)
+  do_rounds(m, qE1, qE0, qT0, qK3, qW0, qW1, qW2, qW3, do_add, do_sha1su0, do_sha1su1)
+  do_rounds(m, qE0, qE1, qT1, qK4, qW1, qW2, qW3, qW0, do_add, do_sha1su0, do_sha1su1)
+  do_rounds(m, qE1, qE0, qT0, qK4, qW2, qW3, qW0, qW1, do_add, do_sha1su0, do_sha1su1)
+
+  do_rounds(p, qE0, qE1, qT1, qK4, qW3, qW0, qW1, qW2, do_add, do_sha1su0, do_sha1su1)
+  beq .Lend
+
+  vld1.8 {qW0-qW1}, [r1]! /* preload */
+  do_rounds(p, qE1, qE0, qT0, qK4, _  , _  , qW2, qW3, do_add, _, do_sha1su1)
+  vrev32.8 qW0, qW0
+  vld1.8 {qW2}, [r1]!
+  vrev32.8 qW1, qW1
+  do_rounds(p, qE0, qE1, qT1, qK4, _  , _  , qW3, _  , do_add, _, _)
+  vld1.8 {qW3}, [r1]!
+  vrev32.8 qW2, qW2
+  do_rounds(p, qE1, qE0, qT0, _, _, _, _, _, _, _, _)
+  vrev32.8 qW3, qW3
+  do_rounds(p, qE0, qE1, qT1, _, _, _, _, _, _, _, _)
+
+  do_add(qT0, qW0, qK1)
+  vadd.u32 qH4, qE0
+  vadd.u32 qABCD, qH0123
+  do_add(qT1, qW1, qK1)
+
+  vmov qH0123, qABCD
+
+  b .Loop
+
+.Lend:
+  do_rounds(p, qE1, qE0, qT0, qK4, _  , _  , qW2, qW3, do_add, _, do_sha1su1)
+  do_rounds(p, qE0, qE1, qT1, qK4, _  , _  , qW3, _  , do_add, _, _)
+  do_rounds(p, qE1, qE0, qT0, _, _, _, _, _, _, _, _)
+  do_rounds(p, qE0, qE1, qT1, _, _, _, _, _, _, _, _)
+
+  vadd.u32 qH4, qE0
+  vadd.u32 qH0123, qABCD
+
+  CLEAR_REG(qW0)
+  CLEAR_REG(qW1)
+  CLEAR_REG(qW2)
+  CLEAR_REG(qW3)
+  CLEAR_REG(qABCD)
+  CLEAR_REG(qE1)
+  CLEAR_REG(qE0)
+
+  vstr sH4, [r0, #16]    /* store h4 */
+  vst1.32 {qH0123}, [r0] /* store h0,h1,h2,h3 */
+
+  CLEAR_REG(qH0123)
+  CLEAR_REG(qH4)
+  vpop {q4-q7}
+
+.Ldo_nothing:
+  mov r0, #0
+  pop {r4,pc}
+.size _gcry_sha1_transform_armv8_ce,.-_gcry_sha1_transform_armv8_ce;
+
+#endif
diff --git a/cipher/sha1.c b/cipher/sha1.c
index d15c2a29..e0b68b28 100644
--- a/cipher/sha1.c
+++ b/cipher/sha1.c
@@ -76,8 +76,18 @@
      && defined(HAVE_GCC_INLINE_ASM_NEON)
 #  define USE_NEON 1
 # endif
-#endif /*ENABLE_NEON_SUPPORT*/
+#endif
 
+/* USE_ARM_CE indicates whether to enable ARMv8 Crypto Extension assembly
+ * code. */
+#undef USE_ARM_CE
+#ifdef ENABLE_ARM_CRYPTO_SUPPORT
+# if defined(HAVE_ARM_ARCH_V6) && defined(__ARMEL__) \
+     && defined(HAVE_COMPATIBLE_GCC_ARM_PLATFORM_AS) \
+     && defined(HAVE_GCC_INLINE_ASM_AARCH32_CRYPTO)
+#  define USE_ARM_CE 1
+# endif
+#endif
 
 /* A macro to test whether P is properly aligned for an u32 type.
    Note that config.h provides a suitable replacement for uintptr_t if
@@ -127,6 +137,9 @@ sha1_init (void *context, unsigned int flags)
 #ifdef USE_NEON
   hd->use_neon = (features & HWF_ARM_NEON) != 0;
 #endif
+#ifdef USE_ARM_CE
+  hd->use_arm_ce = (features & HWF_ARM_SHA1) != 0;
+#endif
   (void)features;
 }
 
@@ -164,13 +177,18 @@ _gcry_sha1_mixblock_init (SHA1_CONTEXT *hd)
 			       } while(0)
 
 
-
 #ifdef USE_NEON
 unsigned int
 _gcry_sha1_transform_armv7_neon (void *state, const unsigned char *data,
                                  size_t nblks);
 #endif
 
+#ifdef USE_ARM_CE
+unsigned int
+_gcry_sha1_transform_armv8_ce (void *state, const unsigned char *data,
+                               size_t nblks);
+#endif
+
 /*
  * Transform NBLOCKS of each 64 bytes (16 32-bit words) at DATA.
  */
@@ -340,6 +358,10 @@ transform (void *ctx, const unsigned char *data, size_t nblks)
     return _gcry_sha1_transform_amd64_ssse3 (&hd->h0, data, nblks)
            + 4 * sizeof(void*) + ASM_EXTRA_STACK;
 #endif
+#ifdef USE_ARM_CE
+  if (hd->use_arm_ce)
+    return _gcry_sha1_transform_armv8_ce (&hd->h0, data, nblks);
+#endif
 #ifdef USE_NEON
   if (hd->use_neon)
     return _gcry_sha1_transform_armv7_neon (&hd->h0, data, nblks)
diff --git a/cipher/sha1.h b/cipher/sha1.h
index 6b87631c..d448fcac 100644
--- a/cipher/sha1.h
+++ b/cipher/sha1.h
@@ -30,6 +30,7 @@ typedef struct
   unsigned int use_avx:1;
   unsigned int use_bmi2:1;
   unsigned int use_neon:1;
+  unsigned int use_arm_ce:1;
 } SHA1_CONTEXT;
 
 
diff --git a/configure.ac b/configure.ac
index 02995715..8e52a344 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2335,6 +2335,7 @@ case "${host}" in
   arm*-*-*)
     # Build with the assembly implementation
     GCRYPT_DIGESTS="$GCRYPT_DIGESTS sha1-armv7-neon.lo"
+    GCRYPT_DIGESTS="$GCRYPT_DIGESTS sha1-armv8-aarch32-ce.lo"
   ;;
 esac