PPC64: Add libmvec SIMD single-precision natural exponent function [BZ #24209]

Passes all tests.

Based off the ./sysdeps/ieee754/dbl-64/e_exp.c implementation,
and thus provides identical results.

Unlike other libmvec functions, this sets the underflow and overflow bits.
The caller can check these flags, and possibly re-run the calculations with
scalar expf to figure out what is causing the overflow or underflow.

Suprisingly the special-case path performs as well as the normal path.
(both of which are vectorized)
Running 20 times over 32MiB
vector: mean 432.263032 MiB/s (sd 0.486733)
scalar: mean 178.646197 MiB/s (sd 0.050013)

Reviewed-by: Tulio Magno Quites Machado Filho <tuliom@linux.ibm.com>

10 files changed, 321 insertions, 11 deletions

diff --git a/NEWS b/NEWS
index 0b873906db..ed2671909e 100644
--- a/NEWS
+++ b/NEWS
@@ -268,6 +268,7 @@ Major new features:
   - single-precision sine: sinf
   - double-precision sincos: sincos
   - single-precision sincos: sincosf
+  - single-precision exp: expf
 
   GCC support for auto-vectorization of functions on PPC64 is not yet
   available. Until that is done, the new vector math functions are
diff --git a/sysdeps/powerpc/bits/math-vector.h b/sysdeps/powerpc/bits/math-vector.h
index 6dc62480fd..bbc9db0dd9 100644
--- a/sysdeps/powerpc/bits/math-vector.h
+++ b/sysdeps/powerpc/bits/math-vector.h
@@ -44,6 +44,8 @@
 #  define __DECL_SIMD_sincos __DECL_SIMD_PPC64
 #  undef __DECL_SIMD_sincosf
 #  define __DECL_SIMD_sincosf __DECL_SIMD_PPC64
+#  undef __DECL_SIMD_expf
+#  define __DECL_SIMD_expf __DECL_SIMD_PPC64
 
 # endif
 #endif
diff --git a/sysdeps/powerpc/powerpc64/fpu/Versions b/sysdeps/powerpc/powerpc64/fpu/Versions
index c083d9f938..7c45e76074 100644
--- a/sysdeps/powerpc/powerpc64/fpu/Versions
+++ b/sysdeps/powerpc/powerpc64/fpu/Versions
@@ -2,5 +2,6 @@ libmvec {
   GLIBC_2.30 {
     _ZGVbN2v_cos; _ZGVbN4v_cosf; _ZGVbN2v_sin; _ZGVbN4v_sinf;
     _ZGVbN2v_log; _ZGVbN4v_logf; _ZGVbN2vvv_sincos; _ZGVbN4vvv_sincosf;
+    _ZGVbN4v_expf;
   }
 }
diff --git a/sysdeps/powerpc/powerpc64/fpu/multiarch/Makefile b/sysdeps/powerpc/powerpc64/fpu/multiarch/Makefile
index 238bfe7fe9..c89025c311 100644
--- a/sysdeps/powerpc/powerpc64/fpu/multiarch/Makefile
+++ b/sysdeps/powerpc/powerpc64/fpu/multiarch/Makefile
@@ -3,10 +3,13 @@ libmvec-sysdep_routines += vec_d_cos2_vsx vec_s_cosf4_vsx \
 			   vec_d_sin2_vsx vec_s_sinf4_vsx \
 			   vec_d_log2_vsx vec_d_log_data \
 			   vec_s_logf4_vsx vec_s_logf_data \
+			   vec_s_expf4_vsx vec_s_exp2f_data \
+			   vec_math_errf \
 			   vec_d_sincos2_vsx vec_s_sincosf4_vsx
 CFLAGS-vec_d_cos2_vsx.c += -mabi=altivec -maltivec -mvsx -mpower8-vector
 CFLAGS-vec_d_log2_vsx.c += -mabi=altivec -maltivec -mvsx -mpower8-vector
 CFLAGS-vec_d_log_data.c += -mabi=altivec -maltivec -mvsx -mpower8-vector
+CFLAGS-vec_math_errf.c += -mabi=altivec -maltivec -mvsx
 CFLAGS-vec_s_cosf4_vsx.c += -mabi=altivec -maltivec -mvsx
 CFLAGS-vec_s_logf4_vsx.c += -mabi=altivec -maltivec -mvsx
 CFLAGS-vec_s_logf_data.c += -mabi=altivec -maltivec -mvsx
@@ -14,6 +17,8 @@ CFLAGS-vec_d_sin2_vsx.c += -mabi=altivec -maltivec -mvsx -mpower8-vector
 CFLAGS-vec_s_sinf4_vsx.c += -mabi=altivec -maltivec -mvsx
 CFLAGS-vec_d_sincos2_vsx.c += -mabi=altivec -maltivec -mvsx -mpower8-vector
 CFLAGS-vec_s_sincosf4_vsx.c += -mabi=altivec -maltivec -mvsx
+CFLAGS-vec_s_expf4_vsx.c += -mabi=altivec -maltivec -mvsx
+CFLAGS-vec_s_exp2f_data.c += -mabi=altivec -maltivec -mvsx
 endif
 
 # Variables for libmvec tests.
@@ -22,7 +27,7 @@ ifeq ($(build-mathvec),yes)
 libmvec-tests += double-vlen2 float-vlen4
 
 double-vlen2-funcs = cos sin sincos log
-float-vlen4-funcs = cos sin sincos log
+float-vlen4-funcs = cos sin sincos log exp
 
 double-vlen2-arch-ext-cflags = -mabi=altivec -maltivec -mvsx -DREQUIRE_VSX
 float-vlen4-arch-ext-cflags = -mabi=altivec -maltivec -mvsx -DREQUIRE_VSX
diff --git a/sysdeps/powerpc/powerpc64/fpu/multiarch/math_config_flt.h b/sysdeps/powerpc/powerpc64/fpu/multiarch/math_config_flt.h
index b5fbe02d91..18f34197e2 100644
--- a/sysdeps/powerpc/powerpc64/fpu/multiarch/math_config_flt.h
+++ b/sysdeps/powerpc/powerpc64/fpu/multiarch/math_config_flt.h
@@ -48,16 +48,38 @@
 /* Round x to nearest int in all rounding modes, ties have to be rounded
    consistently with converttoint so the results match.  If the result
    would be outside of [-2^31, 2^31-1] then the semantics is unspecified.  */
-static inline double_t
-roundtoint (double_t x);
+static inline vector double
+roundtoint (vector double x);
 
 /* Convert x to nearest int in all rounding modes, ties have to be rounded
    consistently with roundtoint.  If the result is not representible in an
    int32_t then the semantics is unspecified.  */
-static inline int32_t
-converttoint (double_t x);
+static inline vector unsigned long long
+converttoint (vector double x);
 #endif
 
+static inline vector unsigned
+vasuint (vector float f)
+{
+  union
+  {
+    vector float f;
+    vector unsigned i;
+  } u = {f};
+  return u.i;
+}
+
+static inline vector float
+vasfloat (vector unsigned i)
+{
+  union
+  {
+    vector unsigned i;
+    vector float f;
+  } u = {i};
+  return u.f;
+}
+
 static inline uint32_t
 asuint (float f)
 {
@@ -80,6 +102,28 @@ asfloat (uint32_t i)
   return u.f;
 }
 
+static inline vector unsigned long long
+vasuint64 (vector double f)
+{
+  union
+  {
+    vector double f;
+    vector unsigned long long i;
+  } u = {f};
+  return u.i;
+}
+
+static inline vector double
+vasdouble (vector unsigned long long i)
+{
+  union
+  {
+    vector unsigned long long i;
+    vector double f;
+  } u = {i};
+  return u.f;
+}
+
 static inline uint64_t
 asuint64 (double f)
 {
@@ -124,12 +168,12 @@ attribute_hidden float __math_invalidf (float);
 #define EXP2F_POLY_ORDER 3
 extern const struct exp2f_data
 {
-  uint64_t tab[1 << EXP2F_TABLE_BITS];
-  double shift_scaled;
-  double poly[EXP2F_POLY_ORDER];
-  double shift;
-  double invln2_scaled;
-  double poly_scaled[EXP2F_POLY_ORDER];
+  long long unsigned tab[1 << EXP2F_TABLE_BITS];
+  vector double shift_scaled;
+  vector double poly[EXP2F_POLY_ORDER];
+  vector double shift;
+  vector double invln2_scaled;
+  vector double poly_scaled[EXP2F_POLY_ORDER];
 } __exp2f_data attribute_hidden;
 
 #define LOGF_TABLE_BITS 4
diff --git a/sysdeps/powerpc/powerpc64/fpu/multiarch/test-float-vlen4-wrappers.c b/sysdeps/powerpc/powerpc64/fpu/multiarch/test-float-vlen4-wrappers.c
index b4721b1dac..37886d1a45 100644
--- a/sysdeps/powerpc/powerpc64/fpu/multiarch/test-float-vlen4-wrappers.c
+++ b/sysdeps/powerpc/powerpc64/fpu/multiarch/test-float-vlen4-wrappers.c
@@ -24,4 +24,5 @@
 VECTOR_WRAPPER (WRAPPER_NAME (cosf), _ZGVbN4v_cosf)
 VECTOR_WRAPPER (WRAPPER_NAME (sinf), _ZGVbN4v_sinf)
 VECTOR_WRAPPER (WRAPPER_NAME (logf), _ZGVbN4v_logf)
+VECTOR_WRAPPER (WRAPPER_NAME (expf), _ZGVbN4v_expf)
 VECTOR_WRAPPER_fFF (WRAPPER_NAME (sincosf), _ZGVbN4vvv_sincosf)
diff --git a/sysdeps/powerpc/powerpc64/fpu/multiarch/vec_math_errf.c b/sysdeps/powerpc/powerpc64/fpu/multiarch/vec_math_errf.c
new file mode 100644
index 0000000000..1d145ffde1
--- /dev/null
+++ b/sysdeps/powerpc/powerpc64/fpu/multiarch/vec_math_errf.c
@@ -0,0 +1,38 @@
+/* Single-precision vector math error handling.
+   Copyright (C) 2019 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include "math_config_flt.h"
+/* NOINLINE prevents fenv semantics breaking optimizations.  */
+NOINLINE static float
+xflowf (uint32_t sign, float y)
+{
+  y = (sign ? -y : y) * y;
+  return y;
+}
+
+attribute_hidden float
+__math_uflowf (uint32_t sign)
+{
+  return xflowf (sign, 0x1p-95f);
+}
+
+attribute_hidden float
+__math_oflowf (uint32_t sign)
+{
+  return xflowf (sign, 0x1p97f);
+}
diff --git a/sysdeps/powerpc/powerpc64/fpu/multiarch/vec_s_exp2f_data.c b/sysdeps/powerpc/powerpc64/fpu/multiarch/vec_s_exp2f_data.c
new file mode 100644
index 0000000000..b2232713a9
--- /dev/null
+++ b/sysdeps/powerpc/powerpc64/fpu/multiarch/vec_s_exp2f_data.c
@@ -0,0 +1,58 @@
+/* Data for exp, rearranged for vector operations
+   Copyright (C) 2019 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#include "math_config_flt.h"
+
+#define N (1 << EXP2F_TABLE_BITS)
+
+const struct exp2f_data __exp2f_data = {
+  /* tab[i] = uint(2^(i/N)) - (i << 52-BITS)
+     used for computing 2^(k/N) for an int |k| < 150 N as
+     double(tab[k%N] + (k << 52-BITS)) */
+  .tab = {
+0x3ff0000000000000, 0x3fefd9b0d3158574, 0x3fefb5586cf9890f, 0x3fef9301d0125b51,
+0x3fef72b83c7d517b, 0x3fef54873168b9aa, 0x3fef387a6e756238, 0x3fef1e9df51fdee1,
+0x3fef06fe0a31b715, 0x3feef1a7373aa9cb, 0x3feedea64c123422, 0x3feece086061892d,
+0x3feebfdad5362a27, 0x3feeb42b569d4f82, 0x3feeab07dd485429, 0x3feea47eb03a5585,
+0x3feea09e667f3bcd, 0x3fee9f75e8ec5f74, 0x3feea11473eb0187, 0x3feea589994cce13,
+0x3feeace5422aa0db, 0x3feeb737b0cdc5e5, 0x3feec49182a3f090, 0x3feed503b23e255d,
+0x3feee89f995ad3ad, 0x3feeff76f2fb5e47, 0x3fef199bdd85529c, 0x3fef3720dcef9069,
+0x3fef5818dcfba487, 0x3fef7c97337b9b5f, 0x3fefa4afa2a490da, 0x3fefd0765b6e4540,
+  },
+  .shift_scaled = {
+    0x1.8p+52 / N,
+    0x1.8p+52 / N },
+  .poly = {
+  {0x1.c6af84b912394p-5, 0x1.c6af84b912394p-5},
+  {0x1.ebfce50fac4f3p-3, 0x1.ebfce50fac4f3p-3},
+  {0x1.62e42ff0c52d6p-1, 0x1.62e42ff0c52d6p-1},
+  },
+  .shift = {
+    0x1.8p+52,
+    0x1.8p+52,
+  },
+  .invln2_scaled = {
+    0x1.71547652b82fep+0 * N,
+    0x1.71547652b82fep+0 * N,
+  },
+  .poly_scaled = {
+  {0x1.c6af84b912394p-5/N/N/N, 0x1.c6af84b912394p-5/N/N/N},
+  {0x1.ebfce50fac4f3p-3/N/N, 0x1.ebfce50fac4f3p-3/N/N},
+  {0x1.62e42ff0c52d6p-1/N, 0x1.62e42ff0c52d6p-1/N},
+  },
+};
diff --git a/sysdeps/powerpc/powerpc64/fpu/multiarch/vec_s_expf4_vsx.c b/sysdeps/powerpc/powerpc64/fpu/multiarch/vec_s_expf4_vsx.c
new file mode 100644
index 0000000000..d118428a66
--- /dev/null
+++ b/sysdeps/powerpc/powerpc64/fpu/multiarch/vec_s_expf4_vsx.c
@@ -0,0 +1,159 @@
+/* Single-precision vector expf function.
+   Copyright (C) 2019 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+/* Based off ./sysdeps/ieee754/flt-32/e_expf.c
+   which was contributed by Szabolcs Nagy of ARM Ltd.  */
+#include <altivec.h>
+#include <math.h>
+
+#include "math_config_flt.h"
+
+typedef vector long long unsigned v64u;
+typedef union {
+  vector unsigned u;
+  vector float f;
+  v64u l;
+  vector double d;
+} u;
+typedef union {
+  double d;
+  int64_t l;
+  unsigned u;
+  float f;
+} us;
+
+#define N (1 << EXP2F_TABLE_BITS)
+#define InvLn2N __exp2f_data.invln2_scaled
+#define T __exp2f_data.tab
+#define C __exp2f_data.poly_scaled
+
+vector float
+_ZGVbN4v_expf (vector float x)
+{
+  u res;
+  u xu;
+  xu.f = x;
+  us c88;
+  c88.f = 88.0f;
+  us inf;
+  inf.f = INFINITY;
+  us ninf;
+  ninf.f = -INFINITY;
+  vector unsigned constants = {(c88.u & 0xfff00000) << 1,
+				ninf.u, inf.u, 0};
+  vector float constants2 = {0x1.62e42ep6f, -0x1.9fe368p6f, 0, 0};
+  vector unsigned zero = {0, 0, 0, 0};
+  vector unsigned v88 = vec_splat (constants, 0);
+  vector unsigned is_special_case
+	= (vector unsigned) vec_cmpge (xu.u << 1, v88);
+  vector unsigned is_special_not_covered = is_special_case;
+  if (__glibc_unlikely (!vec_all_eq (is_special_case, zero)))
+    {
+      vector unsigned inf = vec_splat (constants, 2);
+      vector unsigned is_inf_or_ninf_or_nan
+	= (vector unsigned) vec_cmpge (xu.u, inf);
+      u xpx;
+      xpx.f = x + x;
+      res.u = xpx.u;
+      is_special_not_covered &= ~is_inf_or_ninf_or_nan;
+      vector unsigned ninf = vec_splat (constants, 1);
+      vector unsigned is_ninf
+	= (vector unsigned) vec_cmpeq (xu.u, ninf);
+      res.u = vec_sel (res.u, zero, is_ninf);
+
+      vector float overflow_v = vec_splat (constants2, 0);
+      vector unsigned is_overflow
+	= (vector unsigned) vec_cmpgt (xu.f, overflow_v);
+      if (__glibc_unlikely (!vec_all_eq (is_overflow, zero)))
+	  {
+	  // This branch is because we are generating an
+	  // overflow fp flag
+	  us ofu;
+	  ofu.f = __math_oflowf (0);
+	  vector unsigned of = {ofu.u, ofu.u, ofu.u, ofu.u};
+	  res.u = vec_sel (res.u, of, is_overflow);
+	  is_special_not_covered &= ~is_overflow;
+	}
+      vector float underflow_v = vec_splat (constants2, 1);
+      vector unsigned is_underflow
+	= (vector unsigned) vec_cmplt (xu.f, underflow_v);
+      if (__glibc_unlikely (!vec_all_eq (is_underflow, zero)))
+	{
+	  // This branch is because we are generating an
+	  // underflow fp flag
+	  us ufu;
+	  ufu.f = __math_uflowf (0);
+	  vector unsigned uf = {ufu.u, ufu.u, ufu.u, ufu.u};
+	  res.u = vec_sel (res.u, uf, is_underflow);
+	  is_special_not_covered &= ~is_underflow;
+	}
+     }
+
+#if __GNUC__ >= 8
+  vector double xl = vec_unpackh (x);
+  vector double xr = vec_unpackl (x);
+#else
+  vector double xl = {x[0], x[1]};
+  vector double xr = {x[2], x[3]};
+#endif
+  vector double zl = InvLn2N * xl;
+  vector double zr = InvLn2N * xr;
+#if TOINT_INTRINSICS
+  vector double kdl = roundtoint (zl);
+  vector double kdl = roundtoint (zl);
+  v64u kil = converttoint (zl);
+  v64u kir = converttoint (zr);
+#else
+#define SHIFT __exp2f_data.shift
+  vector double kdl = zl + SHIFT;
+  vector double kdr = zr + SHIFT;
+  u kilu;
+  kilu.d = kdl;
+  u kiru;
+  kiru.d = kdr;
+  v64u kil = kilu.l;
+  v64u kir = kiru.l;
+  kdl -= SHIFT;
+  kdr -= SHIFT;
+#endif
+  vector double rl = zl - kdl;
+  vector double rr = zr - kdr;
+
+  v64u tl = {T[kil[0] % N], T[kil[1] % N]};
+  v64u tr = {T[kir[0] % N], T[kir[1] % N]};
+  tl += (kil << (52 - EXP2F_TABLE_BITS));
+  tr += (kir << (52 - EXP2F_TABLE_BITS));
+  u slu;
+  slu.l = tl;
+  u sru;
+  sru.l = tr;
+  // This cast is obnoxious, but there is no vec_ld for double
+  zl = C[0] * rl + C[1];
+  zr = C[0] * rr + C[1];
+  vector double r2l = rl * rl;
+  vector double r2r = rr * rr;
+  vector double yl = C[2] * rl + 1;
+  vector double yr = C[2] * rr + 1;
+  yl = zl * r2l + yl;
+  yr = zr * r2r + yr;
+  yl = yl * slu.d;
+  yr = yr * sru.d;
+  vector float restmp = {(float) yl[0], (float) yl[1],
+	(float) yr[0], (float) yr[1]};
+  return vec_sel (restmp, res.f,
+	is_special_case & ~is_special_not_covered);
+}
diff --git a/sysdeps/unix/sysv/linux/powerpc/powerpc64/libmvec.abilist b/sysdeps/unix/sysv/linux/powerpc/powerpc64/libmvec.abilist
index bb9f0beea4..bfbd340626 100644
--- a/sysdeps/unix/sysv/linux/powerpc/powerpc64/libmvec.abilist
+++ b/sysdeps/unix/sysv/linux/powerpc/powerpc64/libmvec.abilist
@@ -3,6 +3,7 @@ GLIBC_2.30 _ZGVbN2v_log F
 GLIBC_2.30 _ZGVbN2v_sin F
 GLIBC_2.30 _ZGVbN2vvv_sincos F
 GLIBC_2.30 _ZGVbN4v_cosf F
+GLIBC_2.30 _ZGVbN4v_expf F
 GLIBC_2.30 _ZGVbN4v_logf F
 GLIBC_2.30 _ZGVbN4v_sinf F
 GLIBC_2.30 _ZGVbN4vvv_sincosf F