neon: SetResidualCoeffs

Much faster with aarch64. Still somewhat faster without vmaxv.

C: 3.700s
ArmV7: 3.675
aarch64: 3.600

BUG=b/118740850

Change-Id: I3be852da89633eca4bddce443c87f5e4a2f55868

7 files changed, 75 insertions, 0 deletions

diff --git a/Android.mk b/Android.mk
index fb960f17..8f0cb756 100644
--- a/Android.mk
+++ b/Android.mk
@@ -97,6 +97,7 @@ dsp_enc_srcs := \
     src/dsp/cost.c \
     src/dsp/cost_mips32.c \
     src/dsp/cost_mips_dsp_r2.c \
+    src/dsp/cost_neon.$(NEON) \
     src/dsp/cost_sse2.c \
     src/dsp/enc.c \
     src/dsp/enc_mips32.c \
diff --git a/Makefile.vc b/Makefile.vc
index 0f9c9c1c..8a54559a 100644
--- a/Makefile.vc
+++ b/Makefile.vc
@@ -238,6 +238,7 @@ DSP_ENC_OBJS = \
     $(DIROBJ)\dsp\cost.obj \
     $(DIROBJ)\dsp\cost_mips32.obj \
     $(DIROBJ)\dsp\cost_mips_dsp_r2.obj \
+    $(DIROBJ)\dsp\cost_neon.obj \
     $(DIROBJ)\dsp\cost_sse2.obj \
     $(DIROBJ)\dsp\enc.obj \
     $(DIROBJ)\dsp\enc_mips32.obj \
diff --git a/build.gradle b/build.gradle
index 233562f6..bb91fac0 100644
--- a/build.gradle
+++ b/build.gradle
@@ -173,6 +173,7 @@ model {
             include "cost.c"
             include "cost_mips32.c"
             include "cost_mips_dsp_r2.c"
+            include "cost_neon.$NEON"
             include "cost_sse2.c"
             include "enc.c"
             include "enc_mips32.c"
diff --git a/makefile.unix b/makefile.unix
index ed64fc54..87b6b653 100644
--- a/makefile.unix
+++ b/makefile.unix
@@ -185,6 +185,7 @@ DSP_ENC_OBJS = \
     src/dsp/cost.o \
     src/dsp/cost_mips32.o \
     src/dsp/cost_mips_dsp_r2.o \
+    src/dsp/cost_neon.o \
     src/dsp/cost_sse2.o \
     src/dsp/enc.o \
     src/dsp/enc_mips32.o \
diff --git a/src/dsp/Makefile.am b/src/dsp/Makefile.am
index b2e79948..8cd3e53c 100644
--- a/src/dsp/Makefile.am
+++ b/src/dsp/Makefile.am
@@ -121,6 +121,7 @@ libwebpdsp_sse41_la_CFLAGS = $(AM_CFLAGS) $(SSE41_FLAGS)
 libwebpdsp_sse41_la_LIBADD = libwebpdspdecode_sse41.la
 
 libwebpdsp_neon_la_SOURCES =
+libwebpdsp_neon_la_SOURCES += cost_neon.c
 libwebpdsp_neon_la_SOURCES += enc_neon.c
 libwebpdsp_neon_la_SOURCES += lossless_enc_neon.c
 libwebpdsp_neon_la_CPPFLAGS = $(libwebpdsp_la_CPPFLAGS)
diff --git a/src/dsp/cost.c b/src/dsp/cost.c
index 634ccc20..cc681cdd 100644
--- a/src/dsp/cost.c
+++ b/src/dsp/cost.c
@@ -377,6 +377,7 @@ VP8SetResidualCoeffsFunc VP8SetResidualCoeffs;
 extern void VP8EncDspCostInitMIPS32(void);
 extern void VP8EncDspCostInitMIPSdspR2(void);
 extern void VP8EncDspCostInitSSE2(void);
+extern void VP8EncDspCostInitNEON(void);
 
 WEBP_DSP_INIT_FUNC(VP8EncDspCostInit) {
   VP8GetResidualCost = GetResidualCost_C;
@@ -399,6 +400,11 @@ WEBP_DSP_INIT_FUNC(VP8EncDspCostInit) {
       VP8EncDspCostInitSSE2();
     }
 #endif
+#if defined(WEBP_USE_NEON)
+    if (VP8GetCPUInfo(kNEON)) {
+      VP8EncDspCostInitNEON();
+    }
+#endif
   }
 }
 
diff --git a/src/dsp/cost_neon.c b/src/dsp/cost_neon.c
new file mode 100644
index 00000000..faa6e196
--- /dev/null
+++ b/src/dsp/cost_neon.c
@@ -0,0 +1,64 @@
+// Copyright 2018 Google Inc. All Rights Reserved.
+//
+// Use of this source code is governed by a BSD-style license
+// that can be found in the COPYING file in the root of the source
+// tree. An additional intellectual property rights grant can be found
+// in the file PATENTS. All contributing project authors may
+// be found in the AUTHORS file in the root of the source tree.
+// -----------------------------------------------------------------------------
+//
+// ARM NEON version of cost functions
+
+#include "src/dsp/dsp.h"
+
+#if defined(WEBP_USE_NEON)
+
+#include "src/dsp/neon.h"
+#include "src/enc/cost_enc.h"
+
+static const uint8_t position[16] = { 1, 2,  3,  4,  5,  6,  7,  8,
+                                      9, 10, 11, 12, 13, 14, 15, 16 };
+
+static void SetResidualCoeffs_NEON(const int16_t* const coeffs,
+                                   VP8Residual* const res) {
+  const int16x8_t minus_one = vdupq_n_s16(-1);
+  const int16x8_t coeffs_0 = vld1q_s16(coeffs);
+  const int16x8_t coeffs_1 = vld1q_s16(coeffs + 8);
+  const uint16x8_t eob_0 = vtstq_s16(coeffs_0, minus_one);
+  const uint16x8_t eob_1 = vtstq_s16(coeffs_1, minus_one);
+  const uint8x16_t eob = vcombine_u8(vqmovn_u16(eob_0), vqmovn_u16(eob_1));
+  const uint8x16_t masked = vandq_u8(eob, vld1q_u8(position));
+
+#ifdef __aarch64__
+  res->last = vmaxvq_u8(masked) - 1;
+#else
+  const uint8x8_t eob_8x8 = vmax_u8(vget_low_u8(masked), vget_high_u8(masked));
+  const uint16x8_t eob_16x8 = vmovl_u8(eob_8x8);
+  const uint16x4_t eob_16x4 =
+      vmax_u16(vget_low_u16(eob_16x8), vget_high_u16(eob_16x8));
+  const uint32x4_t eob_32x4 = vmovl_u16(eob_16x4);
+  uint32x2_t eob_32x2 =
+      vmax_u32(vget_low_u32(eob_32x4), vget_high_u32(eob_32x4));
+  eob_32x2 = vpmax_u32(eob_32x2, eob_32x2);
+
+  vst1_lane_s32(&res->last, vreinterpret_s32_u32(eob_32x2), 0);
+  --res->last;
+#endif  // __aarch64__
+
+  res->coeffs = coeffs;
+}
+
+//------------------------------------------------------------------------------
+// Entry point
+
+extern void VP8EncDspCostInitNEON(void);
+
+WEBP_TSAN_IGNORE_FUNCTION void VP8EncDspCostInitNEON(void) {
+  VP8SetResidualCoeffs = SetResidualCoeffs_NEON;
+}
+
+#else  // !WEBP_USE_NEON
+
+WEBP_DSP_INIT_STUB(VP8EncDspCostInitNEON)
+
+#endif  // WEBP_USE_NEON