Bug 1687164 - Introduce NSS_DISABLE_CRYPTO_VSX and disable_crypto_vsx r=bbeurdouche

Currently, NSS assumes that every PowerPC target supports the crypto
and VSX extensions of the PowerPC ABI. However, VSX was only introduced
with ISA version 2.06 and the crypto extensions with ISA version 2.07
and enabling them on older PowerPC targets will result in a SIGILL. Thus,
make their use configurable and enable them by default on ppc64le only.

Differential Revision: https://phabricator.services.mozilla.com/D105354

5 files changed, 112 insertions, 45 deletions

diff --git a/coreconf/Linux.mk b/coreconf/Linux.mk
index 956f0e4a9..70651e28f 100644
--- a/coreconf/Linux.mk
+++ b/coreconf/Linux.mk
@@ -54,6 +54,11 @@ ifeq (,$(filter-out ppc64 ppc64le,$(OS_TEST)))
 ifeq ($(USE_64),1)
 	ARCHFLAG	= -m64
 endif
+ifeq (,$(filter-out ppc ppc64,$(OS_TEST)))
+ifneq ($(NSS_DISABLE_CRYPTO_VSX),0)
+	NSS_DISABLE_CRYPTO_VSX=1
+endif
+endif
 else
 ifeq ($(OS_TEST),alpha)
         OS_REL_CFLAGS   = -D_ALPHA_
diff --git a/coreconf/config.gypi b/coreconf/config.gypi
index 4c8931c6d..951e9abd1 100644
--- a/coreconf/config.gypi
+++ b/coreconf/config.gypi
@@ -107,6 +107,7 @@
     'disable_libpkix%': 1,
     'disable_werror%': 0,
     'disable_altivec%': 0,
+    'disable_crypto_vsx%': 0,
     'disable_arm32_neon%': 0,
     'mozilla_client%': 0,
     'comm_client%': 0,
diff --git a/coreconf/config.mk b/coreconf/config.mk
index 2f7b63896..4ae941c47 100644
--- a/coreconf/config.mk
+++ b/coreconf/config.mk
@@ -231,10 +231,15 @@ DEFINES += -DNSS_DISABLE_ARM32_NEON
 endif
 
 # Avoid building with PowerPC's Altivec acceleration
-ifdef NSS_DISABLE_ALTIVEC
+ifeq ($(NSS_DISABLE_ALTIVEC),1)
 DEFINES += -DNSS_DISABLE_ALTIVEC
 endif
 
+# Avoid building with PowerPC's Crypto and VSX instructions
+ifeq ($(NSS_DISABLE_CRYPTO_VSX),1)
+DEFINES += -DNSS_DISABLE_CRYPTO_VSX
+endif
+
 # This allows all library and tools code to use the util function
 # implementations directly from libnssutil3, rather than the wrappers
 # in libnss3 which are present for binary compatibility only
diff --git a/lib/freebl/Makefile b/lib/freebl/Makefile
index ef93d075f..3d5357297 100644
--- a/lib/freebl/Makefile
+++ b/lib/freebl/Makefile
@@ -760,15 +760,20 @@ $(OBJDIR)/$(PROG_PREFIX)sha256-armv8$(OBJ_SUFFIX): CFLAGS += -march=armv8-a+cryp
 endif
 
 ifeq ($(CPU_ARCH),ppc)
-ifndef NSS_DISABLE_ALTIVEC
-$(OBJDIR)/$(PROG_PREFIX)gcm-ppc$(OBJ_SUFFIX): CFLAGS += -mcrypto -maltivec -mvsx
-$(OBJDIR)/$(PROG_PREFIX)gcm$(OBJ_SUFFIX): CFLAGS += -mcrypto -maltivec -mvsx
-$(OBJDIR)/$(PROG_PREFIX)rijndael$(OBJ_SUFFIX): CFLAGS += -mcrypto -maltivec -mvsx
-$(OBJDIR)/$(PROG_PREFIX)sha512$(OBJ_SUFFIX): CFLAGS += -mcrypto -maltivec -mvsx \
-					-funroll-loops -fpeel-loops
-ifeq ($(OS_TEST),ppc64le)
-$(OBJDIR)/$(PROG_PREFIX)chacha20poly1305-ppc$(OBJ_SUFFIX): CFLAGS += -mcrypto -maltivec -mvsx
-endif # ppc64le
+$(OBJDIR)/$(PROG_PREFIX)sha512$(OBJ_SUFFIX): CFLAGS += -funroll-loops -fpeel-loops
+ifneq ($(NSS_DISABLE_ALTIVEC),1)
+$(OBJDIR)/$(PROG_PREFIX)gcm-ppc$(OBJ_SUFFIX): CFLAGS += -maltivec
+$(OBJDIR)/$(PROG_PREFIX)gcm$(OBJ_SUFFIX): CFLAGS += -maltivec
+$(OBJDIR)/$(PROG_PREFIX)rijndael$(OBJ_SUFFIX): CFLAGS += -maltivec
+$(OBJDIR)/$(PROG_PREFIX)sha512$(OBJ_SUFFIX): CFLAGS += -maltivec
+$(OBJDIR)/$(PROG_PREFIX)chacha20poly1305-ppc$(OBJ_SUFFIX): CFLAGS += -maltivec
+endif
+ifneq ($(NSS_DISABLE_CRYPTO_VSX),1)
+$(OBJDIR)/$(PROG_PREFIX)gcm-ppc$(OBJ_SUFFIX): CFLAGS += -mcrypto -mvsx
+$(OBJDIR)/$(PROG_PREFIX)gcm$(OBJ_SUFFIX): CFLAGS += -mcrypto -mvsx
+$(OBJDIR)/$(PROG_PREFIX)rijndael$(OBJ_SUFFIX): CFLAGS += -mcrypto -mvsx
+$(OBJDIR)/$(PROG_PREFIX)sha512$(OBJ_SUFFIX): CFLAGS += -mcrypto -mvsx
+$(OBJDIR)/$(PROG_PREFIX)chacha20poly1305-ppc$(OBJ_SUFFIX): CFLAGS += -mcrypto -mvsx
 endif
 endif
 
diff --git a/lib/freebl/freebl.gyp b/lib/freebl/freebl.gyp
index 86f05aa33..865f89232 100644
--- a/lib/freebl/freebl.gyp
+++ b/lib/freebl/freebl.gyp
@@ -264,14 +264,25 @@
       'dependencies': [
         '<(DEPTH)/exports.gyp:nss_exports'
       ],
-      'cflags': [
-        '-mcrypto',
-        '-maltivec'
-      ],
-      'cflags_mozilla': [
-        '-mcrypto',
-        '-maltivec'
-      ],
+      'conditions': [
+        [ 'disable_crypto_vsx==0', {
+          'cflags': [
+            '-mcrypto',
+            '-maltivec'
+           ],
+           'cflags_mozilla': [
+             '-mcrypto',
+             '-maltivec'
+           ],
+        }, 'disable_crypto_vsx==1', {
+          'cflags': [
+            '-maltivec'
+          ],
+          'cflags_mozilla': [
+            '-maltivec'
+          ],
+        }]
+      ]
     },
     {
       'target_name': 'gcm-sha512-nodepend-ppc_c_lib',
@@ -282,20 +293,35 @@
       'dependencies': [
         '<(DEPTH)/exports.gyp:nss_exports'
       ],
-      'cflags': [
-        '-mcrypto',
-        '-maltivec',
-        '-mvsx',
-        '-funroll-loops',
-        '-fpeel-loops',
-      ],
-      'cflags_mozilla': [
-        '-mcrypto',
-        '-maltivec',
-        '-mvsx',
-        '-funroll-loops',
-        '-fpeel-loops',
-      ],
+      'conditions': [
+        [ 'disable_crypto_vsx==0', {
+          'cflags': [
+            '-mcrypto',
+            '-maltivec',
+            '-mvsx',
+            '-funroll-loops',
+            '-fpeel-loops'
+           ],
+           'cflags_mozilla': [
+            '-mcrypto',
+            '-maltivec',
+            '-mvsx',
+            '-funroll-loops',
+            '-fpeel-loops'
+           ],
+        }, 'disable_crypto_vsx==1', {
+          'cflags': [
+            '-maltivec',
+            '-funroll-loops',
+            '-fpeel-loops'
+          ],
+          'cflags_mozilla': [
+            '-maltivec',
+            '-funroll-loops',
+            '-fpeel-loops'
+          ],
+        }]
+      ]
     },
     {
       'target_name': 'gcm-sha512-ppc_c_lib',
@@ -306,19 +332,34 @@
       'dependencies': [
         '<(DEPTH)/exports.gyp:nss_exports'
       ],
-      'cflags': [
-        '-mcrypto',
-        '-maltivec',
-        '-mvsx',
-        '-funroll-loops',
-        '-fpeel-loops',
-      ],
-      'cflags_mozilla': [
-        '-mcrypto',
-        '-maltivec',
-        '-mvsx',
-        '-funroll-loops',
-        '-fpeel-loops',
+      'conditions': [
+        [ 'disable_crypto_vsx==0', {
+          'cflags': [
+            '-mcrypto',
+            '-maltivec',
+            '-mvsx',
+            '-funroll-loops',
+            '-fpeel-loops'
+           ],
+           'cflags_mozilla': [
+            '-mcrypto',
+            '-maltivec',
+            '-mvsx',
+            '-funroll-loops',
+            '-fpeel-loops'
+           ],
+        }, 'disable_crypto_vsx==1', {
+          'cflags': [
+            '-maltivec',
+            '-funroll-loops',
+            '-fpeel-loops'
+          ],
+          'cflags_mozilla': [
+            '-maltivec',
+            '-funroll-loops',
+            '-fpeel-loops'
+          ],
+        }]
       ],
       'defines!': [
         'FREEBL_NO_DEPEND',
@@ -432,6 +473,11 @@
             'NSS_DISABLE_ALTIVEC',
           ],
         }],
+        [ 'disable_crypto_vsx==1 and (target_arch=="ppc" or target_arch=="ppc64" or target_arch=="ppc64le")', {
+          'defines!': [
+            'NSS_DISABLE_CRYPTO_VSX',
+          ],
+        }],
         [ 'OS=="linux"', {
           'defines!': [
             'FREEBL_NO_DEPEND',
@@ -500,6 +546,11 @@
             'NSS_DISABLE_ALTIVEC',
           ],
         }],
+        [ 'disable_crypto_vsx==1 and (target_arch=="ppc" or target_arch=="ppc64" or target_arch=="ppc64le")', {
+          'defines!': [
+            'NSS_DISABLE_CRYPTO_VSX',
+          ],
+        }],
         [ 'OS!="linux"', {
           'conditions': [
             [ 'moz_fold_libs==0', {