Bug 1334106 - fuzz mp_invmod, r=ttaubert

Differential Revision: https://nss-review.dev.mozaws.net/D214

14 files changed, 131 insertions, 46 deletions

diff --git a/fuzz/fuzz.gyp b/fuzz/fuzz.gyp
index fc384288a..fee1d01f4 100644
--- a/fuzz/fuzz.gyp
+++ b/fuzz/fuzz.gyp
@@ -245,6 +245,20 @@
       ],
     },
     {
+      'target_name': 'nssfuzz-mpi-invmod',
+      'type': 'executable',
+      'sources': [
+        'mpi_invmod_target.cc',
+      ],
+      'dependencies': [
+        '<(DEPTH)/exports.gyp:nss_exports',
+        'nssfuzz-mpi-base',
+      ],
+      'include_dirs': [
+        '<(DEPTH)/lib/freebl',
+      ],
+    },
+    {
       'target_name': 'nssfuzz-tls-client',
       'type': 'executable',
       'sources': [
@@ -285,6 +299,7 @@
             'nssfuzz-mpi-addmod',
             'nssfuzz-mpi-div',
             'nssfuzz-mpi-expmod',
+            'nssfuzz-mpi-invmod',
             'nssfuzz-mpi-mod',
             'nssfuzz-mpi-mulmod',
             'nssfuzz-mpi-sqr',
diff --git a/fuzz/mpi-invmod.options b/fuzz/mpi-invmod.options
new file mode 100644
index 000000000..a38c2fe33
--- /dev/null
+++ b/fuzz/mpi-invmod.options
@@ -0,0 +1,2 @@
+[libfuzzer]
+max_len = 256
diff --git a/fuzz/mpi_add_target.cc b/fuzz/mpi_add_target.cc
index 2c5247aa9..3ebad370d 100644
--- a/fuzz/mpi_add_target.cc
+++ b/fuzz/mpi_add_target.cc
@@ -14,7 +14,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   if (size < 3) {
     return 0;
   }
-  INIT_NUMBERS
+  INIT_FOUR_NUMBERS
 
   // Compare with OpenSSL addition
   assert(mp_add(&a, &b, &c) == MP_OKAY);
diff --git a/fuzz/mpi_addmod_target.cc b/fuzz/mpi_addmod_target.cc
index efa817983..a7802b62e 100644
--- a/fuzz/mpi_addmod_target.cc
+++ b/fuzz/mpi_addmod_target.cc
@@ -14,7 +14,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   if (size < 3) {
     return 0;
   }
-  INIT_NUMBERS
+  INIT_FOUR_NUMBERS
 
   auto modulus = get_modulus(data, size, ctx);
   // Compare with OpenSSL add mod
diff --git a/fuzz/mpi_div_target.cc b/fuzz/mpi_div_target.cc
index bee3f46d1..08c714ee6 100644
--- a/fuzz/mpi_div_target.cc
+++ b/fuzz/mpi_div_target.cc
@@ -14,7 +14,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   if (size < 3) {
     return 0;
   }
-  INIT_NUMBERS
+  INIT_FOUR_NUMBERS
 
   // We can't divide by 0.
   if (mp_cmp_z(&b) == 0) {
diff --git a/fuzz/mpi_expmod_target.cc b/fuzz/mpi_expmod_target.cc
index bc30169b1..ed31da354 100644
--- a/fuzz/mpi_expmod_target.cc
+++ b/fuzz/mpi_expmod_target.cc
@@ -14,7 +14,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   if (size < 3) {
     return 0;
   }
-  INIT_NUMBERS
+  INIT_FOUR_NUMBERS
 
   auto modulus = get_modulus(data, size, ctx);
   // Compare with OpenSSL exp mod
diff --git a/fuzz/mpi_helper.h b/fuzz/mpi_helper.h
index 8775b533f..17383744b 100644
--- a/fuzz/mpi_helper.h
+++ b/fuzz/mpi_helper.h
@@ -26,7 +26,7 @@ std::tuple<BIGNUM *, mp_int> get_modulus(const uint8_t *data, size_t size,
 
 // Initialise MPI and BN variables
 // XXX: Also silence unused variable warnings for R.
-#define INIT_NUMBERS                     \
+#define INIT_FOUR_NUMBERS                \
   mp_int a, b, c, r;                     \
   mp_int *m1 = nullptr;                  \
   BN_CTX *ctx = BN_CTX_new();            \
@@ -45,6 +45,24 @@ std::tuple<BIGNUM *, mp_int> get_modulus(const uint8_t *data, size_t size,
     (void)(R);                           \
   } while (0);
 
+// Initialise MPI and BN variables
+// XXX: Also silence unused variable warnings for B.
+#define INIT_THREE_NUMBERS        \
+  mp_int a, b, c;                 \
+  BN_CTX *ctx = BN_CTX_new();     \
+  BN_CTX_start(ctx);              \
+  BIGNUM *A = BN_CTX_get(ctx);    \
+  BIGNUM *B = BN_CTX_get(ctx);    \
+  BIGNUM *C = BN_CTX_get(ctx);    \
+  assert(mp_init(&a) == MP_OKAY); \
+  assert(mp_init(&b) == MP_OKAY); \
+  assert(mp_init(&c) == MP_OKAY); \
+  size_t max_size = 4 * size + 1; \
+  parse_input(data, size, A, &a); \
+  do {                            \
+    (void)(B);                    \
+  } while (0);
+
 #define CLEANUP_AND_RETURN \
   mp_clear(&a);            \
   mp_clear(&b);            \
@@ -57,4 +75,12 @@ std::tuple<BIGNUM *, mp_int> get_modulus(const uint8_t *data, size_t size,
   BN_CTX_free(ctx);        \
   return 0;
 
+#define CLEANUP_AND_RETURN_THREE \
+  mp_clear(&a);                  \
+  mp_clear(&b);                  \
+  mp_clear(&c);                  \
+  BN_CTX_end(ctx);               \
+  BN_CTX_free(ctx);              \
+  return 0;
+
 #endif  // mpi_helper_h__
diff --git a/fuzz/mpi_invmod_target.cc b/fuzz/mpi_invmod_target.cc
new file mode 100644
index 000000000..9820af947
--- /dev/null
+++ b/fuzz/mpi_invmod_target.cc
@@ -0,0 +1,70 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * This target fuzzes NSS mpi against openssl bignum.
+ * It therefore requires openssl to be installed.
+ */
+
+#include "mpi_helper.h"
+#include "mpprime.h"
+
+#include <algorithm>
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  // We require at least size 4 to get everything we need from data.
+  if (size < 4) {
+    return 0;
+  }
+
+  INIT_THREE_NUMBERS
+
+  // Make a prime of length size.
+  int count = 0;
+  mp_err res = MP_NO;
+  // mpp_make_prime is so slow :( use something smaller than size.
+  int primeLen = std::max(static_cast<int>(size / 4), 3);
+  uint8_t bp[primeLen];
+  memcpy(bp, data, primeLen);
+  do {
+    bp[0] |= 0x80;            /* set high-order bit */
+    bp[primeLen - 1] |= 0x01; /* set low-order bit  */
+    ++count;
+    assert(mp_read_unsigned_octets(&b, bp, primeLen) == MP_OKAY);
+  } while ((res = mpp_make_prime(&b, primeLen * 8, PR_FALSE, nullptr)) !=
+               MP_YES &&
+           count < 10);
+  if (res != MP_YES) {
+    return 0;
+  }
+
+  // Use the same prime in OpenSSL B
+  char tmp[max_size];
+  mp_toradix(&b, tmp, 16);
+  int tmpLen;
+  assert((tmpLen = BN_hex2bn(&B, tmp)) != 0);
+
+  // Compare with OpenSSL invmod
+  res = mp_invmod(&a, &b, &c);
+  BIGNUM *X = BN_mod_inverse(C, A, B, ctx);
+  if (res != MP_OKAY) {
+    // In case we couldn't compute the inverse, OpenSSL shouldn't be able to
+    // either.
+    assert(X == nullptr);
+  } else {
+    check_equal(C, &c, max_size);
+
+    // Check a * c mod b == 1
+    assert(mp_mulmod(&a, &c, &b, &c) == MP_OKAY);
+    bool eq = mp_cmp_d(&c, 1) == 0;
+    if (!eq) {
+      char cC[max_size];
+      mp_tohex(&c, cC);
+      std::cout << "c = " << std::hex << cC << std::endl;
+    }
+    assert(eq);
+  }
+
+  CLEANUP_AND_RETURN_THREE
+}
diff --git a/fuzz/mpi_mod_target.cc b/fuzz/mpi_mod_target.cc
index 2be1bf081..85c883faf 100644
--- a/fuzz/mpi_mod_target.cc
+++ b/fuzz/mpi_mod_target.cc
@@ -14,7 +14,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   if (size < 3) {
     return 0;
   }
-  INIT_NUMBERS
+  INIT_FOUR_NUMBERS
 
   // We can't divide by 0.
   if (mp_cmp_z(&b) == 0) {
diff --git a/fuzz/mpi_mulmod_target.cc b/fuzz/mpi_mulmod_target.cc
index 7bd4261fa..75585e2d7 100644
--- a/fuzz/mpi_mulmod_target.cc
+++ b/fuzz/mpi_mulmod_target.cc
@@ -14,7 +14,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   if (size < 3) {
     return 0;
   }
-  INIT_NUMBERS
+  INIT_FOUR_NUMBERS
 
   auto modulus = get_modulus(data, size, ctx);
   // Compare with OpenSSL mul mod
diff --git a/fuzz/mpi_sqr_target.cc b/fuzz/mpi_sqr_target.cc
index 1b8504085..b404d624c 100644
--- a/fuzz/mpi_sqr_target.cc
+++ b/fuzz/mpi_sqr_target.cc
@@ -14,16 +14,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   if (size < 2) {
     return 0;
   }
-  mp_int a, c, r;
-  BN_CTX *ctx = BN_CTX_new();
-  BN_CTX_start(ctx);
-  BIGNUM *A = BN_CTX_get(ctx);
-  BIGNUM *C = BN_CTX_get(ctx);
-  assert(mp_init(&a) == MP_OKAY);
-  assert(mp_init(&c) == MP_OKAY);
-  assert(mp_init(&r) == MP_OKAY);
-  size_t max_size = 4 * size + 1;
-  parse_input(data, size, A, &a);
+
+  INIT_THREE_NUMBERS
 
   // Compare with OpenSSL sqr
   assert(mp_sqr(&a, &c) == MP_OKAY);
@@ -31,11 +23,11 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   check_equal(C, &c, max_size);
 
   // Check a * a == a**2
-  assert(mp_mul(&a, &a, &r) == MP_OKAY);
-  bool eq = mp_cmp(&r, &c) == 0;
+  assert(mp_mul(&a, &a, &b) == MP_OKAY);
+  bool eq = mp_cmp(&b, &c) == 0;
   if (!eq) {
     char rC[max_size], cC[max_size], aC[max_size];
-    mp_tohex(&r, rC);
+    mp_tohex(&b, rC);
     mp_tohex(&c, cC);
     mp_tohex(&a, aC);
     std::cout << "a = " << std::hex << aC << std::endl;
@@ -43,11 +35,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
     std::cout << "a ** 2 = " << std::hex << rC << std::endl;
   }
   assert(eq);
-  mp_clear(&a);
-  mp_clear(&c);
-  mp_clear(&r);
-  BN_CTX_end(ctx);
-  BN_CTX_free(ctx);
 
-  return 0;
+  CLEANUP_AND_RETURN_THREE
 }
diff --git a/fuzz/mpi_sqrmod_target.cc b/fuzz/mpi_sqrmod_target.cc
index d3886dacd..ca403b570 100644
--- a/fuzz/mpi_sqrmod_target.cc
+++ b/fuzz/mpi_sqrmod_target.cc
@@ -14,17 +14,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   if (size < 3) {
     return 0;
   }
-  mp_int a, b, c;
-  BN_CTX *ctx = BN_CTX_new();
-  BN_CTX_start(ctx);
-  BIGNUM *A = BN_CTX_get(ctx);
-  BIGNUM *B = BN_CTX_get(ctx);
-  BIGNUM *C = BN_CTX_get(ctx);
-  assert(mp_init(&a) == MP_OKAY);
-  assert(mp_init(&b) == MP_OKAY);
-  assert(mp_init(&c) == MP_OKAY);
-  size_t max_size = 4 * size + 1;
-  parse_input(data, size, A, &a);
+
+  INIT_THREE_NUMBERS
 
   // We can't divide by 0.
   if (mp_cmp_z(&b) == 0) {
@@ -41,11 +32,5 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   (void)BN_mod_sqr(C, A, B, ctx);
   check_equal(C, &c, max_size);
 
-  mp_clear(&a);
-  mp_clear(&b);
-  mp_clear(&c);
-  BN_CTX_end(ctx);
-  BN_CTX_free(ctx);
-
-  return 0;
+  CLEANUP_AND_RETURN_THREE
 }
diff --git a/fuzz/mpi_sub_target.cc b/fuzz/mpi_sub_target.cc
index 1c8a4e90d..da20d74da 100644
--- a/fuzz/mpi_sub_target.cc
+++ b/fuzz/mpi_sub_target.cc
@@ -14,7 +14,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   if (size < 3) {
     return 0;
   }
-  INIT_NUMBERS
+  INIT_FOUR_NUMBERS
 
   // Compare with OpenSSL subtraction
   assert(mp_sub(&a, &b, &c) == MP_OKAY);
diff --git a/fuzz/mpi_submod_target.cc b/fuzz/mpi_submod_target.cc
index d44faceda..26b2c5323 100644
--- a/fuzz/mpi_submod_target.cc
+++ b/fuzz/mpi_submod_target.cc
@@ -14,7 +14,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   if (size < 3) {
     return 0;
   }
-  INIT_NUMBERS
+  INIT_FOUR_NUMBERS
 
   auto modulus = get_modulus(data, size, ctx);
   // Compare with OpenSSL sub mod