Bug 1369091 - check that e > 2 and odd in RSA_NewKey, r=franziskus

Summary: Patch for bug 1369091

Reviewers: franziskus

Reviewed By: franziskus

Bug #: 1369091

Differential Revision: https://phabricator.services.mozilla.com/D165

2 files changed, 58 insertions, 0 deletions

diff --git a/gtests/freebl_gtest/freebl_gtest.gyp b/gtests/freebl_gtest/freebl_gtest.gyp
index aa0810889..21a87c557 100644
--- a/gtests/freebl_gtest/freebl_gtest.gyp
+++ b/gtests/freebl_gtest/freebl_gtest.gyp
@@ -33,6 +33,7 @@
         'dh_unittest.cc',
         'ecl_unittest.cc',
         'ghash_unittest.cc',
+        'rsa_unittest.cc',
         '<(DEPTH)/gtests/common/gtests.cc'
       ],
       'dependencies': [
diff --git a/gtests/freebl_gtest/rsa_unittest.cc b/gtests/freebl_gtest/rsa_unittest.cc
new file mode 100644
index 000000000..c2c435330
--- /dev/null
+++ b/gtests/freebl_gtest/rsa_unittest.cc
@@ -0,0 +1,57 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#include "gtest/gtest.h"
+
+#include <stdint.h>
+
+#include "blapi.h"
+#include "secitem.h"
+
+template <class T>
+struct ScopedDelete {
+  void operator()(T* ptr) {
+    if (ptr) {
+      PORT_FreeArena(ptr->arena, PR_TRUE);
+    }
+  }
+};
+
+typedef std::unique_ptr<RSAPrivateKey, ScopedDelete<RSAPrivateKey>>
+    ScopedRSAPrivateKey;
+
+class RSANewKeyTest : public ::testing::Test {
+ protected:
+  RSAPrivateKey* CreateKeyWithExponent(int keySizeInBits,
+                                       unsigned char publicExponent) {
+    SECItem exp = {siBuffer, 0, 0};
+    unsigned char pubExp[1] = {publicExponent};
+    exp.data = pubExp;
+    exp.len = 1;
+
+    return RSA_NewKey(keySizeInBits, &exp);
+  }
+};
+
+TEST_F(RSANewKeyTest, expOneTest) {
+  ScopedRSAPrivateKey key(CreateKeyWithExponent(2048, 0x01));
+  ASSERT_TRUE(key == nullptr);
+}
+TEST_F(RSANewKeyTest, expTwoTest) {
+  ScopedRSAPrivateKey key(CreateKeyWithExponent(2048, 0x02));
+  ASSERT_TRUE(key == nullptr);
+}
+TEST_F(RSANewKeyTest, expFourTest) {
+  ScopedRSAPrivateKey key(CreateKeyWithExponent(2048, 0x04));
+  ASSERT_TRUE(key == nullptr);
+}
+TEST_F(RSANewKeyTest, WrongKeysizeTest) {
+  ScopedRSAPrivateKey key(CreateKeyWithExponent(2047, 0x03));
+  ASSERT_TRUE(key == nullptr);
+}
+
+TEST_F(RSANewKeyTest, expThreeTest) {
+  ScopedRSAPrivateKey key(CreateKeyWithExponent(2048, 0x03));
+  ASSERT_TRUE(key != nullptr);
+}