Bug 1485864 - improve padding checks in RSA_DecryptBlock, r=mt

Differential Revision: https://phabricator.services.mozilla.com//D10357

1 files changed, 42 insertions, 6 deletions

diff --git a/gtests/freebl_gtest/rsa_unittest.cc b/gtests/freebl_gtest/rsa_unittest.cc
index 5c667a1d1..a1453168f 100644
--- a/gtests/freebl_gtest/rsa_unittest.cc
+++ b/gtests/freebl_gtest/rsa_unittest.cc
@@ -21,7 +21,7 @@ struct ScopedDelete {
 typedef std::unique_ptr<RSAPrivateKey, ScopedDelete<RSAPrivateKey>>
     ScopedRSAPrivateKey;
 
-class RSANewKeyTest : public ::testing::Test {
+class RSATest : public ::testing::Test {
  protected:
   RSAPrivateKey* CreateKeyWithExponent(int keySizeInBits,
                                        unsigned char publicExponent) {
@@ -34,24 +34,24 @@ class RSANewKeyTest : public ::testing::Test {
   }
 };
 
-TEST_F(RSANewKeyTest, expOneTest) {
+TEST_F(RSATest, expOneTest) {
   ScopedRSAPrivateKey key(CreateKeyWithExponent(2048, 0x01));
   ASSERT_TRUE(key == nullptr);
 }
-TEST_F(RSANewKeyTest, expTwoTest) {
+TEST_F(RSATest, expTwoTest) {
   ScopedRSAPrivateKey key(CreateKeyWithExponent(2048, 0x02));
   ASSERT_TRUE(key == nullptr);
 }
-TEST_F(RSANewKeyTest, expFourTest) {
+TEST_F(RSATest, expFourTest) {
   ScopedRSAPrivateKey key(CreateKeyWithExponent(2048, 0x04));
   ASSERT_TRUE(key == nullptr);
 }
-TEST_F(RSANewKeyTest, WrongKeysizeTest) {
+TEST_F(RSATest, WrongKeysizeTest) {
   ScopedRSAPrivateKey key(CreateKeyWithExponent(2047, 0x03));
   ASSERT_TRUE(key == nullptr);
 }
 
-TEST_F(RSANewKeyTest, expThreeTest) {
+TEST_F(RSATest, expThreeTest) {
   ScopedRSAPrivateKey key(CreateKeyWithExponent(2048, 0x03));
 #ifdef NSS_FIPS_DISABLED
   ASSERT_TRUE(key != nullptr);
@@ -59,3 +59,39 @@ TEST_F(RSANewKeyTest, expThreeTest) {
   ASSERT_TRUE(key == nullptr);
 #endif
 }
+
+TEST_F(RSATest, DecryptBlockTestErrors) {
+  unsigned char pubExp[3] = {0x01, 0x00, 0x01};
+  SECItem exp = {siBuffer, pubExp, 3};
+  ScopedRSAPrivateKey key(RSA_NewKey(2048, &exp));
+  ASSERT_TRUE(key);
+  uint8_t out[10] = {0};
+  uint8_t in_small[100] = {0};
+  unsigned int outputLen = 0;
+  unsigned int maxOutputLen = sizeof(out);
+
+  // This should fail because input the same size as the modulus (256).
+  SECStatus rv = RSA_DecryptBlock(key.get(), out, &outputLen, maxOutputLen,
+                                  in_small, sizeof(in_small));
+  EXPECT_EQ(SECFailure, rv);
+
+  uint8_t in[256] = {0};
+  // This should fail because the padding checks will fail.
+  rv = RSA_DecryptBlock(key.get(), out, &outputLen, maxOutputLen, in,
+                        sizeof(in));
+  EXPECT_EQ(SECFailure, rv);
+  // outputLen should be maxOutputLen.
+  EXPECT_EQ(maxOutputLen, outputLen);
+
+  // This should fail because the padding checks will fail.
+  uint8_t out_long[260] = {0};
+  maxOutputLen = sizeof(out_long);
+  rv = RSA_DecryptBlock(key.get(), out_long, &outputLen, maxOutputLen, in,
+                        sizeof(in));
+  EXPECT_EQ(SECFailure, rv);
+  // outputLen should <= 256-11=245.
+  EXPECT_LE(outputLen, 245u);
+  // Everything over 256 must be 0 in the output.
+  uint8_t out_long_test[4] = {0};
+  EXPECT_EQ(0, memcmp(out_long_test, &out_long[256], 4));
+}