Bug 1400591 - ssl3con.c signed/unsigned compilation warnings r=ttaubert

Reviewers: ttaubert

Reviewed By: ttaubert

Bug #: 1400591

Differential Revision: https://phabricator.services.mozilla.com/D93

1 files changed, 28 insertions, 37 deletions

diff --git a/lib/ssl/ssl3con.c b/lib/ssl/ssl3con.c
index 05752716b..79d3fb042 100644
--- a/lib/ssl/ssl3con.c
+++ b/lib/ssl/ssl3con.c
@@ -4324,7 +4324,7 @@ ssl3_ConsumeHandshakeNumber(sslSocket *ss, PRUint32 *num, PRUint32 bytes,
                             PRUint8 **b, PRUint32 *length)
 {
     PRUint8 *buf = *b;
-    int i;
+    PRUint32 i;
 
     PORT_Assert(ss->opt.noLocks || ssl_HaveRecvBufLock(ss));
     PORT_Assert(ss->opt.noLocks || ssl_HaveSSL3HandshakeLock(ss));
@@ -4946,7 +4946,7 @@ ssl3_SendClientHello(sslSocket *ss, sslClientHelloType type)
     sslSessionID *sid;
     ssl3CipherSpec *cwSpec;
     SECStatus rv;
-    int i;
+    unsigned int i;
     int length;
     int num_suites;
     int actual_count = 0;
@@ -7140,11 +7140,11 @@ ssl_HandleDHServerKeyExchange(sslSocket *ss, PRUint8 *b, PRUint32 length)
     }
 
     rv = NSS_OptionGet(NSS_DH_MIN_KEY_SIZE, &minDH);
-    if (rv != SECSuccess) {
+    if (rv != SECSuccess || minDH <= 0) {
         minDH = SSL_DH_MIN_P_BITS;
     }
     dh_p_bits = SECKEY_BigIntegerBitLength(&dh_p);
-    if (dh_p_bits < minDH) {
+    if (dh_p_bits < (unsigned)minDH) {
         errCode = SSL_ERROR_WEAK_SERVER_EPHEMERAL_DH_KEY;
         goto alert_loser;
     }
@@ -8081,8 +8081,8 @@ SECStatus
 ssl3_NegotiateCipherSuite(sslSocket *ss, const SECItem *suites,
                           PRBool initHashes)
 {
-    int j;
-    int i;
+    unsigned int j;
+    unsigned int i;
 
     for (j = 0; j < ssl_V3_SUITES_IMPLEMENTED; j++) {
         ssl3CipherSuiteCfg *suite = &ss->cipherSuites[j];
@@ -8682,15 +8682,6 @@ ssl3_HandleClientHello(sslSocket *ss, PRUint8 *b, PRUint32 length)
         ssl3_DisableNonDTLSSuites(ss);
     }
 
-#ifdef PARANOID
-    /* Look for a matching cipher suite. */
-    j = ssl3_config_match_init(ss);
-    if (j <= 0) {                  /* no ciphers are working/supported by PK11 */
-        errCode = PORT_GetError(); /* error code is already set. */
-        goto alert_loser;
-    }
-#endif
-
     if (ss->version >= SSL_LIBRARY_VERSION_TLS_1_3) {
         rv = tls13_HandleClientHelloPart2(ss, &suites, sid);
     } else {
@@ -8722,7 +8713,7 @@ ssl3_HandleClientHelloPart2(sslSocket *ss,
     SSL3AlertDescription desc = illegal_parameter;
     SECStatus rv;
     unsigned int i;
-    int j;
+    unsigned int j;
 
     /* If we already have a session for this client, be sure to pick the
     ** same cipher suite and compression method we picked before.
@@ -8754,7 +8745,7 @@ ssl3_HandleClientHelloPart2(sslSocket *ss,
                     break;
             }
             PORT_Assert(j > 0);
-            if (j <= 0)
+            if (j == 0)
                 break;
 #ifdef PARANOID
             /* Double check that the cached cipher suite is still enabled,
@@ -8791,8 +8782,7 @@ ssl3_HandleClientHelloPart2(sslSocket *ss,
 
 #ifndef PARANOID
     /* Look for a matching cipher suite. */
-    j = ssl3_config_match_init(ss);
-    if (j <= 0) { /* no ciphers are working/supported by PK11 */
+    if (ssl3_config_match_init(ss) <= 0) {
         desc = internal_error;
         errCode = PORT_GetError(); /* error code is already set. */
         goto alert_loser;
@@ -9694,12 +9684,12 @@ ssl3_SendCertificateRequest(sslSocket *ss)
     PRBool isTLS12;
     const PRUint8 *certTypes;
     SECStatus rv;
-    int length;
+    PRUint32 length;
     SECItem *names;
     unsigned int calen;
     unsigned int nnames;
     SECItem *name;
-    int i;
+    unsigned int i;
     int certTypesLength;
     PRUint8 sigAlgs[MAX_SIGNATURE_SCHEMES * 2];
     unsigned int sigAlgsLength = 0;
@@ -10917,7 +10907,8 @@ ssl3_AuthCertificate(sslSocket *ss)
         }
         if (pubKey) {
             KeyType pubKeyType;
-            PRInt32 minKey;
+            PRUint32 minKey;
+            PRInt32 optval;
             /* This partly fixes Bug 124230 and may cause problems for
              * callers which depend on the old (wrong) behavior. */
             ss->sec.authKeyBits = SECKEY_PublicKeyStrengthInBits(pubKey);
@@ -10928,29 +10919,29 @@ ssl3_AuthCertificate(sslSocket *ss)
                 case rsaPssKey:
                 case rsaOaepKey:
                     rv =
-                        NSS_OptionGet(NSS_RSA_MIN_KEY_SIZE, &minKey);
-                    if (rv !=
-                        SECSuccess) {
-                        minKey =
-                            SSL_RSA_MIN_MODULUS_BITS;
+                        NSS_OptionGet(NSS_RSA_MIN_KEY_SIZE, &optval);
+                    if (rv == SECSuccess && optval > 0) {
+                        minKey = (PRUint32)optval;
+                    } else {
+                        minKey = SSL_RSA_MIN_MODULUS_BITS;
                     }
                     break;
                 case dsaKey:
                     rv =
-                        NSS_OptionGet(NSS_DSA_MIN_KEY_SIZE, &minKey);
-                    if (rv !=
-                        SECSuccess) {
-                        minKey =
-                            SSL_DSA_MIN_P_BITS;
+                        NSS_OptionGet(NSS_DSA_MIN_KEY_SIZE, &optval);
+                    if (rv == SECSuccess && optval > 0) {
+                        minKey = (PRUint32)optval;
+                    } else {
+                        minKey = SSL_DSA_MIN_P_BITS;
                     }
                     break;
                 case dhKey:
                     rv =
-                        NSS_OptionGet(NSS_DH_MIN_KEY_SIZE, &minKey);
-                    if (rv !=
-                        SECSuccess) {
-                        minKey =
-                            SSL_DH_MIN_P_BITS;
+                        NSS_OptionGet(NSS_DH_MIN_KEY_SIZE, &optval);
+                    if (rv == SECSuccess && optval > 0) {
+                        minKey = (PRUint32)optval;
+                    } else {
+                        minKey = SSL_DH_MIN_P_BITS;
                     }
                     break;
                 default: