Make SSL API consistent in using SECStatus as return value for functions

that return only values in that enumeration.  Bug 68097. R&A = relyea.
Modified Files:
 	lib/ssl/ssl.h lib/ssl/sslauth.c lib/ssl/sslsecur.c
 	lib/ssl/sslsnce.c lib/ssl/sslsock.c cmd/selfserv/selfserv.c
 	cmd/strsclnt/strsclnt.c

5 files changed, 80 insertions, 81 deletions

diff --git a/security/nss/lib/ssl/ssl.h b/security/nss/lib/ssl/ssl.h
index 7e434aa33..d5cc34a84 100644
--- a/security/nss/lib/ssl/ssl.h
+++ b/security/nss/lib/ssl/ssl.h
@@ -165,7 +165,7 @@ SSL_IMPORT SECStatus SSL_ResetHandshake(PRFileDesc *fd, PRBool asServer);
 ** Force the handshake for fd to complete immediately.  This blocks until
 ** the complete SSL handshake protocol is finished.
 */
-SSL_IMPORT int SSL_ForceHandshake(PRFileDesc *fd);
+SSL_IMPORT SECStatus SSL_ForceHandshake(PRFileDesc *fd);
 
 /*
 ** Query security status of socket. *on is set to one if security is
@@ -178,9 +178,9 @@ SSL_IMPORT int SSL_ForceHandshake(PRFileDesc *fd);
 ** data is not needed.  All strings returned by this function are owned
 ** by SSL, and will be freed when the socket is closed.
 */
-SSL_IMPORT int SSL_SecurityStatus(PRFileDesc *fd, int *on, char **cipher,
-			      int *keySize, int *secretKeySize,
-			      char **issuer, char **subject);
+SSL_IMPORT SECStatus SSL_SecurityStatus(PRFileDesc *fd, int *on, char **cipher,
+			                int *keySize, int *secretKeySize,
+			                char **issuer, char **subject);
 
 /* Values for "on" */
 #define SSL_SECURITY_STATUS_NOOPT	-1
@@ -203,14 +203,16 @@ SSL_IMPORT CERTCertificate *SSL_PeerCertificate(PRFileDesc *fd);
 ** (because of SSL_REQUIRE_CERTIFICATE in SSL_Enable) to authenticate the
 ** certificate.
 */
-typedef int (*SSLAuthCertificate)(void *arg, PRFileDesc *fd, PRBool checkSig,
-				  PRBool isServer);
-SSL_IMPORT int SSL_AuthCertificateHook(PRFileDesc *fd, SSLAuthCertificate f,
-				   void *arg);
+typedef SECStatus (*SSLAuthCertificate)(void *arg, PRFileDesc *fd, 
+					PRBool checkSig, PRBool isServer);
+
+SSL_IMPORT SECStatus SSL_AuthCertificateHook(PRFileDesc *fd, 
+					     SSLAuthCertificate f,
+				             void *arg);
 
 /* An implementation of the certificate authentication hook */
-SSL_IMPORT int SSL_AuthCertificate(void *arg, PRFileDesc *fd, PRBool checkSig,
-			       PRBool isServer);
+SSL_IMPORT SECStatus SSL_AuthCertificate(void *arg, PRFileDesc *fd, 
+					 PRBool checkSig, PRBool isServer);
 
 /*
  * Prototype for SSL callback to get client auth data from the application.
@@ -219,7 +221,7 @@ SSL_IMPORT int SSL_AuthCertificate(void *arg, PRFileDesc *fd, PRBool checkSig,
  *	pRetCert - pointer to pointer to cert, for return of cert
  *	pRetKey - pointer to key pointer, for return of key
  */
-typedef int (*SSLGetClientAuthData)(void *arg, PRFileDesc *fd,
+typedef SECStatus (*SSLGetClientAuthData)(void *arg, PRFileDesc *fd,
 				    CERTDistNames *caNames,
 				    CERTCertificate **pRetCert,/*return */
 				    SECKEYPrivateKey **pRetKey);/* return */
@@ -231,8 +233,8 @@ typedef int (*SSLGetClientAuthData)(void *arg, PRFileDesc *fd,
  *	f - the application's callback that delivers the key and cert
  *	a - application specific data
  */
-SSL_IMPORT int SSL_GetClientAuthDataHook(PRFileDesc *fd, SSLGetClientAuthData f,
-				     void *a);
+SSL_IMPORT SECStatus SSL_GetClientAuthDataHook(PRFileDesc *fd, 
+			                       SSLGetClientAuthData f, void *a);
 
 
 /*
@@ -240,15 +242,16 @@ SSL_IMPORT int SSL_GetClientAuthDataHook(PRFileDesc *fd, SSLGetClientAuthData f,
  *	fd - the file descriptor for the connection in question
  *	a - pkcs11 application specific data
  */
-SSL_IMPORT int SSL_SetPKCS11PinArg(PRFileDesc *fd, void *a);
+SSL_IMPORT SECStatus SSL_SetPKCS11PinArg(PRFileDesc *fd, void *a);
 
 /*
 ** This is a callback for dealing with server certs that are not authenticated
 ** by the client.  The client app can decide that it actually likes the
 ** cert by some external means and restart the connection.
 */
-typedef int (*SSLBadCertHandler)(void *arg, PRFileDesc *fd);
-SSL_IMPORT int SSL_BadCertHook(PRFileDesc *fd, SSLBadCertHandler f, void *arg);
+typedef SECStatus (*SSLBadCertHandler)(void *arg, PRFileDesc *fd);
+SSL_IMPORT SECStatus SSL_BadCertHook(PRFileDesc *fd, SSLBadCertHandler f, 
+				     void *arg);
 
 /*
 ** Configure ssl for running a secure server. Needs the
@@ -264,7 +267,8 @@ typedef enum {
     kt_kea_size
 } SSLKEAType;
 
-SSL_IMPORT SECStatus SSL_ConfigSecureServer(PRFileDesc *fd, CERTCertificate *cert,
+SSL_IMPORT SECStatus SSL_ConfigSecureServer(
+				PRFileDesc *fd, CERTCertificate *cert,
 				SECKEYPrivateKey *key, SSLKEAType kea);
 
 /*
@@ -275,10 +279,10 @@ SSL_IMPORT SECStatus SSL_ConfigSecureServer(PRFileDesc *fd, CERTCertificate *cer
 ** This version of the function is for use in applications that have only one 
 ** process that uses the cache (even if that process has multiple threads).
 */
-SSL_IMPORT int SSL_ConfigServerSessionIDCache(int      maxCacheEntries,
-					  PRUint32 timeout,
-					  PRUint32 ssl3_timeout,
-				    const char *   directory);
+SSL_IMPORT SECStatus SSL_ConfigServerSessionIDCache(int      maxCacheEntries,
+					            PRUint32 timeout,
+					            PRUint32 ssl3_timeout,
+				              const char *   directory);
 /*
 ** Like SSL_ConfigServerSessionIDCache, with one important difference.
 ** If the application will run multiple processes (as opposed to, or in 
@@ -288,10 +292,10 @@ SSL_IMPORT int SSL_ConfigServerSessionIDCache(int      maxCacheEntries,
 ** This function sets up a Server Session ID (SID) cache that is safe for
 ** access by multiple processes on the same system.
 */
-SSL_IMPORT int SSL_ConfigMPServerSIDCache(int      maxCacheEntries, 
-				      PRUint32 timeout,
-			       	      PRUint32 ssl3_timeout, 
-		                const char *   directory);
+SSL_IMPORT SECStatus SSL_ConfigMPServerSIDCache(int      maxCacheEntries, 
+				                PRUint32 timeout,
+			       	                PRUint32 ssl3_timeout, 
+		                          const char *   directory);
 
 /* environment variable set by SSL_ConfigMPServerSIDCache, and queried by
  * SSL_InheritMPServerSIDCache when envString is NULL.
@@ -310,8 +314,8 @@ SSL_IMPORT SECStatus SSL_InheritMPServerSIDCache(const char * envString);
 ** performing a handshake.
 */
 typedef void (*SSLHandshakeCallback)(PRFileDesc *fd, void *client_data);
-SSL_IMPORT int SSL_HandshakeCallback(PRFileDesc *fd, SSLHandshakeCallback cb,
-				 void *client_data);
+SSL_IMPORT SECStatus SSL_HandshakeCallback(PRFileDesc *fd, 
+			          SSLHandshakeCallback cb, void *client_data);
 
 /*
 ** For the server, request a new handshake.  For the client, begin a new
@@ -321,7 +325,7 @@ SSL_IMPORT int SSL_HandshakeCallback(PRFileDesc *fd, SSLHandshakeCallback cb,
 ** do the much faster session restart handshake.  This will change the 
 ** session keys without doing another private key operation.
 */
-SSL_IMPORT int SSL_ReHandshake(PRFileDesc *fd, PRBool flushCache);
+SSL_IMPORT SECStatus SSL_ReHandshake(PRFileDesc *fd, PRBool flushCache);
 
 #ifdef SSL_DEPRECATED_FUNCTION 
 /* deprecated!
@@ -330,13 +334,13 @@ SSL_IMPORT int SSL_ReHandshake(PRFileDesc *fd, PRBool flushCache);
 ** full handshake will be done.  
 ** This call is equivalent to SSL_ReHandshake(fd, PR_TRUE)
 */
-SSL_IMPORT int SSL_RedoHandshake(PRFileDesc *fd);
+SSL_IMPORT SECStatus SSL_RedoHandshake(PRFileDesc *fd);
 #endif
 
 /*
  * Allow the application to pass a URL or hostname into the SSL library
  */
-SSL_IMPORT int SSL_SetURL(PRFileDesc *fd, const char *url);
+SSL_IMPORT SECStatus SSL_SetURL(PRFileDesc *fd, const char *url);
 
 /*
 ** Return the number of bytes that SSL has waiting in internal buffers.
@@ -347,7 +351,7 @@ SSL_IMPORT int SSL_DataPending(PRFileDesc *fd);
 /*
 ** Invalidate the SSL session associated with fd.
 */
-SSL_IMPORT int SSL_InvalidateSession(PRFileDesc *fd);
+SSL_IMPORT SECStatus SSL_InvalidateSession(PRFileDesc *fd);
 
 /*
 ** Return a SECItem containing the SSL session ID associated with the fd.
@@ -355,7 +359,7 @@ SSL_IMPORT int SSL_InvalidateSession(PRFileDesc *fd);
 SSL_IMPORT SECItem *SSL_GetSessionID(PRFileDesc *fd);
 
 /*
-** Clear out the SSL session cache.
+** Clear out the client's SSL session cache, not the server's session cache.
 */
 SSL_IMPORT void SSL_ClearSessionCache(void);
 
@@ -363,7 +367,7 @@ SSL_IMPORT void SSL_ClearSessionCache(void);
 ** Set peer information so we can correctly look up SSL session later.
 ** You only have to do this if you're tunneling through a proxy.
 */
-SSL_IMPORT int SSL_SetSockPeerID(PRFileDesc *fd, char *peerID);
+SSL_IMPORT SECStatus SSL_SetSockPeerID(PRFileDesc *fd, char *peerID);
 
 /*
 ** Reveal the security information for the peer. 
diff --git a/security/nss/lib/ssl/sslauth.c b/security/nss/lib/ssl/sslauth.c
index e91c04f77..33f188876 100644
--- a/security/nss/lib/ssl/sslauth.c
+++ b/security/nss/lib/ssl/sslauth.c
@@ -59,7 +59,7 @@ CERTCertificate *SSL_PeerCertificate(PRFileDesc *fd)
 }
 
 /* NEED LOCKS IN HERE.  */
-int
+SECStatus
 SSL_SecurityStatus(PRFileDesc *fd, int *op, char **cp, int *kp0, int *kp1,
 		   char **ip, char **sp)
 {
@@ -141,17 +141,17 @@ SSL_SecurityStatus(PRFileDesc *fd, int *op, char **cp, int *kp0, int *kp1,
 	}
     }
 
-    return 0;
+    return SECSuccess;
 }
 
 /************************************************************************/
 
 /* NEED LOCKS IN HERE.  */
-int
+SECStatus
 SSL_AuthCertificateHook(PRFileDesc *s, SSLAuthCertificate func, void *arg)
 {
     sslSocket *ss;
-    int rv;
+    SECStatus rv;
 
     ss = ssl_FindSocket(s);
     if (!ss) {
@@ -161,21 +161,21 @@ SSL_AuthCertificateHook(PRFileDesc *s, SSLAuthCertificate func, void *arg)
     }
 
     if ((rv = ssl_CreateSecurityInfo(ss)) != 0) {
-	return(rv);
+	return rv;
     }
     ss->authCertificate = func;
     ss->authCertificateArg = arg;
 
-    return(0);
+    return SECSuccess;
 }
 
 /* NEED LOCKS IN HERE.  */
-int 
+SECStatus 
 SSL_GetClientAuthDataHook(PRFileDesc *s, SSLGetClientAuthData func,
 			      void *arg)
 {
     sslSocket *ss;
-    int rv;
+    SECStatus rv;
 
     ss = ssl_FindSocket(s);
     if (!ss) {
@@ -189,15 +189,15 @@ SSL_GetClientAuthDataHook(PRFileDesc *s, SSLGetClientAuthData func,
     }
     ss->getClientAuthData = func;
     ss->getClientAuthDataArg = arg;
-    return 0;
+    return SECSuccess;
 }
 
 /* NEED LOCKS IN HERE.  */
-int 
+SECStatus 
 SSL_SetPKCS11PinArg(PRFileDesc *s, void *arg)
 {
     sslSocket *ss;
-    int rv;
+    SECStatus rv;
 
     ss = ssl_FindSocket(s);
     if (!ss) {
@@ -210,7 +210,7 @@ SSL_SetPKCS11PinArg(PRFileDesc *s, void *arg)
 	return rv;
     }
     ss->pkcs11PinArg = arg;
-    return 0;
+    return SECSuccess;
 }
 
 
@@ -218,7 +218,7 @@ SSL_SetPKCS11PinArg(PRFileDesc *s, void *arg)
  * certificate message is received from the peer and the local application
  * has not registered an authCert callback function.
  */
-int
+SECStatus
 SSL_AuthCertificate(void *arg, PRFileDesc *fd, PRBool checkSig, PRBool isServer)
 {
     SECStatus          rv;
diff --git a/security/nss/lib/ssl/sslsecur.c b/security/nss/lib/ssl/sslsecur.c
index 021ff565e..19883c200 100644
--- a/security/nss/lib/ssl/sslsecur.c
+++ b/security/nss/lib/ssl/sslsecur.c
@@ -174,11 +174,7 @@ ssl_Do1stHandshake(sslSocket *ss)
  * Handshake function that blocks.  Used to force a
  * retry on a connection on the next read/write.
  */
-#ifdef macintosh
 static SECStatus
-#else
-static int
-#endif
 AlwaysBlock(sslSocket *ss)
 {
     PORT_SetError(PR_WOULD_BLOCK_ERROR);	/* perhaps redundant. */
@@ -259,11 +255,11 @@ SSL_ResetHandshake(PRFileDesc *s, PRBool asServer)
 ** and then starts new client hello or hello request.
 ** Acquires and releases HandshakeLock.
 */
-int
+SECStatus
 SSL_ReHandshake(PRFileDesc *fd, PRBool flushCache)
 {
     sslSocket *ss;
-    int        rv;
+    SECStatus  rv;
     
     ss = ssl_FindSocket(fd);
     if (!ss) {
@@ -292,7 +288,7 @@ SSL_ReHandshake(PRFileDesc *fd, PRBool flushCache)
     return rv;
 }
 
-int
+SECStatus
 SSL_RedoHandshake(PRFileDesc *fd)
 {
     return SSL_ReHandshake(fd, PR_TRUE);
@@ -301,7 +297,7 @@ SSL_RedoHandshake(PRFileDesc *fd)
 /* Register an application callback to be called when SSL handshake completes.
 ** Acquires and releases HandshakeLock.
 */
-int
+SECStatus
 SSL_HandshakeCallback(PRFileDesc *fd, SSLHandshakeCallback cb,
 		      void *client_data)
 {
@@ -343,35 +339,37 @@ SSL_HandshakeCallback(PRFileDesc *fd, SSLHandshakeCallback cb,
 ** or a fatal error occurs.
 ** Application should use handshake completion callback to tell which. 
 */
-int
+SECStatus
 SSL_ForceHandshake(PRFileDesc *fd)
 {
     sslSocket *ss;
-    int        rv;
+    SECStatus  rv = SECFailure;
 
     ss = ssl_FindSocket(fd);
     if (!ss) {
 	SSL_DBG(("%d: SSL[%d]: bad socket in ForceHandshake",
 		 SSL_GETPID(), fd));
-	return SECFailure;
+	return rv;
     }
 
     /* Don't waste my time */
     if (!ss->useSecurity) 
-    	return 0;
+    	return SECSuccess;
 
     ssl_Get1stHandshakeLock(ss);
 
     if (ss->version >= SSL_LIBRARY_VERSION_3_0) {
+	int gatherResult;
+
     	ssl_GetRecvBufLock(ss);
-	rv = ssl3_GatherCompleteHandshake(ss, 0);
+	gatherResult = ssl3_GatherCompleteHandshake(ss, 0);
 	ssl_ReleaseRecvBufLock(ss);
-	if (rv == 0) {
+	if (gatherResult > 0) {
+	    rv = SECSuccess;
+	} else if (gatherResult == 0) {
 	    PORT_SetError(PR_END_OF_FILE_ERROR);
-	    rv = SECFailure;
-	} else if (rv == SECWouldBlock) {
+	} else if (gatherResult == SECWouldBlock) {
 	    PORT_SetError(PR_WOULD_BLOCK_ERROR);
-	    rv = SECFailure;
 	}
     } else if (!ss->connected) {
 	rv = ssl_Do1stHandshake(ss);
@@ -382,8 +380,6 @@ SSL_ForceHandshake(PRFileDesc *fd)
 
     ssl_Release1stHandshakeLock(ss);
 
-    if (rv > 0) 
-    	rv = SECSuccess;
     return rv;
 }
 
@@ -1097,11 +1093,11 @@ ssl_SecureWrite(sslSocket *ss, const unsigned char *buf, int len)
     return ssl_SecureSend(ss, buf, len, 0);
 }
 
-int
+SECStatus
 SSL_BadCertHook(PRFileDesc *fd, SSLBadCertHandler f, void *arg)
 {
     sslSocket *ss;
-    int rv;
+    SECStatus rv;
     
     ss = ssl_FindSocket(fd);
     if (!ss) {
@@ -1111,23 +1107,23 @@ SSL_BadCertHook(PRFileDesc *fd, SSLBadCertHandler f, void *arg)
     }
 
     if ((rv = ssl_CreateSecurityInfo(ss)) != 0) {
-	return(rv);
+	return rv;
     }
     ss->handleBadCert = f;
     ss->badCertArg = arg;
 
-    return(0);
+    return SECSuccess;
 }
 
 /*
  * Allow the application to pass the url or hostname into the SSL library
  * so that we can do some checking on it.
  */
-int
+SECStatus
 SSL_SetURL(PRFileDesc *fd, const char *url)
 {
     sslSocket *   ss = ssl_FindSocket(fd);
-    int           rv = SECSuccess;
+    SECStatus     rv = SECSuccess;
 
     if (!ss) {
 	SSL_DBG(("%d: SSL[%d]: bad socket in SSLSetURL",
@@ -1164,7 +1160,6 @@ SSL_DataPending(PRFileDesc *fd)
 
     ss = ssl_FindSocket(fd);
 
-
     if (ss && ss->useSecurity) {
 
 	ssl_Get1stHandshakeLock(ss);
@@ -1185,11 +1180,11 @@ SSL_DataPending(PRFileDesc *fd)
     return rv;
 }
 
-int
+SECStatus
 SSL_InvalidateSession(PRFileDesc *fd)
 {
     sslSocket *   ss = ssl_FindSocket(fd);
-    int           rv = SECFailure;
+    SECStatus     rv = SECFailure;
 
     ssl_Get1stHandshakeLock(ss);
     ssl_GetSSL3HandshakeLock(ss);
diff --git a/security/nss/lib/ssl/sslsnce.c b/security/nss/lib/ssl/sslsnce.c
index b3fbd4202..28a2f3812 100644
--- a/security/nss/lib/ssl/sslsnce.c
+++ b/security/nss/lib/ssl/sslsnce.c
@@ -1504,7 +1504,7 @@ InitCertCache(const char *directory)
     return SECFailure;
 }
 
-int
+SECStatus
 SSL_ConfigServerSessionIDCache(	int      maxCacheEntries, 
 				PRUint32 timeout,
 			       	PRUint32 ssl3_timeout, 
@@ -1539,14 +1539,14 @@ SSL_ConfigServerSessionIDCache(	int      maxCacheEntries,
 /* Use this function, instead of SSL_ConfigServerSessionIDCache,
  * if the cache will be shared by multiple processes.
  */
-int
+SECStatus
 SSL_ConfigMPServerSIDCache(	int      maxCacheEntries, 
 				PRUint32 timeout,
 			       	PRUint32 ssl3_timeout, 
 		          const char *   directory)
 {
     char *	envValue;
-    int 	result;
+    SECStatus 	result;
     SECStatus	putEnvFailed;
 
     isMultiProcess = PR_TRUE;
@@ -1898,7 +1898,7 @@ ssl_SetWrappingKey(SSLWrappedSymWrappingKey *wswk)
 #include "ssl.h"
 #include "sslimpl.h"
 
-int
+SECStatus
 SSL_ConfigServerSessionIDCache(	int      maxCacheEntries, 
 				PRUint32 timeout,
 			       	PRUint32 ssl3_timeout, 
@@ -1908,7 +1908,7 @@ SSL_ConfigServerSessionIDCache(	int      maxCacheEntries,
     return SECFailure;
 }
 
-int
+SECStatus
 SSL_ConfigMPServerSIDCache(	int      maxCacheEntries, 
 				PRUint32 timeout,
 			       	PRUint32 ssl3_timeout, 
diff --git a/security/nss/lib/ssl/sslsock.c b/security/nss/lib/ssl/sslsock.c
index 28967ace5..860cf365e 100644
--- a/security/nss/lib/ssl/sslsock.c
+++ b/security/nss/lib/ssl/sslsock.c
@@ -1264,7 +1264,7 @@ ssl_GetSockName(PRFileDesc *fd, PRNetAddr *name)
     return (PRStatus)(*ss->ops->getsockname)(ss, name);
 }
 
-int PR_CALLBACK
+SECStatus PR_CALLBACK
 SSL_SetSockPeerID(PRFileDesc *fd, char *peerID)
 {
     sslSocket *ss;
@@ -1277,7 +1277,7 @@ SSL_SetSockPeerID(PRFileDesc *fd, char *peerID)
     }
 
     ss->peerID = PORT_Strdup(peerID);
-    return 0;
+    return SECSuccess;
 }
 
 static PRInt16 PR_CALLBACK