Fixed Integer overflows in clnt_vc_create and clnt_dg_createlibtirpc-1-0-4-rc2

There exits a possibility of an integer overflow in the
clnt_vc_create @ src/clnt_vc.c : 217 and
clnt_dg_create @ src/clnt_dg.c : 176.

In clnt_dg_create, the integer dtbsize is multiplied
with the size of the cond_t structure to get the total
number of bytes to be allocated.

The integer dtbsize is the value returned by a call to __rpc_dtbsize.

163: int cv_allocsz;
164: size_t fd_allocsz;
165: int dtbsize = __rpc_dtbsize();

176: cv_allocsz = dtbsize * sizeof (cond_t);

On a 32 bit version, the integer dtbsize is multiplied with the value 48.
The value returned by __rpc_dtbsize is the hard limit on the maximum
number of file descriptors which is 2^20 by default in my Ubuntu 16.04.

If this hard limit was increased to a value greater than 2^27,
this multiplication would overflow and result in a value smaller
than the expected size.

I understand that changing the hard limit on the maximum value of file
descriptors requires root privilege. But it would be reasonable
to double check this value before using it in calculations.

Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=1600284

From: Jayakrishna Menon <jkrshnmenon@gmail.com>

Signed-off-by: Steve Dickson <steved@redhat.com>

2 files changed, 27 insertions, 5 deletions

diff --git a/src/clnt_dg.c b/src/clnt_dg.c
index 04a2aba..eb5467f 100644
--- a/src/clnt_dg.c
+++ b/src/clnt_dg.c
@@ -160,15 +160,22 @@ clnt_dg_create(fd, svcaddr, program, version, sendsz, recvsz)
 	thr_sigsetmask(SIG_SETMASK, &newmask, &mask);
 	mutex_lock(&clnt_fd_lock);
 	if (dg_fd_locks == (int *) NULL) {
-		int cv_allocsz;
-		size_t fd_allocsz;
-		int dtbsize = __rpc_dtbsize();
+		size_t cv_allocsz, fd_allocsz;
+		unsigned int dtbsize = __rpc_dtbsize();
+
+		if ( (size_t) dtbsize > SIZE_MAX/sizeof(cond_t)) {
+			mutex_unlock(&clnt_fd_lock);
+			thr_sigsetmask(SIG_SETMASK, &(mask), NULL);
+			errno = EOVERFLOW;
+			goto err1;
+		}
 
 		fd_allocsz = dtbsize * sizeof (int);
 		dg_fd_locks = (int *) mem_alloc(fd_allocsz);
 		if (dg_fd_locks == (int *) NULL) {
 			mutex_unlock(&clnt_fd_lock);
 			thr_sigsetmask(SIG_SETMASK, &(mask), NULL);
+			errno = ENOMEM;
 			goto err1;
 		} else
 			memset(dg_fd_locks, '\0', fd_allocsz);
@@ -180,6 +187,7 @@ clnt_dg_create(fd, svcaddr, program, version, sendsz, recvsz)
 			dg_fd_locks = (int *) NULL;
 			mutex_unlock(&clnt_fd_lock);
 			thr_sigsetmask(SIG_SETMASK, &(mask), NULL);
+			errno = ENOMEM;
 			goto err1;
 		} else {
 			int i;
diff --git a/src/clnt_vc.c b/src/clnt_vc.c
index 6098c3a..3d775c7 100644
--- a/src/clnt_vc.c
+++ b/src/clnt_vc.c
@@ -63,6 +63,7 @@
 #include <string.h>
 #include <unistd.h>
 #include <signal.h>
+#include <stdint.h>
 
 #include <rpc/rpc.h>
 #include "rpc_com.h"
@@ -201,14 +202,25 @@ clnt_vc_create(fd, raddr, prog, vers, sendsz, recvsz)
 	thr_sigsetmask(SIG_SETMASK, &newmask, &mask);
 	mutex_lock(&clnt_fd_lock);
 	if (vc_fd_locks == (int *) NULL) {
-		int cv_allocsz, fd_allocsz;
-		int dtbsize = __rpc_dtbsize();
+		size_t cv_allocsz, fd_allocsz;
+		unsigned int dtbsize = __rpc_dtbsize();
+		struct rpc_createerr *ce = &get_rpc_createerr();
+
+		if ( (size_t) dtbsize > SIZE_MAX/sizeof(cond_t)) {
+			mutex_unlock(&clnt_fd_lock);
+			thr_sigsetmask(SIG_SETMASK, &(mask), NULL);
+			ce->cf_stat = RPC_SYSTEMERROR;
+			ce->cf_error.re_errno = EOVERFLOW;
+			goto err;
+		}
 
 		fd_allocsz = dtbsize * sizeof (int);
 		vc_fd_locks = (int *) mem_alloc(fd_allocsz);
 		if (vc_fd_locks == (int *) NULL) {
 			mutex_unlock(&clnt_fd_lock);
 			thr_sigsetmask(SIG_SETMASK, &(mask), NULL);
+			ce->cf_stat = RPC_SYSTEMERROR;
+			ce->cf_error.re_errno = ENOMEM;
 			goto err;
 		} else
 			memset(vc_fd_locks, '\0', fd_allocsz);
@@ -221,6 +233,8 @@ clnt_vc_create(fd, raddr, prog, vers, sendsz, recvsz)
 			vc_fd_locks = (int *) NULL;
 			mutex_unlock(&clnt_fd_lock);
 			thr_sigsetmask(SIG_SETMASK, &(mask), NULL);
+			ce->cf_stat = RPC_SYSTEMERROR;
+			ce->cf_error.re_errno = ENOMEM;
 			goto err;
 		} else {
 			int i;