Fix possible integer overflows and cases of undefined behavior.

* src/avail.c (avail_comp): Prevent integer overflow.
(gdbm_avail_table_valid_p): Likewise.
* src/gdbmdefs.h (off_t_sum_ok): Change return type.
Return false if any argument is negative.
* src/gdbmopen.c (compute_directory_size): Prevent integer overflow.
* src/hash.c (_gdbm_hash): Treat dptr elements as unsigned
integers.

5 files changed, 17 insertions, 9 deletions

diff --git a/NEWS b/NEWS
index fe36106..f4741e0 100644
--- a/NEWS
+++ b/NEWS
@@ -1,4 +1,4 @@
-GNU dbm NEWS -- history of user-visible changes. 2021-03-21
+GNU dbm NEWS -- history of user-visible changes. 2021-03-23
 Copyright (C) 1990-2021 Free Software Foundation, Inc.
 See the end of file for copying conditions.
 
@@ -54,9 +54,9 @@ See https://puszcza.gnu.org.ua/bugs/?401
 
 ** Fix spurious error from gdbm_dump and gdbm_export
 
-The functions incorrectly reported as error the GDBM_ITEM_NOT_FOUND
-status, which is reported when upon normal termination of iteration
-over the database keys.
+The functions incorrectly treated as error the GDBM_ITEM_NOT_FOUND
+status, which is reported upon normal termination of iteration
+over database keys.
 
 ** Make sure gdbm_sync always returns a meaningful value
 See https://puszcza.gnu.org.ua/bugs/?400
diff --git a/src/avail.c b/src/avail.c
index 50c3a44..6c69144 100644
--- a/src/avail.c
+++ b/src/avail.c
@@ -24,7 +24,11 @@ avail_comp (void const *a, void const *b)
 {
   avail_elem const *ava = a;
   avail_elem const *avb = b;
-  return ava->av_size - avb->av_size;
+  if (ava->av_size < avb->av_size)
+    return -1;
+  else if (ava->av_size > avb->av_size)
+    return 1;
+  return 0;
 }
 
 /* Returns true if the avail array AV[0]@COUNT is valid.
@@ -53,6 +57,7 @@ gdbm_avail_table_valid_p (GDBM_FILE dbf, avail_elem *av, int count)
   for (i = 0; i < count; i++, p++)
     {
       if (!(p->av_adr >= dbf->header->bucket_size
+	    && off_t_sum_ok (p->av_adr, p->av_size)
 	    && p->av_adr + p->av_size <= dbf->header->next_block))
 	return 0;
       if (p->av_size < prev)
diff --git a/src/gdbmdefs.h b/src/gdbmdefs.h
index c6df13d..e746ac3 100644
--- a/src/gdbmdefs.h
+++ b/src/gdbmdefs.h
@@ -34,11 +34,12 @@
 /* Maximum value for off_t */
 #define OFF_T_MAX SIGNED_TYPE_MAXIMUM (off_t)
 
-/* Return true if A can be added to B without integer overflow */
-static inline off_t
+/* Return true if both A and B are non-negative offsets and A can be added
+   to B without integer overflow */
+static inline int
 off_t_sum_ok (off_t a, off_t b)
 {
-  return OFF_T_MAX - a >= b;
+  return a >= 0 && b >= 0 && OFF_T_MAX - a >= b;
 }
 
 /* The type definitions are next.  */
diff --git a/src/gdbmopen.c b/src/gdbmopen.c
index ca26792..39ec928 100644
--- a/src/gdbmopen.c
+++ b/src/gdbmopen.c
@@ -39,6 +39,8 @@ compute_directory_size (blksize_t block_size,
   int dir_size = 8 * sizeof (off_t);
   int dir_bits = 3;
 
+  if (block_size > INT_MAX / 2)
+    block_size = INT_MAX / 2;
   while (dir_size < block_size && dir_bits < GDBM_HASH_BITS - 3)
     {
       dir_size <<= 1;
diff --git a/src/hash.c b/src/hash.c
index 52fbc82..97402f7 100644
--- a/src/hash.c
+++ b/src/hash.c
@@ -35,7 +35,7 @@ _gdbm_hash (datum key)
   /* Set the initial value from key. */
   value = 0x238F13AFu * key.dsize;
   for (index = 0; index < key.dsize; index++)
-    value = (value + (key.dptr[index] << ((unsigned) index * 5 % 24))) & 0x7FFFFFFF;
+    value = (value + (((unsigned)key.dptr[index]) << ((unsigned) index * 5 % 24))) & 0x7FFFFFFF;
 
   value = (1103515243u * value + 12345) & 0x7FFFFFFF;