shared: avoid compiler warning for nm_utils_get_next_realloc_size() returning huge sizes

On s390x (gcc-8.3.1-5.1.el8.s390x) the compiler warns that we don't
pass size larger than 2^63-1 to malloc. With LTO enabled, it is also
quite adamant in detecting that with nm_utils_get_next_realloc_size().

Optimally, we would disable this useless warning with "-Wno-alloc-size-larger-than",
but that seems not to work. So add a workaround in code :(

It's hard to actually workaround the warning while handling all kinds of
sizes. The only simple solution is to no handle such huge cases and only
assert.

    In function 'nm_secret_mem_realloc',
        inlined from '_nm_str_buf_ensure_size' at shared/nm-glib-aux/nm-shared-utils.c:5316:31:
    shared/nm-glib-aux/nm-secret-utils.h:180:17: error: argument 1 value '18446744073709551615' exceeds maximum object size 9223372036854775807 [-Werror=alloc-size-larger-than=]
             m_new = g_malloc(new_len);
                     ^
    shared/nm-glib-aux/nm-secret-utils.h: In function '_nm_str_buf_ensure_size':
    /usr/include/glib-2.0/glib/gmem.h:78:10: note: in a call to allocation function 'g_malloc' declared here
     gpointer g_malloc         (gsize  n_bytes) G_GNUC_MALLOC G_GNUC_ALLOC_SIZE(1);
              ^
    lto1: all warnings being treated as errors

2 files changed, 49 insertions, 19 deletions

diff --git a/shared/nm-glib-aux/tests/test-shared-general.c b/shared/nm-glib-aux/tests/test-shared-general.c
index 39f958e4b3..3907bf8dd7 100644
--- a/shared/nm-glib-aux/tests/test-shared-general.c
+++ b/shared/nm-glib-aux/tests/test-shared-general.c
@@ -763,25 +763,42 @@ test_nm_utils_get_next_realloc_size(void)
         }
 
         {
-            const gsize requested      = requested0;
-            const gsize reserved_true  = nm_utils_get_next_realloc_size(TRUE, requested);
-            const gsize reserved_false = nm_utils_get_next_realloc_size(FALSE, requested);
+            const gsize requested = requested0;
+            gsize       reserved_true;
+            gsize       reserved_false;
+            bool        truncated_true  = FALSE;
+            bool        truncated_false = FALSE;
+
+            if (sizeof(gsize) > 4 && requested > SIZE_MAX / 2u - 24u) {
+                reserved_false  = G_MAXSSIZE;
+                truncated_false = TRUE;
+            } else
+                reserved_false = nm_utils_get_next_realloc_size(FALSE, requested);
+
+            if (sizeof(gsize) > 4 && requested > SIZE_MAX - 0x1000u - 24u) {
+                reserved_true  = G_MAXSSIZE;
+                truncated_true = TRUE;
+            } else
+                reserved_true = nm_utils_get_next_realloc_size(TRUE, requested);
 
             g_assert_cmpuint(reserved_true, >, 0);
             g_assert_cmpuint(reserved_false, >, 0);
-            g_assert_cmpuint(reserved_true, >=, requested);
-            g_assert_cmpuint(reserved_false, >=, requested);
-            g_assert_cmpuint(reserved_false, >=, reserved_true);
+            if (!truncated_true)
+                g_assert_cmpuint(reserved_true, >=, requested);
+            if (!truncated_false)
+                g_assert_cmpuint(reserved_false, >=, requested);
+            if (!truncated_true && !truncated_false)
+                g_assert_cmpuint(reserved_false, >=, reserved_true);
 
             if (i < G_N_ELEMENTS(test_data)) {
-                g_assert_cmpuint(reserved_true, ==, test_data[i].reserved_true);
-                g_assert_cmpuint(reserved_false, ==, test_data[i].reserved_false);
+                if (!truncated_true)
+                    g_assert_cmpuint(reserved_true, ==, test_data[i].reserved_true);
+                if (!truncated_false)
+                    g_assert_cmpuint(reserved_false, ==, test_data[i].reserved_false);
             }
 
             /* reserved_false is generally the next power of two - 24. */
-            if (reserved_false == G_MAXSIZE)
-                g_assert_cmpuint(requested, >, G_MAXSIZE / 2u - 24u);
-            else {
+            if (!truncated_false) {
                 g_assert_cmpuint(reserved_false, <=, G_MAXSIZE - 24u);
                 if (reserved_false >= 40) {
                     const gsize _pow2 = reserved_false + 24u;
@@ -799,9 +816,7 @@ test_nm_utils_get_next_realloc_size(void)
             }
 
             /* reserved_true is generally the next 4k border - 24. */
-            if (reserved_true == G_MAXSIZE)
-                g_assert_cmpuint(requested, >, G_MAXSIZE - 0x1000u - 24u);
-            else {
+            if (!truncated_true) {
                 g_assert_cmpuint(reserved_true, <=, G_MAXSIZE - 24u);
                 if (reserved_true > 8168u) {
                     const gsize page_border = reserved_true + 24u;
diff --git a/shared/nm-std-aux/nm-std-utils.c b/shared/nm-std-aux/nm-std-utils.c
index cc82e03d55..9bf08e02ad 100644
--- a/shared/nm-std-aux/nm-std-utils.c
+++ b/shared/nm-std-aux/nm-std-utils.c
@@ -5,6 +5,8 @@
 #include "nm-std-utils.h"
 
 #include <stdint.h>
+#include <assert.h>
+#include <limits.h>
 
 /*****************************************************************************/
 
@@ -50,7 +52,7 @@ nm_utils_get_next_realloc_size(bool true_realloc, size_t requested)
          * We get thus sizes of 104, 232, 488, 1000, 2024, 4072, 8168... */
 
         if (NM_UNLIKELY(requested > SIZE_MAX / 2u - 24u))
-            return SIZE_MAX;
+            goto out_huge;
 
         x = requested + 24u;
         n = 128u;
@@ -63,13 +65,26 @@ nm_utils_get_next_realloc_size(bool true_realloc, size_t requested)
         return n - 24u;
     }
 
-    if (NM_UNLIKELY(requested > SIZE_MAX - 0x1000u - 24u))
-        return SIZE_MAX;
-
     /* For large allocations (with !true_realloc) we allocate memory in chunks of
      * 4K (- 24 bytes extra), assuming that the memory gets mmapped and thus
      * realloc() is efficient by just reordering pages. */
     n = ((requested + (0x0FFFu + 24u)) & ~((size_t) 0x0FFFu)) - 24u;
-    nm_assert(n >= requested);
+
+    if (NM_UNLIKELY(n < requested)) {
+        /* overflow happened. */
+        goto out_huge;
+    }
+
     return n;
+
+out_huge:
+    if (sizeof(size_t) > 4u) {
+        /* on s390x (64 bit), gcc with LTO can complain that the size argument to
+         * malloc must not be larger than 9223372036854775807.
+         *
+         * Work around that by returning SSIZE_MAX. It should be plenty still! */
+        assert(requested <= (size_t) SSIZE_MAX);
+        return (size_t) SSIZE_MAX;
+    }
+    return SIZE_MAX;
 }