avoid integer overflow in Perl_av_extend_guts()

There were two issues; first the 'overextend' algorithm (add a fifth of
the current size to the requested size) could overflow,
and secondly MEM_WRAP_CHECK_1() was being called with newmax+1,
which could overflow if newmax happened to equal SSize_t_MAX.

e.g.

    $a[0x7fffffffffffffff] = 1
    $a[5] = 1; $a[0x7fffffffffffffff] = 1

could produce under ASan:

    av.c:133:16: runtime error: signed integer overflow: 9223372036854775807 + 1 cannot be represented in type 'long'
    av.c:170:7: runtime error: signed integer overflow: 9223372036854775807 + 1 cannot be represented in type 'long'

1 files changed, 13 insertions, 3 deletions

diff --git a/av.c b/av.c
index 3de7b83b2c..53690d5cbb 100644
--- a/av.c
+++ b/av.c
@@ -130,14 +130,23 @@ Perl_av_extend_guts(pTHX_ AV *av, SSize_t key, SSize_t *maxp, SV ***allocp,
 		if (key <= newmax) 
 		    goto resized;
 #endif 
-		newmax = key + *maxp / 5;
+                /* overflow-safe version of newmax = key + *maxp/5 */
+		newmax = *maxp / 5;
+                newmax = (key > SSize_t_MAX - newmax)
+                            ? SSize_t_MAX : key + newmax;
 	      resize:
 		{
 #ifdef PERL_MALLOC_WRAP /* Duplicated in pp_hot.c */
 		    static const char oom_array_extend[] =
 			"Out of memory during array extend";
 #endif
-		    MEM_WRAP_CHECK_1(newmax+1, SV*, oom_array_extend);
+                    /* it should really be newmax+1 here, but if newmax
+                     * happens to equal SSize_t_MAX, then newmax+1 is
+                     * undefined. This means technically we croak one
+                     * index lower than we should in theory; in practice
+                     * its unlikely the system has SSize_t_MAX/sizeof(SV*)
+                     * bytes to spare! */
+		    MEM_WRAP_CHECK_1(newmax, SV*, oom_array_extend);
 		}
 #ifdef STRESS_REALLOC
 		{
@@ -167,7 +176,8 @@ Perl_av_extend_guts(pTHX_ AV *av, SSize_t key, SSize_t *maxp, SV ***allocp,
 		    static const char oom_array_extend[] =
 			"Out of memory during array extend";
 #endif
-		    MEM_WRAP_CHECK_1(newmax+1, SV*, oom_array_extend);
+                    /* see comment above about newmax+1*/
+		    MEM_WRAP_CHECK_1(newmax, SV*, oom_array_extend);
 		}
 		Newx(*allocp, newmax+1, SV*);
 		ary = *allocp + 1;