regcomp.c: Prevent integer overflow from nested regex quantifiers.

(CVE-2020-10543) On 32bit systems the size calculations for nested regular
expression quantifiers could overflow causing heap memory corruption.

Fixes: Perl/perl5-security#125
(cherry picked from commit 208dea486fa24081cbc0cf05fa5a15c802e2bc68)

1 files changed, 6 insertions, 0 deletions

diff --git a/regcomp.c b/regcomp.c
index e1da15a77c..dd18add1db 100644
--- a/regcomp.c
+++ b/regcomp.c
@@ -5181,6 +5181,12 @@ S_study_chunk(pTHX_ RExC_state_t *pRExC_state, regnode **scanp,
 		    (void)ReREFCNT_inc(RExC_rx_sv);
 		}
 
+                if ( ( minnext > 0 && mincount >= SSize_t_MAX / minnext )
+                    || min >= SSize_t_MAX - minnext * mincount )
+                {
+                    FAIL("Regexp out of space");
+                }
+
 		min += minnext * mincount;
 		is_inf_internal |= deltanext == SSize_t_MAX
                          || (maxcount == REG_INFTY && minnext + deltanext > 0);