diff options
author | molsonkiko <46202915+molsonkiko@users.noreply.github.com> | 2023-03-26 18:32:40 -0700 |
---|---|---|
committer | molsonkiko <46202915+molsonkiko@users.noreply.github.com> | 2023-03-26 18:32:40 -0700 |
commit | 988283a7a18ad15f00db28902c643f653dfd7278 (patch) | |
tree | 59be70bdabf8766cefed4bd973877fbbd12651d9 /numpy/f2py/tests | |
parent | 09c23ef73c839d3a7f31e755f40f2f06b9791b7f (diff) | |
download | numpy-988283a7a18ad15f00db28902c643f653dfd7278.tar.gz |
make time tests more resilient to random noise
Diffstat (limited to 'numpy/f2py/tests')
-rw-r--r-- | numpy/f2py/tests/test_crackfortran.py | 23 |
1 files changed, 16 insertions, 7 deletions
diff --git a/numpy/f2py/tests/test_crackfortran.py b/numpy/f2py/tests/test_crackfortran.py index 449251435..23965087d 100644 --- a/numpy/f2py/tests/test_crackfortran.py +++ b/numpy/f2py/tests/test_crackfortran.py @@ -290,18 +290,23 @@ class TestNameArgsPatternBacktracking: def test_nameargspattern_backtracking(self, adversary): '''address ReDOS vulnerability: https://github.com/numpy/numpy/issues/23338''' - last_time = 0. + last_median = 0. trials_per_count = 128 start_reps, end_reps = 15, 25 + times_median_doubled = 0 for ii in range(start_reps, end_reps): repeated_adversary = adversary * ii - total_time = 0 + times = [] for _ in range(trials_per_count): t0 = time.perf_counter() mtch = nameargspattern.search(repeated_adversary) - total_time += (time.perf_counter() - t0) + times.append(time.perf_counter() - t0) + # We should use a measure of time that's resilient to outliers. + # Times jump around a lot due to the CPU's scheduler. + median = np.median(times) assert not mtch - # if the adversary is capped with @)@, it becomes acceptable. + # if the adversary is capped with @)@, it becomes acceptable + # according to the old version of the regex. # that should still be true. good_version_of_adversary = repeated_adversary + '@)@' assert nameargspattern.search(good_version_of_adversary) @@ -309,8 +314,12 @@ class TestNameArgsPatternBacktracking: # the hallmark of exponentially catastrophic backtracking # is that runtime doubles for every added instance of # the problematic pattern. - assert total_time < 1.9 * last_time + times_median_doubled += median > 2 * last_median # also try to rule out non-exponential but still bad cases # arbitrarily, we should set a hard limit of 10ms as too slow - assert total_time < trials_per_count * 0.01 - last_time = total_time
\ No newline at end of file + assert median < trials_per_count * 0.01 + last_median = median + # we accept that maybe the median might double once, due to + # the CPU scheduler acting weird or whatever. More than that + # seems suspicious. + assert times_median_doubled < 2
\ No newline at end of file |