regress: introduce TT_RETRIABLE

We have some tests that has false-positive due to real/CPU time bound,
but they are pretty generic and we do not want to skip them by default.

TT_RETRIABLE is the flag that will indicate tinytest to retry the test
in case of failure, use it to avoid next possible false-positives:
- real time-related
- CPU time-related
Since I guess it is better to see/grepping RETRYING messages over
ignoring completely failed builds.

No configuration switch for number of retries was done on purpose (only
3 retries and no more).

And this is how it looks BTW:
  $ gcc ../test/tinytest_demo.c ../test/tinytest.c
  $ ./a.out --verbose --no-fork
  demo/timeout_retry
  demo/timeout_retry:
    FAIL ../test/tinytest_demo.c:201: assert(i != 1): 1 vs 1
    [timeout_retry FAILED]

    [RETRYING timeout_retry (3)]
  demo/timeout_retry:
           OK ../test/tinytest_demo.c:201: assert(i != 1): 2 vs 1
           OK ../test/tinytest_demo.c:213: assert(t2-t1 >= 4): 5 vs 4
           OK ../test/tinytest_demo.c:215: assert(t2-t1 <= 6): 5 vs 6
  1 tests ok.  (0 skipped)

1 files changed, 29 insertions, 8 deletions

diff --git a/test/tinytest.c b/test/tinytest.c
index 3a8e3310..a27a906a 100644
--- a/test/tinytest.c
+++ b/test/tinytest.c
@@ -253,15 +253,12 @@ testcase_run_one(const struct testgroup_t *group,
 	}
 
 	if (outcome == OK) {
-		++n_ok;
 		if (opt_verbosity>0 && !opt_forked)
 			puts(opt_verbosity==1?"OK":"");
 	} else if (outcome == SKIP) {
-		++n_skipped;
 		if (opt_verbosity>0 && !opt_forked)
 			puts("SKIPPED");
 	} else {
-		++n_bad;
 		if (!opt_forked)
 			printf("\n  [%s FAILED]\n", testcase->name);
 	}
@@ -428,11 +425,35 @@ tinytest_main(int c, const char **v, struct testgroup_t *groups)
 #endif
 
 	++in_tinytest_main;
-	for (i=0; groups[i].prefix; ++i)
-		for (j=0; groups[i].cases[j].name; ++j)
-			if (groups[i].cases[j].flags & TT_ENABLED_)
-				testcase_run_one(&groups[i],
-						 &groups[i].cases[j]);
+	for (i = 0; groups[i].prefix; ++i) {
+		struct testgroup_t *group = &groups[i];
+		for (j = 0; group->cases[j].name; ++j) {
+			struct testcase_t *testcase = &group->cases[j];
+			int test_attempts = 3;
+			int test_ret_err;
+
+			if (!(testcase->flags & TT_ENABLED_))
+				continue;
+
+			for (;;) {
+				test_ret_err = testcase_run_one(group, testcase);
+
+				if (test_ret_err == OK)
+					break;
+				if (!(testcase->flags & TT_RETRIABLE))
+					break;
+				printf("\n  [RETRYING %s (%i)]\n", testcase->name, test_attempts);
+				if (!test_attempts--)
+					break;
+			}
+
+			switch (test_ret_err) {
+				case OK:   ++n_ok;      break;
+				case SKIP: ++n_skipped; break;
+				default:   ++n_bad;     break;
+			}
+		}
+	}
 
 	--in_tinytest_main;