errors: be more robust in handling unexpected fatal errors

Introduce a new error level, ERR_CRITICAL, beyond which we will
minimize the amount of code that will be executed before we die; in
particular don't execute any memory allocations, and if we somehow end
up recursing, abort() immediately.

Basically, "less than panic, more than fatal."

At this point this level is used by nasm_alloc_failed().

Signed-off-by: H. Peter Anvin <hpa@zytor.com>

2 files changed, 3 insertions, 19 deletions

diff --git a/nasmlib/alloc.c b/nasmlib/alloc.c
index b0d623a2..e25e0e0a 100644
--- a/nasmlib/alloc.c
+++ b/nasmlib/alloc.c
@@ -42,25 +42,9 @@
 
 size_t _nasm_last_string_size;
 
-no_return nasm_alloc_failed(void)
+fatal_func nasm_alloc_failed(void)
 {
-    /* If nasm_fatal() gets us back here, then croak hard */
-    static bool already_here = false;
-    FILE *errfile;
-
-    if (likely(!already_here)) {
-        already_here = true;
-        nasm_fatal("out of memory!");
-    }
-
-    errfile = error_file;
-    if (!errfile)
-        error_file = stderr;
-
-    fprintf(error_file, "nasm: out of memory!\n");
-    fflush(error_file);
-    fflush(NULL);
-    abort();
+    nasm_critical("out of memory!");
 }
 
 void *nasm_malloc(size_t size)
diff --git a/nasmlib/alloc.h b/nasmlib/alloc.h
index 1b896585..9b8ad192 100644
--- a/nasmlib/alloc.h
+++ b/nasmlib/alloc.h
@@ -36,7 +36,7 @@
 
 #include "compiler.h"
 
-no_return nasm_alloc_failed(void);
+fatal_func nasm_alloc_failed(void);
 
 static inline void *validate_ptr(void *p)
 {