preproc: add support for builtin include modules (%use)

Add a builtin equivalent to the %include directive called %use.
%use includes a standard macro file compiled into the binary; these
come from the macros/ directory in the source code.

The idea here is to be able to provide optional macro packages with
the distribution, without adding complex host filesystem dependencies.

10 files changed, 252 insertions, 72 deletions

diff --git a/Makefile.in b/Makefile.in
index e6447a8c..98d2b4f1 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -114,9 +114,9 @@ version.sed: version version.pl
 # This source file is generated from the standard macros file
 # `standard.mac' by another Perl script. Again, it's part of the
 # standard distribution.
-
-macros.c: macros.pl standard.mac version.mac
-	$(PERL) $(srcdir)/macros.pl $(srcdir)/standard.mac version.mac
+macros.c: macros.pl standard.mac version.mac $(srcdir)/macros/*.mac
+	$(PERL) $(srcdir)/macros.pl $(srcdir)/standard.mac version.mac \
+		$(srcdir)/macros/*.mac
 
 # These source files are generated from regs.dat by yet another
 # perl script.
@@ -261,7 +261,8 @@ lib/snprintf.$(O): lib/snprintf.c compiler.h config.h nasmlib.h
 lib/vsnprintf.$(O): lib/vsnprintf.c compiler.h config.h nasmlib.h
 listing.$(O): listing.c compiler.h config.h insnsi.h listing.h nasm.h \
  nasmlib.h regs.h version.h
-macros.$(O): macros.c compiler.h config.h insnsi.h tables.h
+macros.$(O): macros.c compiler.h config.h hashtbl.h insnsi.h nasmlib.h \
+ tables.h
 nasm.$(O): nasm.c assemble.h compiler.h config.h eval.h float.h insns.h \
  insnsi.h labels.h listing.h nasm.h nasmlib.h outform.h parser.h pptok.h \
  preproc.h raa.h regs.h saa.h stdscan.h tokens.h version.h
diff --git a/Mkfiles/msvc.mak b/Mkfiles/msvc.mak
index 62ef1ce4..14581bef 100644
--- a/Mkfiles/msvc.mak
+++ b/Mkfiles/msvc.mak
@@ -207,7 +207,7 @@ lib/snprintf.$(O): lib/snprintf.c compiler.h nasmlib.h
 lib/vsnprintf.$(O): lib/vsnprintf.c compiler.h nasmlib.h
 listing.$(O): listing.c compiler.h insnsi.h listing.h nasm.h nasmlib.h \
  regs.h version.h
-macros.$(O): macros.c compiler.h insnsi.h tables.h
+macros.$(O): macros.c compiler.h hashtbl.h insnsi.h nasmlib.h tables.h
 nasm.$(O): nasm.c assemble.h compiler.h eval.h float.h insns.h insnsi.h \
  labels.h listing.h nasm.h nasmlib.h outform.h parser.h pptok.h preproc.h \
  raa.h regs.h saa.h stdscan.h tokens.h version.h
diff --git a/Mkfiles/netware.mak b/Mkfiles/netware.mak
index 6384714e..aa84016b 100644
--- a/Mkfiles/netware.mak
+++ b/Mkfiles/netware.mak
@@ -147,7 +147,7 @@ snprintf.o: snprintf.c compiler.h config.h nasmlib.h
 vsnprintf.o: vsnprintf.c compiler.h config.h nasmlib.h
 listing.o: listing.c compiler.h config.h insnsi.h listing.h nasm.h nasmlib.h \
  regs.h version.h
-macros.o: macros.c compiler.h config.h insnsi.h tables.h
+macros.o: macros.c compiler.h config.h hashtbl.h insnsi.h nasmlib.h tables.h
 nasm.o: nasm.c assemble.h compiler.h config.h eval.h float.h insns.h \
  insnsi.h labels.h listing.h nasm.h nasmlib.h outform.h parser.h pptok.h \
  preproc.h raa.h regs.h saa.h stdscan.h tokens.h version.h
diff --git a/Mkfiles/openwcom.mak b/Mkfiles/openwcom.mak
index de586960..c56ed53c 100644
--- a/Mkfiles/openwcom.mak
+++ b/Mkfiles/openwcom.mak
@@ -236,7 +236,7 @@ lib\snprintf.$(O): lib\snprintf.c compiler.h nasmlib.h
 lib\vsnprintf.$(O): lib\vsnprintf.c compiler.h nasmlib.h
 listing.$(O): listing.c compiler.h insnsi.h listing.h nasm.h nasmlib.h &
  regs.h version.h
-macros.$(O): macros.c compiler.h insnsi.h tables.h
+macros.$(O): macros.c compiler.h hashtbl.h insnsi.h nasmlib.h tables.h
 nasm.$(O): nasm.c assemble.h compiler.h eval.h float.h insns.h insnsi.h &
  labels.h listing.h nasm.h nasmlib.h outform.h parser.h pptok.h preproc.h &
  raa.h regs.h saa.h stdscan.h tokens.h version.h
diff --git a/Mkfiles/owlinux.mak b/Mkfiles/owlinux.mak
index 4dd5eefd..d2224914 100644
--- a/Mkfiles/owlinux.mak
+++ b/Mkfiles/owlinux.mak
@@ -246,7 +246,7 @@ lib/snprintf.$(O): lib/snprintf.c compiler.h nasmlib.h
 lib/vsnprintf.$(O): lib/vsnprintf.c compiler.h nasmlib.h
 listing.$(O): listing.c compiler.h insnsi.h listing.h nasm.h nasmlib.h \
  regs.h version.h
-macros.$(O): macros.c compiler.h insnsi.h tables.h
+macros.$(O): macros.c compiler.h hashtbl.h insnsi.h nasmlib.h tables.h
 nasm.$(O): nasm.c assemble.h compiler.h eval.h float.h insns.h insnsi.h \
  labels.h listing.h nasm.h nasmlib.h outform.h parser.h pptok.h preproc.h \
  raa.h regs.h saa.h stdscan.h tokens.h version.h
diff --git a/macros.pl b/macros.pl
index 7bba2b19..c22185b9 100755
--- a/macros.pl
+++ b/macros.pl
@@ -1,4 +1,4 @@
-#!/usr/bin/perl -w
+#!/usr/bin/perl
 #
 # macros.pl   produce macros.c from standard.mac
 #
@@ -7,42 +7,58 @@
 # redistributable under the license given in the file "LICENSE"
 # distributed in the NASM archive.
 
-use strict;
+require 'phash.ph';
 
 my $fname;
 my $line = 0;
 my $index      = 0;
-my $tasm_count;
+my $tasm_count = 0;
 
-undef $tasm_count;
+open(OUT,">macros.c") or die "unable to open macros.c\n";
 
-open(OUTPUT,">macros.c") or die "unable to open macros.c\n";
+print OUT "/*\n";
+print OUT " * Do not edit - this file auto-generated by macros.pl from:\n";
+print OUT " * ", join(' ', @ARGV), "\n";
+print OUT " */\n";
+print OUT "\n";
+print OUT "#include \"tables.h\"\n";
+print OUT "#include \"nasmlib.h\"\n";
+print OUT "#include \"hashtbl.h\"\n";
+print OUT "\n";
+print OUT "const char * const nasm_stdmac[] = {";
 
-print OUTPUT "/*\n";
-print OUTPUT " * Do not edit - this file auto-generated by macros.pl from:\n";
-print OUTPUT " * ", join(' ', @ARGV), "\n";
-print OUTPUT " */\n";
-print OUTPUT "\n";
-print OUTPUT "#include \"tables.h\"\n";
-print OUTPUT "\n";
-print OUTPUT "const char * const nasm_stdmac[] = {";
+my $nmodule = 0;
+my @module_list   = ();
+my %module_number = ();
+my %module_index  = ();
+my $module;
 
 foreach $fname ( @ARGV ) {
     open(INPUT,$fname) or die "unable to open $fname\n";
-    print OUTPUT "\n    /* From $fname */\n";
+    print OUT "\n    /* From $fname */\n";
     while (<INPUT>) {
 	$line++;
 	chomp;
 	if (m/^\s*\*END\*TASM\*MACROS\*\s*$/) {
 	    $tasm_count = $index;
-	    print OUTPUT "    /* End of TASM macros */\n";
+	    print OUT "    /* End of TASM macros */\n";
+	} elsif (m/^USE:\s*(\S+)\s*$/) {
+	    $module = $1;
+	    if (defined($module_number{$module})) {
+		die "$0: $fname: duplicate module: $module\n";
+	    }
+	    printf OUT "        /* %4d */ NULL,\n", $index++;
+	    print OUT "    /* %use $module */\n";
+	    push(@module_list, $module);
+	    $module_number{$module} = $nmodule++;
+	    $module_index{$module}  = $index;
 	} elsif (m/^\s*((\s*([^\"\';\s]+|\"[^\"]*\"|\'[^\']*\'))*)\s*(;.*)?$/) {
 	    $_ = $1;
+	    s/(\s)\s+/$1/g;
 	    s/\\/\\\\/g;
 	    s/"/\\"/g;
 	    if (length > 0) {
-		print OUTPUT "        \"$_\",\n";
-		$index++;
+		printf OUT "        /* %4d */ \"%s\",\n", $index++, $_;
 	    }
 	} else {
 	    die "$fname:$line:  error unterminated quote";
@@ -50,8 +66,70 @@ foreach $fname ( @ARGV ) {
     }
     close(INPUT);
 }
-print OUTPUT "\n    NULL\n};\n\n";
-$tasm_count = $index unless ( defined($tasm_count) );
-print OUTPUT "const char * const * nasm_stdmac_after_tasm = ",
-    "&nasm_stdmac[$tasm_count];\n";
-close(OUTPUT);
+printf OUT "        /* %4d */ NULL\n};\n\n", $index++;
+print OUT "const char * const * const nasm_stdmac_after_tasm = ",
+    "&nasm_stdmac[$tasm_count];\n\n";
+
+my @hashinfo = gen_perfect_hash(\%module_number);
+if (!@hashinfo) {
+    die "$0: no hash found\n";
+}
+# Paranoia...
+verify_hash_table(\%module_number, \@hashinfo);
+my ($n, $sv, $g) = @hashinfo;
+die if ($n & ($n-1));
+
+print OUT "const char * const *nasm_stdmac_find_module(const char *module)\n";
+print OUT "{\n";
+print OUT "    static const struct {\n";
+print OUT "         const char *module;\n";
+print OUT "         const char * const *macros;\n";
+print OUT "    } modules[$nmodule] = {\n";
+foreach $module (@module_list) {
+    printf OUT "        { \"%s\", nasm_stdmac+%d },\n",
+	$module, $module_index{$module};
+}
+print OUT "    };\n";
+
+# Put a large value in unused slots.  This makes it extremely unlikely
+# that any combination that involves unused slot will pass the range test.
+# This speeds up rejection of unrecognized tokens, i.e. identifiers.
+print OUT "#define UNUSED 16383\n";
+
+print OUT "    static const int16_t hash1[$n] = {\n";
+for ($i = 0; $i < $n; $i++) {
+    my $h = ${$g}[$i*2+0];
+    print OUT "        ", defined($h) ? $h : 'UNUSED', ",\n";
+}
+print OUT "    };\n";
+
+print OUT "    static const int16_t hash2[$n] = {\n";
+for ($i = 0; $i < $n; $i++) {
+    my $h = ${$g}[$i*2+1];
+    print OUT "        ", defined($h) ? $h : 'UNUSED', ",\n";
+}
+print OUT "    };\n";
+
+print OUT  "    uint32_t k1, k2;\n";
+print OUT  "    uint64_t crc;\n";
+# For correct overflow behavior, "ix" should be unsigned of the same
+# width as the hash arrays.
+print OUT  "    uint16_t ix;\n";
+print OUT  "\n";
+
+printf OUT "    crc = crc64i(UINT64_C(0x%08x%08x), module);\n",
+    $$sv[0], $$sv[1];
+print  OUT "    k1 = (uint32_t)crc;\n";
+print  OUT "    k2 = (uint32_t)(crc >> 32);\n";
+print  OUT "\n";
+printf OUT "    ix = hash1[k1 & 0x%x] + hash2[k2 & 0x%x];\n", $n-1, $n-1;
+printf OUT "    if (ix >= %d)\n", scalar(@module_list);
+print OUT  "        return NULL;\n";
+print OUT  "\n";
+print OUT  "    if (nasm_stricmp(modules[ix].module, module))\n";
+print OUT  "        return NULL;\n";
+print OUT  "\n";
+print OUT  "    return modules[ix].macros;\n";
+print OUT  "}\n";
+
+close(OUT);
diff --git a/macros/altreg.mac b/macros/altreg.mac
new file mode 100644
index 00000000..83d69679
--- /dev/null
+++ b/macros/altreg.mac
@@ -0,0 +1,74 @@
+;;
+;; altreg.mac
+;;
+;; Alternate register names for 64-bit mode
+;;
+
+USE: altreg
+
+;;
+;; Intel documents R8L-R15L instead of R8B-R15B
+;; (Warning: this may confuse people with an AT&T-style assembly
+;; background, where "r8l" means R8D, etc.)
+;;
+%idefine r8l  r8b
+%idefine r9l  r9b
+%idefine r10l r10b
+%idefine r11l r11b
+%idefine r12l r12b
+%idefine r13l r13b
+%idefine r14l r14b
+%idefine r15l r15b
+
+;;
+;; Numeric register names for the lower 8 registers
+;;
+%idefine r0  rax
+%idefine r1  rcx
+%idefine r2  rdx
+%idefine r3  rbx
+%idefine r4  rsp
+%idefine r5  rbp
+%idefine r6  rsi
+%idefine r7  rdi
+
+%idefine r0d eax
+%idefine r1d ecx
+%idefine r2d edx
+%idefine r3d ebx
+%idefine r4d esp
+%idefine r5d ebp
+%idefine r6d esi
+%idefine r7d edi
+
+%idefine r0w ax
+%idefine r1w cx
+%idefine r2w dx
+%idefine r3w bx
+%idefine r4w sp
+%idefine r5w bp
+%idefine r6w si
+%idefine r7w di
+
+%idefine r0b al
+%idefine r1b cl
+%idefine r2b dl
+%idefine r3b bl
+%idefine r4b spl
+%idefine r5b bpl
+%idefine r6b sil
+%idefine r7b dil
+
+%idefine r0l al
+%idefine r1l cl
+%idefine r2l dl
+%idefine r3l bl
+%idefine r4l spl
+%idefine r5l bpl
+%idefine r6l sil
+%idefine r7l dil
+
+%idefine r0h ah
+%idefine r1h ch
+%idefine r2h dh
+%idefine r3h bh
diff --git a/pptok.dat b/pptok.dat
index 78a975fd..a30dd7e0 100644
--- a/pptok.dat
+++ b/pptok.dat
@@ -47,5 +47,6 @@
 %strlen
 %substr
 %undef
+%use
 %warning
 %xdefine
diff --git a/preproc.c b/preproc.c
index 8859005b..525cce45 100644
--- a/preproc.c
+++ b/preproc.c
@@ -333,6 +333,7 @@ static StrList **dephead, **deptail; /* Dependency list */
 static uint64_t unique;    /* unique identifier numbers */
 
 static Line *predef = NULL;
+static bool do_predef;
 
 static ListGen *list;
 
@@ -640,41 +641,41 @@ static char *read_line(void)
     int bufsize, continued_count;
 
     if (stdmacpos) {
-        if (*stdmacpos) {
-            char *ret = nasm_strdup(*stdmacpos++);
-            if (!*stdmacpos && any_extrastdmac) {
-                stdmacpos = extrastdmac;
-                any_extrastdmac = false;
-                return ret;
-            }
-            /*
-             * Nasty hack: here we push the contents of `predef' on
-             * to the top-level expansion stack, since this is the
-             * most convenient way to implement the pre-include and
-             * pre-define features.
-             */
-            if (!*stdmacpos) {
-                Line *pd, *l;
-                Token *head, **tail, *t;
-
-                for (pd = predef; pd; pd = pd->next) {
-                    head = NULL;
-                    tail = &head;
-                    for (t = pd->first; t; t = t->next) {
-                        *tail = new_Token(NULL, t->type, t->text, 0);
-                        tail = &(*tail)->next;
-                    }
-                    l = nasm_malloc(sizeof(Line));
-                    l->next = istk->expansion;
-                    l->first = head;
-                    l->finishes = false;
-                    istk->expansion = l;
-                }
-            }
-            return ret;
-        } else {
-            stdmacpos = NULL;
-        }
+	char *ret = nasm_strdup(*stdmacpos++);
+	if (!*stdmacpos) {
+	    /* This was the last of the standard macro chain... */
+	    stdmacpos = NULL;
+	    if (any_extrastdmac) {
+		stdmacpos = extrastdmac;
+		any_extrastdmac = false;
+	    } else if (do_predef) {
+		Line *pd, *l;
+		Token *head, **tail, *t;
+
+		/*
+		 * Nasty hack: here we push the contents of
+		 * `predef' on to the top-level expansion stack,
+		 * since this is the most convenient way to
+		 * implement the pre-include and pre-define
+		 * features.
+		 */
+		for (pd = predef; pd; pd = pd->next) {
+		    head = NULL;
+		    tail = &head;
+		    for (t = pd->first; t; t = t->next) {
+			*tail = new_Token(NULL, t->type, t->text, 0);
+			tail = &(*tail)->next;
+		    }
+		    l = nasm_malloc(sizeof(Line));
+		    l->next = istk->expansion;
+		    l->first = head;
+		    l->finishes = false;
+		    istk->expansion = l;
+		}
+		do_predef = false;
+	    }
+	}
+	return ret;
     }
 
     bufsize = BUF_DELTA;
@@ -1151,7 +1152,7 @@ static int ppscan(void *private_data, struct tokenval *tokval)
 	bq = tline->text[0];
         tokval->t_charptr = tline->text;
         tokval->t_inttwo = nasm_unquote(tline->text, &ep);
-	
+
 	if (ep[0] != bq || ep[1] != '\0')
 	    return tokval->t_type = TOKEN_ERRSTR;
 	else
@@ -1523,7 +1524,7 @@ static bool if_condition(Token * tline, enum preproc_token ct)
             if (t->type == TOK_STRING) {
 		size_t l1 = nasm_unquote(t->text, NULL);
 		size_t l2 = nasm_unquote(tt->text, NULL);
-		
+
 		if (l1 != l2) {
 		    j = false;
 		    break;
@@ -2095,7 +2096,7 @@ static int do_directive(Token * tline)
     case PP_INCLUDE:
 	t = tline->next = expand_smacro(tline->next);
         skip_white_(t);
-	
+
         if (!t || (t->type != TOK_STRING &&
                        t->type != TOK_INTERNAL_STRING)) {
             error(ERR_NONFATAL, "`%%include' expects a file name");
@@ -2127,6 +2128,29 @@ static int do_directive(Token * tline)
 	free_tlist(origline);
         return DIRECTIVE_FOUND;
 
+    case PP_USE:
+	t = tline->next = expand_smacro(tline->next);
+	skip_white_(t);
+
+        if (!t || (t->type != TOK_STRING &&
+		   t->type != TOK_INTERNAL_STRING &&
+		   t->type != TOK_ID)) {
+            error(ERR_NONFATAL, "`%%use' expects a module name");
+            free_tlist(origline);
+            return DIRECTIVE_FOUND;     /* but we did _something_ */
+	}
+        if (t->next)
+            error(ERR_WARNING,
+                  "trailing garbage after `%%use' ignored");
+	p = t->text;
+	if (t->type == TOK_STRING)
+	    nasm_unquote(p, NULL);
+	stdmacpos = nasm_stdmac_find_module(p);
+	if (!stdmacpos)
+	    error(ERR_NONFATAL, "unknown `%%use' module: %s", p);
+	free_tlist(origline);
+        return DIRECTIVE_FOUND;
+
     case PP_PUSH:
         tline = tline->next;
         skip_white_(tline);
@@ -2198,7 +2222,7 @@ static int do_directive(Token * tline)
             p = detoken(tline, false);
 	    error(severity, "%s: %s",  pp_directives[i], p);
 	    nasm_free(p);
-	}	    
+	}
         free_tlist(origline);
         break;
     }
@@ -2798,7 +2822,7 @@ static int do_directive(Token * tline)
     {
 	int64_t a1, a2;
 	size_t len;
-	
+
 	casesense = true;
 
         tline = tline->next;
@@ -3952,7 +3976,8 @@ pp_reset(char *file, int apass, efunc errfunc, evalfunc eval,
     } else {
         stdmacpos = nasm_stdmac_after_tasm;
     }
-    any_extrastdmac = (extrastdmac != NULL);
+    any_extrastdmac = extrastdmac && *extrastdmac;
+    do_predef = true;
     list = listgen;
     evaluate = eval;
     pass = apass;
diff --git a/tables.h b/tables.h
index 0ac0ae6f..f4d9a774 100644
--- a/tables.h
+++ b/tables.h
@@ -15,7 +15,8 @@
 
 /* macros.c */
 extern const char * const nasm_stdmac[];
-extern const char * const * nasm_stdmac_after_tasm;
+extern const char * const * const nasm_stdmac_after_tasm;
+const char * const *nasm_stdmac_find_module(const char *);
 
 /* --- From insns.dat via insns.pl: --- */