Stage22: allow builtin concatenation outside $@

15 files changed, 423 insertions, 341 deletions

diff --git a/Makefile.am b/Makefile.am
index a67b4159..7d123e55 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -32,7 +32,9 @@ MAINTAINERCLEANFILES = COPYING INSTALL Makefile.in aclocal.m4 \
 	m4/gnulib-tool.m4 missing stamp-h.in
 
 ACLOCAL_AMFLAGS = -I m4
-DISTCHECK_CONFIGURE_FLAGS = --enable-changeword
+## Enough users install GNU M4 as gm4 that we make sure 'make installcheck'
+## will handle that, prior to making a release.
+DISTCHECK_CONFIGURE_FLAGS = --enable-changeword --program-prefix=g
 
 BUILT_SOURCES = $(top_srcdir)/.version
 $(top_srcdir)/.version:
diff --git a/NEWS b/NEWS
index 73adcf26..41d70737 100644
--- a/NEWS
+++ b/NEWS
@@ -12,9 +12,19 @@ Foundation, Inc.
    required by POSIX.  The manual mentions a way to restore the LIFO order
    present in earlier GNU M4 versions.
 
+** The `defn' builtin now warns when operating on an undefined macro name.
+   To simulate 1.4.x behavior, use:
+     pushdef(`defn', `ifdef(`$1', `builtin(`defn', `$1')')')
+
 ** Enhance the `ifdef', `ifelse', and `shift' builtins, as well as all
    user macros, to transparently handle builtin tokens generated by `defn'.
 
+** Allow the concatenation of builtin macros with arbitrary text in
+   several contexts, via the `defn' builtin or argument expansion, rather
+   than warning and converting the builtin token to an empty string.
+   However, it is still not possible to use a concatenated builtin when
+   defining a macro.
+
 ** Enhance the `defn', `dumpdef', `ifdef', `popdef', `traceon', `traceoff',
    and `undefine' macros to warn when encountering a builtin token in the
    context of a macro name, rather than acting on the empty string.  This
diff --git a/checks/Makefile.in b/checks/Makefile.in
index 6d93144e..0e1091e5 100644
--- a/checks/Makefile.in
+++ b/checks/Makefile.in
@@ -1,6 +1,7 @@
 ## Makefile.in - template for building Makefile for M4 testsuite.
 ##
-## Copyright (C) 1992, 1993, 1994, 2006, 2007 Free Software Foundation, Inc.
+## Copyright (C) 1992, 1993, 1994, 2006, 2007, 2008 Free Software
+## Foundation, Inc.
 ##
 ## This file is part of GNU M4.
 ##
@@ -17,10 +18,11 @@
 ## You should have received a copy of the GNU General Public License
 ## along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+@SET_MAKE@
 PACKAGE = @PACKAGE@
 VERSION = @VERSION@
 
-SHELL = /bin/sh
+SHELL = @SHELL@
 
 bindir = @bindir@
 exec_prefix = @exec_prefix@
@@ -28,6 +30,7 @@ prefix = @prefix@
 srcdir = @srcdir@
 VPATH = @srcdir@
 PATH_SEPARATOR = @PATH_SEPARATOR@
+program_transform_name = @program_transform_name@
 
 # Should be GNU awk, for the get-them script.
 AWK = @AWK@
@@ -57,7 +60,8 @@ check: $(srcdir)/stamp-checks
 
 installcheck: $(srcdir)/stamp-checks
 	PATH='$(bindir)'"$(PATH_SEPARATOR)"$$PATH; export PATH; \
-	$(srcdir)/check-them -I $(srcdir)/../examples $(CHECKS)
+	$(srcdir)/check-them -I $(srcdir)/../examples \
+	-m "`echo m4 | sed '$(program_transform_name)'`" $(CHECKS)
 
 tags:
 
diff --git a/checks/check-them b/checks/check-them
index 9fca39b5..7fba1d6f 100755
--- a/checks/check-them
+++ b/checks/check-them
@@ -2,9 +2,6 @@
 # Check GNU m4 against examples from the manual source.
 # Copyright (C) 1992, 2006, 2007, 2008 Free Software Foundation, Inc.
 
-# Sanity check what we are testing
-m4 --version
-
 # Clean up temp files on exit
 pwd=`pwd`
 tmp=m4-tmp.$$
@@ -29,13 +26,27 @@ skipped=
 strip_needed=false
 diffopts=-c
 
+# Find out where the examples live.
+examples=.
+if test "x$1" = x-I ; then
+  examples="$2"
+  shift; shift
+fi
+
+# Find out how to run m4.
+m4=m4
+if test "x$1" = x-m ; then
+  m4="$2"
+  shift; shift
+fi
+
 # Find out how the executable prints argv[0]
-m4=`m4 --help | sed -e 's/Usage: \(.*\) \[OPTION.*/\1/' \
-    -e 's/\\\\/\\\\\\\\/g' -e 1q`
+m4name=`"$m4" --help | sed -e 's/Usage: \(.*\) \[OPTION.*/\1/' \
+  -e 's/\\\\/\\\\\\\\/g' -e 1q`
 
 # Find out if we should strip \r in the output
-m4 --version > $out
-m4 --version | tr -d '\015' > $xout
+"$m4" --version | tee $out
+"$m4" --version | tr -d '\015' > $xout
 if cmp -s $out $xout; then
   :
 else
@@ -43,13 +54,6 @@ else
   strip_needed=:
 fi
 
-# Find out where the examples live.
-examples=.
-if test "x$1" = x-I ; then
-  examples="$2"
-  shift; shift
-fi
-
 # Find out if diff supports useful options.
 if diff -u /dev/null /dev/null 2>/dev/null ; then
   diffopts="-u"
@@ -68,7 +72,7 @@ do
   echo "Checking $file"
   options=`sed -ne '3s/^dnl @ extra options: //p;3q' "$file"`
   sed -e '/^dnl @/d' -e '/^\^D$/q' "$file" \
-    | LC_MESSAGES=C M4PATH=$examples m4 -d $options - >$out 2>$err
+    | LC_MESSAGES=C M4PATH=$examples "$m4" -d $options - >$out 2>$err
   stat=$?
 
   xstat=`sed -ne '2s/^dnl @ expected status: //p;2q' "$file"`
@@ -88,7 +92,7 @@ do
 
   xoutfile=`sed -n 's/^dnl @ expected output: //p' "$file"`
   if test -z "$xoutfile" ; then
-    sed -e '/^dnl @result{}/!d' -e 's///' -e "s|\.\./examples|$examples|" \
+    sed -e '/^dnl @result{}/!d' -e 's///' -e "s|examples/|$examples/|" \
       "$file" > $xout
   else
     cp "$examples/$xoutfile" $xout
@@ -96,10 +100,11 @@ do
 
   xerrfile=`sed -n 's/^dnl @ expected error: //p' "$file"`
   if test -z "$xerrfile" ; then
-    sed '/^dnl @error{}/!d; s///; '"s|^m4:|$m4:|; s|\.\./examples|$examples|" \
+    sed '/^dnl @error{}/!d
+	 s///; '"s|^m4:|$m4name:|; s|examples/|$examples/|" \
       "$file" > $xerr
   else
-    sed "s|^m4:|$m4:|; s|\.\./examples|$examples|" \
+    sed "s|^m4:|$m4name:|; s|examples/|$examples/|" \
       "$examples/$xerrfile" > $xerr
   fi
 
diff --git a/doc/Makefile.am b/doc/Makefile.am
index ded347f6..b1b7eac8 100644
--- a/doc/Makefile.am
+++ b/doc/Makefile.am
@@ -1,6 +1,6 @@
 ## Makefile.am - template for generating Makefile via Automake
 ##
-## Copyright (C) 2006, 2007 Free Software Foundation, Inc.
+## Copyright (C) 2006, 2007, 2008 Free Software Foundation, Inc.
 ##
 ## This file is part of GNU M4.
 ##
@@ -21,22 +21,22 @@
 
 info_TEXINFOS = m4.texinfo
 m4_TEXINFOS = fdl.texi gpl-3.0.texi
-man_MANS = m4.1
+man_MANS = $(srcdir)/m4.1
 EXTRA_DIST = $(man_MANS) gendocs_template
 MAINTAINERCLEANFILES = $(man_MANS) gendocs_template
 SUFFIXES = .1
+HELP2MAN = $(SHELL) $(top_srcdir)/build-aux/missing --run help2man
 
-# Depend on configure.ac for version, m4.c for usage text.  Do not depend on
+# Depend on ../.version for version, m4.c for usage text.  Do not depend on
 # built m4 executable, since not everyone has help2man or perl.
 # Build the man page once in the srcdir, rather than in every VPATH build
 # dir, to match how automake builds info pages.  This is safe for 'make
 # distcheck' since it is distributed pre-built.
-$(srcdir)/m4.1: $(top_srcdir)/configure.ac $(top_srcdir)/src/m4.c
+$(srcdir)/m4.1: $(top_srcdir)/.version $(top_srcdir)/src/m4.c
 	@if test -x ../src/m4$(EXEEXT) ; then \
 	  echo "Updating man page m4.1" ; \
-	  $(SHELL) $(top_srcdir)/build-aux/missing --run \
-	    help2man --name="macro processor" --source=FSF \
-	      --info-page=m4 --output=$@ ../src/m4$(EXEEXT) ; \
+	  $(HELP2MAN) --name="macro processor" --source='$(PACKAGE_STRING)' \
+	    --info-page=m4 --output=$@ ../src/m4$(EXEEXT) ; \
 	else \
 	  echo "WARNING: The \`man' page \`$@' cannot be updated yet."; \
 	  echo "         Retry once the program executable is ready."; \
diff --git a/doc/m4.texinfo b/doc/m4.texinfo
index 98eba85d..8d83fb0f 100644
--- a/doc/m4.texinfo
+++ b/doc/m4.texinfo
@@ -2111,17 +2111,14 @@ the builtin @code{defn}:
 @deffn Builtin defn (@var{name}@dots{})
 Expands to the @emph{quoted definition} of each @var{name}.  If an
 argument is not a defined macro, the expansion for that argument is
-empty.
+empty and triggers a warning.
 
 If @var{name} is a user-defined macro, the quoted definition is simply
-the quoted expansion text.  If, instead, there is only one @var{name}
-and it is a builtin, the
+the quoted expansion text.  If, instead, @var{name} is a builtin, the
 expansion is a special token, which points to the builtin's internal
-definition.  This token is only meaningful as the second argument to
+definition.  This token meaningful primarily as the second argument to
 @code{define} (and @code{pushdef}), and is silently converted to an
-empty string in most other contexts.  Using multiple @var{name} to
-combine a builtin with anything else is not supported; a warning is
-issued and the builtin is omitted from the final expansion.
+empty string in many other contexts.
 
 The macro @code{defn} is recognized only with parameters.
 @end deffn
@@ -2280,14 +2277,18 @@ bar
 @result{}0
 @end example
 
-Also note that as of M4 1.6, @code{defn} with multiple arguments can
-join text with builtin tokens.  However, when collecting macro
-arguments, a builtin token is preserved only when it occurs in
-isolation.  A future version of @acronym{GNU} M4 may lift this
-restriction.
+A warning is issued if @var{name} is undefined.  Also note that as of M4
+1.6, @code{defn} with multiple arguments can join text with builtin
+tokens.  However, when defining a macro via @code{define} or
+@code{pushdef}, a warning is issued and the builtin token ignored if the
+builtin token does not occur in isolation.  A future version of
+@acronym{GNU} M4 may lift this restriction.
 
 @example
 $ @kbd{m4 -d}
+defn(`foo')
+@error{}m4:stdin:1: Warning: defn: undefined macro `foo'
+@result{}
 define(`a', `A')define(`AA', `b')
 @result{}
 traceon(`defn', `define')
@@ -2297,29 +2298,28 @@ defn(`a', `divnum', `a')
 @result{}AA
 define(`mydivnum', defn(`divnum', `divnum'))mydivnum
 @error{}m4trace: -2- defn(`divnum', `divnum') -> `<divnum><divnum>'
-@error{}m4:stdin:4: Warning: define: cannot concatenate builtin `divnum'
-@error{}m4:stdin:4: Warning: define: cannot concatenate builtin `divnum'
-@error{}m4trace: -1- define(`mydivnum', `')
+@error{}m4:stdin:5: Warning: define: cannot concatenate builtins
+@error{}m4trace: -1- define(`mydivnum', `<divnum><divnum>')
 @result{}
-traceoff(`defn', `define')
+traceoff(`defn', `define')dumpdef(`mydivnum')
+@error{}mydivnum:@tabchar{}`'
 @result{}
 define(`mydivnum', defn(`divnum')defn(`divnum'))mydivnum
-@error{}m4:stdin:6: Warning: define: cannot concatenate builtin `divnum'
-@error{}m4:stdin:6: Warning: define: cannot concatenate builtin `divnum'
+@error{}m4:stdin:7: Warning: define: cannot concatenate builtins
 @result{}
 define(`mydivnum', defn(`divnum')`a')mydivnum
-@error{}m4:stdin:7: Warning: define: cannot concatenate builtin `divnum'
+@error{}m4:stdin:8: Warning: define: cannot concatenate builtins
 @result{}A
 define(`mydivnum', `a'defn(`divnum'))mydivnum
-@error{}m4:stdin:8: Warning: define: cannot concatenate builtin `divnum'
+@error{}m4:stdin:9: Warning: define: cannot concatenate builtins
 @result{}A
 define(`q', ``$@@'')
 @result{}
 define(`foo', q(`a', defn(`divnum')))foo
-@error{}m4:stdin:10: Warning: define: cannot concatenate builtins
-@result{}foo
+@error{}m4:stdin:11: Warning: define: cannot concatenate builtins
+@result{}a,
 ifdef(`foo', `yes', `no')
-@result{}no
+@result{}yes
 @end example
 
 @node Pushdef
@@ -2760,6 +2760,63 @@ foo(`a', `b', `c')
 @result{}arguments:3
 @end example
 
+Since m4 is a macro language, it is even possible to write a macro that
+makes defining blind macros easier:
+
+@deffn Composite define_blind (@var{name}, @ovar{value})
+Defines @var{name} as a blind macro, such that @var{name} will expand to
+@var{value} only when given explicit arguments.  @var{value} should not
+be the result of @code{defn} (@pxref{Defn}).  This macro is only
+recognized with parameters, and results in an empty string.
+@end deffn
+
+Defining a macro to define another macro can be a bit tricky.  We want
+to use a literal @samp{$#} in the argument to the nested @code{define}.
+However, if @samp{$} and @samp{#} are adjacent in the definition of
+@code{define_blind}, then it would be expanded as the number of
+arguments to @code{define_blind} rather than the intended number of
+arguments to @var{name}.  The solution is to pass the difficult
+characters through extra arguments to a helper macro
+@code{_define_blind}.
+
+As for the limitation against using @code{defn}, there are two reasons.
+If a macro was previously defined with @code{define_blind}, then it can
+safely be renamed to a new blind macro using plain @code{define}; using
+@code{define_blind} to rename it just adds another layer of
+@code{ifelse}, occupying memory and slowing down execution.  And if a
+macro is a builtin, then it would result in an attempt to define a macro
+consisting of both text and a builtin token; this is not supported, and
+the builtin token is flattened to an empty string.
+
+With that explanation, here's the definition, and some sample usage.
+Notice that @code{define_blind} is itself a blind macro.
+
+@example
+$ @kbd{m4 -d}
+define(`define_blind', `ifelse(`$#', `0', ``$0'',
+`_$0(`$1', `$2', `$'`#', `$'`0')')')
+@result{}
+define(`_define_blind', `define(`$1',
+`ifelse(`$3', `0', ``$4'', `$2')')')
+@result{}
+define_blind
+@result{}define_blind
+define_blind(`foo', `arguments were $*')
+@result{}
+foo
+@result{}foo
+foo(`bar')
+@result{}arguments were bar
+define(`blah', defn(`foo'))
+@result{}
+blah
+@result{}blah
+blah(`a', `b')
+@result{}arguments were a,b
+defn(`blah')
+@result{}ifelse(`$#', `0', ``$0'', `arguments were $*')
+@end example
+
 @cindex multibranches
 @cindex switch statement
 @cindex case statement
@@ -2795,6 +2852,8 @@ ifelse(defn(`defn'), defn(`divnum'), `yes', `no')
 @result{}no
 ifelse(defn(`defn'), defn(`defn'), `yes', `no')
 @result{}yes
+ifelse(defn(`defn', `divnum'), defn(`defn')defn(`divnum'), `yes', `no')
+@result{}yes
 define(`foo', ifelse(`', `', defn(`divnum')))
 @result{}
 foo
@@ -2859,8 +2918,6 @@ ifelse(`-01234567890123456789', `-'e(long)`-', `yes', `no')
 @result{}no
 @end example
 
-@comment It would be nice to allow concatenation of builtins without
-@comment using $@ handling.
 @example
 define(`e', `$@@')define(`q', ``$@@'')define(`u', `$*')
 @result{}
@@ -2870,16 +2927,12 @@ cmp(`defn(`defn')', `defn(`d')')
 @result{}yes
 cmp(`defn(`defn')', ``<defn>'')
 @result{}no
-cmp(`q(defn(`defn'))', `q(defn(`d'))')-fixme
-@error{}m4:stdin:5: Warning: ifelse: cannot quote builtin
-@error{}m4:stdin:5: Warning: ifelse: cannot quote builtin
-@result{}yes-fixme
-cmp(`q(defn(`defn'))', `q(`<defn>')')-fixme
-@error{}m4:stdin:6: Warning: ifelse: cannot quote builtin
-@result{}no-fixme
-cmp(`q(defn(`defn'))', ``'')-fixme
-@error{}m4:stdin:7: Warning: ifelse: cannot quote builtin
-@result{}no-fixme
+cmp(`q(defn(`defn'))', `q(defn(`d'))')
+@result{}yes
+cmp(`q(defn(`defn'))', `q(`<defn>')')
+@result{}no
+cmp(`q(defn(`defn'))', ``'')
+@result{}no
 cmp(`q(`1', `2', defn(`defn'))', `q(`1', `2', defn(`d'))')
 @result{}yes
 cmp(`q(`1', `2', defn(`defn'))', `q(`1', `2', `<defn>')')
@@ -3772,7 +3825,17 @@ debugmode()
 foo
 @error{}m4trace: -1- foo -> `FOO'
 @result{}FOO
+debugmode(`V')
+@result{}
+foo(`ignored')
+@error{}m4trace:stdin:6: -1- id 6: foo ...
+@error{}m4trace:stdin:6: -1- id 6: foo(`ignored') -> ???
+@error{}m4trace:stdin:6: -1- id 6: foo(...) -> `FOO'
+@result{}FOO
 debugmode
+@error{}m4trace:stdin:7: -1- id 7: debugmode ...
+@error{}m4trace:stdin:7: -1- id 7: debugmode -> ???
+@error{}
 @result{}
 foo
 @error{}m4trace: -1- foo
@@ -3780,7 +3843,7 @@ foo
 debugmode(`+l')
 @result{}
 foo
-@error{}m4trace:8: -1- foo
+@error{}m4trace:10: -1- foo
 @result{}FOO
 @end example
 
@@ -3803,6 +3866,23 @@ indir(`echo', defn(`changequote'))
 @result{}
 @end example
 
+This example shows the effects of the debug flags that are not related
+to macro tracing.
+
+@comment examples
+@comment options: -dip
+@example
+$ @kbd{m4 -dip -I examples}
+@error{}m4debug: input read from stdin
+include(`foo')dnl
+@error{}m4debug: path search for `foo' found `examples/foo'
+@error{}m4debug: input read from examples/foo
+@result{}bar
+@error{}m4debug: input reverted to stdin, line 1
+^D
+@error{}m4debug: input exhausted
+@end example
+
 @node Debug Output
 @section Saving debugging output
 
@@ -4470,7 +4550,7 @@ define(`bar
 ', defn(`__file__'))
 @result{}
 include(`foo')
-@result{}../examples/foo
+@result{}examples/foo
 define(`bar
 ', defn(`__line__'))
 @result{}
@@ -4720,7 +4800,7 @@ than computing the builtin token up front, as is done for @code{bar}.
 m4wrap(`define(`foo', defn(`divnum'))foo
 ')
 @result{}
-m4wrap(`define(`bar', ')m4wrap(defn(`divnum'))m4wrap(`)bar
+m4wrap(`define(`bar', 'defn(`divnum')`)bar
 ')
 @result{}
 ^D
@@ -4955,7 +5035,7 @@ ifdef(`__unix__', ,
 ')m4exit(`77')')dnl
 changequote(`[', `]')dnl
 syscmd([echo 'divert(1)hi
-format(%1000000d, 1)' | m4 | sed 1q])dnl
+format(%1000000d, 1)' | ]__program__[ | sed -n 1p])dnl
 @result{}hi
 sysval
 @result{}0
@@ -5718,6 +5798,8 @@ ifelse(format(`%.1A', `1.999'), `0X1.0P+1', `success',
        format(`%.1A', `1.999'), `0X2.0P+0', `success',
        format(`%.1A', `1.999'))
 @result{}success
+format(`%g', `0xa.P+1')
+@result{}20
 @end example
 
 Using the @code{forloop} macro defined earlier (@pxref{Forloop}), this
@@ -6483,11 +6565,13 @@ sysval
 @c the expansion.
 
 @example
-syscmd(`rm -f foo??????')sysval
+syscmd(`rm -rf foodir')sysval
 @result{}0
-len(mkstemp(`fooXXXXX'))
-@result{}9
-syscmd(`rm foo??????')sysval
+syscmd(`mkdir foodir')sysval
+@result{}0
+len(mkstemp(`foodir/fooXXXXX'))
+@result{}16
+syscmd(`rm -r foodir')sysval
 @result{}0
 @end example
 
@@ -6606,7 +6690,7 @@ foo
 @result{}foo called at stdin:2
 include(`incl.m4')
 @result{}Include file start
-@result{}foo called at ../examples/incl.m4:2
+@result{}foo called at examples/incl.m4:2
 @result{}Include file end
 @result{}
 @end example
diff --git a/src/Makefile.am b/src/Makefile.am
index 4d061c71..f2c15244 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -1,6 +1,6 @@
 ## Makefile.am - template for generating Makefile via Automake
 ##
-## Copyright (C) 2006, 2007 Free Software Foundation, Inc.
+## Copyright (C) 2006, 2007, 2008 Free Software Foundation, Inc.
 ##
 ## This file is part of GNU M4.
 ##
@@ -27,4 +27,4 @@ macro.c output.c path.c symtab.c
 if STACKOVF
 m4_SOURCES += stackovf.c
 endif
-m4_LDADD = ../lib/libm4.a $(LIBM4_LIBDEPS)
+m4_LDADD = ../lib/libm4.a $(LIBM4_LIBDEPS) $(POW_LIB)
diff --git a/src/builtin.c b/src/builtin.c
index 497e2b91..142f0dc6 100644
--- a/src/builtin.c
+++ b/src/builtin.c
@@ -675,12 +675,9 @@ define_macro (int argc, macro_arguments *argv, symbol_lookup mode)
     {
     case TOKEN_COMP:
       m4_warn (0, me, _("cannot concatenate builtins"));
-      // TODO fall through instead
-      break;
-
+      /* fallthru */
     case TOKEN_TEXT:
-      // TODO flatten TOKEN_COMP value, or support concatenation of builtins
-      define_user_macro (ARG (1), ARG_LEN (1), ARG (2), mode);
+      define_user_macro (ARG (1), ARG_LEN (1), arg_text (argv, 2, true), mode);
       break;
 
     case TOKEN_FUNC:
@@ -763,7 +760,7 @@ static void
 m4_ifelse (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *me = ARG (0);
-  int index;
+  int i;
 
   if (argc == 2 || bad_argc (me, argc, 3, -1))
     return;
@@ -771,14 +768,14 @@ m4_ifelse (struct obstack *obs, int argc, macro_arguments *argv)
     /* Diagnose excess arguments if 5, 8, 11, etc., actual arguments.  */
     bad_argc (me, argc, 0, argc - 2);
 
-  index = 1;
+  i = 1;
   argc--;
 
   while (true)
     {
-      if (arg_equal (argv, index, index + 1))
+      if (arg_equal (argv, i, i + 1))
 	{
-	  push_arg (obs, argv, index + 2);
+	  push_arg (obs, argv, i + 2);
 	  return;
 	}
       switch (argc)
@@ -788,12 +785,12 @@ m4_ifelse (struct obstack *obs, int argc, macro_arguments *argv)
 
 	case 4:
 	case 5:
-	  push_arg (obs, argv, index + 3);
+	  push_arg (obs, argv, i + 3);
 	  return;
 
 	default:
 	  argc -= 3;
-	  index += 3;
+	  i += 3;
 	}
     }
 }
@@ -1008,7 +1005,10 @@ m4_defn (struct obstack *obs, int argc, macro_arguments *argv)
 	}
       s = lookup_symbol (ARG (i), SYMBOL_LOOKUP);
       if (s == NULL)
-	continue;
+	{
+	  m4_warn (0, me, _("undefined macro `%s'"), ARG (i));
+	  continue;
+	}
 
       switch (SYMBOL_TYPE (s))
 	{
@@ -1042,10 +1042,10 @@ m4_defn (struct obstack *obs, int argc, macro_arguments *argv)
 
 /* Helper macros for readability.  */
 #if UNIX || defined WEXITSTATUS
-# define M4SYSVAL_EXITBITS(status)                       \
-   (WIFEXITED (status) ? WEXITSTATUS (status) : 0)
-# define M4SYSVAL_TERMSIGBITS(status)                    \
-   (WIFSIGNALED (status) ? WTERMSIG (status) << 8 : 0)
+# define M4SYSVAL_EXITBITS(status)			 \
+  (WIFEXITED (status) ? WEXITSTATUS (status) : 0)
+# define M4SYSVAL_TERMSIGBITS(status)			 \
+  (WIFSIGNALED (status) ? WTERMSIG (status) << 8 : 0)
 
 #else /* !UNIX && !defined WEXITSTATUS */
 /* Platforms such as mingw do not support the notion of reporting
@@ -1976,7 +1976,7 @@ m4_format (struct obstack *obs, int argc, macro_arguments *argv)
 {
   if (bad_argc (ARG (0), argc, 1, -1))
     return;
-  format (obs, argc, argv);
+  expand_format (obs, argc, argv);
 }
 
 /*-------------------------------------------------------------------------.
diff --git a/src/debug.c b/src/debug.c
index 4a4e915d..fde6c499 100644
--- a/src/debug.c
+++ b/src/debug.c
@@ -248,7 +248,7 @@ trace_format (const char *fmt, ...)
   char ch;
   int d;
   const char *s;
-  int maxlen;
+  size_t maxlen;
 
   va_start (args, fmt);
 
@@ -260,7 +260,7 @@ trace_format (const char *fmt, ...)
       if (ch == '\0')
 	break;
 
-      maxlen = INT_MAX;
+      maxlen = SIZE_MAX;
       switch (*fmt++)
 	{
 	case 'B':
@@ -294,7 +294,7 @@ trace_format (const char *fmt, ...)
 	  break;
 	}
 
-      if (obstack_print (&trace, s, SIZE_MAX, &maxlen))
+      if (shipout_string_trunc (&trace, s, SIZE_MAX, &maxlen))
 	break;
     }
 
@@ -364,7 +364,7 @@ trace_pre (const char *name, int id, macro_arguments *argv)
 
   if (arg_argc (argv) > 1 && (debug_level & DEBUG_TRACE_ARGS))
     {
-      int len = max_debug_argument_length;
+      size_t len = max_debug_argument_length;
       trace_format ("(");
       arg_print (&trace, argv, 1,
 		 (debug_level & DEBUG_TRACE_QUOTE) ? &curr_quote : NULL,
@@ -401,31 +401,3 @@ trace_post (const char *name, int id, macro_arguments *argv,
     trace_format (" -> %l%B%r", expanded);
   trace_flush ();
 }
-
-/* Dump the string STR of length LEN to the obstack OBS.  If LEN is
-   SIZE_MAX, use strlen (STR) instead.  If MAX_LEN is non-NULL,
-   truncate the dump at MAX_LEN bytes and return true if MAX_LEN was
-   reached; otherwise, return false and update MAX_LEN as
-   appropriate.  */
-bool
-obstack_print (struct obstack *obs, const char *str, size_t len, int *max_len)
-{
-  int max = max_len ? *max_len : INT_MAX;
-
-  if (len == SIZE_MAX)
-    len = strlen (str);
-  if (len < max)
-    {
-      obstack_grow (obs, str, len);
-      max -= len;
-    }
-  else
-    {
-      obstack_grow (obs, str, max);
-      obstack_grow (obs, "...", 3);
-      max = 0;
-    }
-  if (max_len)
-    *max_len = max;
-  return max == 0;
-}
diff --git a/src/format.c b/src/format.c
index 6808ad5d..717a0702 100644
--- a/src/format.c
+++ b/src/format.c
@@ -122,7 +122,7 @@ arg_double (const char *me, const char *str)
 `------------------------------------------------------------------*/
 
 void
-format (struct obstack *obs, int argc, macro_arguments *argv)
+expand_format (struct obstack *obs, int argc, macro_arguments *argv)
 {
   const char *me = ARG (0);		/* Macro name.  */
   const char *f;			/* Format control string.  */
@@ -130,7 +130,7 @@ format (struct obstack *obs, int argc, macro_arguments *argv)
   char fstart[] = "%'+- 0#*.*hhd";	/* Current format spec.  */
   char *p;				/* Position within fstart.  */
   unsigned char c;			/* A simple character.  */
-  int index = 0;			/* Index within argc used so far.  */
+  int i = 0;				/* Index within argc used so far.  */
   bool valid_format = true;		/* True if entire format string ok.  */
 
   /* Flags.  */
@@ -161,7 +161,7 @@ format (struct obstack *obs, int argc, macro_arguments *argv)
   char *str;			/* Malloc'd buffer of formatted text.  */
   enum {CHAR, INT, LONG, DOUBLE, STR} datatype;
 
-  f = fmt = ARG_STR (index, argc, argv);
+  f = fmt = ARG_STR (i, argc, argv);
   memset (ok, 0, sizeof ok);
   while (true)
     {
@@ -170,7 +170,7 @@ format (struct obstack *obs, int argc, macro_arguments *argv)
 	  if (c == '\0')
 	    {
 	      if (valid_format)
-		bad_argc (me, argc, index, index);
+		bad_argc (me, argc, i, i);
 	      return;
 	    }
 	  obstack_1grow (obs, c);
@@ -250,7 +250,7 @@ format (struct obstack *obs, int argc, macro_arguments *argv)
       *p++ = '*';
       if (*fmt == '*')
 	{
-	  width = ARG_INT (index, argc, argv);
+	  width = ARG_INT (i, argc, argv);
 	  fmt++;
 	}
       else
@@ -270,7 +270,7 @@ format (struct obstack *obs, int argc, macro_arguments *argv)
 	  ok['c'] = 0;
 	  if (*(++fmt) == '*')
 	    {
-	      prec = ARG_INT (index, argc, argv);
+	      prec = ARG_INT (i, argc, argv);
 	      ++fmt;
 	    }
 	  else
@@ -359,27 +359,27 @@ format (struct obstack *obs, int argc, macro_arguments *argv)
 	{
 	case CHAR:
 	  str = asnprintf (base, &len, fstart, width,
-			   ARG_INT (index, argc, argv));
+			   ARG_INT (i, argc, argv));
 	  break;
 
 	case INT:
 	  str = asnprintf (base, &len, fstart, width, prec,
-			   ARG_INT (index, argc, argv));
+			   ARG_INT (i, argc, argv));
 	  break;
 
 	case LONG:
 	  str = asnprintf (base, &len, fstart, width, prec,
-			   ARG_LONG (index, argc, argv));
+			   ARG_LONG (i, argc, argv));
 	  break;
 
 	case DOUBLE:
 	  str = asnprintf (base, &len, fstart, width, prec,
-			   ARG_DOUBLE (index, argc, argv));
+			   ARG_DOUBLE (i, argc, argv));
 	  break;
 
 	case STR:
 	  str = asnprintf (base, &len, fstart, width, prec,
-			   ARG_STR (index, argc, argv));
+			   ARG_STR (i, argc, argv));
 	  break;
 
 	default:
diff --git a/src/input.c b/src/input.c
index cdb7d034..f5c2deeb 100644
--- a/src/input.c
+++ b/src/input.c
@@ -235,14 +235,14 @@ make_text_link (struct obstack *obs, token_chain **start, token_chain **end)
 /*-------------------------------------------------------------------.
 | push_file () pushes an input file on the input stack, saving the   |
 | current file name and line number.  If next is non-NULL, this push |
-| invalidates a call to push_string_init (), whose storage is        |
-| consequently released.  If CLOSE, then close FP after EOF is       |
-| detected.  TITLE is used as the location for text parsed from the  |
-| file (not necessarily the file name).                              |
+| invalidates a call to push_string_init (), whose storage is	     |
+| consequently released.  If CLOSE_WHEN_DONE, then close FP after    |
+| EOF is detected.  TITLE is used as the location for text parsed    |
+| from the file (not necessarily the file name).		     |
 `-------------------------------------------------------------------*/
 
 void
-push_file (FILE *fp, const char *title, bool close)
+push_file (FILE *fp, const char *title, bool close_when_done)
 {
   input_block *i;
 
@@ -263,7 +263,7 @@ push_file (FILE *fp, const char *title, bool close)
 
   i->u.u_f.fp = fp;
   i->u.u_f.end = false;
-  i->u.u_f.close = close;
+  i->u.u_f.close = close_when_done;
   i->u.u_f.advance = start_of_input_line;
   output_current_line = -1;
 
@@ -343,18 +343,18 @@ push_string_init (void)
 | rather than copying everything consecutively onto the input stack.  |
 | Must be called between push_string_init and push_string_finish.     |
 |                                                                     |
-| If TOKEN contains text, then convert the current input block into   |
-| a chain if it is not one already, and add the contents of TOKEN as  |
-| a new link in the chain.  LEVEL describes the current expansion     |
-| level, or -1 if TOKEN is composite, its contents reside entirely    |
-| on the current_input stack, and TOKEN lives in temporary storage.   |
-| If TOKEN is a simple string, then it belongs to the current macro   |
-| expansion.  If TOKEN is composite, then each text link has a level  |
-| of -1 if it belongs to the current macro expansion, otherwise it    |
-| is a back-reference where level tracks which stack it came from.    |
-| The resulting input block chain contains links with a level of -1   |
-| if the text belongs to the input stack, otherwise the level where   |
-| the back-reference comes from.				      |
+| Convert the current input block into a chain if it is not one	      |
+| already, and add the contents of TOKEN as a new link in the chain.  |
+| LEVEL describes the current expansion level, or -1 if TOKEN is      |
+| composite, its contents reside entirely on the current_input	      |
+| stack, and TOKEN lives in temporary storage.  If TOKEN is a simple  |
+| string, then it belongs to the current macro expansion.  If TOKEN   |
+| is composite, then each text link has a level of -1 if it belongs   |
+| to the current macro expansion, otherwise it is a back-reference    |
+| where level tracks which stack it came from.  The resulting input   |
+| block chain contains links with a level of -1 if the text belongs   |
+| to the input stack, otherwise the level where the back-reference    |
+| comes from.							      |
 |                                                                     |
 | Return true only if a reference was created to the contents of      |
 | TOKEN, in which case, LEVEL was non-negative and the lifetime of    |
@@ -660,7 +660,7 @@ pop_input (bool cleanup)
 	return false;
       if (debug_level & DEBUG_TRACE_INPUT)
 	{
-	  if (tmp)
+	  if (tmp != &input_eof)
 	    DEBUG_MESSAGE2 ("input reverted to %s, line %d",
 			    tmp->file, tmp->line);
 	  else
@@ -737,14 +737,14 @@ pop_wrapup (void)
 void
 input_print (struct obstack *obs, const input_block *input)
 {
-  int maxlen = max_debug_argument_length;
+  size_t maxlen = max_debug_argument_length;
   token_chain *chain;
 
   assert (input);
   switch (input->type)
     {
     case INPUT_STRING:
-      obstack_print (obs, input->u.u_s.str, input->u.u_s.len, &maxlen);
+      shipout_string_trunc (obs, input->u.u_s.str, input->u.u_s.len, &maxlen);
       break;
     case INPUT_FILE:
       obstack_grow (obs, "<file: ", strlen ("<file: "));
@@ -758,8 +758,8 @@ input_print (struct obstack *obs, const input_block *input)
 	  switch (chain->type)
 	    {
 	    case CHAIN_STR:
-	      if (obstack_print (obs, chain->u.u_s.str, chain->u.u_s.len,
-				 &maxlen))
+	      if (shipout_string_trunc (obs, chain->u.u_s.str,
+					chain->u.u_s.len, &maxlen))
 		return;
 	      break;
 	    case CHAIN_FUNC:
@@ -1062,19 +1062,35 @@ skip_line (const char *name)
 | When next_token() sees a builtin token with peek_input, this	    |
 | retrieves the value of the function pointer, stores it in TD, and |
 | consumes the input so the caller does not need to do next_char.   |
-| If TD is NULL, discard the token instead.			    |
+| If OBS, TD will be converted to a composite token using storage   |
+| from OBS as necessary; otherwise, if TD is NULL, the builtin is   |
+| discarded.                                                        |
 `------------------------------------------------------------------*/
 
 static void
-init_macro_token (token_data *td)
+init_macro_token (struct obstack *obs, token_data *td)
 {
   token_chain *chain;
 
   assert (isp->type == INPUT_CHAIN);
   chain = isp->u.u_c.chain;
   assert (!chain->quote_age && chain->type == CHAIN_FUNC && chain->u.func);
-  if (td)
+  if (obs)
     {
+      assert (td);
+      if (TOKEN_DATA_TYPE (td) == TOKEN_VOID)
+	{
+	  TOKEN_DATA_TYPE (td) = TOKEN_COMP;
+	  td->u.u_c.chain = td->u.u_c.end = NULL;
+	  td->u.u_c.wrapper = false;
+	  td->u.u_c.has_func = true;
+	}
+      assert (TOKEN_DATA_TYPE (td) == TOKEN_COMP);
+      append_macro (obs, chain->u.func, &td->u.u_c.chain, &td->u.u_c.end);
+    }
+  else if (td)
+    {
+      assert (TOKEN_DATA_TYPE (td) == TOKEN_VOID);
       TOKEN_DATA_TYPE (td) = TOKEN_FUNC;
       TOKEN_DATA_FUNC (td) = chain->u.func;
     }
@@ -1599,7 +1615,7 @@ next_token (token_data *td, int *line, struct obstack *obs, bool allow_argv,
     }
   if (ch == CHAR_MACRO)
     {
-      init_macro_token (td);
+      init_macro_token (obs, td);
 #ifdef DEBUG_INPUT
       xfprintf (stderr, "next_token -> MACDEF (%s)\n",
 		find_builtin_by_addr (TOKEN_DATA_FUNC (td))->name);
@@ -1635,10 +1651,7 @@ next_token (token_data *td, int *line, struct obstack *obs, bool allow_argv,
 			      _("end of file in comment"));
 	  if (ch == CHAR_MACRO)
 	    {
-	      // TODO support concatenation of builtins
-	      m4_warn_at_line (0, file, *line, caller,
-			       _("cannot comment builtin"));
-	      init_macro_token (NULL);
+	      init_macro_token (obs, obs ? td : NULL);
 	      continue;
 	    }
 	  if (MATCH (ch, curr_comm.str2, true))
@@ -1734,35 +1747,8 @@ next_token (token_data *td, int *line, struct obstack *obs, bool allow_argv,
 			      _("end of file in string"));
 
 	  if (ch == CHAR_MACRO)
-	    {
-	      // TODO support concatenation of builtins
-	      if (obstack_object_size (obs_td) == 0
-		  && TOKEN_DATA_TYPE (td) == TOKEN_VOID)
-		{
-		  assert (quote_level == 1);
-		  init_macro_token (td);
-		  ch = peek_input (false);
-		  if (MATCH (ch, curr_quote.str2, false))
-		    {
-#ifdef DEBUG_INPUT
-		      const builtin *bp
-			= find_builtin_by_addr (TOKEN_DATA_FUNC (td));
-		      xfprintf (stderr, "next_token -> MACDEF (%s)\n",
-				bp->name);
-#endif
-		      ch = next_char (false, false);
-		      MATCH (ch, curr_quote.str2, true);
-		      return TOKEN_MACDEF;
-		    }
-		  TOKEN_DATA_TYPE (td) = TOKEN_VOID;
-		}
-	      else
-		init_macro_token (NULL);
-	      m4_warn_at_line (0, file, *line, caller,
-			       _("cannot quote builtin"));
-	      continue;
-	    }
-	  if (ch == CHAR_QUOTE)
+	    init_macro_token (obs, obs ? td : NULL);
+	  else if (ch == CHAR_QUOTE)
 	    append_quote_token (obs, td);
 	  else if (MATCH (ch, curr_quote.str2, true))
 	    {
diff --git a/src/m4.c b/src/m4.c
index 0ace6dcd..2ad82c2b 100644
--- a/src/m4.c
+++ b/src/m4.c
@@ -47,7 +47,7 @@ int no_gnu_extensions = 0;
 int prefix_all_builtins = 0;
 
 /* Max length of arguments in trace output (-lsize).  */
-int max_debug_argument_length = INT_MAX;
+size_t max_debug_argument_length = SIZE_MAX;
 
 /* Suppress warnings about missing arguments.  */
 int suppress_warnings = 0;
@@ -572,9 +572,10 @@ main (int argc, char *const *argv, char *const *envp)
 	break;
 
       case 'l':
-	max_debug_argument_length = atoi (optarg);
-	if (max_debug_argument_length <= 0)
-	  max_debug_argument_length = INT_MAX;
+	{
+	  long tmp = strtol (optarg, NULL, 10);
+	  max_debug_argument_length = tmp <= 0 ? SIZE_MAX : (size_t) tmp;
+	}
 	break;
 
       case 'o':
diff --git a/src/m4.h b/src/m4.h
index cdad61a6..2b728872 100644
--- a/src/m4.h
+++ b/src/m4.h
@@ -129,7 +129,7 @@ extern int debug_level;			/* -d */
 extern size_t hash_table_size;		/* -H */
 extern int no_gnu_extensions;		/* -G */
 extern int prefix_all_builtins;		/* -P */
-extern int max_debug_argument_length;	/* -l */
+extern size_t max_debug_argument_length;/* -l */
 extern int suppress_warnings;		/* -Q */
 extern int warning_status;		/* -E */
 extern int nesting_limit;		/* -L */
@@ -182,7 +182,7 @@ extern FILE *debug;
 #define DEBUG_TRACE_CALLID 0x200
 
 /* V: very verbose --  print everything */
-#define DEBUG_TRACE_VERBOSE 0x377
+#define DEBUG_TRACE_VERBOSE 0x3FF
 /* default flags -- equiv: aeq */
 #define DEBUG_TRACE_DEFAULT 0x007
 
@@ -249,7 +249,6 @@ void trace_pre (const char *, int, macro_arguments *);
 void trace_post (const char *, int, macro_arguments *,
 		 const input_block *);
 
-bool obstack_print (struct obstack *, const char *, size_t, int *);
 
 /* File: input.c  --- lexical definitions.  */
 
@@ -266,7 +265,7 @@ enum token_type
   TOKEN_COMMA,	/* Active character `,', TOKEN_TEXT.  */
   TOKEN_CLOSE,	/* Active character `)', TOKEN_TEXT.  */
   TOKEN_SIMPLE,	/* Any other single character, TOKEN_TEXT.  */
-  TOKEN_MACDEF,	/* A macro's definition (see "defn"), TOKEN_FUNC.  */
+  TOKEN_MACDEF,	/* A builtin macro, TOKEN_FUNC or TOKEN_COMP.  */
   TOKEN_ARGV	/* A series of parameters, TOKEN_COMP.  */
 };
 
@@ -434,7 +433,8 @@ extern int output_current_line;
 void output_init (void);
 void output_exit (void);
 void output_text (const char *, int);
-void shipout_text (struct obstack *, const char *, int, int);
+void divert_text (struct obstack *, const char *, int, int);
+bool shipout_string_trunc (struct obstack *, const char *, size_t, size_t *);
 void make_diversion (int);
 void insert_diversion (int);
 void insert_file (FILE *);
@@ -503,7 +503,7 @@ size_t adjust_refcount (int, bool);
 bool arg_adjust_refcount (macro_arguments *, bool);
 unsigned int arg_argc (macro_arguments *);
 token_data_type arg_type (macro_arguments *, unsigned int);
-const char *arg_text (macro_arguments *, unsigned int);
+const char *arg_text (macro_arguments *, unsigned int, bool);
 bool arg_equal (macro_arguments *, unsigned int, unsigned int);
 bool arg_empty (macro_arguments *, unsigned int);
 size_t arg_len (macro_arguments *, unsigned int);
@@ -511,7 +511,7 @@ builtin_func *arg_func (macro_arguments *, unsigned int);
 struct obstack *arg_scratch (void);
 bool arg_print (struct obstack *, macro_arguments *, unsigned int,
 		const string_pair *, bool, token_chain **, const char *,
-		int *, bool);
+		size_t *, bool);
 macro_arguments *make_argv_ref (macro_arguments *, const char *, size_t,
 				bool, bool);
 void push_arg (struct obstack *, macro_arguments *, unsigned int);
@@ -522,7 +522,7 @@ void wrap_args (macro_arguments *);
 
 /* Grab the text at argv index I.  Assumes macro_argument *argv is in
    scope, and aborts if the argument is not text.  */
-#define ARG(i) arg_text (argv, i)
+#define ARG(i) arg_text (argv, i, false)
 
 /* Grab the text length at argv index I.  Assumes macro_argument *argv
    is in scope, and aborts if the argument is not text.  */
@@ -587,7 +587,7 @@ bool evaluate (const char *, const char *, int32_t *);
 
 /* File: format.c  --- printf like formatting.  */
 
-void format (struct obstack *, int, macro_arguments *);
+void expand_format (struct obstack *, int, macro_arguments *);
 
 /* File: freeze.c --- frozen state files.  */
 
diff --git a/src/macro.c b/src/macro.c
index 5b99e677..4dcbdf3e 100644
--- a/src/macro.c
+++ b/src/macro.c
@@ -305,10 +305,10 @@ expand_token (struct obstack *obs, token_type t, token_data *td, int line,
 	      && peek_token () != TOKEN_OPEN))
 	{
 #ifdef ENABLE_CHANGEWORD
-	  shipout_text (obs, TOKEN_DATA_ORIG_TEXT (td),
-			TOKEN_DATA_LEN (td), line);
+	  divert_text (obs, TOKEN_DATA_ORIG_TEXT (td),
+		       TOKEN_DATA_LEN (td), line);
 #else
-	  shipout_text (obs, TOKEN_DATA_TEXT (td), TOKEN_DATA_LEN (td), line);
+	  divert_text (obs, TOKEN_DATA_TEXT (td), TOKEN_DATA_LEN (td), line);
 #endif /* !ENABLE_CHANGEWORD */
 	  /* The word just appended is unquoted, but the heuristics of
 	     safe_quote are applicable.  */
@@ -324,23 +324,11 @@ expand_token (struct obstack *obs, token_type t, token_data *td, int line,
       assert (!"expand_token");
       abort ();
     }
-  shipout_text (obs, TOKEN_DATA_TEXT (td), TOKEN_DATA_LEN (td), line);
+  divert_text (obs, TOKEN_DATA_TEXT (td), TOKEN_DATA_LEN (td), line);
   return result;
 }
 
 
-/*---------------------------------------------------------------.
-| Helper function to print warning about concatenating FUNC with |
-| text.                                                          |
-`---------------------------------------------------------------*/
-static void
-warn_builtin_concat (const char *caller, builtin_func *func)
-{
-  const builtin *bp = find_builtin_by_addr (func);
-  assert (bp);
-  m4_warn (0, caller, _("cannot concatenate builtin `%s'"), bp->name);
-}
-
 /*-------------------------------------------------------------------.
 | This function parses one argument to a macro call.  It expects the |
 | first left parenthesis or the separating comma to have been read   |
@@ -383,15 +371,10 @@ expand_argument (struct obstack *obs, token_data *argp, const char *caller)
 	case TOKEN_CLOSE:
 	  if (paren_level == 0)
 	    {
-	      size_t len = obstack_object_size (obs);
-	      if (TOKEN_DATA_TYPE (argp) == TOKEN_FUNC)
-		{
-		  if (!len)
-		    return t == TOKEN_COMMA;
-		  warn_builtin_concat (caller, TOKEN_DATA_FUNC (argp));
-		}
+	      assert (TOKEN_DATA_TYPE (argp) != TOKEN_FUNC);
 	      if (TOKEN_DATA_TYPE (argp) != TOKEN_COMP)
 		{
+		  size_t len = obstack_object_size (obs);
 		  TOKEN_DATA_TYPE (argp) = TOKEN_TEXT;
 		  if (len)
 		    {
@@ -404,7 +387,16 @@ expand_argument (struct obstack *obs, token_data *argp, const char *caller)
 		  TOKEN_DATA_QUOTE_AGE (argp) = age;
 		}
 	      else
-		make_text_link (obs, NULL, &argp->u.u_c.end);
+		{
+		  make_text_link (obs, NULL, &argp->u.u_c.end);
+		  if (argp->u.u_c.chain == argp->u.u_c.end
+		      && argp->u.u_c.chain->type == CHAIN_FUNC)
+		    {
+		      builtin_func *func = argp->u.u_c.chain->u.func;
+		      TOKEN_DATA_TYPE (argp) = TOKEN_FUNC;
+		      TOKEN_DATA_FUNC (argp) = func;
+		    }
+		}
 	      return t == TOKEN_COMMA;
 	    }
 	  /* fallthru */
@@ -427,14 +419,13 @@ expand_argument (struct obstack *obs, token_data *argp, const char *caller)
 
 	case TOKEN_WORD:
 	case TOKEN_STRING:
+	case TOKEN_MACDEF:
 	  if (!expand_token (obs, t, &td, line, first))
 	    age = 0;
 	  if (TOKEN_DATA_TYPE (&td) == TOKEN_COMP)
 	    {
 	      if (TOKEN_DATA_TYPE (argp) != TOKEN_COMP)
 		{
-		  if (TOKEN_DATA_TYPE (argp) == TOKEN_FUNC)
-		    warn_builtin_concat (caller, TOKEN_DATA_FUNC (argp));
 		  TOKEN_DATA_TYPE (argp) = TOKEN_COMP;
 		  argp->u.u_c.chain = td.u.u_c.chain;
 		  argp->u.u_c.wrapper = argp->u.u_c.has_func = false;
@@ -450,22 +441,6 @@ expand_argument (struct obstack *obs, token_data *argp, const char *caller)
 	    }
 	  break;
 
-	case TOKEN_MACDEF:
-	  if (TOKEN_DATA_TYPE (argp) == TOKEN_VOID
-	      && obstack_object_size (obs) == 0)
-	    {
-	      TOKEN_DATA_TYPE (argp) = TOKEN_FUNC;
-	      TOKEN_DATA_FUNC (argp) = TOKEN_DATA_FUNC (&td);
-	    }
-	  else
-	    {
-	      if (TOKEN_DATA_TYPE (argp) == TOKEN_FUNC)
-		warn_builtin_concat (caller, TOKEN_DATA_FUNC (argp));
-	      warn_builtin_concat (caller, TOKEN_DATA_FUNC (&td));
-	      TOKEN_DATA_TYPE (argp) = TOKEN_TEXT;
-	    }
-	  break;
-
 	case TOKEN_ARGV:
 	  assert (paren_level == 0 && TOKEN_DATA_TYPE (argp) == TOKEN_VOID
 		  && obstack_object_size (obs) == 0
@@ -798,6 +773,8 @@ arg_adjust_refcount (macro_arguments *argv, bool increase)
 		  if (chain->u.u_s.level >= 0)
 		    adjust_refcount (chain->u.u_s.level, increase);
 		  break;
+		case CHAIN_FUNC:
+		  break;
 		case CHAIN_ARGV:
 		  assert (chain->u.u_a.argv->inuse);
 		  arg_adjust_refcount (chain->u.u_a.argv, increase);
@@ -814,30 +791,30 @@ arg_adjust_refcount (macro_arguments *argv, bool increase)
 }
 
 
-/* Given ARGV, return the token_data that contains argument INDEX;
-   INDEX must be > 0, < argv->argc.  If LEVEL is non-NULL, *LEVEL is
-   set to the obstack level that contains the token (which is not
-   necessarily the level of ARGV).  If FLATTEN, avoid returning a
-   builtin function.  */
+/* Given ARGV, return the token_data that contains argument ARG; ARG
+   must be > 0, < argv->argc.  If LEVEL is non-NULL, *LEVEL is set to
+   the obstack level that contains the token (which is not necessarily
+   the level of ARGV).  If FLATTEN, avoid returning a builtin
+   function.  */
 static token_data *
-arg_token (macro_arguments *argv, unsigned int index, int *level, bool flatten)
+arg_token (macro_arguments *argv, unsigned int arg, int *level, bool flatten)
 {
   unsigned int i;
-  token_data *token;
+  token_data *token = NULL;
 
-  assert (index && index < argv->argc);
+  assert (arg && arg < argv->argc);
   if (level)
     *level = argv->level;
   flatten |= argv->flatten;
   if (!argv->wrapper)
     {
-      token = argv->array[index - 1];
+      token = argv->array[arg - 1];
       if (flatten && TOKEN_DATA_TYPE (token) == TOKEN_FUNC)
 	token = &empty_token;
       return token;
     }
 
-  /* Must cycle through all tokens, until we find index, since a ref
+  /* Must cycle through all tokens, until we find arg, since a ref
      may occupy multiple indices.  */
   for (i = 0; i < argv->arraylen; i++)
     {
@@ -846,18 +823,18 @@ arg_token (macro_arguments *argv, unsigned int index, int *level, bool flatten)
 	{
 	  token_chain *chain = token->u.u_c.chain;
 	  assert (!chain->next && chain->type == CHAIN_ARGV);
-	  if (index <= (chain->u.u_a.argv->argc - chain->u.u_a.index
-			- chain->u.u_a.skip_last))
+	  if (arg <= (chain->u.u_a.argv->argc - chain->u.u_a.index
+		      - chain->u.u_a.skip_last))
 	    {
 	      token = arg_token (chain->u.u_a.argv,
-				 chain->u.u_a.index - 1 + index, level,
+				 chain->u.u_a.index - 1 + arg, level,
 				 flatten || chain->u.u_a.flatten);
 	      break;
 	    }
-	  index -= (chain->u.u_a.argv->argc - chain->u.u_a.index
-		    - chain->u.u_a.skip_last);
+	  arg -= (chain->u.u_a.argv->argc - chain->u.u_a.index
+		  - chain->u.u_a.skip_last);
 	}
-      else if (--index == 0)
+      else if (--arg == 0)
 	break;
     }
   return token;
@@ -893,17 +870,17 @@ arg_argc (macro_arguments *argv)
   return argv->argc;
 }
 
-/* Given ARGV, return the type of argument INDEX.  Index 0 is always
-   text, and indices beyond argc are likewise treated as text.  */
+/* Given ARGV, return the type of argument ARG.  Arg 0 is always text,
+   and indices beyond argc are likewise treated as text.  */
 token_data_type
-arg_type (macro_arguments *argv, unsigned int index)
+arg_type (macro_arguments *argv, unsigned int arg)
 {
   token_data_type type;
   token_data *token;
 
-  if (argv->flatten || !argv->has_func || index == 0 || index >= argv->argc)
+  if (argv->flatten || !argv->has_func || arg == 0 || arg >= argv->argc)
     return TOKEN_TEXT;
-  token = arg_token (argv, index, NULL, false);
+  token = arg_token (argv, arg, NULL, false);
   type = TOKEN_DATA_TYPE (token);
   if (type == TOKEN_COMP && !token->u.u_c.has_func)
     type = TOKEN_TEXT;
@@ -912,22 +889,23 @@ arg_type (macro_arguments *argv, unsigned int index)
   return type;
 }
 
-/* Given ARGV, return the text at argument INDEX.  Abort if the
-   argument is not text.  Index 0 is always text, and indices beyond
-   argc return the empty string.  The result is always NUL-terminated,
-   even if it includes embedded NUL characters.  */
+/* Given ARGV, return the text at argument ARG.  Abort if the argument
+   is not text.  Arg 0 is always text, and indices beyond argc return
+   the empty string.  If FLATTEN, builtins are ignored.  The result is
+   always NUL-terminated, even if it includes embedded NUL
+   characters.  */
 const char *
-arg_text (macro_arguments *argv, unsigned int index)
+arg_text (macro_arguments *argv, unsigned int arg, bool flatten)
 {
   token_data *token;
   token_chain *chain;
   struct obstack *obs; /* Scratch space; cleaned at end of macro_expand.  */
 
-  if (index == 0)
+  if (arg == 0)
     return argv->argv0;
-  if (index >= argv->argc)
+  if (arg >= argv->argc)
     return "";
-  token = arg_token (argv, index, NULL, false);
+  token = arg_token (argv, arg, NULL, flatten);
   switch (TOKEN_DATA_TYPE (token))
     {
     case TOKEN_TEXT:
@@ -942,13 +920,18 @@ arg_text (macro_arguments *argv, unsigned int index)
 	    case CHAIN_STR:
 	      obstack_grow (obs, chain->u.u_s.str, chain->u.u_s.len);
 	      break;
+	    case CHAIN_FUNC:
+	      if (flatten)
+		break;
+	      assert (!"arg_text");
+	      abort ();
 	    case CHAIN_ARGV:
-	      assert (!chain->u.u_a.has_func || argv->flatten);
+	      assert (!chain->u.u_a.has_func || flatten || argv->flatten);
 	      arg_print (obs, chain->u.u_a.argv, chain->u.u_a.index,
 			 quote_cache (NULL, chain->quote_age,
 				      chain->u.u_a.quotes),
-			 argv->flatten || chain->u.u_a.flatten, NULL, NULL,
-			 NULL, false);
+			 flatten || argv->flatten || chain->u.u_a.flatten,
+			 NULL, NULL, NULL, false);
 	      break;
 	    default:
 	      assert (!"arg_text");
@@ -969,7 +952,7 @@ arg_text (macro_arguments *argv, unsigned int index)
 /* Given ARGV, compare text arguments INDEXA and INDEXB for equality.
    Both indices must be non-zero and less than argc.  Return true if
    the arguments contain the same contents; often more efficient than
-   strcmp (arg_text (argv, indexa), arg_text (argv, indexb)) == 0.  */
+   strcmp (arg_text (argv, a, 1), arg_text (argv, b, 1)) == 0.  */
 bool
 arg_equal (macro_arguments *argv, unsigned int indexa, unsigned int indexb)
 {
@@ -1115,33 +1098,33 @@ arg_equal (macro_arguments *argv, unsigned int indexa, unsigned int indexb)
   return ca == cb;
 }
 
-/* Given ARGV, return true if argument INDEX is the empty string.
-   This gives the same result as comparing arg_len against 0, but is
-   often faster.  */
+/* Given ARGV, return true if argument ARG is the empty string.  This
+   gives the same result as comparing arg_len against 0, but is often
+   faster.  */
 bool
-arg_empty (macro_arguments *argv, unsigned int index)
+arg_empty (macro_arguments *argv, unsigned int arg)
 {
-  if (index == 0)
+  if (arg == 0)
     return argv->argv0_len == 0;
-  if (index >= argv->argc)
+  if (arg >= argv->argc)
     return true;
-  return arg_token (argv, index, NULL, false) == &empty_token;
+  return arg_token (argv, arg, NULL, false) == &empty_token;
 }
 
-/* Given ARGV, return the length of argument INDEX.  Abort if the
+/* Given ARGV, return the length of argument ARG.  Abort if the
    argument is not text.  Indices beyond argc return 0.  */
 size_t
-arg_len (macro_arguments *argv, unsigned int index)
+arg_len (macro_arguments *argv, unsigned int arg)
 {
   token_data *token;
   token_chain *chain;
   size_t len;
 
-  if (index == 0)
+  if (arg == 0)
     return argv->argv0_len;
-  if (index >= argv->argc)
+  if (arg >= argv->argc)
     return 0;
-  token = arg_token (argv, index, NULL, false);
+  token = arg_token (argv, arg, NULL, false);
   switch (TOKEN_DATA_TYPE (token))
     {
     case TOKEN_TEXT:
@@ -1200,14 +1183,14 @@ arg_len (macro_arguments *argv, unsigned int index)
   abort ();
 }
 
-/* Given ARGV, return the builtin function referenced by argument
-   INDEX.  Abort if it is not a builtin in isolation.  */
+/* Given ARGV, return the builtin function referenced by argument ARG.
+   Abort if it is not a builtin in isolation.  */
 builtin_func *
-arg_func (macro_arguments *argv, unsigned int index)
+arg_func (macro_arguments *argv, unsigned int arg)
 {
   token_data *token;
 
-  token = arg_token (argv, index, NULL, false);
+  token = arg_token (argv, arg, NULL, false);
   assert (TOKEN_DATA_TYPE (token) == TOKEN_FUNC);
   return TOKEN_DATA_FUNC (token);
 }
@@ -1223,7 +1206,7 @@ arg_scratch (void)
 }
 
 /* Dump a representation of ARGV to the obstack OBS, starting with
-   argument INDEX.  If QUOTES is non-NULL, each argument is displayed
+   argument ARG.  If QUOTES is non-NULL, each argument is displayed
    with those quotes.  If FLATTEN, builtins are converted to empty
    quotes; if CHAINP, *CHAINP is updated with macro tokens; otherwise,
    builtins are represented by their name.  Separate arguments with
@@ -1236,9 +1219,9 @@ arg_scratch (void)
    arguments may be discarded.  MAX_LEN and CHAINP may not both be
    specified.  */
 bool
-arg_print (struct obstack *obs, macro_arguments *argv, unsigned int index,
+arg_print (struct obstack *obs, macro_arguments *argv, unsigned int arg,
 	   const string_pair *quotes, bool flatten, token_chain **chainp,
-	   const char *sep, int *max_len, bool quote_each)
+	   const char *sep, size_t *max_len, bool quote_each)
 {
   int len = max_len ? *max_len : INT_MAX;
   unsigned int i;
@@ -1249,32 +1232,37 @@ arg_print (struct obstack *obs, macro_arguments *argv, unsigned int index,
   size_t sep_len;
   size_t *plen = quote_each ? NULL : &len;
 
+  flatten |= argv->flatten;
   if (chainp)
     assert (!max_len && *chainp);
   if (!sep)
     sep = ",";
   sep_len = strlen (sep);
-  for (i = index; i < argv->argc; i++)
+  for (i = arg; i < argv->argc; i++)
     {
       if (quote_each && max_len)
 	len = *max_len;
-      if (use_sep && obstack_print (obs, sep, sep_len, plen))
+      if (use_sep && shipout_string_trunc (obs, sep, sep_len, plen))
 	return true;
       use_sep = true;
       token = arg_token (argv, i, NULL, flatten);
       switch (TOKEN_DATA_TYPE (token))
 	{
 	case TOKEN_TEXT:
-	  if (quotes && obstack_print (obs, quotes->str1, quotes->len1, plen))
+	  if (quotes && shipout_string_trunc (obs, quotes->str1, quotes->len1,
+					      plen))
 	    return true;
-	  if (obstack_print (obs, TOKEN_DATA_TEXT (token),
-			     TOKEN_DATA_LEN (token), &len) && !quote_each)
+	  if (shipout_string_trunc (obs, TOKEN_DATA_TEXT (token),
+				    TOKEN_DATA_LEN (token), &len)
+	      && !quote_each)
 	    return true;
-	  if (quotes && obstack_print (obs, quotes->str2, quotes->len2, plen))
+	  if (quotes && shipout_string_trunc (obs, quotes->str2, quotes->len2,
+					      plen))
 	    return true;
 	  break;
 	case TOKEN_COMP:
-	  if (quotes && obstack_print (obs, quotes->str1, quotes->len1, plen))
+	  if (quotes && shipout_string_trunc (obs, quotes->str1, quotes->len1,
+					      plen))
 	    return true;
 	  chain = token->u.u_c.chain;
 	  done = false;
@@ -1283,8 +1271,8 @@ arg_print (struct obstack *obs, macro_arguments *argv, unsigned int index,
 	      switch (chain->type)
 		{
 		case CHAIN_STR:
-		  if (obstack_print (obs, chain->u.u_s.str, chain->u.u_s.len,
-				     &len))
+		  if (shipout_string_trunc (obs, chain->u.u_s.str,
+					    chain->u.u_s.len, &len))
 		    done = true;
 		  break;
 		case CHAIN_FUNC:
@@ -1306,7 +1294,8 @@ arg_print (struct obstack *obs, macro_arguments *argv, unsigned int index,
 	    }
 	  if (done && !quote_each)
 	    return true;
-	  if (quotes && obstack_print (obs, quotes->str2, quotes->len2, plen))
+	  if (quotes && shipout_string_trunc (obs, quotes->str2, quotes->len2,
+					      plen))
 	    return true;
 	  break;
 	case TOKEN_FUNC:
@@ -1326,18 +1315,18 @@ arg_print (struct obstack *obs, macro_arguments *argv, unsigned int index,
 }
 
 /* Populate the new TOKEN as a wrapper to ARGV, starting with argument
-   INDEX.  Allocate any data on OBS, owned by a given expansion LEVEL.
+   ARG.  Allocate any data on OBS, owned by a given expansion LEVEL.
    FLATTEN determines whether to allow builtins, and QUOTES determines
    whether all arguments are quoted.  Return TOKEN when successful,
    NULL when wrapping ARGV is trivially empty.  */
 static token_data *
 make_argv_ref_token (token_data *token, struct obstack *obs, int level,
-		     macro_arguments *argv, unsigned int index, bool flatten,
+		     macro_arguments *argv, unsigned int arg, bool flatten,
 		     const string_pair *quotes)
 {
   token_chain *chain;
 
-  if (index >= argv->argc)
+  if (arg >= argv->argc)
     return NULL;
   TOKEN_DATA_TYPE (token) = TOKEN_COMP;
   token->u.u_c.chain = token->u.u_c.end = NULL;
@@ -1354,13 +1343,13 @@ make_argv_ref_token (token_data *token, struct obstack *obs, int level,
 	       && argv->array[i]->u.u_c.wrapper)
 	      || level >= 0)
 	    break;
-	  if (index == 1)
+	  if (arg == 1)
 	    {
 	      push_arg_quote (obs, argv, i + 1, quotes);
 	      obstack_1grow (obs, ',');
 	    }
 	  else
-	    index--;
+	    arg--;
 	}
       assert (i < argv->arraylen);
       if (i + 1 == argv->arraylen)
@@ -1371,11 +1360,11 @@ make_argv_ref_token (token_data *token, struct obstack *obs, int level,
 	  assert (!chain->next && chain->type == CHAIN_ARGV
 		  && !chain->u.u_a.skip_last);
 	  argv = chain->u.u_a.argv;
-	  index += chain->u.u_a.index - 1;
+	  arg += chain->u.u_a.index - 1;
 	}
       else
 	{
-	  index += i;
+	  arg += i;
 	  break;
 	}
     }
@@ -1393,7 +1382,7 @@ make_argv_ref_token (token_data *token, struct obstack *obs, int level,
   chain->type = CHAIN_ARGV;
   chain->quote_age = argv->quote_age;
   chain->u.u_a.argv = argv;
-  chain->u.u_a.index = index;
+  chain->u.u_a.index = arg;
   chain->u.u_a.flatten = flatten;
   chain->u.u_a.has_func = argv->has_func;
   chain->u.u_a.comma = false;
@@ -1416,12 +1405,12 @@ make_argv_ref (macro_arguments *argv, const char *argv0, size_t argv0_len,
   macro_arguments *new_argv;
   token_data *token;
   token_data *new_token;
-  unsigned int index = skip ? 2 : 1;
+  unsigned int i = skip ? 2 : 1;
   struct obstack *obs = arg_scratch ();
 
   new_token = (token_data *) obstack_alloc (obs, sizeof *token);
-  token = make_argv_ref_token (new_token, obs, expansion_level - 1, argv,
-			       index, flatten, NULL);
+  token = make_argv_ref_token (new_token, obs, expansion_level - 1, argv, i,
+			       flatten, NULL);
   if (!token)
     {
       obstack_free (obs, new_token);
@@ -1444,7 +1433,7 @@ make_argv_ref (macro_arguments *argv, const char *argv0, size_t argv0_len,
       new_argv->flatten = flatten;
       new_argv->has_func = argv->has_func;
     }
-  new_argv->argc = argv->argc - (index - 1);
+  new_argv->argc = argv->argc - (i - 1);
   new_argv->inuse = false;
   new_argv->argv0 = argv0;
   new_argv->argv0_len = argv0_len;
@@ -1453,33 +1442,32 @@ make_argv_ref (macro_arguments *argv, const char *argv0, size_t argv0_len,
   return new_argv;
 }
 
-/* Push argument INDEX from ARGV onto the expansion stack OBS for
+/* Push argument ARG from ARGV onto the expansion stack OBS for
    rescanning.  */
 void
-push_arg (struct obstack *obs, macro_arguments *argv, unsigned int index)
+push_arg (struct obstack *obs, macro_arguments *argv, unsigned int arg)
 {
-  if (index == 0)
+  if (arg == 0)
     {
       /* Always push copy of arg 0, since its lifetime is not
 	 guaranteed beyond expand_macro.  */
       obstack_grow (obs, argv->argv0, argv->argv0_len);
       return;
     }
-  if (index >= argv->argc)
+  if (arg >= argv->argc)
     return;
-  push_arg_quote (obs, argv, index, NULL);
+  push_arg_quote (obs, argv, arg, NULL);
 }
 
-/* Push argument INDEX from ARGV onto the expansion stack OBS for
-   rescanning.  INDEX must be > 0, < argc.  QUOTES determines any
-   quote delimiters that were in effect when the reference was
-   created.  */
+/* Push argument ARG from ARGV onto the expansion stack OBS for
+   rescanning.  ARG must be > 0, < argc.  QUOTES determines any quote
+   delimiters that were in effect when the reference was created.  */
 void
-push_arg_quote (struct obstack *obs, macro_arguments *argv, unsigned int index,
+push_arg_quote (struct obstack *obs, macro_arguments *argv, unsigned int arg,
 		const string_pair *quotes)
 {
   int level;
-  token_data *token = arg_token (argv, index, &level, false);
+  token_data *token = arg_token (argv, arg, &level, false);
 
   if (quotes)
     obstack_grow (obs, quotes->str1, quotes->len1);
diff --git a/src/output.c b/src/output.c
index d252d74e..ee1907b2 100644
--- a/src/output.c
+++ b/src/output.c
@@ -121,10 +121,10 @@ cmp_diversion_CB (const void *elt1, const void *elt2)
 static bool
 threshold_diversion_CB (const void *elt, const void *threshold)
 {
-  const m4_diversion *div = (const m4_diversion *) elt;
+  const m4_diversion *diversion = (const m4_diversion *) elt;
   /* No need to worry about overflow, since we don't create diversions
      with negative divnum.  */
-  return div->divnum >= *(const int *) threshold;
+  return diversion->divnum >= *(const int *) threshold;
 }
 
 void
@@ -198,7 +198,8 @@ m4_tmpname (int divnum)
       obstack_1grow (&diversion_storage, '4');
       obstack_1grow (&diversion_storage, '-');
       offset = obstack_object_size (&diversion_storage);
-      buffer = obstack_alloc (&diversion_storage, INT_BUFSIZE_BOUND (divnum));
+      buffer = (char *) obstack_alloc (&diversion_storage,
+				       INT_BUFSIZE_BOUND (divnum));
     }
   if (snprintf (&buffer[offset], INT_BUFSIZE_BOUND (divnum), "%d", divnum) < 0)
     m4_error (EXIT_FAILURE, errno, NULL,
@@ -385,8 +386,8 @@ make_room_for (int length)
 	}
 
       /* The current buffer may be safely reallocated.  */
-      output_diversion->u.buffer
-	= xrealloc (output_diversion->u.buffer, (size_t) wanted_size);
+      output_diversion->u.buffer = xrealloc (output_diversion->u.buffer,
+					     (size_t) wanted_size);
 
       total_buffer_size += wanted_size - output_diversion->size;
       output_diversion->size = wanted_size;
@@ -469,7 +470,7 @@ output_text (const char *text, int length)
 `--------------------------------------------------------------------*/
 
 void
-shipout_text (struct obstack *obs, const char *text, int length, int line)
+divert_text (struct obstack *obs, const char *text, int length, int line)
 {
   static bool start_of_output_line = true;
   const char *cursor;
@@ -571,6 +572,35 @@ shipout_text (struct obstack *obs, const char *text, int length, int line)
 	}
     }
 }
+
+/* Dump the string STR of length LEN to the obstack OBS.  If LEN is
+   SIZE_MAX, use strlen (STR) instead.  If MAX_LEN is non-NULL,
+   truncate the dump at MAX_LEN bytes and return true if MAX_LEN was
+   reached; otherwise, return false and update MAX_LEN as
+   appropriate.  */
+bool
+shipout_string_trunc (struct obstack *obs, const char *str, size_t len,
+		      size_t *max_len)
+{
+  size_t max = max_len ? *max_len : INT_MAX;
+
+  if (len == SIZE_MAX)
+    len = strlen (str);
+  if (len < max)
+    {
+      obstack_grow (obs, str, len);
+      max -= len;
+    }
+  else
+    {
+      obstack_grow (obs, str, max);
+      obstack_grow (obs, "...", 3);
+      max = 0;
+    }
+  if (max_len)
+    *max_len = max;
+  return max == 0;
+}
 
 /* Functions for use by diversions.  */