move easy lexer state variables into PL_parser

(where "easy" == "only appear in toke.c")

p4raw-id: //depot/perl@29655

11 files changed, 164 insertions, 272 deletions

diff --git a/embed.fnc b/embed.fnc
index 36d57e647a..fecb528618 100644
--- a/embed.fnc
+++ b/embed.fnc
@@ -957,6 +957,7 @@ p	|int	|yyerror	|NN const char* s
 p	|int	|yylex
 p	|int	|yyparse
 p	|void	|parser_free	|NN const yy_parser *
+p	|yy_parser*|parser_dup	|NN const yy_parser *proto|NN CLONE_PARAMS* param
 p	|int	|yywarn		|NN const char* s
 #if defined(MYMALLOC)
 Ap	|void	|dump_mstats	|NN char* s
diff --git a/embed.h b/embed.h
index 09fbb8fbf3..4dc5616972 100644
--- a/embed.h
+++ b/embed.h
@@ -977,6 +977,7 @@
 #define yylex			Perl_yylex
 #define yyparse			Perl_yyparse
 #define parser_free		Perl_parser_free
+#define parser_dup		Perl_parser_dup
 #define yywarn			Perl_yywarn
 #endif
 #if defined(MYMALLOC)
@@ -3185,6 +3186,7 @@
 #define yylex()			Perl_yylex(aTHX)
 #define yyparse()		Perl_yyparse(aTHX)
 #define parser_free(a)		Perl_parser_free(aTHX_ a)
+#define parser_dup(a,b)		Perl_parser_dup(aTHX_ a,b)
 #define yywarn(a)		Perl_yywarn(aTHX_ a)
 #endif
 #if defined(MYMALLOC)
diff --git a/embedvar.h b/embedvar.h
index 7fe5e295a3..566c2ff196 100644
--- a/embedvar.h
+++ b/embedvar.h
@@ -195,7 +195,6 @@
 #define PL_egid			(vTHX->Iegid)
 #define PL_encoding		(vTHX->Iencoding)
 #define PL_endav		(vTHX->Iendav)
-#define PL_endwhite		(vTHX->Iendwhite)
 #define PL_envgv		(vTHX->Ienvgv)
 #define PL_errgv		(vTHX->Ierrgv)
 #define PL_error_count		(vTHX->Ierror_count)
@@ -207,7 +206,6 @@
 #define PL_exitlist		(vTHX->Iexitlist)
 #define PL_exitlistlen		(vTHX->Iexitlistlen)
 #define PL_expect		(vTHX->Iexpect)
-#define PL_faketokens		(vTHX->Ifaketokens)
 #define PL_fdpid		(vTHX->Ifdpid)
 #define PL_filemode		(vTHX->Ifilemode)
 #define PL_forkprocess		(vTHX->Iforkprocess)
@@ -241,22 +239,7 @@
 #define PL_lastfd		(vTHX->Ilastfd)
 #define PL_laststatval		(vTHX->Ilaststatval)
 #define PL_laststype		(vTHX->Ilaststype)
-#define PL_lasttoke		(vTHX->Ilasttoke)
-#define PL_lex_brackets		(vTHX->Ilex_brackets)
-#define PL_lex_brackstack	(vTHX->Ilex_brackstack)
-#define PL_lex_casemods		(vTHX->Ilex_casemods)
-#define PL_lex_casestack	(vTHX->Ilex_casestack)
-#define PL_lex_defer		(vTHX->Ilex_defer)
-#define PL_lex_dojoin		(vTHX->Ilex_dojoin)
-#define PL_lex_expect		(vTHX->Ilex_expect)
-#define PL_lex_formbrack	(vTHX->Ilex_formbrack)
-#define PL_lex_inpat		(vTHX->Ilex_inpat)
-#define PL_lex_inwhat		(vTHX->Ilex_inwhat)
-#define PL_lex_op		(vTHX->Ilex_op)
-#define PL_lex_repl		(vTHX->Ilex_repl)
-#define PL_lex_starts		(vTHX->Ilex_starts)
 #define PL_lex_state		(vTHX->Ilex_state)
-#define PL_lex_stuff		(vTHX->Ilex_stuff)
 #define PL_lineary		(vTHX->Ilineary)
 #define PL_linestart		(vTHX->Ilinestart)
 #define PL_linestr		(vTHX->Ilinestr)
@@ -280,17 +263,13 @@
 #define PL_minus_n		(vTHX->Iminus_n)
 #define PL_minus_p		(vTHX->Iminus_p)
 #define PL_modglobal		(vTHX->Imodglobal)
-#define PL_multi_close		(vTHX->Imulti_close)
 #define PL_multi_end		(vTHX->Imulti_end)
-#define PL_multi_open		(vTHX->Imulti_open)
-#define PL_multi_start		(vTHX->Imulti_start)
 #define PL_my_cxt_keys		(vTHX->Imy_cxt_keys)
 #define PL_my_cxt_list		(vTHX->Imy_cxt_list)
 #define PL_my_cxt_size		(vTHX->Imy_cxt_size)
 #define PL_nexttoke		(vTHX->Inexttoke)
 #define PL_nexttype		(vTHX->Inexttype)
 #define PL_nextval		(vTHX->Inextval)
-#define PL_nextwhite		(vTHX->Inextwhite)
 #define PL_nice_chunk		(vTHX->Inice_chunk)
 #define PL_nice_chunk_size	(vTHX->Inice_chunk_size)
 #define PL_nomemok		(vTHX->Inomemok)
@@ -315,21 +294,18 @@
 #define PL_padix_floor		(vTHX->Ipadix_floor)
 #define PL_parser		(vTHX->Iparser)
 #define PL_patchlevel		(vTHX->Ipatchlevel)
-#define PL_pending_ident	(vTHX->Ipending_ident)
 #define PL_perl_destruct_level	(vTHX->Iperl_destruct_level)
 #define PL_perldb		(vTHX->Iperldb)
 #define PL_perlio		(vTHX->Iperlio)
 #define PL_pidstatus		(vTHX->Ipidstatus)
 #define PL_ppid			(vTHX->Ippid)
 #define PL_preambleav		(vTHX->Ipreambleav)
-#define PL_preambled		(vTHX->Ipreambled)
 #define PL_preprocess		(vTHX->Ipreprocess)
 #define PL_profiledata		(vTHX->Iprofiledata)
 #define PL_psig_name		(vTHX->Ipsig_name)
 #define PL_psig_pend		(vTHX->Ipsig_pend)
 #define PL_psig_ptr		(vTHX->Ipsig_ptr)
 #define PL_ptr_table		(vTHX->Iptr_table)
-#define PL_realtokenstart	(vTHX->Irealtokenstart)
 #define PL_reentrant_buffer	(vTHX->Ireentrant_buffer)
 #define PL_reentrant_retint	(vTHX->Ireentrant_retint)
 #define PL_regex_pad		(vTHX->Iregex_pad)
@@ -348,7 +324,6 @@
 #define PL_sig_pending		(vTHX->Isig_pending)
 #define PL_sighandlerp		(vTHX->Isighandlerp)
 #define PL_signals		(vTHX->Isignals)
-#define PL_skipwhite		(vTHX->Iskipwhite)
 #define PL_sort_RealCmp		(vTHX->Isort_RealCmp)
 #define PL_splitstr		(vTHX->Isplitstr)
 #define PL_srand_called		(vTHX->Isrand_called)
@@ -360,7 +335,6 @@
 #define PL_stdingv		(vTHX->Istdingv)
 #define PL_strtab		(vTHX->Istrtab)
 #define PL_sub_generation	(vTHX->Isub_generation)
-#define PL_sublex_info		(vTHX->Isublex_info)
 #define PL_subline		(vTHX->Isubline)
 #define PL_subname		(vTHX->Isubname)
 #define PL_sv_arenaroot		(vTHX->Isv_arenaroot)
@@ -373,12 +347,6 @@
 #define PL_sys_intern		(vTHX->Isys_intern)
 #define PL_taint_warn		(vTHX->Itaint_warn)
 #define PL_tainting		(vTHX->Itainting)
-#define PL_thisclose		(vTHX->Ithisclose)
-#define PL_thismad		(vTHX->Ithismad)
-#define PL_thisopen		(vTHX->Ithisopen)
-#define PL_thisstuff		(vTHX->Ithisstuff)
-#define PL_thistoken		(vTHX->Ithistoken)
-#define PL_thiswhite		(vTHX->Ithiswhite)
 #define PL_threadhook		(vTHX->Ithreadhook)
 #define PL_tokenbuf		(vTHX->Itokenbuf)
 #define PL_uid			(vTHX->Iuid)
@@ -493,7 +461,6 @@
 #define PL_Iegid		PL_egid
 #define PL_Iencoding		PL_encoding
 #define PL_Iendav		PL_endav
-#define PL_Iendwhite		PL_endwhite
 #define PL_Ienvgv		PL_envgv
 #define PL_Ierrgv		PL_errgv
 #define PL_Ierror_count		PL_error_count
@@ -505,7 +472,6 @@
 #define PL_Iexitlist		PL_exitlist
 #define PL_Iexitlistlen		PL_exitlistlen
 #define PL_Iexpect		PL_expect
-#define PL_Ifaketokens		PL_faketokens
 #define PL_Ifdpid		PL_fdpid
 #define PL_Ifilemode		PL_filemode
 #define PL_Iforkprocess		PL_forkprocess
@@ -539,22 +505,7 @@
 #define PL_Ilastfd		PL_lastfd
 #define PL_Ilaststatval		PL_laststatval
 #define PL_Ilaststype		PL_laststype
-#define PL_Ilasttoke		PL_lasttoke
-#define PL_Ilex_brackets	PL_lex_brackets
-#define PL_Ilex_brackstack	PL_lex_brackstack
-#define PL_Ilex_casemods	PL_lex_casemods
-#define PL_Ilex_casestack	PL_lex_casestack
-#define PL_Ilex_defer		PL_lex_defer
-#define PL_Ilex_dojoin		PL_lex_dojoin
-#define PL_Ilex_expect		PL_lex_expect
-#define PL_Ilex_formbrack	PL_lex_formbrack
-#define PL_Ilex_inpat		PL_lex_inpat
-#define PL_Ilex_inwhat		PL_lex_inwhat
-#define PL_Ilex_op		PL_lex_op
-#define PL_Ilex_repl		PL_lex_repl
-#define PL_Ilex_starts		PL_lex_starts
 #define PL_Ilex_state		PL_lex_state
-#define PL_Ilex_stuff		PL_lex_stuff
 #define PL_Ilineary		PL_lineary
 #define PL_Ilinestart		PL_linestart
 #define PL_Ilinestr		PL_linestr
@@ -578,17 +529,13 @@
 #define PL_Iminus_n		PL_minus_n
 #define PL_Iminus_p		PL_minus_p
 #define PL_Imodglobal		PL_modglobal
-#define PL_Imulti_close		PL_multi_close
 #define PL_Imulti_end		PL_multi_end
-#define PL_Imulti_open		PL_multi_open
-#define PL_Imulti_start		PL_multi_start
 #define PL_Imy_cxt_keys		PL_my_cxt_keys
 #define PL_Imy_cxt_list		PL_my_cxt_list
 #define PL_Imy_cxt_size		PL_my_cxt_size
 #define PL_Inexttoke		PL_nexttoke
 #define PL_Inexttype		PL_nexttype
 #define PL_Inextval		PL_nextval
-#define PL_Inextwhite		PL_nextwhite
 #define PL_Inice_chunk		PL_nice_chunk
 #define PL_Inice_chunk_size	PL_nice_chunk_size
 #define PL_Inomemok		PL_nomemok
@@ -613,21 +560,18 @@
 #define PL_Ipadix_floor		PL_padix_floor
 #define PL_Iparser		PL_parser
 #define PL_Ipatchlevel		PL_patchlevel
-#define PL_Ipending_ident	PL_pending_ident
 #define PL_Iperl_destruct_level	PL_perl_destruct_level
 #define PL_Iperldb		PL_perldb
 #define PL_Iperlio		PL_perlio
 #define PL_Ipidstatus		PL_pidstatus
 #define PL_Ippid		PL_ppid
 #define PL_Ipreambleav		PL_preambleav
-#define PL_Ipreambled		PL_preambled
 #define PL_Ipreprocess		PL_preprocess
 #define PL_Iprofiledata		PL_profiledata
 #define PL_Ipsig_name		PL_psig_name
 #define PL_Ipsig_pend		PL_psig_pend
 #define PL_Ipsig_ptr		PL_psig_ptr
 #define PL_Iptr_table		PL_ptr_table
-#define PL_Irealtokenstart	PL_realtokenstart
 #define PL_Ireentrant_buffer	PL_reentrant_buffer
 #define PL_Ireentrant_retint	PL_reentrant_retint
 #define PL_Iregex_pad		PL_regex_pad
@@ -646,7 +590,6 @@
 #define PL_Isig_pending		PL_sig_pending
 #define PL_Isighandlerp		PL_sighandlerp
 #define PL_Isignals		PL_signals
-#define PL_Iskipwhite		PL_skipwhite
 #define PL_Isort_RealCmp	PL_sort_RealCmp
 #define PL_Isplitstr		PL_splitstr
 #define PL_Isrand_called	PL_srand_called
@@ -658,7 +601,6 @@
 #define PL_Istdingv		PL_stdingv
 #define PL_Istrtab		PL_strtab
 #define PL_Isub_generation	PL_sub_generation
-#define PL_Isublex_info		PL_sublex_info
 #define PL_Isubline		PL_subline
 #define PL_Isubname		PL_subname
 #define PL_Isv_arenaroot	PL_sv_arenaroot
@@ -671,12 +613,6 @@
 #define PL_Isys_intern		PL_sys_intern
 #define PL_Itaint_warn		PL_taint_warn
 #define PL_Itainting		PL_tainting
-#define PL_Ithisclose		PL_thisclose
-#define PL_Ithismad		PL_thismad
-#define PL_Ithisopen		PL_thisopen
-#define PL_Ithisstuff		PL_thisstuff
-#define PL_Ithistoken		PL_thistoken
-#define PL_Ithiswhite		PL_thiswhite
 #define PL_Ithreadhook		PL_threadhook
 #define PL_Itokenbuf		PL_tokenbuf
 #define PL_Iuid			PL_uid
diff --git a/intrpvar.h b/intrpvar.h
index 660b716e54..a9e0f46690 100644
--- a/intrpvar.h
+++ b/intrpvar.h
@@ -173,7 +173,6 @@ PERLVAR(Ioldname,	char *)		/* what to preserve mode on */
 PERLVAR(IArgv,		char **)	/* stuff to free from do_aexec, vfork safe */
 PERLVAR(ICmd,		char *)		/* stuff to free from do_aexec, vfork safe */
 PERLVARI(Igensym,	I32,	0)	/* next symbol for getsym() to define */
-PERLVAR(Ipreambled,	bool)
 PERLVAR(Ipreambleav,	AV *)
 PERLVARI(Ilaststatval,	int,	-1)
 PERLVARI(Ilaststype,	I32,	OP_STAT)
@@ -227,8 +226,6 @@ PERLVARI(Iin_clean_objs,bool,    FALSE)	/* from sv.c */
 PERLVARI(Iin_clean_all,	bool,    FALSE)	/* from sv.c */
 
 PERLVAR(Ilinestart,	char *)		/* beg. of most recently read line */
-PERLVAR(Ipending_ident,	char)		/* pending identifier lookup */
-PERLVAR(Isublex_info,	SUBLEXINFO)	/* from toke.c */
 
 PERLVAR(Iuid,		Uid_t)		/* current real user id */
 PERLVAR(Ieuid,		Uid_t)		/* current effective user id */
@@ -282,36 +279,10 @@ PERLVARI(Icshlen,	I32,	0)
 #endif
 
 PERLVAR(Ilex_state,	U32)		/* next token is determined */
-PERLVAR(Ilex_defer,	U32)		/* state after determined token */
-PERLVAR(Ilex_expect,	int)		/* expect after determined token */
-PERLVAR(Ilex_brackets,	I32)		/* bracket count */
-PERLVAR(Ilex_formbrack,	I32)		/* bracket count at outer format level */
-PERLVAR(Ilex_casemods,	I32)		/* casemod count */
-PERLVAR(Ilex_dojoin,	I32)		/* doing an array interpolation */
-PERLVAR(Ilex_starts,	I32)		/* how many interps done on level */
-PERLVAR(Ilex_stuff,	SV *)		/* runtime pattern from m// or s/// */
-PERLVAR(Ilex_repl,	SV *)		/* runtime replacement from s/// */
-PERLVAR(Ilex_op,	OP *)		/* extra info to pass back on op */
-PERLVAR(Ilex_inpat,	OP *)		/* in pattern $) and $| are special */
-PERLVAR(Ilex_inwhat,	I32)		/* what kind of quoting are we in */
-PERLVAR(Ilex_brackstack,char *)		/* what kind of brackets to pop */
-PERLVAR(Ilex_casestack,	char *)		/* what kind of case mods in effect */
 
 /* What we know when we're in LEX_KNOWNEXT state. */
 #ifdef PERL_MAD
 PERLVARA(Inexttoke,5,	NEXTTOKE)	/* value of next token, if any */
-PERLVAR(Ilasttoke,	I32)
-PERLVAR(Irealtokenstart,I32)
-PERLVAR(Ifaketokens,	I32)
-PERLVAR(Ithismad,	MADPROP *)
-PERLVAR(Ithistoken,	SV *)
-PERLVAR(Ithisopen,	SV *)
-PERLVAR(Ithisstuff,	SV *)
-PERLVAR(Ithisclose,	SV *)
-PERLVAR(Ithiswhite,	SV *)
-PERLVAR(Inextwhite,	SV *)
-PERLVAR(Iskipwhite,	SV *)
-PERLVAR(Iendwhite,	SV *)
 PERLVAR(Icurforce,	I32)
 #else
 PERLVARA(Inextval,5,	YYSTYPE)	/* value of next token, if any */
@@ -326,10 +297,7 @@ PERLVAR(Ioldoldbufptr,	char *)
 PERLVAR(Ibufend,	char *)
 PERLVARI(Iexpect,int,	XSTATE)		/* how to interpret ambiguous tokens */
 
-PERLVAR(Imulti_start,	I32)		/* 1st line of multi-line string */
 PERLVAR(Imulti_end,	I32)		/* last line of multi-line string */
-PERLVAR(Imulti_open,	I32)		/* delimiter of said string */
-PERLVAR(Imulti_close,	I32)		/* delimiter of said string */
 
 PERLVAR(Ierror_count,	I32)		/* how many errors so far, max 10 */
 PERLVAR(Isubline,	I32)		/* line this subroutine began on */
diff --git a/parser.h b/parser.h
index 89d1c6cdaf..4bb225412f 100644
--- a/parser.h
+++ b/parser.h
@@ -21,6 +21,9 @@ typedef struct {
 } yy_stack_frame;
 
 typedef struct yy_parser {
+
+    /* parser state */
+
     struct yy_parser *old_parser; /* previous value of PL_parser */
     int		    yychar;	/* The lookahead symbol.  */
     YYSTYPE	    yylval;	/* value of lookahead symbol, set by yylex() */
@@ -32,6 +35,44 @@ typedef struct yy_parser {
     int		    yylen;	/* length of active reduction */
     yy_stack_frame  *stack;	/* base of stack */
     yy_stack_frame  *ps;	/* current stack frame */
+
+    /* lexer state */
+
+    I32		lex_brackets;	/* bracket count */
+    I32		lex_casemods;	/* casemod count */
+    char	*lex_brackstack;/* what kind of brackets to pop */
+    char	*lex_casestack;	/* what kind of case mods in effect */
+    U32		lex_defer;	/* state after determined token */
+    I32		lex_dojoin;	/* doing an array interpolation */
+    int		lex_expect;	/* expect after determined token */
+    I32		lex_formbrack;	/* bracket count at outer format level */
+    OP		*lex_inpat;	/* in pattern $) and $| are special */
+    I32		lex_inwhat;	/* what kind of quoting are we in */
+    OP		*lex_op;	/* extra info to pass back on op */
+    SV		*lex_repl;	/* runtime replacement from s/// */
+    I32		lex_starts;	/* how many interps done on level */
+    SV		*lex_stuff;	/* runtime pattern from m// or s/// */
+    I32		multi_start;	/* 1st line of multi-line string */
+    I32		multi_open;	/* delimiter of said string */
+    I32		multi_close;	/* delimiter of said string */
+    char	pending_ident;	/* pending identifier lookup */
+    bool	preambled;
+    SUBLEXINFO	sublex_info;
+
+#ifdef PERL_MAD
+    SV		*endwhite;
+    I32		faketokens;
+    I32		lasttoke;
+    SV		*nextwhite;
+    I32		realtokenstart;
+    SV		*skipwhite;
+    SV		*thisclose;
+    MADPROP *	thismad;
+    SV		*thisopen;
+    SV		*thisstuff;
+    SV		*thistoken;
+    SV		*thiswhite;
+#endif
 } yy_parser;
     
 
diff --git a/perl.h b/perl.h
index 1b9f6ce2f4..9d1c1b188c 100644
--- a/perl.h
+++ b/perl.h
@@ -3242,7 +3242,6 @@ typedef        struct crypt_data {     /* straight from /usr/include/crypt.h */
 #  define YYTOKENTYPE
 #endif
 #include "perly.h"
-#include "parser.h"
 
 #ifdef PERL_MAD
 struct nexttoken {
@@ -3279,6 +3278,8 @@ struct _sublex_info {
     char *super_bufend;	/* PL_bufend that was */
 };
 
+#include "parser.h"
+
 typedef struct magic_state MGS;	/* struct magic_state defined in mg.c */
 
 struct scan_data_t;		/* Used in S_* functions in regcomp.c */
diff --git a/perlapi.h b/perlapi.h
index f21407cfe3..3189d1f5b6 100644
--- a/perlapi.h
+++ b/perlapi.h
@@ -268,8 +268,6 @@ END_EXTERN_C
 #define PL_encoding		(*Perl_Iencoding_ptr(aTHX))
 #undef  PL_endav
 #define PL_endav		(*Perl_Iendav_ptr(aTHX))
-#undef  PL_endwhite
-#define PL_endwhite		(*Perl_Iendwhite_ptr(aTHX))
 #undef  PL_envgv
 #define PL_envgv		(*Perl_Ienvgv_ptr(aTHX))
 #undef  PL_errgv
@@ -292,8 +290,6 @@ END_EXTERN_C
 #define PL_exitlistlen		(*Perl_Iexitlistlen_ptr(aTHX))
 #undef  PL_expect
 #define PL_expect		(*Perl_Iexpect_ptr(aTHX))
-#undef  PL_faketokens
-#define PL_faketokens		(*Perl_Ifaketokens_ptr(aTHX))
 #undef  PL_fdpid
 #define PL_fdpid		(*Perl_Ifdpid_ptr(aTHX))
 #undef  PL_filemode
@@ -360,38 +356,8 @@ END_EXTERN_C
 #define PL_laststatval		(*Perl_Ilaststatval_ptr(aTHX))
 #undef  PL_laststype
 #define PL_laststype		(*Perl_Ilaststype_ptr(aTHX))
-#undef  PL_lasttoke
-#define PL_lasttoke		(*Perl_Ilasttoke_ptr(aTHX))
-#undef  PL_lex_brackets
-#define PL_lex_brackets		(*Perl_Ilex_brackets_ptr(aTHX))
-#undef  PL_lex_brackstack
-#define PL_lex_brackstack	(*Perl_Ilex_brackstack_ptr(aTHX))
-#undef  PL_lex_casemods
-#define PL_lex_casemods		(*Perl_Ilex_casemods_ptr(aTHX))
-#undef  PL_lex_casestack
-#define PL_lex_casestack	(*Perl_Ilex_casestack_ptr(aTHX))
-#undef  PL_lex_defer
-#define PL_lex_defer		(*Perl_Ilex_defer_ptr(aTHX))
-#undef  PL_lex_dojoin
-#define PL_lex_dojoin		(*Perl_Ilex_dojoin_ptr(aTHX))
-#undef  PL_lex_expect
-#define PL_lex_expect		(*Perl_Ilex_expect_ptr(aTHX))
-#undef  PL_lex_formbrack
-#define PL_lex_formbrack	(*Perl_Ilex_formbrack_ptr(aTHX))
-#undef  PL_lex_inpat
-#define PL_lex_inpat		(*Perl_Ilex_inpat_ptr(aTHX))
-#undef  PL_lex_inwhat
-#define PL_lex_inwhat		(*Perl_Ilex_inwhat_ptr(aTHX))
-#undef  PL_lex_op
-#define PL_lex_op		(*Perl_Ilex_op_ptr(aTHX))
-#undef  PL_lex_repl
-#define PL_lex_repl		(*Perl_Ilex_repl_ptr(aTHX))
-#undef  PL_lex_starts
-#define PL_lex_starts		(*Perl_Ilex_starts_ptr(aTHX))
 #undef  PL_lex_state
 #define PL_lex_state		(*Perl_Ilex_state_ptr(aTHX))
-#undef  PL_lex_stuff
-#define PL_lex_stuff		(*Perl_Ilex_stuff_ptr(aTHX))
 #undef  PL_lineary
 #define PL_lineary		(*Perl_Ilineary_ptr(aTHX))
 #undef  PL_linestart
@@ -438,14 +404,8 @@ END_EXTERN_C
 #define PL_minus_p		(*Perl_Iminus_p_ptr(aTHX))
 #undef  PL_modglobal
 #define PL_modglobal		(*Perl_Imodglobal_ptr(aTHX))
-#undef  PL_multi_close
-#define PL_multi_close		(*Perl_Imulti_close_ptr(aTHX))
 #undef  PL_multi_end
 #define PL_multi_end		(*Perl_Imulti_end_ptr(aTHX))
-#undef  PL_multi_open
-#define PL_multi_open		(*Perl_Imulti_open_ptr(aTHX))
-#undef  PL_multi_start
-#define PL_multi_start		(*Perl_Imulti_start_ptr(aTHX))
 #undef  PL_my_cxt_keys
 #define PL_my_cxt_keys		(*Perl_Imy_cxt_keys_ptr(aTHX))
 #undef  PL_my_cxt_list
@@ -458,8 +418,6 @@ END_EXTERN_C
 #define PL_nexttype		(*Perl_Inexttype_ptr(aTHX))
 #undef  PL_nextval
 #define PL_nextval		(*Perl_Inextval_ptr(aTHX))
-#undef  PL_nextwhite
-#define PL_nextwhite		(*Perl_Inextwhite_ptr(aTHX))
 #undef  PL_nice_chunk
 #define PL_nice_chunk		(*Perl_Inice_chunk_ptr(aTHX))
 #undef  PL_nice_chunk_size
@@ -508,8 +466,6 @@ END_EXTERN_C
 #define PL_parser		(*Perl_Iparser_ptr(aTHX))
 #undef  PL_patchlevel
 #define PL_patchlevel		(*Perl_Ipatchlevel_ptr(aTHX))
-#undef  PL_pending_ident
-#define PL_pending_ident	(*Perl_Ipending_ident_ptr(aTHX))
 #undef  PL_perl_destruct_level
 #define PL_perl_destruct_level	(*Perl_Iperl_destruct_level_ptr(aTHX))
 #undef  PL_perldb
@@ -522,8 +478,6 @@ END_EXTERN_C
 #define PL_ppid			(*Perl_Ippid_ptr(aTHX))
 #undef  PL_preambleav
 #define PL_preambleav		(*Perl_Ipreambleav_ptr(aTHX))
-#undef  PL_preambled
-#define PL_preambled		(*Perl_Ipreambled_ptr(aTHX))
 #undef  PL_preprocess
 #define PL_preprocess		(*Perl_Ipreprocess_ptr(aTHX))
 #undef  PL_profiledata
@@ -536,8 +490,6 @@ END_EXTERN_C
 #define PL_psig_ptr		(*Perl_Ipsig_ptr_ptr(aTHX))
 #undef  PL_ptr_table
 #define PL_ptr_table		(*Perl_Iptr_table_ptr(aTHX))
-#undef  PL_realtokenstart
-#define PL_realtokenstart	(*Perl_Irealtokenstart_ptr(aTHX))
 #undef  PL_reentrant_buffer
 #define PL_reentrant_buffer	(*Perl_Ireentrant_buffer_ptr(aTHX))
 #undef  PL_reentrant_retint
@@ -574,8 +526,6 @@ END_EXTERN_C
 #define PL_sighandlerp		(*Perl_Isighandlerp_ptr(aTHX))
 #undef  PL_signals
 #define PL_signals		(*Perl_Isignals_ptr(aTHX))
-#undef  PL_skipwhite
-#define PL_skipwhite		(*Perl_Iskipwhite_ptr(aTHX))
 #undef  PL_sort_RealCmp
 #define PL_sort_RealCmp		(*Perl_Isort_RealCmp_ptr(aTHX))
 #undef  PL_splitstr
@@ -598,8 +548,6 @@ END_EXTERN_C
 #define PL_strtab		(*Perl_Istrtab_ptr(aTHX))
 #undef  PL_sub_generation
 #define PL_sub_generation	(*Perl_Isub_generation_ptr(aTHX))
-#undef  PL_sublex_info
-#define PL_sublex_info		(*Perl_Isublex_info_ptr(aTHX))
 #undef  PL_subline
 #define PL_subline		(*Perl_Isubline_ptr(aTHX))
 #undef  PL_subname
@@ -624,18 +572,6 @@ END_EXTERN_C
 #define PL_taint_warn		(*Perl_Itaint_warn_ptr(aTHX))
 #undef  PL_tainting
 #define PL_tainting		(*Perl_Itainting_ptr(aTHX))
-#undef  PL_thisclose
-#define PL_thisclose		(*Perl_Ithisclose_ptr(aTHX))
-#undef  PL_thismad
-#define PL_thismad		(*Perl_Ithismad_ptr(aTHX))
-#undef  PL_thisopen
-#define PL_thisopen		(*Perl_Ithisopen_ptr(aTHX))
-#undef  PL_thisstuff
-#define PL_thisstuff		(*Perl_Ithisstuff_ptr(aTHX))
-#undef  PL_thistoken
-#define PL_thistoken		(*Perl_Ithistoken_ptr(aTHX))
-#undef  PL_thiswhite
-#define PL_thiswhite		(*Perl_Ithiswhite_ptr(aTHX))
 #undef  PL_threadhook
 #define PL_threadhook		(*Perl_Ithreadhook_ptr(aTHX))
 #undef  PL_tokenbuf
diff --git a/perly.c b/perly.c
index 36f78a346b..af09290581 100644
--- a/perly.c
+++ b/perly.c
@@ -195,7 +195,7 @@ S_clear_yystack(pTHX_  const yy_parser *parser)
     yy_stack_frame *ps     = parser->ps;
     int i;
 
-    if (ps == parser->stack)
+    if (!parser->stack || ps == parser->stack)
 	return;
 
     YYDPRINTF ((Perl_debug_log, "clearing the parse stack\n"));
diff --git a/proto.h b/proto.h
index 4f99f58755..b908cf2f60 100644
--- a/proto.h
+++ b/proto.h
@@ -2600,6 +2600,10 @@ PERL_CALLCONV int	Perl_yyparse(pTHX);
 PERL_CALLCONV void	Perl_parser_free(pTHX_ const yy_parser *)
 			__attribute__nonnull__(pTHX_1);
 
+PERL_CALLCONV yy_parser*	Perl_parser_dup(pTHX_ const yy_parser *proto, CLONE_PARAMS* param)
+			__attribute__nonnull__(pTHX_1)
+			__attribute__nonnull__(pTHX_2);
+
 PERL_CALLCONV int	Perl_yywarn(pTHX_ const char* s)
 			__attribute__nonnull__(pTHX_1);
 
diff --git a/sv.c b/sv.c
index e78fd2edda..6dd53b7b89 100644
--- a/sv.c
+++ b/sv.c
@@ -9479,6 +9479,70 @@ ptr_table_* functions.
 #define SAVEPV(p)	((p) ? savepv(p) : NULL)
 #define SAVEPVN(p,n)	((p) ? savepvn(p,n) : NULL)
 
+/* clone a parser */
+
+yy_parser *
+Perl_parser_dup(pTHX_ const yy_parser *proto, CLONE_PARAMS* param)
+{
+    yy_parser *parser;
+
+    if (!proto)
+	return NULL;
+
+    Newxz(parser, 1, yy_parser);
+
+    parser->yyerrstatus = 0;
+    parser->yychar = YYEMPTY;		/* Cause a token to be read.  */
+
+    /* XXX these not yet duped */
+    parser->old_parser = NULL;
+    parser->stack = NULL;
+    parser->ps = NULL;
+    parser->stack_size = 0;
+    /* XXX parser->stack->state = 0; */
+
+    /* XXX eventually, just Copy() most of the parser struct ? */
+
+    parser->lex_brackets = proto->lex_brackets;
+    parser->lex_casemods = proto->lex_casemods;
+    parser->lex_brackstack = savepvn(proto->lex_brackstack,
+		    (proto->lex_brackets < 120 ? 120 : proto->lex_brackets));
+    parser->lex_casestack = savepvn(proto->lex_casestack,
+		    (proto->lex_casemods < 12 ? 12 : proto->lex_casemods));
+    parser->lex_defer	= proto->lex_defer;
+    parser->lex_dojoin	= proto->lex_dojoin;
+    parser->lex_expect	= proto->lex_expect;
+    parser->lex_formbrack = proto->lex_formbrack;
+    parser->lex_inpat	= proto->lex_inpat;
+    parser->lex_inwhat	= proto->lex_inwhat;
+    parser->lex_op	= proto->lex_op;
+    parser->lex_repl	= sv_dup_inc(proto->lex_repl, param);
+    parser->lex_starts	= proto->lex_starts;
+    parser->lex_stuff	= sv_dup_inc(proto->lex_stuff, param);
+    parser->multi_close	= proto->multi_close;
+    parser->multi_open	= proto->multi_open;
+    parser->multi_start	= proto->multi_start;
+    parser->pending_ident = proto->pending_ident;
+    parser->preambled	= proto->preambled;
+    parser->sublex_info	= proto->sublex_info; /* XXX not quite right */
+
+#ifdef PERL_MAD
+    parser->endwhite	= proto->endwhite;
+    parser->faketokens	= proto->faketokens;
+    parser->lasttoke	= proto->lasttoke;
+    parser->nextwhite	= proto->nextwhite;
+    parser->realtokenstart = proto->realtokenstart;
+    parser->skipwhite	= proto->skipwhite;
+    parser->thisclose	= proto->thisclose;
+    parser->thismad	= proto->thismad;
+    parser->thisopen	= proto->thisopen;
+    parser->thisstuff	= proto->thisstuff;
+    parser->thistoken	= proto->thistoken;
+    parser->thiswhite	= proto->thiswhite;
+#endif
+    return parser;
+}
+
 
 /* duplicate a file handle */
 
@@ -11022,7 +11086,6 @@ perl_clone_using(PerlInterpreter *proto_perl, UV flags,
     PL_Argv		= NULL;
     PL_Cmd		= NULL;
     PL_gensym		= proto_perl->Igensym;
-    PL_preambled	= proto_perl->Ipreambled;
     PL_preambleav	= av_dup_inc(proto_perl->Ipreambleav, param);
     PL_laststatval	= proto_perl->Ilaststatval;
     PL_laststype	= proto_perl->Ilaststype;
@@ -11102,38 +11165,12 @@ perl_clone_using(PerlInterpreter *proto_perl, UV flags,
     PL_cshname		= proto_perl->Icshname; /* XXX never deallocated */
 #endif
 
+    PL_parser		= parser_dup(proto_perl->Iparser, param);
+
     PL_lex_state	= proto_perl->Ilex_state;
-    PL_lex_defer	= proto_perl->Ilex_defer;
-    PL_lex_expect	= proto_perl->Ilex_expect;
-    PL_lex_formbrack	= proto_perl->Ilex_formbrack;
-    PL_lex_dojoin	= proto_perl->Ilex_dojoin;
-    PL_lex_starts	= proto_perl->Ilex_starts;
-    PL_lex_stuff	= sv_dup_inc(proto_perl->Ilex_stuff, param);
-    PL_lex_repl		= sv_dup_inc(proto_perl->Ilex_repl, param);
-    PL_lex_op		= proto_perl->Ilex_op;
-    PL_lex_inpat	= proto_perl->Ilex_inpat;
-    PL_lex_inwhat	= proto_perl->Ilex_inwhat;
-    PL_lex_brackets	= proto_perl->Ilex_brackets;
-    i = (PL_lex_brackets < 120 ? 120 : PL_lex_brackets);
-    PL_lex_brackstack	= SAVEPVN(proto_perl->Ilex_brackstack,i);
-    PL_lex_casemods	= proto_perl->Ilex_casemods;
-    i = (PL_lex_casemods < 12 ? 12 : PL_lex_casemods);
-    PL_lex_casestack	= SAVEPVN(proto_perl->Ilex_casestack,i);
 
 #ifdef PERL_MAD
     Copy(proto_perl->Inexttoke, PL_nexttoke, 5, NEXTTOKE);
-    PL_lasttoke		= proto_perl->Ilasttoke;
-    PL_realtokenstart	= proto_perl->Irealtokenstart;
-    PL_faketokens	= proto_perl->Ifaketokens;
-    PL_thismad		= proto_perl->Ithismad;
-    PL_thistoken	= proto_perl->Ithistoken;
-    PL_thisopen		= proto_perl->Ithisopen;
-    PL_thisstuff	= proto_perl->Ithisstuff;
-    PL_thisclose	= proto_perl->Ithisclose;
-    PL_thiswhite	= proto_perl->Ithiswhite;
-    PL_nextwhite	= proto_perl->Inextwhite;
-    PL_skipwhite	= proto_perl->Iskipwhite;
-    PL_endwhite		= proto_perl->Iendwhite;
     PL_curforce		= proto_perl->Icurforce;
 #else
     Copy(proto_perl->Inextval, PL_nextval, 5, YYSTYPE);
@@ -11151,15 +11188,10 @@ perl_clone_using(PerlInterpreter *proto_perl, UV flags,
     i = proto_perl->Ilinestart - SvPVX_const(proto_perl->Ilinestr);
     PL_linestart	= SvPVX(PL_linestr) + (i < 0 ? 0 : i);
     PL_bufend		= SvPVX(PL_linestr) + SvCUR(PL_linestr);
-    PL_pending_ident	= proto_perl->Ipending_ident;
-    PL_sublex_info	= proto_perl->Isublex_info;	/* XXX not quite right */
 
     PL_expect		= proto_perl->Iexpect;
 
-    PL_multi_start	= proto_perl->Imulti_start;
     PL_multi_end	= proto_perl->Imulti_end;
-    PL_multi_open	= proto_perl->Imulti_open;
-    PL_multi_close	= proto_perl->Imulti_close;
 
     PL_error_count	= proto_perl->Ierror_count;
     PL_subline		= proto_perl->Isubline;
diff --git a/toke.c b/toke.c
index 0a63cb648d..1655918c92 100644
--- a/toke.c
+++ b/toke.c
@@ -28,6 +28,44 @@
 /* YYINITDEPTH -- initial size of the parser's stacks.  */
 #define YYINITDEPTH 200
 
+/* XXX temporary backwards compatibility */
+#define PL_lex_brackets		(PL_parser->lex_brackets)
+#define PL_lex_brackstack	(PL_parser->lex_brackstack)
+#define PL_lex_casemods		(PL_parser->lex_casemods)
+#define PL_lex_casestack        (PL_parser->lex_casestack)
+#define PL_lex_defer		(PL_parser->lex_defer)
+#define PL_lex_dojoin		(PL_parser->lex_dojoin)
+#define PL_lex_expect		(PL_parser->lex_expect)
+#define PL_lex_formbrack        (PL_parser->lex_formbrack)
+#define PL_lex_inpat		(PL_parser->lex_inpat)
+#define PL_lex_inwhat		(PL_parser->lex_inwhat)
+#define PL_lex_op		(PL_parser->lex_op)
+#define PL_lex_repl		(PL_parser->lex_repl)
+#define PL_lex_starts		(PL_parser->lex_starts)
+#define PL_lex_stuff		(PL_parser->lex_stuff)
+#define PL_multi_start		(PL_parser->multi_start)
+#define PL_multi_open		(PL_parser->multi_open)
+#define PL_multi_close		(PL_parser->multi_close)
+#define PL_pending_ident        (PL_parser->pending_ident)
+#define PL_preambled		(PL_parser->preambled)
+#define PL_sublex_info		(PL_parser->sublex_info)
+
+#ifdef PERL_MAD
+#  define PL_endwhite		(PL_parser->endwhite)
+#  define PL_faketokens		(PL_parser->faketokens)
+#  define PL_lasttoke		(PL_parser->lasttoke)
+#  define PL_nextwhite		(PL_parser->nextwhite)
+#  define PL_realtokenstart	(PL_parser->realtokenstart)
+#  define PL_skipwhite		(PL_parser->skipwhite)
+#  define PL_thisclose		(PL_parser->thisclose)
+#  define PL_thismad		(PL_parser->thismad)
+#  define PL_thisopen		(PL_parser->thisopen)
+#  define PL_thisstuff		(PL_parser->thisstuff)
+#  define PL_thistoken		(PL_parser->thistoken)
+#  define PL_thiswhite		(PL_parser->thiswhite)
+#endif
+
+
 static const char ident_too_long[] = "Identifier too long";
 static const char commaless_variable_list[] = "comma-less variable list";
 
@@ -571,6 +609,8 @@ S_cr_textfilter(pTHX_ int idx, SV *sv, int maxlen)
 }
 #endif
 
+
+
 /*
  * Perl_lex_start
  * Initialize variables.  Uses the Perl save_stack to save its state (for
@@ -587,7 +627,7 @@ Perl_lex_start(pTHX_ SV *line)
 
     /* create and initialise a parser */
 
-    Newx(parser, 1, yy_parser);
+    Newxz(parser, 1, yy_parser);
     parser->old_parser = PL_parser;
     PL_parser = parser;
 
@@ -601,35 +641,19 @@ Perl_lex_start(pTHX_ SV *line)
 
     /* initialise lexer state */
 
-    SAVEI32(PL_lex_dojoin);
     SAVEI32(PL_lex_brackets);
     SAVEI32(PL_lex_casemods);
-    SAVEI32(PL_lex_starts);
     SAVEI32(PL_lex_state);
-    SAVEVPTR(PL_lex_inpat);
-    SAVEI32(PL_lex_inwhat);
 #ifdef PERL_MAD
     if (PL_lex_state == LEX_KNOWNEXT) {
-	I32 toke = PL_lasttoke;
+	I32 toke = parser->old_parser->lasttoke;
  	while (--toke >= 0) {
 	    SAVEI32(PL_nexttoke[toke].next_type);
 	    SAVEVPTR(PL_nexttoke[toke].next_val);
 	    if (PL_madskills)
 		SAVEVPTR(PL_nexttoke[toke].next_mad);
 	}
-	SAVEI32(PL_lasttoke);
-    }
-    SAVESPTR(PL_endwhite);
-    SAVESPTR(PL_thistoken);
-    SAVESPTR(PL_thiswhite);
-    SAVESPTR(PL_nextwhite);
-    SAVESPTR(PL_thisopen);
-    SAVESPTR(PL_thisclose);
-    SAVESPTR(PL_thisstuff);
-    SAVEVPTR(PL_thismad);
-    SAVEI32(PL_realtokenstart);
-    SAVEI32(PL_faketokens);
-    SAVESPTR(PL_skipwhite);
+    }
     SAVEI32(PL_curforce);
 #else
     if (PL_lex_state == LEX_KNOWNEXT) {
@@ -650,70 +674,17 @@ Perl_lex_start(pTHX_ SV *line)
     SAVEPPTR(PL_last_uni);
     SAVEPPTR(PL_linestart);
     SAVESPTR(PL_linestr);
-    SAVEGENERICPV(PL_lex_brackstack);
-    SAVEGENERICPV(PL_lex_casestack);
     SAVEDESTRUCTOR_X(restore_rsfp, PL_rsfp);
-    SAVESPTR(PL_lex_stuff);
-    SAVEI32(PL_lex_defer);
-    SAVEI32(PL_sublex_info.sub_inwhat);
-    SAVEI32(PL_sublex_info.super_state);
-    SAVEVPTR(PL_sublex_info.sub_op);
-    SAVEPPTR(PL_sublex_info.super_bufptr);
-    SAVEPPTR(PL_sublex_info.super_bufend);
-    SAVESPTR(PL_lex_repl);
     SAVEINT(PL_expect);
-    SAVEINT(PL_lex_expect);
-    SAVEI32(PL_lex_formbrack);
-    SAVEVPTR(PL_lex_op);
-    SAVEI32(PL_multi_close);
-    SAVEI32(PL_multi_open);
-    SAVEI32(PL_multi_start);
-    SAVEI8(PL_pending_ident);
-    SAVEBOOL(PL_preambled);
 
     PL_lex_state = LEX_NORMAL;
-    PL_lex_defer = 0;
     PL_expect = XSTATE;
-    PL_lex_brackets = 0;
-    Newx(PL_lex_brackstack, 120, char);
-    Newx(PL_lex_casestack, 12, char);
-    PL_lex_casemods = 0;
-    *PL_lex_casestack = '\0';
-    PL_lex_dojoin = 0;
-    PL_lex_starts = 0;
-    PL_lex_stuff = NULL;
-    PL_lex_repl = NULL;
-    PL_lex_inpat = 0;
-#ifdef PERL_MAD
-    PL_lasttoke = 0;
-    PL_endwhite = NULL;
-    PL_faketokens = 0;
-    PL_nextwhite = NULL;
-    PL_realtokenstart = 0;
-    PL_skipwhite = NULL;
-    PL_thisclose = NULL;
-    PL_thisopen = NULL;
-    PL_thisstuff = NULL;
-    PL_thistoken = NULL;
-    PL_thiswhite = NULL;
-    PL_thismad = NULL;
-#else
+    Newx(parser->lex_brackstack, 120, char);
+    Newx(parser->lex_casestack, 12, char);
+    *parser->lex_casestack = '\0';
+#ifndef PERL_MAD
     PL_nexttoke = 0;
 #endif
-    PL_lex_inwhat = 0;
-    PL_sublex_info.sub_inwhat = 0;
-    PL_sublex_info.super_state = 0;
-    PL_sublex_info.sub_op = NULL;
-    PL_sublex_info.super_bufptr = NULL;
-    PL_sublex_info.super_bufend = NULL;
-    PL_lex_expect = 0;
-    PL_lex_formbrack = 0;
-    PL_lex_op = NULL;
-    PL_multi_close = 0;
-    PL_multi_open = 0;
-    PL_multi_start = 0;
-    PL_pending_ident = '\0';
-    PL_preambled = FALSE;
 
     if (line) {
 	s = SvPV_const(line, len);