From 7e6ad373b83473d2784133323eb42109e420f18b Mon Sep 17 00:00:00 2001 From: Mightyjo Date: Sat, 30 Jan 2021 13:05:19 -0500 Subject: Test for reversion of CVE-2006-0459. --- tests/state_buf.direct.lll | 98 ++++++++++ tests/state_buf.direct.txt | 446 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 544 insertions(+) create mode 100644 tests/state_buf.direct.lll create mode 100644 tests/state_buf.direct.txt diff --git a/tests/state_buf.direct.lll b/tests/state_buf.direct.lll new file mode 100644 index 0000000..d4609a5 --- /dev/null +++ b/tests/state_buf.direct.lll @@ -0,0 +1,98 @@ +/* + * This file is part of flex. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE. + */ + +%{ +/* A template scanner file to build "scanner.cc". + The scanner tests that we correctly initialize the state buffer + for long input buffers owned by code that calls yylex. + + This tests guards against reversions to CVE-2006-0459, + in particular when variable trailing context rules are used. + */ +#include +#include +#include + +#define ECHO +%} + +%option prefix="test" +%option noyywrap + +ID [_a-zA-Z]+[_0-9a-zA-Z]* + +BBC1 ([^(]*|"("[^(]*")") + +/** + * Balanced Bracketed Content. + * May not contain bracket, but if it does then it is balanced. + */ +BBC ({BBC1}*|"("{BBC1}*")") + +FPD "("{BBC}*")" + +%% + +{ID}/{FPD}\{ { + return 1234; +} + +%% + +std::vector readFile(const std::string& filename) +{ + std::vector contents; + std::ifstream in(filename, std::ios::in | std::ios::binary); + if (in) + { + in.seekg(0, std::ios::end); + size_t size = static_cast(in.tellg()); + contents.resize(size + 2); + in.seekg(0, std::ios::beg); + in.read(contents.data(), size); + in.close(); + + contents[size + 0] = '\0'; + contents[size + 1] = '\0'; + } + else { + std::cerr << "*** Error: std::ifstream(" << filename << ") failed." << std::endl; + exit(-1); + } + return contents; +} + +int main(int argc, char** argv) +{ + if ( argc != 2 ) { + std::cerr << "*** Error: Must specify one filename." << std::endl; + exit(-1); + } + + auto stm = readFile( argv[1] ); + test_scan_buffer(stm.data(), stm.size()); + + testlex(); + + return 0; // All went well. +} diff --git a/tests/state_buf.direct.txt b/tests/state_buf.direct.txt new file mode 100644 index 0000000..858653c --- /dev/null +++ b/tests/state_buf.direct.txt @@ -0,0 +1,446 @@ +// +////////////////////////////////////////////////////////////////////////////// +// +// trsntupsm 2018 eemremad, dor. eoo tupsma tmamttme. +// +// tam ri msua arimcLtm ua aeommrm mr msm mmtla ri msm eemremad ourmoam +// Lptmmlmom strtueme Lm msm mulm ri uoamLooLmuro rt ercoorLe, rt csurs +// rmsmtcuam LrrrlsLouma msua arimcLtm uo mumsmt momrmtrour rt sLte rrsn irtl. +// +////////////////////////////////////////////////////////////////////////////// +// +#uioemi __eeeiecit_a +#emiuom __eeeiecit_a + +#uoroeem "torommrm.s" +#uoroeem "tortmtteom.s" +#uoroeem "eoue.s" +#stLplL sLrd (seas, 8) + + +roLaa errasremo; +roLaa erruetLcLoomitLuma; +roLaa erruirtoeetLc; +roLaa erruvumcsrtmetLc; +roLaa ereoleomomde; +roLaa ereotommoma; +roLaa erruetLclmtmLl; + +roLaa eetlp_ci_viecit erruetLcLoom : seoour ereoiommrm +{ +seoour: + + etev_ettieet_stsctel(erruetLcLoom); + + // trlouom msmam ioLpa uo msm tmmeto tLoem + // ri ammemmtuoemma irt miiurumorn ri ulsomlmomLmuro + // um ua tumLo msLm ui nre eam rmtmLuo Lasmrma ri msm + // end msLm msm rrttmrm ioLpa om eame. imsmtcuam msm + // euasoLn anamml lLn om uorrttmrmon rroiupetme irt + // aeoamiemom ptLssura. ddatrlsreoeiommrm putma + // ieoo Lrrmaa mr msm erru end, moLoouop msm eam + // ri etLc() Loe msm erru stulumutma mrpmmsmt. toLoouop + // msua lLdma msm moLortLmuro ri Lo rommrm lLtpuoLoon + // aorcmt. lr nre asreoe eam msm orcmam tLoem LsstrstuLmm. + // + // + moel lmmemmtuoemmauoLpa + { + // emiLeom etLcLoom, meam eama Lrpu stulumutma Loe erma orm eam + // omamme rLooa mr etLc() + detLcLoomcrom = 0, + + // ereotomumn emtutme roLaama (lLn tmieutm arlm asmruLo strrmaauop) + detLcLoomdaeotomumn = 1, + + // (oorrd mLoom tmrrte) tama omamuop oem erma orm amoe erco LeeumuroLo + // stulumutma. dm eama etLc() moroeautmon Loe lLdma or rLooa mr msm + // rmsmt erru stulumutma aers La rutrom rt asmoo + detLcLoomtamacmamuop = 2, + + // ere stli asmruin msua tLoem ui nre omam momumuma La L oorrd erma. + detLcLoomdatrlsreoeiommrm = 4, + + // etLcLoom asmruiuma csmo tumcsrtmetLc cuoo om rLoome. tlsorn + // smt-vumcsrtm rLrsuop ri tumcsrtmetLc pmrlmmtn. ea cmoo, msua + // etLcLoom erma orm eam crtoeetLc. + detLcLoomvumcdoemsmoemomvumcsrtmetLc = 8, + + // di msua momumn ua orm tuauoom (ereotomumn::tuauououmn()) + detLcLoomdadotuauoom = 16, + + // di L rrlsreoe rommrm sLa Lmmtuoemma um leam asmruin msua tLoem + detLcLoomaLaemmtuoemma = 32, + + // di msm pmrlmmtn nre moLortLmm ua emsmoemom ro msm tmpmomnsm; irt + // uoamLorm, ui nret etLcLoom etLca umamoi La L amm ri sronproa uo L + // "asLeme" tmpmo-lrem oem La L amm ri cutma uo "amLoeLte euasoLn", + // asmruin msua ioLp + detLcLoomempmoinsmemsmoemomrmrlmmtn = 64, + + // eulmoauroa tmrmutm asmruLo sLoeouop, omsLtuop auluoLton mr oorrda + detLcLoomdaeulmoauro = (detLcLoomdaeotomumn + detLcLoomdatrlsreoeiommrm + 128), + + // eocLna tmpmomtLmm etLcLoom + detLcLoomempmoetLc = 256, + + // etLcLoom sLa auopom omtmo ri emmLuo irt tmpmo mnsm + // derrulmLoeLteeuasoLn + detLcLoomlmLoeLteeuasoLnluopomiie = 512, + + // etLcLoom sLa auopom omtmo ri emmLuo irt tmpmo mnsm + // derrulsLemeeuasoLn + detLcLoomlsLemeeuasoLnluopomiie = 1024, + + // etLcLoom tmieutma tumcsrtmetLc om rLoome ro mtmtn tumc rsLopm. + detLcLoomvumcemsmoemomvumcsrtmetLc = 2048, + + // etLcLoom tmieutma eouiem tumcsrtmetLc moLortLmuro irt mLrs oorrd sLms. + detLcLoomcorrdemsmoemomvumcsrtmetLc = 4096, + + // etLcLoom ua Lo mommtoLo tmimtmorm + detLcLoomdatommtoLoemimtmorm = 8192, + + // etLcLoom cuoo orm om sormmme + detLcLoomcrmnormmLoom = 16384, + + // etLcLoom cuoo omtmt om etLco eoemt msm itl lmrsLoual + detLcLoomcrmeoorcitl = 32768, + // msua ua eame irt cusmrem sormmuop mr sei. + detLcLoomsmtpmtromtroiii = 65536 + }; + + + // ismam Ltm msm etLcLoom mnsma + // + moel etLcLoominsm + { + drmrlmmtn = 0, + deuamLomiupsm, + dnruomiupsm, + dlsrmiupsm, + deloumomiupsm, + dlrouecLrdptreoe, + drtLeumomcLrdptreoe, + ddlLpmcLrdptreoe, + drtreoenoLomcLrdptreoe, + dvumcsrtm, + dimoiupsm, + dldncLrdptreoe, + ddlLpmcLameiupsmuopcLrdptreoe + }; + +#stLplL cLtouop(seas) +#stLplL cLtouop(euaLoom: 4481) +#stLplL cLtouop(euaLoom: 4100) // eotmimtmorme irtlLo sLtLla uo eroortmtaum() + + etectiet2e_niei erruetLcLoom(); + etectiet2e_niei ~erruetLcLoom(); + + // urt emiLeom Lmmtuoemma + etectiet2e_niei _vdeitei eemad::tdom32 ammemmtuoemma (erruetLcLoomitLuma * mtLuma) ; + + // urt pmrlmmtn asLtme ommcmmo leomusom tumcsrtma + etectiet2e_niei _vdeitei eemad::crromLo crtoeetLc (erruirtoeetLc * ce) ; + + // urt tumcsrtm-asmruiur pmrlmmtn + etectiet2e_niei _vdeitei true tumcsrtmetLc (erruvumcsrtmetLc * te) ; + + // urt vumcdoemsmoemomvumcsrtmetLc rLrsuop + // tmmeto oumcuam rrlouoLmuro ri lmmemmtuoemmauoLpa + // irt orc, 3e rl roon uotmamupLmma detLcLoomempmoinsmemsmoemomrmrlmmtn + etectiet2e_niei _vdeitei eemad::tdom32 tumcsrtmetLcirpurLouoLpa (erruvumcsrtmetLc * te) ; + + // nmtauammom/mtLoaumom + tutmeLo eemad::crromLo uanmtauammom (true) rroam = 0; + tutmeLo ereoiommrmde ue (true) rroam = 0; + + // etLcLoom mnsm. + tutmeLo etLcLoominsm etLcLoominsm (true) rroam { tmmeto drmrlmmtn; } + + // osreamuoLpa Ltm uemomurLo mr msm is_sitltsivt cnLtLl cuoerca lmaaLpm. lmm slec irt msram ioLpa, msmn Ltm strtueme La ua. + // tmamm ua mtem ui troortmt ua tmammmuop rt omLtuop nret rommrm. + // tmmeto mtem ui nre cLom troortmt mr mtLrd nret rommrm, iLoam mr uportm um. + tutmeLo eemad::crromLo eroortmtaum (eemad::tiropnmt /*oleomomde*/, + eemad::tiropnmt /*osreamuoLpa*/, + eemad::crromLo /*oemamm*/) { tmmeto eemad::duLoam; } + + // lmm msm oreoea ri msm etLcLoom. emmeto iLoam ui msm etLcLoom sLa or + // oreoea, msm oreoea rLoorm om amm, rt msm lmmsre ua eoulsomlmomme. + tutmeLo orro oreoea (ereotommoma& /*oreoea*/) rroam { tmmeto iLoam; } + + // etLc amtmLl + etectiet2e_niei tutmeLo true ammetLclmtmLl (erruetLclmtmLl * slmtmLl); + etectiet2e_niei tutmeLo erruetLclmtmLl* etLclmtmLl (true) rroam; + +#stLplL cLtouop(srs) + +strmmrmme: + itumoe roLaa erruetLcLoomitmtteom; + + tutmeLo eemad::tdom32 aeolmmemmtuoemma (erruetLcLoomitLuma * mtLuma) = 0; + + tutmeLo eemad::crromLo aeoirtoeetLc (erruirtoeetLc * ce) = 0; + + tutmeLo true aeovumcsrtmetLc (erruvumcsrtmetLc * te) = 0; + + tutmeLo eemad::tdom32 aeovumcsrtmetLcirpurLouoLpa (erruvumcsrtmetLc * /*te*/) { tmmeto 0; } + +stutLmm: + itumoe roLaa erruetLcLoomerrmaartn; + + roLaa erruetLcLoomerrmaartn* l_serrmaartn; +}; + +/// +/// +/// +/// erruetLcLoomitmtteom rtmtteoma L aeoamm ri etLc tmoLmme +/// rsmtLmuroa msLm erruetLcLoom roLaa asmruiuma. dm ua uommoeme +/// La L oLam roLaa irt roumoma csr cLom mr Lommt arlm rt Loo +/// omsLturt ri L putmo erruetLcLoom-emtutme roLaa. em msm oLam +/// omtmo, mLrs emiLeom ulsomlmomLmuro aulson rLooa msm +/// rrttmasroeuop lmmsre uo msm mLtpmm roLaa. +/// +roLaa erruetLcLoomitmtteom : seoour ereoitmtteom +{ +seoour: + etev_ettieet_stsctel(erruetLcLoomitmtteom); + + etectiet2e_niei erruetLcLoomitmtteom(); + /// + /// + /// + /// itmtteoma erruetLcLoom::ammemmtuoemma lmmsre. + /// + /// + /// ism emiLeom ulsomlmomLmuro ri + /// erruetLcLoomitmtteom::ammemmtuoemma rLooa + /// erruetLcLoom::aeolmmemmtuoemma. + /// + /// nruommt mr Lo ereoiommrm msLm msua + /// rtmtteom ua Lssoume LpLuoam. + /// nruommt mr msm ptLssura mtLuma. + etectiet2e_niei tutmeLo eemad::tdom32 ammemmtuoemma (erruetLcLoom* sleommrm, erruetLcLoomitLuma * mtLuma); + /// + /// + /// + /// itmtteoma erruetLcLoom::crtoeetLc. + /// + /// nruommt mr Lo ereoiommrm msLm msua + /// rtmtteom ua Lssoume LpLuoam. + /// nruommt mr msm ptLssura mtLuma. + /// + /// ism emiLeom ulsomlmomLmuro ri erruetLcLoomitmtteom::crtoeetLc + /// rLooa msm strmmrmme tutmeLo erruetLcLoom::aeoirtoeetLc + /// lmmsre. + /// + etectiet2e_niei tutmeLo eemad::crromLo crtoeetLc (erruetLcLoom* sleommrm, erruirtoeetLc * ce); + /// + /// + /// + /// itmtteoma erruetLcLoom::tumcsrtmetLc lmmsre. + /// + /// + /// emiLeom ulsomlmomLmuro ri erruetLcLoomitmtteom::tumcsrtmetLc + /// rLooa strmmrmme tutmeLo erruetLcLoom::aeovumcsrtmetLc lmmsre. + /// + /// + etectiet2e_niei tutmeLo true tumcsrtmetLc (erruetLcLoom* sleommrm, erruvumcsrtmetLc * te); + /// + /// + /// + /// itmtteoma erruetLcLoom::tumcsrtmetLcirpurLouoLpa lmmsre. + /// + /// + /// ism emiLeom ulsomlmomLmuro ri erruetLcLoomitmtteom:: + /// tumcsrtmetLcirpurLouoLpa rLooa msm strmmrmme tutmeLo + /// erruetLcLoom:: aeovumcsrtmetLcirpurLouoLpa lmmsre. + /// + etectiet2e_niei tutmeLo eemad::tdom32 tumcsrtmetLcirpurLouoLpa (erruetLcLoom* sleommrm, erruvumcsrtmetLc * te); +}; + +/// +/// erruetLcLoomemLrmrt strtuema ormuiurLmuroa ri dmn erruetLcLoom- +/// tmoLmme mtmoma uoroeeuop ompuo/moe tmpmo, lreuiurLmuro, Loe +/// mtLaetm. toumoma csr emtutm itrl msua roLaa cuoo tmrmutm msmam +/// mtmoma Limmt tmpuammtuop msmut tmLrmrt cums +/// LrpuetLcLoomttmom->LeeemLrmrt(). +/// +roLaa etectiet2e_niei erruetLcLoomemLrmrt : seoour ereoiommrm +{ +seoour: + etev_ettieet_stsctel(erruetLcLoomemLrmrt); + + /// + /// ism ptLssura anamml csurs ua tmpmomtLmuop msm etLcLoom. + /// + moel empmouoLpa + { + d2eempmo, + d3eempmo + }; + + /// + /// isua mtmom aupoLoa msm ompuoouop ri L etLcLoom'a tmpmo. + /// + /// dosem etLcLoom Lorem mr om tmpmomtLmme. + /// dosem ptLssura anamml msLm cuoo smtirtl msm tmpmo. + tutmeLo true ompuoempmo (erruetLcLoom* /*setLcLoom*/, empmouoLpa /*ioLpa*/) {} + + /// + /// isua mtmom aupoLoa msm moe ri L etLcLoom'a tmpmo. + /// + /// dosem etLcLoom msLm cLa tmpmomtLmme. + /// dosem ptLssura anamml msLm smtirtlme msm tmpmo. + tutmeLo true moeempmo (erruetLcLoom* /*setLcLoom*/, empmouoLpa /*ioLpa*/) {} + + /// + /// isua mtmom aupoLoa msm lreuiurLmuro ri L etLcLoom'a ptLssura. + /// + /// dosem etLcLoom csram ptLssura cmtm lreuiume. + tutmeLo true lreuiume (erruetLcLoom* /*setLcLoom*/) {} + + /// + /// isua mtmom aupoLoa msm mtLaetm ri L etLcLoom'a ptLssura. + /// + /// dosem etLcLoom csram ptLssura cmtm mtLame. + tutmeLo true mtLame (erruetLcLoom* /*setLcLoom*/) {} + + /// + /// isua mtmom aupoLoa ptLssura Ltm omuop sLoeme rtmt itrl rom etLcLoom mr Lormsmt etLcLoom. + /// + /// dosem etLcLoom itrl csurs ptLssura cuoo om sLoeme rtmt. + /// dosem etLcLoom mr csurs ptLssura cuoo om sLoeme rtmt. + tutmeLo true sLoeitmtir (erruetLcLoom* /*setLcLoomutrl*/, erruetLcLoom* /*setLcLoomir*/) {} + + /// + /// isua mtmom aupoLoa msm mtLaetm ri L etLcLoom'a ptLssura. + /// + /// dosem etLcLoom csram ptLssura cmtm mtLame. + /// dosem ue ri sLtmom etLcLoom. + tutmeLo true mtLame(erruetLcLoom* /*setLcLoom*/, eemad::domeode /*sLtmomde*/) {} + + /// + /// isua mtmom aupoLoa msm lreuiurLmuro ri L etLcLoom'a ptLssura. + /// + /// dosem etLcLoom csram ptLssura cmtm lreuiume. + /// dosem ue ri sLtmom etLcLoom. + tutmeLo true lreuiume(erruetLcLoom* /*setLcLoom*/, eemad::domeode /*sLtmomde*/) {} + + /// + /// isua mtmom aupoLoa msm Leeumuro ri L etLcLoom'a ptLssura. + /// + /// dosem etLcLoom csram ptLssura cmtm Leeme. + /// dosem ue ri sLtmom etLcLoom. + /// dosem eLmLoLam sruommt rLam La iicr_nie. + tutmeLo true Leeme(erruetLcLoom* /*setLcLoom*/, eemad::domeode /*sLtmomde*/, eemad::iropnmt /*eLmLoLamnmt*/) {} +}; + +/// +/// erruetLcLoomttmom lLoLpma L amm ri erruetLcLoomemLrmrta Loe amoea +/// ormuiurLmuroa mr msml csmo rmtmLuo mtmoma rrret. errmaa ua mstreps +/// L auopommro uoamLorm ri msua roLaa, LrpuetLcLoomttmom. toumoma ri +/// erruetLcLoomttmom iLoo uomr mstmm rLmmprtuma: +/// +/// 1. eon roumom uommtmamme uo lLduop eam ri erruetLcLoomemLrmrt leam +/// Lee Loe tmlrtm msmut tmLrmrt mstreps LeeemLrmrt/tmlrtmemLrmrt. +/// +/// 2. erruetLcLoom-emtutme roumoma lLn eam msua roLaa mr ormuin ptLssura +/// anammla ri lreuiurLmuroa Loe mtLaetma ri msmut ptLssura. +/// +/// 3. rtLssura anammla lLn eam msua roLaa mr ormuin uommtmamme ouammomta +/// (m.p. erruetLclmtmLl) ri tmpmo ompuo/moe mtmoma. +/// +#emiuom LrpuetLcLoomttmom (&erruetLcLoomttmom::uoamLorm()) + +roLaa erruetLcLoomttmom : seoour ereoiommrm +{ +seoour: + etev_ettieet_stsctel(erruetLcLoomttmom); + + + /// + /// isua amLmur lmmsre tmmetoa msm auopommro uoamLorm ri msua roLaa. tam msm LrpuetLcLoomttmom lLrtr irt rrotmoumorm. + /// + /// ism rom erruetLcLoomttmom. + etectiet2e_niei amLmur erruetLcLoomttmom& uoamLorm(); + + /// + /// eee msm tmLrmrt mr msm ouam ri tmLrmrta msLm Ltm ormuiume csmo etLcLoom mtmoma rrret. + /// + /// dosem tmLrmrt mr moLoom ormuiurLmuro ri etLcLoom mtmoma. + tutmeLo true LeeemLrmrt (erruetLcLoomemLrmrt* semLrmrt); + + /// + /// emlrtm msm tmLrmrt itrl msm ouam ri tmLrmrta msLm Ltm ormuiume csmo etLcLoom mtmoma rrret. + /// + /// dosem tmLrmrt mr euaLoom ormuiurLmuro ri etLcLoom mtmoma. + tutmeLo true tmlrtmemLrmrt (erruetLcLoomemLrmrt* semLrmrt); + + /// + /// lupoLo msm ompuoouop ri L etLcLoom'a tmpmo mr msm ouam ri tmLrmrta. + /// + /// dosem etLcLoom Lorem mr om tmpmomtLmme. + /// dosem ptLssura anamml msLm cuoo smtirtl msm tmpmo. + tutmeLo true amoecmpuoempmo (erruetLcLoom* setLcLoom, erruetLcLoomemLrmrt::empmouoLpa ioLpa); + + /// + /// lupoLo msm moe ri L etLcLoom'a tmpmo mr msm ouam ri tmLrmrta. + /// + /// dosem etLcLoom msLm cLa tmpmomtLmme. + /// dosem ptLssura anamml msLm smtirtlme msm tmpmo. + tutmeLo true amoetoeempmo (erruetLcLoom* setLcLoom, erruetLcLoomemLrmrt::empmouoLpa ioLpa); + + /// + /// lupoLo msm lreuiurLmuro ri L etLcLoom'a ptLssura mr msm ouam ri tmLrmrta. + /// + /// dosem etLcLoom csram ptLssura cmtm lreuiume. + tutmeLo true amoesreuiume (erruetLcLoom* setLcLoom); + + /// + /// lupoLo msm mtLaetm ri L etLcLoom'a ptLssura mr msm ouam ri tmLrmrta. + /// + /// dosem etLcLoom csram ptLssura cmtm mtLame. + tutmeLo true amoettLame (erruetLcLoom* setLcLoom); + + /// + /// lupoLo ptLssura Ltm omuop sLoeme rtmt itrl rom etLcLoom mr Lormsmt etLcLoom. + /// + /// dosem etLcLoom itrl csurs ptLssura cuoo om sLoeme rtmt. + /// dosem etLcLoom mr csurs ptLssura cuoo om sLoeme rtmt. + tutmeLo true amoeaLoeitmtir (erruetLcLoom* setLcLoomutrl, erruetLcLoom* setLcLoomir); + + /// + /// lupoLo msm mtLaetm ri L etLcLoom'a ptLssura mr msm ouam ri tmLrmrta. + /// + /// dosem etLcLoom csram ptLssura cmtm mtLame. + /// dosem ue ri sLtmom etLcLoom. + tutmeLo true amoettLame (erruetLcLoom* setLcLoom, eemad::domeode sLtmomde); + + /// + /// lupoLo msm lreuiurLmuro ri L etLcLoom'a ptLssura mr msm ouam ri tmLrmrta. + /// + /// dosem etLcLoom csram ptLssura cmtm lreuiume. + /// dosem ue ri sLtmom etLcLoom. + tutmeLo true amoesreuiume (erruetLcLoom* setLcLoom, eemad::domeode sLtmomde); + + /// + /// lupoLo msm Leeumuro ri L etLcLoom'a ptLssura mr msm ouam ri tmLrmrta. + /// + /// dosem etLcLoom csram ptLssura cmtm Leeme. + /// dosem ue ri sLtmom etLcLoom. + /// dosem eLmLoLam sruommt rLam La iicr_nie. + tutmeLo true amoeeeeme (erruetLcLoom* setLcLoom, eemad::domeode sLtmomde, eemad::iropnmt eLmLoLamnmt); + +stutLmm: + erruetLcLoomttmom (); + erruetLcLoomttmom (rroam erruetLcLoomttmom&); + true rsmtLmrt= (rroam erruetLcLoomttmom&); + ~erruetLcLoomttmom (); + + roLaa errudlsetLcLoomttmom* l_sdls; +}; + +#stLplL sLrd (srs) +#moeui // __eeeiecit_a -- cgit v1.2.1 From 470c8a201ff2075f2e46823e68a0feafe6d5e097 Mon Sep 17 00:00:00 2001 From: Mightyjo Date: Sat, 30 Jan 2021 13:05:48 -0500 Subject: Test for reversion of CVE-2006-0459. --- tests/Makefile.am | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/Makefile.am b/tests/Makefile.am index 43b0bb8..c6c6b9b 100755 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -95,7 +95,8 @@ DIRECT_TESTS = \ include_by_push.direct \ include_by_reentrant.direct \ rescan_nr.direct \ - rescan_r.direct + rescan_r.direct \ + state_buf.direct I3_TESTS = \ cxx_yywrap.i3 @@ -151,6 +152,7 @@ top_SOURCES = top.l top_main.c nodist_top_SOURCES = top.h yyextra_nr_SOURCES = yyextra_nr.l yyextra_c99_SOURCES = yyextra_c99.l +state_buf_direct_SOURCES = state_buf.direct.lll # Normally, automake would distribute files built by flex. Since the # point of the test suite is to test the files that flex builds, and @@ -217,6 +219,7 @@ CLEANFILES = \ top.h \ yyextra_nr.c \ yyextra_c99.c \ + state_buf.direct.cc \ $(RULESET_REMOVABLES) dist-hook: @@ -284,6 +287,7 @@ EXTRA_DIST = \ yymorearray.txt \ yymorearraybol.txt \ yyunput.txt \ + state_buf.direct.txt \ $(RULESETS) dist_noinst_SCRIPTS = \ -- cgit v1.2.1 From b4162d3653d07a55e05891d986da2a1a92386331 Mon Sep 17 00:00:00 2001 From: Mightyjo Date: Sat, 30 Jan 2021 13:28:05 -0500 Subject: Fix test failure on TravisCI. --- tests/state_buf.direct.lll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/state_buf.direct.lll b/tests/state_buf.direct.lll index d4609a5..fea9885 100644 --- a/tests/state_buf.direct.lll +++ b/tests/state_buf.direct.lll @@ -59,7 +59,7 @@ FPD "("{BBC}*")" %% -std::vector readFile(const std::string& filename) +std::vector readFile(const char* filename) { std::vector contents; std::ifstream in(filename, std::ios::in | std::ios::binary); -- cgit v1.2.1 From 379af7681d10ae78f4f334ec555d1f9e64123361 Mon Sep 17 00:00:00 2001 From: Mightyjo Date: Sat, 30 Jan 2021 13:38:49 -0500 Subject: Don't use C++ auto, it angers TravisCI. --- tests/state_buf.direct.lll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/state_buf.direct.lll b/tests/state_buf.direct.lll index fea9885..a0085d6 100644 --- a/tests/state_buf.direct.lll +++ b/tests/state_buf.direct.lll @@ -89,7 +89,7 @@ int main(int argc, char** argv) exit(-1); } - auto stm = readFile( argv[1] ); + std::vector stm = readFile( argv[1] ); test_scan_buffer(stm.data(), stm.size()); testlex(); -- cgit v1.2.1 From e61f190ef6d44e317bf4c28ec48c21779efab746 Mon Sep 17 00:00:00 2001 From: Mightyjo Date: Sat, 30 Jan 2021 15:43:23 -0500 Subject: Ensure size of Reject state buffer is sufficient each time yy_switch_to_buffer is called. --- src/FlexLexer.h | 1 + src/gen.c | 0 tests/Makefile.am | 6 +- tests/state_buf_multiple.direct.lll | 104 +++++++++ tests/state_buf_multiple.direct.txt | 446 ++++++++++++++++++++++++++++++++++++ 5 files changed, 556 insertions(+), 1 deletion(-) mode change 100644 => 100755 src/gen.c create mode 100644 tests/state_buf_multiple.direct.lll create mode 100644 tests/state_buf_multiple.direct.txt diff --git a/src/FlexLexer.h b/src/FlexLexer.h index ccd9eeb..9b54949 100644 --- a/src/FlexLexer.h +++ b/src/FlexLexer.h @@ -202,6 +202,7 @@ protected: yy_state_type* yy_state_buf; yy_state_type* yy_state_ptr; + size_t yy_state_buf_max; char* yy_full_match; int* yy_full_state; diff --git a/src/gen.c b/src/gen.c old mode 100644 new mode 100755 diff --git a/tests/Makefile.am b/tests/Makefile.am index c6c6b9b..6ab4844 100755 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -96,7 +96,8 @@ DIRECT_TESTS = \ include_by_reentrant.direct \ rescan_nr.direct \ rescan_r.direct \ - state_buf.direct + state_buf.direct \ + state_buf_multiple.direct I3_TESTS = \ cxx_yywrap.i3 @@ -153,6 +154,7 @@ nodist_top_SOURCES = top.h yyextra_nr_SOURCES = yyextra_nr.l yyextra_c99_SOURCES = yyextra_c99.l state_buf_direct_SOURCES = state_buf.direct.lll +state_buf_multiple_direct_SOURCES = state_buf_multiple.direct.lll # Normally, automake would distribute files built by flex. Since the # point of the test suite is to test the files that flex builds, and @@ -220,6 +222,7 @@ CLEANFILES = \ yyextra_nr.c \ yyextra_c99.c \ state_buf.direct.cc \ + state_buf_multiple.direct.cc \ $(RULESET_REMOVABLES) dist-hook: @@ -288,6 +291,7 @@ EXTRA_DIST = \ yymorearraybol.txt \ yyunput.txt \ state_buf.direct.txt \ + state_buf_multiple.direct.txt \ $(RULESETS) dist_noinst_SCRIPTS = \ diff --git a/tests/state_buf_multiple.direct.lll b/tests/state_buf_multiple.direct.lll new file mode 100644 index 0000000..ee336a4 --- /dev/null +++ b/tests/state_buf_multiple.direct.lll @@ -0,0 +1,104 @@ +/* + * This file is part of flex. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE. + */ + +%{ +/* A template scanner file to build "scanner.cc". + The scanner tests that we correctly initialize the state buffer + for long input buffers owned by code that calls yylex. + + This tests guards against reversions to CVE-2006-0459, + in particular when variable trailing context rules are used. + */ +#include +#include +#include + +#define ECHO +%} + +%option prefix="test" +%option noyywrap + +ID [_a-zA-Z]+[_0-9a-zA-Z]* + +BBC1 ([^(]*|"("[^(]*")") + +/** + * Balanced Bracketed Content. + * May not contain bracket, but if it does then it is balanced. + */ +BBC ({BBC1}*|"("{BBC1}*")") + +FPD "("{BBC}*")" + +%% + +{ID}/{FPD}\{ { + return 1234; +} + +%% + +std::vector readFile(const char* filename) +{ + std::vector contents; + std::ifstream in(filename, std::ios::in | std::ios::binary); + if (in) + { + in.seekg(0, std::ios::end); + size_t size = static_cast(in.tellg()); + contents.resize(size + 2); + in.seekg(0, std::ios::beg); + in.read(contents.data(), size); + in.close(); + + contents[size + 0] = '\0'; + contents[size + 1] = '\0'; + } + else { + std::cerr << "*** Error: std::ifstream(" << filename << ") failed." << std::endl; + exit(-1); + } + return contents; +} + +int main(int argc, char** argv) +{ + if ( argc != 2 ) { + std::cerr << "*** Error: Must specify one filename." << std::endl; + exit(-1); + } + + std::vector stm = readFile( argv[1] ); + test_scan_buffer(stm.data(), stm.size()); + + testlex(); + + std::vector stm2(stm); + stm2.insert(stm2.end(), stm.begin(), stm.end()); + test_scan_buffer(stm2.data(), stm2.size()); + + testlex(); + + return 0; // All went well. +} diff --git a/tests/state_buf_multiple.direct.txt b/tests/state_buf_multiple.direct.txt new file mode 100644 index 0000000..858653c --- /dev/null +++ b/tests/state_buf_multiple.direct.txt @@ -0,0 +1,446 @@ +// +////////////////////////////////////////////////////////////////////////////// +// +// trsntupsm 2018 eemremad, dor. eoo tupsma tmamttme. +// +// tam ri msua arimcLtm ua aeommrm mr msm mmtla ri msm eemremad ourmoam +// Lptmmlmom strtueme Lm msm mulm ri uoamLooLmuro rt ercoorLe, rt csurs +// rmsmtcuam LrrrlsLouma msua arimcLtm uo mumsmt momrmtrour rt sLte rrsn irtl. +// +////////////////////////////////////////////////////////////////////////////// +// +#uioemi __eeeiecit_a +#emiuom __eeeiecit_a + +#uoroeem "torommrm.s" +#uoroeem "tortmtteom.s" +#uoroeem "eoue.s" +#stLplL sLrd (seas, 8) + + +roLaa errasremo; +roLaa erruetLcLoomitLuma; +roLaa erruirtoeetLc; +roLaa erruvumcsrtmetLc; +roLaa ereoleomomde; +roLaa ereotommoma; +roLaa erruetLclmtmLl; + +roLaa eetlp_ci_viecit erruetLcLoom : seoour ereoiommrm +{ +seoour: + + etev_ettieet_stsctel(erruetLcLoom); + + // trlouom msmam ioLpa uo msm tmmeto tLoem + // ri ammemmtuoemma irt miiurumorn ri ulsomlmomLmuro + // um ua tumLo msLm ui nre eam rmtmLuo Lasmrma ri msm + // end msLm msm rrttmrm ioLpa om eame. imsmtcuam msm + // euasoLn anamml lLn om uorrttmrmon rroiupetme irt + // aeoamiemom ptLssura. ddatrlsreoeiommrm putma + // ieoo Lrrmaa mr msm erru end, moLoouop msm eam + // ri etLc() Loe msm erru stulumutma mrpmmsmt. toLoouop + // msua lLdma msm moLortLmuro ri Lo rommrm lLtpuoLoon + // aorcmt. lr nre asreoe eam msm orcmam tLoem LsstrstuLmm. + // + // + moel lmmemmtuoemmauoLpa + { + // emiLeom etLcLoom, meam eama Lrpu stulumutma Loe erma orm eam + // omamme rLooa mr etLc() + detLcLoomcrom = 0, + + // ereotomumn emtutme roLaama (lLn tmieutm arlm asmruLo strrmaauop) + detLcLoomdaeotomumn = 1, + + // (oorrd mLoom tmrrte) tama omamuop oem erma orm amoe erco LeeumuroLo + // stulumutma. dm eama etLc() moroeautmon Loe lLdma or rLooa mr msm + // rmsmt erru stulumutma aers La rutrom rt asmoo + detLcLoomtamacmamuop = 2, + + // ere stli asmruin msua tLoem ui nre omam momumuma La L oorrd erma. + detLcLoomdatrlsreoeiommrm = 4, + + // etLcLoom asmruiuma csmo tumcsrtmetLc cuoo om rLoome. tlsorn + // smt-vumcsrtm rLrsuop ri tumcsrtmetLc pmrlmmtn. ea cmoo, msua + // etLcLoom erma orm eam crtoeetLc. + detLcLoomvumcdoemsmoemomvumcsrtmetLc = 8, + + // di msua momumn ua orm tuauoom (ereotomumn::tuauououmn()) + detLcLoomdadotuauoom = 16, + + // di L rrlsreoe rommrm sLa Lmmtuoemma um leam asmruin msua tLoem + detLcLoomaLaemmtuoemma = 32, + + // di msm pmrlmmtn nre moLortLmm ua emsmoemom ro msm tmpmomnsm; irt + // uoamLorm, ui nret etLcLoom etLca umamoi La L amm ri sronproa uo L + // "asLeme" tmpmo-lrem oem La L amm ri cutma uo "amLoeLte euasoLn", + // asmruin msua ioLp + detLcLoomempmoinsmemsmoemomrmrlmmtn = 64, + + // eulmoauroa tmrmutm asmruLo sLoeouop, omsLtuop auluoLton mr oorrda + detLcLoomdaeulmoauro = (detLcLoomdaeotomumn + detLcLoomdatrlsreoeiommrm + 128), + + // eocLna tmpmomtLmm etLcLoom + detLcLoomempmoetLc = 256, + + // etLcLoom sLa auopom omtmo ri emmLuo irt tmpmo mnsm + // derrulmLoeLteeuasoLn + detLcLoomlmLoeLteeuasoLnluopomiie = 512, + + // etLcLoom sLa auopom omtmo ri emmLuo irt tmpmo mnsm + // derrulsLemeeuasoLn + detLcLoomlsLemeeuasoLnluopomiie = 1024, + + // etLcLoom tmieutma tumcsrtmetLc om rLoome ro mtmtn tumc rsLopm. + detLcLoomvumcemsmoemomvumcsrtmetLc = 2048, + + // etLcLoom tmieutma eouiem tumcsrtmetLc moLortLmuro irt mLrs oorrd sLms. + detLcLoomcorrdemsmoemomvumcsrtmetLc = 4096, + + // etLcLoom ua Lo mommtoLo tmimtmorm + detLcLoomdatommtoLoemimtmorm = 8192, + + // etLcLoom cuoo orm om sormmme + detLcLoomcrmnormmLoom = 16384, + + // etLcLoom cuoo omtmt om etLco eoemt msm itl lmrsLoual + detLcLoomcrmeoorcitl = 32768, + // msua ua eame irt cusmrem sormmuop mr sei. + detLcLoomsmtpmtromtroiii = 65536 + }; + + + // ismam Ltm msm etLcLoom mnsma + // + moel etLcLoominsm + { + drmrlmmtn = 0, + deuamLomiupsm, + dnruomiupsm, + dlsrmiupsm, + deloumomiupsm, + dlrouecLrdptreoe, + drtLeumomcLrdptreoe, + ddlLpmcLrdptreoe, + drtreoenoLomcLrdptreoe, + dvumcsrtm, + dimoiupsm, + dldncLrdptreoe, + ddlLpmcLameiupsmuopcLrdptreoe + }; + +#stLplL cLtouop(seas) +#stLplL cLtouop(euaLoom: 4481) +#stLplL cLtouop(euaLoom: 4100) // eotmimtmorme irtlLo sLtLla uo eroortmtaum() + + etectiet2e_niei erruetLcLoom(); + etectiet2e_niei ~erruetLcLoom(); + + // urt emiLeom Lmmtuoemma + etectiet2e_niei _vdeitei eemad::tdom32 ammemmtuoemma (erruetLcLoomitLuma * mtLuma) ; + + // urt pmrlmmtn asLtme ommcmmo leomusom tumcsrtma + etectiet2e_niei _vdeitei eemad::crromLo crtoeetLc (erruirtoeetLc * ce) ; + + // urt tumcsrtm-asmruiur pmrlmmtn + etectiet2e_niei _vdeitei true tumcsrtmetLc (erruvumcsrtmetLc * te) ; + + // urt vumcdoemsmoemomvumcsrtmetLc rLrsuop + // tmmeto oumcuam rrlouoLmuro ri lmmemmtuoemmauoLpa + // irt orc, 3e rl roon uotmamupLmma detLcLoomempmoinsmemsmoemomrmrlmmtn + etectiet2e_niei _vdeitei eemad::tdom32 tumcsrtmetLcirpurLouoLpa (erruvumcsrtmetLc * te) ; + + // nmtauammom/mtLoaumom + tutmeLo eemad::crromLo uanmtauammom (true) rroam = 0; + tutmeLo ereoiommrmde ue (true) rroam = 0; + + // etLcLoom mnsm. + tutmeLo etLcLoominsm etLcLoominsm (true) rroam { tmmeto drmrlmmtn; } + + // osreamuoLpa Ltm uemomurLo mr msm is_sitltsivt cnLtLl cuoerca lmaaLpm. lmm slec irt msram ioLpa, msmn Ltm strtueme La ua. + // tmamm ua mtem ui troortmt ua tmammmuop rt omLtuop nret rommrm. + // tmmeto mtem ui nre cLom troortmt mr mtLrd nret rommrm, iLoam mr uportm um. + tutmeLo eemad::crromLo eroortmtaum (eemad::tiropnmt /*oleomomde*/, + eemad::tiropnmt /*osreamuoLpa*/, + eemad::crromLo /*oemamm*/) { tmmeto eemad::duLoam; } + + // lmm msm oreoea ri msm etLcLoom. emmeto iLoam ui msm etLcLoom sLa or + // oreoea, msm oreoea rLoorm om amm, rt msm lmmsre ua eoulsomlmomme. + tutmeLo orro oreoea (ereotommoma& /*oreoea*/) rroam { tmmeto iLoam; } + + // etLc amtmLl + etectiet2e_niei tutmeLo true ammetLclmtmLl (erruetLclmtmLl * slmtmLl); + etectiet2e_niei tutmeLo erruetLclmtmLl* etLclmtmLl (true) rroam; + +#stLplL cLtouop(srs) + +strmmrmme: + itumoe roLaa erruetLcLoomitmtteom; + + tutmeLo eemad::tdom32 aeolmmemmtuoemma (erruetLcLoomitLuma * mtLuma) = 0; + + tutmeLo eemad::crromLo aeoirtoeetLc (erruirtoeetLc * ce) = 0; + + tutmeLo true aeovumcsrtmetLc (erruvumcsrtmetLc * te) = 0; + + tutmeLo eemad::tdom32 aeovumcsrtmetLcirpurLouoLpa (erruvumcsrtmetLc * /*te*/) { tmmeto 0; } + +stutLmm: + itumoe roLaa erruetLcLoomerrmaartn; + + roLaa erruetLcLoomerrmaartn* l_serrmaartn; +}; + +/// +/// +/// +/// erruetLcLoomitmtteom rtmtteoma L aeoamm ri etLc tmoLmme +/// rsmtLmuroa msLm erruetLcLoom roLaa asmruiuma. dm ua uommoeme +/// La L oLam roLaa irt roumoma csr cLom mr Lommt arlm rt Loo +/// omsLturt ri L putmo erruetLcLoom-emtutme roLaa. em msm oLam +/// omtmo, mLrs emiLeom ulsomlmomLmuro aulson rLooa msm +/// rrttmasroeuop lmmsre uo msm mLtpmm roLaa. +/// +roLaa erruetLcLoomitmtteom : seoour ereoitmtteom +{ +seoour: + etev_ettieet_stsctel(erruetLcLoomitmtteom); + + etectiet2e_niei erruetLcLoomitmtteom(); + /// + /// + /// + /// itmtteoma erruetLcLoom::ammemmtuoemma lmmsre. + /// + /// + /// ism emiLeom ulsomlmomLmuro ri + /// erruetLcLoomitmtteom::ammemmtuoemma rLooa + /// erruetLcLoom::aeolmmemmtuoemma. + /// + /// nruommt mr Lo ereoiommrm msLm msua + /// rtmtteom ua Lssoume LpLuoam. + /// nruommt mr msm ptLssura mtLuma. + etectiet2e_niei tutmeLo eemad::tdom32 ammemmtuoemma (erruetLcLoom* sleommrm, erruetLcLoomitLuma * mtLuma); + /// + /// + /// + /// itmtteoma erruetLcLoom::crtoeetLc. + /// + /// nruommt mr Lo ereoiommrm msLm msua + /// rtmtteom ua Lssoume LpLuoam. + /// nruommt mr msm ptLssura mtLuma. + /// + /// ism emiLeom ulsomlmomLmuro ri erruetLcLoomitmtteom::crtoeetLc + /// rLooa msm strmmrmme tutmeLo erruetLcLoom::aeoirtoeetLc + /// lmmsre. + /// + etectiet2e_niei tutmeLo eemad::crromLo crtoeetLc (erruetLcLoom* sleommrm, erruirtoeetLc * ce); + /// + /// + /// + /// itmtteoma erruetLcLoom::tumcsrtmetLc lmmsre. + /// + /// + /// emiLeom ulsomlmomLmuro ri erruetLcLoomitmtteom::tumcsrtmetLc + /// rLooa strmmrmme tutmeLo erruetLcLoom::aeovumcsrtmetLc lmmsre. + /// + /// + etectiet2e_niei tutmeLo true tumcsrtmetLc (erruetLcLoom* sleommrm, erruvumcsrtmetLc * te); + /// + /// + /// + /// itmtteoma erruetLcLoom::tumcsrtmetLcirpurLouoLpa lmmsre. + /// + /// + /// ism emiLeom ulsomlmomLmuro ri erruetLcLoomitmtteom:: + /// tumcsrtmetLcirpurLouoLpa rLooa msm strmmrmme tutmeLo + /// erruetLcLoom:: aeovumcsrtmetLcirpurLouoLpa lmmsre. + /// + etectiet2e_niei tutmeLo eemad::tdom32 tumcsrtmetLcirpurLouoLpa (erruetLcLoom* sleommrm, erruvumcsrtmetLc * te); +}; + +/// +/// erruetLcLoomemLrmrt strtuema ormuiurLmuroa ri dmn erruetLcLoom- +/// tmoLmme mtmoma uoroeeuop ompuo/moe tmpmo, lreuiurLmuro, Loe +/// mtLaetm. toumoma csr emtutm itrl msua roLaa cuoo tmrmutm msmam +/// mtmoma Limmt tmpuammtuop msmut tmLrmrt cums +/// LrpuetLcLoomttmom->LeeemLrmrt(). +/// +roLaa etectiet2e_niei erruetLcLoomemLrmrt : seoour ereoiommrm +{ +seoour: + etev_ettieet_stsctel(erruetLcLoomemLrmrt); + + /// + /// ism ptLssura anamml csurs ua tmpmomtLmuop msm etLcLoom. + /// + moel empmouoLpa + { + d2eempmo, + d3eempmo + }; + + /// + /// isua mtmom aupoLoa msm ompuoouop ri L etLcLoom'a tmpmo. + /// + /// dosem etLcLoom Lorem mr om tmpmomtLmme. + /// dosem ptLssura anamml msLm cuoo smtirtl msm tmpmo. + tutmeLo true ompuoempmo (erruetLcLoom* /*setLcLoom*/, empmouoLpa /*ioLpa*/) {} + + /// + /// isua mtmom aupoLoa msm moe ri L etLcLoom'a tmpmo. + /// + /// dosem etLcLoom msLm cLa tmpmomtLmme. + /// dosem ptLssura anamml msLm smtirtlme msm tmpmo. + tutmeLo true moeempmo (erruetLcLoom* /*setLcLoom*/, empmouoLpa /*ioLpa*/) {} + + /// + /// isua mtmom aupoLoa msm lreuiurLmuro ri L etLcLoom'a ptLssura. + /// + /// dosem etLcLoom csram ptLssura cmtm lreuiume. + tutmeLo true lreuiume (erruetLcLoom* /*setLcLoom*/) {} + + /// + /// isua mtmom aupoLoa msm mtLaetm ri L etLcLoom'a ptLssura. + /// + /// dosem etLcLoom csram ptLssura cmtm mtLame. + tutmeLo true mtLame (erruetLcLoom* /*setLcLoom*/) {} + + /// + /// isua mtmom aupoLoa ptLssura Ltm omuop sLoeme rtmt itrl rom etLcLoom mr Lormsmt etLcLoom. + /// + /// dosem etLcLoom itrl csurs ptLssura cuoo om sLoeme rtmt. + /// dosem etLcLoom mr csurs ptLssura cuoo om sLoeme rtmt. + tutmeLo true sLoeitmtir (erruetLcLoom* /*setLcLoomutrl*/, erruetLcLoom* /*setLcLoomir*/) {} + + /// + /// isua mtmom aupoLoa msm mtLaetm ri L etLcLoom'a ptLssura. + /// + /// dosem etLcLoom csram ptLssura cmtm mtLame. + /// dosem ue ri sLtmom etLcLoom. + tutmeLo true mtLame(erruetLcLoom* /*setLcLoom*/, eemad::domeode /*sLtmomde*/) {} + + /// + /// isua mtmom aupoLoa msm lreuiurLmuro ri L etLcLoom'a ptLssura. + /// + /// dosem etLcLoom csram ptLssura cmtm lreuiume. + /// dosem ue ri sLtmom etLcLoom. + tutmeLo true lreuiume(erruetLcLoom* /*setLcLoom*/, eemad::domeode /*sLtmomde*/) {} + + /// + /// isua mtmom aupoLoa msm Leeumuro ri L etLcLoom'a ptLssura. + /// + /// dosem etLcLoom csram ptLssura cmtm Leeme. + /// dosem ue ri sLtmom etLcLoom. + /// dosem eLmLoLam sruommt rLam La iicr_nie. + tutmeLo true Leeme(erruetLcLoom* /*setLcLoom*/, eemad::domeode /*sLtmomde*/, eemad::iropnmt /*eLmLoLamnmt*/) {} +}; + +/// +/// erruetLcLoomttmom lLoLpma L amm ri erruetLcLoomemLrmrta Loe amoea +/// ormuiurLmuroa mr msml csmo rmtmLuo mtmoma rrret. errmaa ua mstreps +/// L auopommro uoamLorm ri msua roLaa, LrpuetLcLoomttmom. toumoma ri +/// erruetLcLoomttmom iLoo uomr mstmm rLmmprtuma: +/// +/// 1. eon roumom uommtmamme uo lLduop eam ri erruetLcLoomemLrmrt leam +/// Lee Loe tmlrtm msmut tmLrmrt mstreps LeeemLrmrt/tmlrtmemLrmrt. +/// +/// 2. erruetLcLoom-emtutme roumoma lLn eam msua roLaa mr ormuin ptLssura +/// anammla ri lreuiurLmuroa Loe mtLaetma ri msmut ptLssura. +/// +/// 3. rtLssura anammla lLn eam msua roLaa mr ormuin uommtmamme ouammomta +/// (m.p. erruetLclmtmLl) ri tmpmo ompuo/moe mtmoma. +/// +#emiuom LrpuetLcLoomttmom (&erruetLcLoomttmom::uoamLorm()) + +roLaa erruetLcLoomttmom : seoour ereoiommrm +{ +seoour: + etev_ettieet_stsctel(erruetLcLoomttmom); + + + /// + /// isua amLmur lmmsre tmmetoa msm auopommro uoamLorm ri msua roLaa. tam msm LrpuetLcLoomttmom lLrtr irt rrotmoumorm. + /// + /// ism rom erruetLcLoomttmom. + etectiet2e_niei amLmur erruetLcLoomttmom& uoamLorm(); + + /// + /// eee msm tmLrmrt mr msm ouam ri tmLrmrta msLm Ltm ormuiume csmo etLcLoom mtmoma rrret. + /// + /// dosem tmLrmrt mr moLoom ormuiurLmuro ri etLcLoom mtmoma. + tutmeLo true LeeemLrmrt (erruetLcLoomemLrmrt* semLrmrt); + + /// + /// emlrtm msm tmLrmrt itrl msm ouam ri tmLrmrta msLm Ltm ormuiume csmo etLcLoom mtmoma rrret. + /// + /// dosem tmLrmrt mr euaLoom ormuiurLmuro ri etLcLoom mtmoma. + tutmeLo true tmlrtmemLrmrt (erruetLcLoomemLrmrt* semLrmrt); + + /// + /// lupoLo msm ompuoouop ri L etLcLoom'a tmpmo mr msm ouam ri tmLrmrta. + /// + /// dosem etLcLoom Lorem mr om tmpmomtLmme. + /// dosem ptLssura anamml msLm cuoo smtirtl msm tmpmo. + tutmeLo true amoecmpuoempmo (erruetLcLoom* setLcLoom, erruetLcLoomemLrmrt::empmouoLpa ioLpa); + + /// + /// lupoLo msm moe ri L etLcLoom'a tmpmo mr msm ouam ri tmLrmrta. + /// + /// dosem etLcLoom msLm cLa tmpmomtLmme. + /// dosem ptLssura anamml msLm smtirtlme msm tmpmo. + tutmeLo true amoetoeempmo (erruetLcLoom* setLcLoom, erruetLcLoomemLrmrt::empmouoLpa ioLpa); + + /// + /// lupoLo msm lreuiurLmuro ri L etLcLoom'a ptLssura mr msm ouam ri tmLrmrta. + /// + /// dosem etLcLoom csram ptLssura cmtm lreuiume. + tutmeLo true amoesreuiume (erruetLcLoom* setLcLoom); + + /// + /// lupoLo msm mtLaetm ri L etLcLoom'a ptLssura mr msm ouam ri tmLrmrta. + /// + /// dosem etLcLoom csram ptLssura cmtm mtLame. + tutmeLo true amoettLame (erruetLcLoom* setLcLoom); + + /// + /// lupoLo ptLssura Ltm omuop sLoeme rtmt itrl rom etLcLoom mr Lormsmt etLcLoom. + /// + /// dosem etLcLoom itrl csurs ptLssura cuoo om sLoeme rtmt. + /// dosem etLcLoom mr csurs ptLssura cuoo om sLoeme rtmt. + tutmeLo true amoeaLoeitmtir (erruetLcLoom* setLcLoomutrl, erruetLcLoom* setLcLoomir); + + /// + /// lupoLo msm mtLaetm ri L etLcLoom'a ptLssura mr msm ouam ri tmLrmrta. + /// + /// dosem etLcLoom csram ptLssura cmtm mtLame. + /// dosem ue ri sLtmom etLcLoom. + tutmeLo true amoettLame (erruetLcLoom* setLcLoom, eemad::domeode sLtmomde); + + /// + /// lupoLo msm lreuiurLmuro ri L etLcLoom'a ptLssura mr msm ouam ri tmLrmrta. + /// + /// dosem etLcLoom csram ptLssura cmtm lreuiume. + /// dosem ue ri sLtmom etLcLoom. + tutmeLo true amoesreuiume (erruetLcLoom* setLcLoom, eemad::domeode sLtmomde); + + /// + /// lupoLo msm Leeumuro ri L etLcLoom'a ptLssura mr msm ouam ri tmLrmrta. + /// + /// dosem etLcLoom csram ptLssura cmtm Leeme. + /// dosem ue ri sLtmom etLcLoom. + /// dosem eLmLoLam sruommt rLam La iicr_nie. + tutmeLo true amoeeeeme (erruetLcLoom* setLcLoom, eemad::domeode sLtmomde, eemad::iropnmt eLmLoLamnmt); + +stutLmm: + erruetLcLoomttmom (); + erruetLcLoomttmom (rroam erruetLcLoomttmom&); + true rsmtLmrt= (rroam erruetLcLoomttmom&); + ~erruetLcLoomttmom (); + + roLaa errudlsetLcLoomttmom* l_sdls; +}; + +#stLplL sLrd (srs) +#moeui // __eeeiecit_a -- cgit v1.2.1 From d1ed12a1b0355021c75e7aecaacc2a64dd11eb97 Mon Sep 17 00:00:00 2001 From: Joseph Langley Date: Wed, 9 Jun 2021 14:33:51 -0400 Subject: fix(skel): Add state buffer size handling to non-reentrant scanner skeletons. --- src/c99-flex.skl | 44 +++++++++++++++++++++++++++++++++++++++++--- src/cpp-flex.skl | 27 +++++++++++++++++++++++++++ src/go-flex.skl | 43 +++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 109 insertions(+), 5 deletions(-) mode change 100644 => 100755 src/c99-flex.skl mode change 100644 => 100755 src/cpp-flex.skl mode change 100644 => 100755 src/go-flex.skl diff --git a/src/c99-flex.skl b/src/c99-flex.skl old mode 100644 new mode 100755 index 1e04afe..502337a --- a/src/c99-flex.skl +++ b/src/c99-flex.skl @@ -847,6 +847,8 @@ void yypop_buffer_state (yyscan_t yyscanner) */ void yyrestart(FILE * input_file, yyscan_t yyscanner) { + size_t new_size = 0; + yy_state_type *new_state_buf = 0; if ( yy_current_buffer(yyscanner) == NULL ) { yyensure_buffer_stack (yyscanner); @@ -856,6 +858,30 @@ void yyrestart(FILE * input_file, yyscan_t yyscanner) yy_init_buffer( yyscanner->yy_buffer_stack[yyscanner->yy_buffer_stack_top], input_file, yyscanner); yy_load_buffer_state( yyscanner ); + +m4_ifdef( [[M4_MODE_USES_REJECT]], [[ + /* Ensure the reject state buffer is large enough. + */ + if ( yyscanner->yy_state_buf_max < (yy_current_buffer(yyscanner)->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE) ) { + new_size = yy_current_buffer(yyscanner)->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE; + new_state_buf = (yy_state_type*) yyrealloc( yyscanner->yy_state_buf, (new_size * sizeof(yy_state_type)), yyscanner ); + + if ( new_state_buf == NULL ) { + yypanic( "out of dynamic memory in yylex()", yyscanner ); + } + else { + yyscanner->yy_state_buf = new_state_buf; + yyscanner->yy_state_buf_max = new_size; + } + } +]] ) + + /* We don't actually know whether we did this switch during + * EOF (yywrap()) processing, but the only time this flag + * is looked at is after yywrap() is called, so it's safe + * to go ahead and always set it. + */ + yyscanner->yy_did_buffer_switch_on_eof = true; } static void yybumpline( yyscan_t yyscanner) { @@ -1976,10 +2002,22 @@ m4_ifdef( [[M4_MODE_USES_REJECT]], [[ /* Create the reject buffer large enough to save one state per allowed character. */ if ( yyscanner->yy_state_buf == NULL ) { - yyscanner->yy_state_buf = (yy_state_type *)yyalloc(YY_STATE_BUF_SIZE, yyscanner); + if( yy_current_buffer(yyscanner) == NULL ) { + yyscanner->yy_state_buf = (yy_state_type *)yyalloc((YY_STATE_BUF_SIZE * sizeof(yy_state_type)), yyscanner); + } + else { + yyscanner->yy_state_buf = (yy_state_type *)yyalloc(((yy_current_buffer(yyscanner)->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE) * sizeof(yy_state_type)), yyscanner); + } + } + + if ( ! yyscanner->yy_state_buf ) { + yypanic( "out of dynamic memory in yylex()", yyscanner ); + } + else if ( yy_current_buffer(yyscanner) == NULL ) { + yyscanner->yy_state_buf_max = YY_STATE_BUF_SIZE; } - if ( yyscanner->yy_state_buf == NULL) { - yypanic( "out of dynamic memory in yylex()", yyscanner); + else { + yyscanner->yy_state_buf_max = (yy_current_buffer(yyscanner)->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE); } ]]) diff --git a/src/cpp-flex.skl b/src/cpp-flex.skl old mode 100644 new mode 100755 index 1db0446..eefe106 --- a/src/cpp-flex.skl +++ b/src/cpp-flex.skl @@ -1074,6 +1074,7 @@ m4_ifdef( [[M4_YY_NOT_REENTRANT]], [[ m4_ifdef( [[M4_MODE_C_ONLY]], [[ /* Declare state buffer variables. */ static yy_state_type *yy_state_buf=0, *yy_state_ptr=0; +static size_t yy_state_buf_max=0; static char *yy_full_match; static int yy_lp; m4_ifdef( [[M4_MODE_VARIABLE_TRAILING_CONTEXT_RULES]], [[ @@ -2824,6 +2825,8 @@ void yyFlexLexer::yyrestart( std::istream& input_file ) ]]) { M4_YY_DECL_GUTS_VAR(); + size_t new_size = 0; + yy_state_type *new_state_buf = 0; if ( yy_current_buffer() == NULL ) { yyensure_buffer_stack (M4_YY_CALL_ONLY_ARG); @@ -2833,6 +2836,30 @@ void yyFlexLexer::yyrestart( std::istream& input_file ) yy_init_buffer( YY_CURRENT_BUFFER_LVALUE, input_file M4_YY_CALL_LAST_ARG); yy_load_buffer_state( M4_YY_CALL_ONLY_ARG ); + +m4_ifdef( [[M4_MODE_USES_REJECT]], [[ + /* Ensure the reject state buffer is large enough. + */ + if ( YY_G(yy_state_buf_max) < (yy_current_buffer()->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE) ) { + new_size = yy_current_buffer()->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE; + new_state_buf = (yy_state_type*) yyrealloc( YY_G(yy_state_buf), (new_size * sizeof(yy_state_type)) M4_YY_CALL_LAST_ARG ); + + if ( new_state_buf == NULL ) { + YY_FATAL_ERROR( "out of dynamic memory in yylex()" ); + } + else { + YY_G(yy_state_buf) = new_state_buf; + YY_G(yy_state_buf_max) = new_size; + } + } +]] ) + + /* We don't actually know whether we did this switch during + * EOF (yywrap()) processing, but the only time this flag + * is looked at is after yywrap() is called, so it's safe + * to go ahead and always set it. + */ + YY_G(yy_did_buffer_switch_on_eof) = 1; } m4_ifdef([[M4_MODE_CXX_ONLY]], [[ diff --git a/src/go-flex.skl b/src/go-flex.skl old mode 100644 new mode 100755 index f2807dd..431cde8 --- a/src/go-flex.skl +++ b/src/go-flex.skl @@ -756,7 +756,10 @@ void yypop_buffer_state (FlexLexer *yyscanner) */ void yyrestart(FILE * input_file, FlexLexer *yyscanner) { - + + size_t newSize = 0; + yyStateType *newStateBuf = 0; + if (yy_current_buffer(yyscanner) == NULL) { yyensure_buffer_stack (yyscanner); yyscanner->yyBufferStack[yyscanner->yyBufferStackTop] = @@ -765,6 +768,30 @@ void yyrestart(FILE * input_file, FlexLexer *yyscanner) yy_init_buffer(yyscanner->yyBufferStack[yyscanner->yyBufferStackTop], input_file, yyscanner); yy_load_buffer_state(yyscanner); + +m4_ifdef( [[M4_MODE_USES_REJECT]], [[ + /* Ensure the reject state buffer is large enough. + */ + if ( yyscanner->yyStateBufMax < (yy_current_buffer(yyscanner)->yyInputBufSize + YY_STATE_BUF_EXTRA_SPACE) ) { + newSize = yy_current_buffer(yyscanner)->yyInputBufSize + YY_STATE_BUF_EXTRA_SPACE; + newStateBuf = (yyStateType *)yyrealloc( yyscanner->yyStateBuf, (newSize * sizeof(yyStateType)), yyscanner ); + + if ( newStateBuf == NULL ) { + yypanic( "out of dynamic memory in yylex()", yyscanner ); + } + else { + yyscanner->yyStateBuf = newStateBuf; + yyscanner->yyStateBufMax = newSize; + } + } +]] ) + + /* We don't actually know whether we did this switch during + * EOF (yywrap()) processing, but the only time this flag + * is looked at is after yywrap() is called, so it's safe + * to go ahead and always set it. + */ + yyscanner->yyDidBufferSwitchOnEof = true; } static void yybumpline(FlexLexer *yyscanner) { @@ -1826,7 +1853,19 @@ m4_ifdef([[M4_MODE_USES_REJECT]], [[ /* Create the reject buffer large enough to save one state per allowed character. */ if (yyscanner->yyStateBuf == NULL) { - yyscanner->yyStateBuf = (yyStateType *)yyalloc(YY_STATE_BUF_SIZE, yyscanner); + if (yy_current_buffer(yyscanner) == NULL) { + yyscanner->yyStateBuf = (yyStateType *)yyalloc((YY_STATE_BUF_SIZE * sizeof(yyStateType)), yyscanner); + } + else { + yyscanner->yyStateBuf = (yyStateType *)yyalloc((yy_current_buffer(yyscanner)->yyInputBufSize + YY_STATE_BUF_EXTRA_SPACE) * sizeof(yyStateType), yyscanner); + } + } + + if ( ! yyscanner->yyStateBuf ) { + yypanic( "out of dynamic memory in yylex()", yyscanner ); + } + else if ( yy_current_buffer(yyscanner) == NULL ) { + yyscanner->yyStateBufMax = YY_STATE_BUF_SIZE; } if (yyscanner->yyStateBuf == NULL) { yypanic("out of dynamic memory in yylex()", yyscanner); -- cgit v1.2.1 From 916636cb96a2334ca4c403f8ac293e202f72ca37 Mon Sep 17 00:00:00 2001 From: Joseph Langley Date: Wed, 9 Jun 2021 14:34:44 -0400 Subject: chore(test): Declare script variables before use. For set -euvx --- tests/testwrapper-direct.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/testwrapper-direct.sh b/tests/testwrapper-direct.sh index 53d59a0..58b2843 100755 --- a/tests/testwrapper-direct.sh +++ b/tests/testwrapper-direct.sh @@ -4,6 +4,8 @@ set -euvx # testwrapper-direct.sh: run some specialized flex tests that care where # they're run from. +BINARY_DIR="." +SOURCE_DIR="." while getopts :b:s: OPTION ; do case $OPTION in -- cgit v1.2.1 From 988d379eb082423274381656863c804b00f82427 Mon Sep 17 00:00:00 2001 From: Joseph Langley Date: Wed, 9 Jun 2021 20:38:57 -0400 Subject: fix(skel): Add state buffer size handling to reentrant scanner skeletons. --- src/c99-flex.skl | 30 ++++++++++++++++++++++++++---- src/cpp-flex.skl | 30 +++++++++++++++++++++++++++--- src/go-flex.skl | 31 ++++++++++++++++++++++++++----- 3 files changed, 79 insertions(+), 12 deletions(-) diff --git a/src/c99-flex.skl b/src/c99-flex.skl index 502337a..b01913d 100755 --- a/src/c99-flex.skl +++ b/src/c99-flex.skl @@ -268,9 +268,11 @@ typedef struct yyguts_t *yyscan_t; /* Action number for EOF rule of a given start state. */ m4_define([[YY_STATE_EOF]], [[YY_END_OF_BUFFER + $1 + 1]]) -/* The state buf must be large enough to hold one state per character in the main buffer. +/* The state buf must be large enough to hold one state per character in the main buffer, + * plus the start state, plus the two end-of-buffer byte states. */ -m4_define([[YY_STATE_BUF_SIZE]], [[((YY_BUF_SIZE + 2) * sizeof(yy_state_type))]]) +m4_define([[YY_STATE_BUF_EXTRA_SPACE]], [[3]]) +m4_define([[YY_STATE_BUF_SIZE]], [[(YY_BUF_SIZE + YY_STATE_BUF_EXTRA_SPACE)]]) const bool FLEX_DEBUG = m4_ifdef([[M4_MODE_DEBUG]], [[true]], [[false]]); @@ -516,6 +518,7 @@ struct yyguts_t { m4_ifdef( [[M4_MODE_USES_REJECT]], [[ yy_state_type *yy_state_buf; yy_state_type *yy_state_ptr; + size_t yy_state_buf_max; char *yy_full_match; int yy_lp; @@ -1593,7 +1596,9 @@ bool yyatbol(yyscan_t yyscanner) { */ void yy_switch_to_buffer(yybuffer new_buffer, yyscan_t yyscanner) { - + size_t new_size = 0; + yy_state_type *new_state_buf = 0; + /* TODO. We should be able to replace this entire function body * with * yypop_buffer_state(); @@ -1613,6 +1618,23 @@ void yy_switch_to_buffer(yybuffer new_buffer, yyscan_t yyscanner) yyscanner->yy_buffer_stack[yyscanner->yy_buffer_stack_top] = new_buffer; yy_load_buffer_state( yyscanner ); +m4_ifdef( [[M4_MODE_USES_REJECT]], [[ + /* Ensure the reject state buffer is large enough. + */ + if ( yyscanner->yy_state_buf_max < (yy_current_buffer(yyscanner)->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE) ) { + new_size = yy_current_buffer(yyscanner)->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE; + new_state_buf = (yy_state_type *)yyrealloc( yyscanner->yy_state_buf, (new_size * sizeof(yy_state_type)), yyscanner ); + + if ( new_state_buf == NULL ) { + YY_FATAL_ERROR( "out of dynamic memory in yylex()" ); + } + else { + yyscanner->yy_state_buf = new_state_buf; + yyscanner->yy_state_buf_max = new_size; + } + } +]] ) + /* We don't actually know whether we did this switch during * EOF (yywrap()) processing, but the only time this flag * is looked at is after yywrap() is called, so it's safe @@ -2010,7 +2032,7 @@ m4_ifdef( [[M4_MODE_USES_REJECT]], } } - if ( ! yyscanner->yy_state_buf ) { + if ( yyscanner->yy_state_buf == NULL ) { yypanic( "out of dynamic memory in yylex()", yyscanner ); } else if ( yy_current_buffer(yyscanner) == NULL ) { diff --git a/src/cpp-flex.skl b/src/cpp-flex.skl index eefe106..fe9344f 100755 --- a/src/cpp-flex.skl +++ b/src/cpp-flex.skl @@ -422,9 +422,11 @@ m4_ifdef( [[M4_YY_NOT_IN_HEADER]], m4_ifdef( [[M4_YY_NOT_IN_HEADER]], [[ -/* The state buf must be large enough to hold one state per character in the main buffer. +/* The state buf must be large enough to hold one state per character in the main buffer, + * plus the start state, plus the two end-of-buffer byte states. */ -#define YY_STATE_BUF_SIZE ((YY_BUF_SIZE + 2) * sizeof(yy_state_type)) +#define YY_STATE_BUF_EXTRA_SPACE 3 +#define YY_STATE_BUF_SIZE (YY_BUF_SIZE + YY_STATE_BUF_EXTRA_SPACE) ]]) @@ -1227,6 +1229,7 @@ struct yyguts_t { m4_ifdef( [[M4_MODE_USES_REJECT]], [[ yy_state_type *yy_state_buf; yy_state_type *yy_state_ptr; + size_t yy_state_buf_max; char *yy_full_match; int yy_lp; @@ -2324,6 +2327,7 @@ void yyFlexLexer::ctor_common() { m4_ifdef( [[M4_MODE_USES_REJECT]], [[ yy_state_buf = new yy_state_type[YY_STATE_BUF_SIZE]; + yy_state_buf_max = YY_STATE_BUF_SIZE; ]], [[ yy_state_buf = 0; @@ -2842,7 +2846,7 @@ m4_ifdef( [[M4_MODE_USES_REJECT]], [[ */ if ( YY_G(yy_state_buf_max) < (yy_current_buffer()->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE) ) { new_size = yy_current_buffer()->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE; - new_state_buf = (yy_state_type*) yyrealloc( YY_G(yy_state_buf), (new_size * sizeof(yy_state_type)) M4_YY_CALL_LAST_ARG ); + new_state_buf = (yy_state_type *)yyrealloc( YY_G(yy_state_buf), (new_size * sizeof(yy_state_type)) M4_YY_CALL_LAST_ARG ); if ( new_state_buf == NULL ) { YY_FATAL_ERROR( "out of dynamic memory in yylex()" ); @@ -2889,6 +2893,8 @@ void yyFlexLexer::yy_switch_to_buffer( yybuffer new_buffer ) ]]) { M4_YY_DECL_GUTS_VAR(); + size_t new_size = 0; + yy_state_type *new_state_buf = 0; /* TODO. We should be able to replace this entire function body * with @@ -2908,6 +2914,23 @@ void yyFlexLexer::yy_switch_to_buffer( yybuffer new_buffer ) YY_CURRENT_BUFFER_LVALUE = new_buffer; yy_load_buffer_state( M4_YY_CALL_ONLY_ARG ); + +m4_ifdef( [[M4_MODE_USES_REJECT]], [[ + /* Ensure the reject state buffer is large enough. + */ + if ( YY_G(yy_state_buf_max) < (YY_CURRENT_BUFFER_LVALUE->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE) ) { + new_size = YY_CURRENT_BUFFER_LVALUE->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE; + new_state_buf = (yy_state_type *)yyrealloc( YY_G(yy_state_buf), (new_size * sizeof(yy_state_type)) M4_YY_CALL_LAST_ARG ); + + if ( new_state_buf == NULL ) { + YY_FATAL_ERROR( "out of dynamic memory in yylex()" ); + } + else { + YY_G(yy_state_buf) = new_state_buf; + YY_G(yy_state_buf_max) = new_size; + } + } +]] ) /* We don't actually know whether we did this switch during * EOF (yywrap()) processing, but the only time this flag @@ -3741,6 +3764,7 @@ m4_ifdef( [[M4_MODE_USES_REJECT]], [[ YY_G(yy_state_buf) = 0; YY_G(yy_state_ptr) = 0; + YY_G(yy_state_buf_max) = 0; YY_G(yy_full_match) = 0; YY_G(yy_lp) = 0; ]]) diff --git a/src/go-flex.skl b/src/go-flex.skl index 431cde8..e9ce1f1 100755 --- a/src/go-flex.skl +++ b/src/go-flex.skl @@ -194,9 +194,11 @@ m4_define([[YY_SC_TO_UI]], [[((YY_CHAR)($1))]]) /* Action number for EOF rule of a given start state. */ m4_define([[YY_STATE_EOF]], [[YY_END_OF_BUFFER + $1 + 1]]) -/* The state buf must be large enough to hold one state per character in the main buffer. +/* The state buf must be large enough to hold one state per character in the main buffer, + * plus the start state, plus the two end-of-buffer byte states. */ -m4_define([[YY_STATE_BUF_SIZE]], [[((flexInputBufferSize + 2) * sizeof(yyStateType))]]) +m4_define([[YY_STATE_BUF_EXTRA_SPACE]], [[3]]) +m4_define([[YY_STATE_BUF_SIZE]], [[(YY_BUF_SIZE + YY_STATE_BUF_EXTRA_SPACE)]]) const bool FLEX_DEBUG = m4_ifdef([[M4_MODE_DEBUG]], [[true]], [[false]]); @@ -1444,6 +1446,8 @@ bool yyatbol(FlexLexer *yyscanner) { */ void yy_switch_to_buffer(yybuffer newBuffer, FlexLexer *yyscanner) { + size_t newSize = 0; + yyStateType *newStateBuf = 0; /* TODO. We should be able to replace this entire function body * with @@ -1463,6 +1467,23 @@ void yy_switch_to_buffer(yybuffer newBuffer, FlexLexer *yyscanner) yyscanner->yyBufferStack[yyscanner->yyBufferStackTop] = newBuffer; yy_load_buffer_state(yyscanner); + +m4_ifdef( [[M4_MODE_USES_REJECT]], [[ + /* Ensure the reject state buffer is large enough. + */ + if ( yyscanner->yyStateBufMax < (yy_current_buffer(yyscanner)->yyInputBufSize + YY_STATE_BUF_EXTRA_SPACE) ) { + newSize = yy_current_buffer(yyscanner)->yyInputBufSize + YY_STATE_BUF_EXTRA_SPACE; + newStateBuf = (yyStateType *)yyrealloc( yyscanner->yyStateBuf, (new_size * sizeof(yyStateType)), yyscanner ); + + if ( newStateBuf == NULL ) { + YY_FATAL_ERROR( "out of dynamic memory in yylex()" ); + } + else { + yyscanner->yyStateBuf = newStateBuf; + yyscanner->yyStateBufMax = newSize; + } + } +]] ) /* We don't actually know whether we did this switch during * EOF (yywrap()) processing, but the only time this flag @@ -1861,14 +1882,14 @@ m4_ifdef([[M4_MODE_USES_REJECT]], } } - if ( ! yyscanner->yyStateBuf ) { + if ( yyscanner->yyStateBuf == NULL ) { yypanic( "out of dynamic memory in yylex()", yyscanner ); } else if ( yy_current_buffer(yyscanner) == NULL ) { yyscanner->yyStateBufMax = YY_STATE_BUF_SIZE; } - if (yyscanner->yyStateBuf == NULL) { - yypanic("out of dynamic memory in yylex()", yyscanner); + else { + yyscanner->yyStateBufMax = (yy_current_buffer(yyscanner)->yyInputBufSize + YY_STATE_BUF_EXTRA_SPACE); } ]]) -- cgit v1.2.1 From 1d128474a79a1e781a67076c751ab773c11c65e2 Mon Sep 17 00:00:00 2001 From: Joseph Langley Date: Wed, 9 Jun 2021 20:45:19 -0400 Subject: fix(skel): Use yypanic in place of YY_FATAL_ERROR. yypanic() is new in the c99 and go skeletons. --- src/c99-flex.skl | 2 +- src/go-flex.skl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/c99-flex.skl b/src/c99-flex.skl index b01913d..9f48e3c 100755 --- a/src/c99-flex.skl +++ b/src/c99-flex.skl @@ -1626,7 +1626,7 @@ m4_ifdef( [[M4_MODE_USES_REJECT]], [[ new_state_buf = (yy_state_type *)yyrealloc( yyscanner->yy_state_buf, (new_size * sizeof(yy_state_type)), yyscanner ); if ( new_state_buf == NULL ) { - YY_FATAL_ERROR( "out of dynamic memory in yylex()" ); + yypanic( "out of dynamic memory in yylex()", yyscanner ); } else { yyscanner->yy_state_buf = new_state_buf; diff --git a/src/go-flex.skl b/src/go-flex.skl index e9ce1f1..9eb42d4 100755 --- a/src/go-flex.skl +++ b/src/go-flex.skl @@ -1476,7 +1476,7 @@ m4_ifdef( [[M4_MODE_USES_REJECT]], [[ newStateBuf = (yyStateType *)yyrealloc( yyscanner->yyStateBuf, (new_size * sizeof(yyStateType)), yyscanner ); if ( newStateBuf == NULL ) { - YY_FATAL_ERROR( "out of dynamic memory in yylex()" ); + yypanic( "out of dynamic memory in yylex()", yyscanner ); } else { yyscanner->yyStateBuf = newStateBuf; -- cgit v1.2.1 From 50819bb82a87e4e310690ce5a90ab798c50e2093 Mon Sep 17 00:00:00 2001 From: Joseph Langley Date: Wed, 9 Jun 2021 21:13:14 -0400 Subject: fix(skel): Fix porting errors from the cpp to the c99 and go skeletons. --- src/c99-flex.skl | 1 + src/go-flex.skl | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/c99-flex.skl b/src/c99-flex.skl index 9f48e3c..18af12b 100755 --- a/src/c99-flex.skl +++ b/src/c99-flex.skl @@ -2415,6 +2415,7 @@ m4_ifdef( [[M4_MODE_USES_REJECT]], [[ yyscanner->yy_state_buf = 0; yyscanner->yy_state_ptr = 0; + yyscanner->yy_state_buf_max = 0; yyscanner->yy_full_match = 0; yyscanner->yy_lp = 0; ]]) diff --git a/src/go-flex.skl b/src/go-flex.skl index 9eb42d4..f05bb51 100755 --- a/src/go-flex.skl +++ b/src/go-flex.skl @@ -198,7 +198,7 @@ m4_define([[YY_STATE_EOF]], [[YY_END_OF_BUFFER + $1 + 1]]) * plus the start state, plus the two end-of-buffer byte states. */ m4_define([[YY_STATE_BUF_EXTRA_SPACE]], [[3]]) -m4_define([[YY_STATE_BUF_SIZE]], [[(YY_BUF_SIZE + YY_STATE_BUF_EXTRA_SPACE)]]) +m4_define([[YY_STATE_BUF_SIZE]], [[(flexInputBufferSize + YY_STATE_BUF_EXTRA_SPACE)]]) const bool FLEX_DEBUG = m4_ifdef([[M4_MODE_DEBUG]], [[true]], [[false]]); @@ -449,6 +449,7 @@ m4_ifdef([[M4_MODE_YYTEXT_IS_ARRAY]], [[ m4_ifdef([[M4_MODE_USES_REJECT]], [[ yyStateType *yyStateBuf; yyStateType *yyStatePtr; + size_t yyStateBufMax; char *yyFullMatch; int yyLp; m4_ifdef([[M4_MODE_VARIABLE_TRAILING_CONTEXT_RULES]], [[m4_dnl @@ -1473,7 +1474,7 @@ m4_ifdef( [[M4_MODE_USES_REJECT]], [[ */ if ( yyscanner->yyStateBufMax < (yy_current_buffer(yyscanner)->yyInputBufSize + YY_STATE_BUF_EXTRA_SPACE) ) { newSize = yy_current_buffer(yyscanner)->yyInputBufSize + YY_STATE_BUF_EXTRA_SPACE; - newStateBuf = (yyStateType *)yyrealloc( yyscanner->yyStateBuf, (new_size * sizeof(yyStateType)), yyscanner ); + newStateBuf = (yyStateType *)yyrealloc( yyscanner->yyStateBuf, (newSize * sizeof(yyStateType)), yyscanner ); if ( newStateBuf == NULL ) { yypanic( "out of dynamic memory in yylex()", yyscanner ); @@ -2264,6 +2265,7 @@ m4_ifdef([[M4_MODE_USES_REJECT]], [[ yyscanner->yyStateBuf = NULL; yyscanner->yyStatePtr = NULL; + yyscanner->yyStateBufMax = 0; yyscanner->yyFullMatch = NULL; yyscanner->yyLp = 0; ]]) -- cgit v1.2.1 From ab783a0b996a91cae2522a7b37edecf142e88f34 Mon Sep 17 00:00:00 2001 From: Joseph Langley Date: Wed, 9 Jun 2021 21:36:20 -0400 Subject: chore(make): Ensure distcheck cleans up directories. --- src/Makefile.am | 1 + tests/Makefile.am | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index fada4a7..ce21a4d 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -130,6 +130,7 @@ endif dist-hook: scan.l flex$(EXEEXT) chmod u+w $(distdir) && \ + chmod u+w $(distdir)/scan.c && \ ./flex$(EXEEXT) -o scan.c $< && \ mv -f scan.c $(distdir) diff --git a/tests/Makefile.am b/tests/Makefile.am index 6ab4844..e9f3f49 100755 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -228,9 +228,9 @@ CLEANFILES = \ dist-hook: chmod u+w $(distdir) && \ for file in $(CLEANFILES) ; do \ - rm -f $(distdir)/$$file \ - chmod u+w $(distdir)/$$file \ - ; done + chmod u+w $(distdir)/$$file 2>/dev/null; \ + rm -f $(distdir)/$$file; \ + done EXTRA_DIST = \ README \ -- cgit v1.2.1 From 8100144eb4e2b5e82476a958ab9690ebccb54ee6 Mon Sep 17 00:00:00 2001 From: Joseph Langley Date: Fri, 2 Jul 2021 17:53:29 -0400 Subject: chore(perm): Fix permissions 755->644 --- src/Makefile.am | 0 src/c99-flex.skl | 0 src/cpp-flex.skl | 0 src/gen.c | 0 src/go-flex.skl | 0 tests/Makefile.am | 0 tests/ruleset.am | 0 7 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 src/Makefile.am mode change 100755 => 100644 src/c99-flex.skl mode change 100755 => 100644 src/cpp-flex.skl mode change 100755 => 100644 src/gen.c mode change 100755 => 100644 src/go-flex.skl mode change 100755 => 100644 tests/Makefile.am mode change 100755 => 100644 tests/ruleset.am diff --git a/src/Makefile.am b/src/Makefile.am old mode 100755 new mode 100644 diff --git a/src/c99-flex.skl b/src/c99-flex.skl old mode 100755 new mode 100644 diff --git a/src/cpp-flex.skl b/src/cpp-flex.skl old mode 100755 new mode 100644 diff --git a/src/gen.c b/src/gen.c old mode 100755 new mode 100644 diff --git a/src/go-flex.skl b/src/go-flex.skl old mode 100755 new mode 100644 diff --git a/tests/Makefile.am b/tests/Makefile.am old mode 100755 new mode 100644 diff --git a/tests/ruleset.am b/tests/ruleset.am old mode 100755 new mode 100644 -- cgit v1.2.1 From 79da5dfa0fb837f97b1f1d71377a83efa90c8081 Mon Sep 17 00:00:00 2001 From: Joseph Langley Date: Fri, 2 Jul 2021 18:46:52 -0400 Subject: fix(skel): Init state buffer correctly State buffer initialization in cpp-flex.skl got lost during a rebase. Added it back in from c99-flex.skl and made improvements suggested by smcpeak. --- src/c99-flex.skl | 37 ++++++++++++++----------------------- src/cpp-flex.skl | 23 ++++++++++++----------- src/go-flex.skl | 37 ++++++++++++++----------------------- 3 files changed, 40 insertions(+), 57 deletions(-) diff --git a/src/c99-flex.skl b/src/c99-flex.skl index 18af12b..317c038 100644 --- a/src/c99-flex.skl +++ b/src/c99-flex.skl @@ -2020,29 +2020,6 @@ m4_ifdef( [[]], m4_ifdef([[YY_USER_INIT]], [[YY_USER_INIT]]) -m4_ifdef( [[M4_MODE_USES_REJECT]], -[[ - /* Create the reject buffer large enough to save one state per allowed character. */ - if ( yyscanner->yy_state_buf == NULL ) { - if( yy_current_buffer(yyscanner) == NULL ) { - yyscanner->yy_state_buf = (yy_state_type *)yyalloc((YY_STATE_BUF_SIZE * sizeof(yy_state_type)), yyscanner); - } - else { - yyscanner->yy_state_buf = (yy_state_type *)yyalloc(((yy_current_buffer(yyscanner)->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE) * sizeof(yy_state_type)), yyscanner); - } - } - - if ( yyscanner->yy_state_buf == NULL ) { - yypanic( "out of dynamic memory in yylex()", yyscanner ); - } - else if ( yy_current_buffer(yyscanner) == NULL ) { - yyscanner->yy_state_buf_max = YY_STATE_BUF_SIZE; - } - else { - yyscanner->yy_state_buf_max = (yy_current_buffer(yyscanner)->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE); - } -]]) - if ( yyscanner->yy_start == 0 ) { yyscanner->yy_start = 1; /* first start state */ } @@ -2057,6 +2034,20 @@ m4_ifdef( [[M4_MODE_USES_REJECT]], yyscanner->yy_buffer_stack[yyscanner->yy_buffer_stack_top] = yy_create_buffer( yyscanner->yyin_r, YY_BUF_SIZE, yyscanner); } + +m4_ifdef( [[M4_MODE_USES_REJECT]], +[[ + /* Create the reject buffer large enough to save one state per allowed character. + * If the reject buffer already exists, keep using it. + */ + if ( yyscanner->yy_state_buf == NULL ) { + yyscanner->yy_state_buf = (yy_state_type *)yyalloc(((yy_current_buffer(yyscanner)->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE) * sizeof(yy_state_type)), yyscanner); + if ( yyscanner->yy_state_buf == NULL ) { + yypanic( "out of dynamic memory in yylex()", yyscanner ); + } + yyscanner->yy_state_buf_max = (yy_current_buffer(yyscanner)->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE); + } +]]) yy_load_buffer_state( yyscanner ); } diff --git a/src/cpp-flex.skl b/src/cpp-flex.skl index fe9344f..10738ba 100644 --- a/src/cpp-flex.skl +++ b/src/cpp-flex.skl @@ -1880,17 +1880,6 @@ m4_ifdef( [[]], YY_USER_INIT; #endif -m4_ifdef( [[M4_MODE_USES_REJECT]], -[[ - /* Create the reject buffer large enough to save one state per allowed character. */ - if ( ! YY_G(yy_state_buf) ) { - YY_G(yy_state_buf) = (yy_state_type *)yyalloc(YY_STATE_BUF_SIZE M4_YY_CALL_LAST_ARG); - } - if ( ! YY_G(yy_state_buf) ) { - YY_FATAL_ERROR( "out of dynamic memory in yylex()" ); - } -]]) - if ( ! YY_G(yy_start) ) { YY_G(yy_start) = 1; /* first start state */ } @@ -1915,6 +1904,18 @@ m4_ifdef([[M4_MODE_CXX_ONLY]], [[ YY_CURRENT_BUFFER_LVALUE = yy_create_buffer( yyin, YY_BUF_SIZE M4_YY_CALL_LAST_ARG); } + +m4_ifdef( [[M4_MODE_USES_REJECT]], +[[ + /* Create the reject buffer large enough to save one state per allowed character. */ + if ( ! YY_G(yy_state_buf) ) { + YY_G(yy_state_buf) = (yy_state_type *)yyalloc( ((YY_CURRENT_BUFFER_LVALUE->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE) * sizeof(yy_state_type)) M4_YY_CALL_LAST_ARG); + if ( ! YY_G(yy_state_buf) ) { + YY_FATAL_ERROR( "out of dynamic memory in yylex()" ); + } + YY_G(yy_state_buf_max) = (YY_CURRENT_BUFFER_LVALUE->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE); + } +]]) yy_load_buffer_state( M4_YY_CALL_ONLY_ARG ); } diff --git a/src/go-flex.skl b/src/go-flex.skl index f05bb51..53f0b90 100644 --- a/src/go-flex.skl +++ b/src/go-flex.skl @@ -1871,29 +1871,6 @@ m4_ifdef([[]], m4_ifdef([[YY_USER_INIT]], [[YY_USER_INIT]]) -m4_ifdef([[M4_MODE_USES_REJECT]], -[[ - /* Create the reject buffer large enough to save one state per allowed character. */ - if (yyscanner->yyStateBuf == NULL) { - if (yy_current_buffer(yyscanner) == NULL) { - yyscanner->yyStateBuf = (yyStateType *)yyalloc((YY_STATE_BUF_SIZE * sizeof(yyStateType)), yyscanner); - } - else { - yyscanner->yyStateBuf = (yyStateType *)yyalloc((yy_current_buffer(yyscanner)->yyInputBufSize + YY_STATE_BUF_EXTRA_SPACE) * sizeof(yyStateType), yyscanner); - } - } - - if ( yyscanner->yyStateBuf == NULL ) { - yypanic( "out of dynamic memory in yylex()", yyscanner ); - } - else if ( yy_current_buffer(yyscanner) == NULL ) { - yyscanner->yyStateBufMax = YY_STATE_BUF_SIZE; - } - else { - yyscanner->yyStateBufMax = (yy_current_buffer(yyscanner)->yyInputBufSize + YY_STATE_BUF_EXTRA_SPACE); - } -]]) - if (yyscanner->yyStart == 0) { yyscanner->yyStart = 1; /* first start state */ } @@ -1909,6 +1886,20 @@ m4_ifdef([[M4_MODE_USES_REJECT]], yy_create_buffer(yyscanner->yyin, flexInputBufferSize, yyscanner); } +m4_ifdef([[M4_MODE_USES_REJECT]], +[[ + /* Create the reject buffer large enough to save one state per allowed character. + * If the reject buffer already exists, keep using it. + */ + if (yyscanner->yyStateBuf == NULL) { + yyscanner->yyStateBuf = (yyStateType *)yyalloc((yy_current_buffer(yyscanner)->yyInputBufSize + YY_STATE_BUF_EXTRA_SPACE) * sizeof(yyStateType), yyscanner); + if ( yyscanner->yyStateBuf == NULL ) { + yypanic( "out of dynamic memory in yylex()", yyscanner ); + } + yyscanner->yyStateBufMax = (yy_current_buffer(yyscanner)->yyInputBufSize + YY_STATE_BUF_EXTRA_SPACE); + } +]]) + yy_load_buffer_state(yyscanner); } -- cgit v1.2.1 From 5cccc82072c425bd5c7c2a0082225c7f01e4f67e Mon Sep 17 00:00:00 2001 From: Joseph Langley Date: Fri, 2 Jul 2021 18:49:38 -0400 Subject: chore(skel): Improve comment at state buffer init. --- src/cpp-flex.skl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) mode change 100644 => 100755 src/cpp-flex.skl diff --git a/src/cpp-flex.skl b/src/cpp-flex.skl old mode 100644 new mode 100755 index 10738ba..25e4327 --- a/src/cpp-flex.skl +++ b/src/cpp-flex.skl @@ -1907,7 +1907,9 @@ m4_ifdef([[M4_MODE_CXX_ONLY]], [[ m4_ifdef( [[M4_MODE_USES_REJECT]], [[ - /* Create the reject buffer large enough to save one state per allowed character. */ + /* Create the reject buffer large enough to save one state per allowed character. + * If the reject buffer already exists, keep using it. + */ if ( ! YY_G(yy_state_buf) ) { YY_G(yy_state_buf) = (yy_state_type *)yyalloc( ((YY_CURRENT_BUFFER_LVALUE->yy_buf_size + YY_STATE_BUF_EXTRA_SPACE) * sizeof(yy_state_type)) M4_YY_CALL_LAST_ARG); if ( ! YY_G(yy_state_buf) ) { -- cgit v1.2.1 From fceec5d95a206745bf100c16f34f68201a9a03fd Mon Sep 17 00:00:00 2001 From: Joseph Langley Date: Fri, 2 Jul 2021 18:52:14 -0400 Subject: chore(skel): Fix permissions on cpp-flex.skl --- src/cpp-flex.skl | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 src/cpp-flex.skl diff --git a/src/cpp-flex.skl b/src/cpp-flex.skl old mode 100755 new mode 100644 -- cgit v1.2.1